From 15f21e21366415cc1acf77359d0cb6662b553a13 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow <7102778+kylehounslow@users.noreply.github.com> Date: Tue, 5 May 2026 12:00:21 -0700 Subject: [PATCH 1/7] feat(compat): add vendor compatibility translation edge (#224) Adds compat/ package and docker-compose.compat.yml overlay that accepts telemetry from Datadog, Jaeger (legacy wire protocol), and Splunk HEC agents and forwards OTLP to the base observability-stack collector. Includes customer-facing documentation under docs/starlight-docs and a Writing Tenets section in CONTRIBUTING.md. The overlay is opt-in and follows the existing INCLUDE_COMPOSE_* activation pattern. The base collector, its config, and all existing pipelines are unchanged. Activation: echo 'INCLUDE_COMPOSE_COMPAT=docker-compose.compat.yml' >> .env docker compose up -d Compat collector pipelines: - traces: [datadog, jaeger] - metrics: [datadog, statsd, splunk_hec] - logs: [datadog, splunk_hec] Modern OpenTelemetry SDK applications bypass the compat hop and send OTLP directly to the base collector on 4317/4318. Public documentation added under /docs/send-data/from-vendor/: - index.md (overview + architecture + decision table) - datadog.md, jaeger.md, splunk.md (per-vendor migration guides) - From Vendor Agents sidebar entry added via astro.config.mjs - Cross-link added from /docs/send-data/ overview Validated end-to-end with real vendor SDKs: - Datadog: dd-trace-py + FastAPI + patch_all() -> 42 spans in OpenSearch with service.name, service.version, deployment.environment.name preserved; instrumentationScope Datadog; parent/child chains intact - Splunk HEC: splunk_handler (Python logging) -> 5 log records in logs-otel-v1-* with com.splunk.source/sourcetype/index preserved - Jaeger OTLP: jaegertracing/example-hotrod demo -> 40 spans, 6-service topology, visible in APM and Discover Traces Starlight docs build validated (115 pages, all internal links resolve). SignalFx is not supported. The upstream signalfxreceiver is deprecated with explicit guidance to migrate to OTLP. Signed-off-by: Kyle Hounslow --- CONTRIBUTING.md | 48 ++++++++ compat/README.md | 102 +++++++++++++++ compat/collector/README.md | 48 ++++++++ compat/collector/config.compat.yaml | 78 ++++++++++++ compat/vendors/datadog/README.md | 21 ++++ compat/vendors/jaeger/README.md | 24 ++++ compat/vendors/splunk/README.md | 23 ++++ docker-compose.compat.yml | 60 +++++++++ docker-compose.yml | 1 + docs/starlight-docs/astro.config.mjs | 4 + .../docs/send-data/from-vendor/datadog.md | 116 ++++++++++++++++++ .../docs/send-data/from-vendor/index.md | 87 +++++++++++++ .../docs/send-data/from-vendor/jaeger.md | 79 ++++++++++++ .../docs/send-data/from-vendor/splunk.md | 99 +++++++++++++++ .../src/content/docs/send-data/index.md | 4 + 15 files changed, 794 insertions(+) create mode 100644 compat/README.md create mode 100644 compat/collector/README.md create mode 100644 compat/collector/config.compat.yaml create mode 100644 compat/vendors/datadog/README.md create mode 100644 compat/vendors/jaeger/README.md create mode 100644 compat/vendors/splunk/README.md create mode 100644 docker-compose.compat.yml create mode 100644 docs/starlight-docs/src/content/docs/send-data/from-vendor/datadog.md create mode 100644 docs/starlight-docs/src/content/docs/send-data/from-vendor/index.md create mode 100644 docs/starlight-docs/src/content/docs/send-data/from-vendor/jaeger.md create mode 100644 docs/starlight-docs/src/content/docs/send-data/from-vendor/splunk.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e67bb00..190676aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -398,6 +398,7 @@ AI coding assistants are welcome to contribute! When contributing as an AI agent - Note performance considerations - Reference relevant specifications - Keep comments up to date +- Avoid narrative prose and internal team voice (e.g., "we deliberately...") — comments are for future maintainers, not decision history ### Examples @@ -407,6 +408,53 @@ AI coding assistants are welcome to contribute! When contributing as an AI agent - Follow language-specific conventions - Test examples before committing +### Writing Tenets (for agents) + +Applies to all user-facing documentation: READMEs, public docs under `docs/`, migration guides, and PR descriptions. + +**Framing** + +- No us-vs-them. Write "Point your agents at observability-stack," not "at us." In open-source projects, the reader is part of the same community. +- Don't leak internal conversation into docs. Design-doc voice ("this is the canonical case where...", "we deliberately omitted...") belongs in PR discussion or design documents, not user-facing artifacts. +- Factual, not promotional. Avoid marketing phrases like "does ONE thing," "zero drift risk," or "honest limits." +- Acknowledge nuance via asides (`:::note` in Starlight docs) or italic notes, not prose digressions. + +**Maintenance hygiene** + +- Don't pin version strings. Link to `main` of upstream repos (e.g., `opentelemetry-collector-contrib/tree/main/receiver/...`), not to a specific tag. Version pins go stale. +- Don't duplicate source code in docs. Config YAML, pipeline definitions, and translation tables drift from the real source. Link to the source file instead. +- Don't maintain per-vendor translation tables beyond 3–5 canonical well-known fields. Defer to upstream receiver READMEs and vendor documentation. Positioning this repo as a schema authority creates permanent maintenance burden. +- Repo READMEs should link to public docs, not duplicate them. One source of truth per content type. + +**Accuracy** + +- Verify specific claims before writing them. Dates, version numbers, protocol behavior, UI terminology — check primary sources. +- If a claim cannot be verified from primary sources, phrase it more vaguely. "Modern versions support X" beats "as of v1.42, X is supported" when the version claim is unverified. +- Check existing conventions. Before using a UI name or terminology, grep the rest of the docs to see what other pages call it. +- Run the documentation build (`npm run build` in `docs/starlight-docs`) before committing doc changes. Verify internal links are valid. + +**Public-doc page structure** + +Pages for users migrating TO observability-stack should cover, in order: + +1. Action-oriented lead (one sentence — what the reader can do) +2. Decision table when multiple paths exist ("Do I need this?" / "Which path applies?") +3. Configuration — concrete environment variables, example config, code snippet per path +4. Verify step — one-command check that it's working +5. What lands in OpenSearch — concrete example of end state (field names, index patterns) +6. Caveats — real observed gotchas surfaced from validation, not theoretical ones +7. Not covered — honest scope boundaries +8. References — upstream sources, vendor docs + +**Repo READMEs** are for contributors, not migrators. Keep them short (20–40 lines for leaf READMEs; 100 max for overview). Link out to public docs for user-facing content. Include repo-local context only: config file paths, upstream receiver links, local dev workflow commands. + +**Caveats from real validation are more trustworthy than theoretical ones.** When end-to-end testing reveals a gotcha (e.g., an attribute gets overwritten, a field doesn't translate), document it in the caveats section. Lead with what the user will see, not why it happens. + +**Scope discipline** + +- Prune aggressively when in doubt. Deletion is cheaper than maintenance. +- Don't commit working files — audit tables, compatibility matrices, session notes, TODO lists, WIP drafts. If it's not useful to a future reader with no context, it's not a docs artifact. + ## Community ### Getting Help diff --git a/compat/README.md b/compat/README.md new file mode 100644 index 00000000..071a3ac5 --- /dev/null +++ b/compat/README.md @@ -0,0 +1,102 @@ +# Vendor Compatibility Overlay + +User-facing migration guides: https://observability.opensearch.org/docs/send-data/from-vendor/ + +This overlay adds a dedicated OpenTelemetry Collector that accepts Datadog, Jaeger, and Splunk HEC wire protocols, translates them to OTLP, and forwards to the base collector. No application code changes are required — vendor agents are pointed at observability-stack by changing an endpoint URL. + +## Do I need this overlay? + +| If your apps emit... | Need this overlay? | +|----------------------|--------------------| +| OpenTelemetry OTLP (gRPC or HTTP) | No. Send directly to the base collector on 4317 or 4318. | +| Datadog (dd-trace-*, DogStatsD) | Yes. | +| Jaeger native wire protocol (`jaeger-client-*`) | Yes. | +| Splunk HEC | Yes. | +| Jaeger via modern OpenTelemetry SDK + OTLP | No. | + +## Architecture + +``` +vendor agents ──▶ otel-collector-compat ──OTLP──▶ otel-collector (base, unchanged) ──▶ Data Prepper / Prometheus ──▶ OpenSearch + (this overlay) + +OTLP apps ─────────────────────────────────▶ (direct to base, no compat hop) +``` + +The compat collector uses upstream [`opentelemetry-collector-contrib`](https://github.com/open-telemetry/opentelemetry-collector-contrib) receivers. All enrichment and downstream routing happens in the base pipeline — the compat config is purely ingest + forward. + +## Activation + +```bash +echo "INCLUDE_COMPOSE_COMPAT=docker-compose.compat.yml" >> .env +docker compose up -d +``` + +Adds `otel-collector-compat` and the bundled Jaeger `hotrod` demo to the stack. + +### Verify it's running + +```bash +docker compose ps otel-collector-compat +curl -sI http://localhost:8126/info # HTTP 200 = Datadog receiver is live +``` + +## Supported vendors + +| Vendor | Receiver(s) | Default ports | Repo notes | +|--------|-------------|---------------|------------| +| Datadog | `datadogreceiver`, `statsdreceiver` | 8126/tcp, 8125/udp | [vendors/datadog/](vendors/datadog/) | +| Jaeger (legacy wire protocol) | `jaegerreceiver` | 14250/tcp, 14268/tcp | [vendors/jaeger/](vendors/jaeger/) | +| Splunk HEC | `splunkhecreceiver` | 8088/tcp | [vendors/splunk/](vendors/splunk/) | + +User-facing migration guides live at https://observability.opensearch.org/docs/send-data/from-vendor/. + +SignalFx is not supported. The upstream `signalfxreceiver` is deprecated with explicit guidance to migrate to OTLP. + +## Deployment modes + +Each vendor supports greenfield, side-by-side, and full-replacement modes. See the public migration guide for each vendor for specifics. + +## Port customization + +Ports are remappable via environment variables. Useful when a real vendor agent already occupies the default port on the host. + +| Variable | Default | Receiver | +|----------|---------|----------| +| `COMPAT_DATADOG_APM_PORT` | 8126 | Datadog trace-agent | +| `COMPAT_DATADOG_STATSD_PORT` | 8125 | DogStatsD | +| `COMPAT_JAEGER_GRPC_PORT` | 14250 | Jaeger gRPC | +| `COMPAT_JAEGER_THRIFT_HTTP_PORT` | 14268 | Jaeger Thrift HTTP | +| `COMPAT_SPLUNK_HEC_PORT` | 8088 | Splunk HEC | +| `COMPAT_COLLECTOR_MEMORY_LIMIT` | 256M | Compat collector memory limit | + +## Attribute translation + +Each receiver translates vendor-specific data to the OpenTelemetry data model. Translation behavior is defined by the upstream receivers. For schema details, consult: + +- The upstream receiver READMEs under [`opentelemetry-collector-contrib/receiver/`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver) +- The vendor's own instrumentation and tagging documentation + +## Directory layout + +``` +compat/ +├── README.md ← this file +├── collector/ +│ ├── config.compat.yaml ← compat collector config +│ └── README.md ← compat collector design notes +└── vendors/ + ├── datadog/README.md ← developer notes + link to migration guide + ├── jaeger/README.md + └── splunk/README.md + +docker-compose.compat.yml ← overlay service definitions +``` + +## Adding a vendor + +1. Create `vendors//README.md` with a link to the (forthcoming) public migration guide and developer notes (receiver used, config location, quick local test). +2. Add the receiver stanza to `collector/config.compat.yaml` and wire it into the appropriate pipeline(s). +3. Add port mappings to `docker-compose.compat.yml`. +4. Add a page at `docs/starlight-docs/src/content/docs/send-data/from-vendor/.md` with the user migration guide. +5. Verify end-to-end: send vendor-format data → confirm it lands in OpenSearch. diff --git a/compat/collector/README.md b/compat/collector/README.md new file mode 100644 index 00000000..b280c6bc --- /dev/null +++ b/compat/collector/README.md @@ -0,0 +1,48 @@ +# Compat Collector + +Developer notes for `otel-collector-compat` and [`config.compat.yaml`](./config.compat.yaml). + +For migration guides and usage, see the public docs: https://observability.opensearch.org/docs/send-data/from-vendor/ + +## Role + +Accepts vendor wire protocols on their native ports, translates to the OpenTelemetry data model via upstream [`opentelemetry-collector-contrib`](https://github.com/open-telemetry/opentelemetry-collector-contrib) receivers, and forwards OTLP to the base collector. All enrichment, filtering, and downstream routing (Data Prepper, Prometheus) happens in the base collector config. + +``` +vendor apps ──▶ otel-collector-compat ──OTLP──▶ otel-collector (base) + [config.compat.yaml] [unchanged] +``` + +The compat config contains only receivers, the `batch` processor, and an OTLP exporter pointed at the base collector. No transforms. + +## Receivers + +| Receiver | Upstream | +|----------|----------| +| `datadog` | [`datadogreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/datadogreceiver) | +| `statsd` | [`statsdreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/statsdreceiver) | +| `jaeger` | [`jaegerreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jaegerreceiver) | +| `splunk_hec` | [`splunkhecreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/splunkhecreceiver) | + +Pipeline wiring lives in [`config.compat.yaml`](./config.compat.yaml) under `service.pipelines`. + +## Local dev workflow + +Edit `config.compat.yaml`, then: + +```bash +docker compose restart otel-collector-compat +docker compose logs -f otel-collector-compat +``` + +The `debug` exporter is wired into every pipeline. To see what's flowing through, bump its verbosity to `detailed`: + +```yaml +exporters: + debug: + verbosity: detailed +``` + +## Resource limits + +Default memory limit: 256MB (`COMPAT_COLLECTOR_MEMORY_LIMIT`). Idle usage is typically under 100MB. diff --git a/compat/collector/config.compat.yaml b/compat/collector/config.compat.yaml new file mode 100644 index 00000000..73b4c18e --- /dev/null +++ b/compat/collector/config.compat.yaml @@ -0,0 +1,78 @@ +# config.compat.yaml +# +# Edge collector config. Translates vendor wire protocols to OTLP and forwards +# to the base collector, which handles enrichment and downstream export. +# +# vendor apps ──▶ otel-collector-compat ──OTLP──▶ otel-collector (base) +# (this config) (unchanged) + +receivers: + # Datadog trace-agent protocol. Also accepts metrics and logs endpoints; + # all three are wired into the service pipelines below. + # See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/datadogreceiver + datadog: + endpoint: 0.0.0.0:8126 + read_timeout: 60s + trace_id_cache_size: 100 + + # StatsD / DogStatsD over UDP. + # See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/statsdreceiver + statsd: + endpoint: 0.0.0.0:8125 + aggregation_interval: 60s + enable_metric_type: true + is_monotonic_counter: false + + # Jaeger native wire protocol (Thrift HTTP + gRPC). For legacy + # jaeger-client-* applications. Modern Jaeger apps emit OTLP and send + # directly to the base collector on 4317/4318 without this hop. + # See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jaegerreceiver + jaeger: + protocols: + grpc: + endpoint: 0.0.0.0:14250 + thrift_http: + endpoint: 0.0.0.0:14268 + + # Splunk HTTP Event Collector (logs and metrics). + # See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/splunkhecreceiver + splunk_hec: + endpoint: 0.0.0.0:8088 + +processors: + # Batch spans/events to reduce RPC volume against the base collector. + batch: + timeout: 10s + send_batch_size: 1024 + +exporters: + # Forward to the base collector. Downstream routing (Data Prepper, + # Prometheus, etc.) is configured there. + otlp_grpc: + endpoint: otel-collector:4317 + tls: + insecure: true + + debug: + verbosity: basic + +service: + pipelines: + traces: + receivers: [datadog, jaeger] + processors: [batch] + exporters: [otlp_grpc, debug] + + metrics: + receivers: [datadog, statsd, splunk_hec] + processors: [batch] + exporters: [otlp_grpc, debug] + + logs: + receivers: [datadog, splunk_hec] + processors: [batch] + exporters: [otlp_grpc, debug] + + telemetry: + logs: + level: info diff --git a/compat/vendors/datadog/README.md b/compat/vendors/datadog/README.md new file mode 100644 index 00000000..4338da07 --- /dev/null +++ b/compat/vendors/datadog/README.md @@ -0,0 +1,21 @@ +# Datadog + +Migration guide and user-facing documentation: https://observability.opensearch.org/docs/send-data/from-vendor/datadog/ + +## Receivers used + +- [`datadogreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/datadogreceiver) — traces, metrics, logs (8126/tcp) +- [`statsdreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/statsdreceiver) — DogStatsD (8125/udp) + +Config: [`../../collector/config.compat.yaml`](../../collector/config.compat.yaml) — `datadog:` and `statsd:` receiver blocks. + +## Quick local test + +```bash +# DogStatsD metric +echo "test.metric:1|c|#env:dev" | nc -u -w1 localhost 8125 + +# HEC-style trace payload (msgpack) — see upstream receiver README for format details +``` + +Traces are best exercised by pointing a `dd-trace-*` SDK application at `localhost:8126`. diff --git a/compat/vendors/jaeger/README.md b/compat/vendors/jaeger/README.md new file mode 100644 index 00000000..4ed4381f --- /dev/null +++ b/compat/vendors/jaeger/README.md @@ -0,0 +1,24 @@ +# Jaeger + +Migration guide and user-facing documentation: https://observability.opensearch.org/docs/send-data/from-vendor/jaeger/ + +Jaeger is an open-source CNCF project rather than a proprietary vendor. This directory exists because users migrating from Jaeger deployments typically look for it alongside other vendor integrations. + +## Receiver used + +- [`jaegerreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jaegerreceiver) — Jaeger native wire protocol (Thrift HTTP on 14268/tcp, gRPC on 14250/tcp) + +Config: [`../../collector/config.compat.yaml`](../../collector/config.compat.yaml) — `jaeger:` receiver block. + +Modern Jaeger apps emit OTLP natively and bypass this overlay. The bundled `hotrod` demo (port 8080) is configured this way and exercises the base collector directly. + +## Quick local test + +```bash +# Trigger the bundled hotrod demo (OTLP path) +curl http://localhost:8080/dispatch?customer=123 + +# View traces at http://localhost:5601 → Trace Analytics +``` + +The legacy Thrift HTTP path is harder to exercise without a `jaeger-client-*` app. See the upstream receiver README for wire format details. diff --git a/compat/vendors/splunk/README.md b/compat/vendors/splunk/README.md new file mode 100644 index 00000000..2fcab066 --- /dev/null +++ b/compat/vendors/splunk/README.md @@ -0,0 +1,23 @@ +# Splunk HEC + +Migration guide and user-facing documentation: https://observability.opensearch.org/docs/send-data/from-vendor/splunk/ + +## Receiver used + +- [`splunkhecreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/splunkhecreceiver) — Splunk HTTP Event Collector (8088/tcp) + +Config: [`../../collector/config.compat.yaml`](../../collector/config.compat.yaml) — `splunk_hec:` receiver block. + +Wired into the logs and metrics pipelines. Traces are not wired in the default compat config. + +## Quick local test + +```bash +curl -X POST http://localhost:8088/services/collector \ + -H "Authorization: Splunk any-token" \ + -H "Content-Type: application/json" \ + -d '{"event":"hello","sourcetype":"manual","source":"curl"}' +# {"text": "Success", "code": 0} +``` + +Events land in OpenSearch under `logs-otel-v1-*`. View via Discover Logs in OpenSearch Dashboards. diff --git a/docker-compose.compat.yml b/docker-compose.compat.yml new file mode 100644 index 00000000..2c711522 --- /dev/null +++ b/docker-compose.compat.yml @@ -0,0 +1,60 @@ +# docker-compose.compat.yml +# +# Vendor compatibility overlay. Adds an edge OTel Collector that accepts +# Datadog, Jaeger, and Splunk HEC wire protocols and forwards OTLP to the +# base collector. +# +# Activation: +# echo "INCLUDE_COMPOSE_COMPAT=docker-compose.compat.yml" >> .env +# docker compose up -d +# +# vendor apps ──▶ otel-collector-compat ──OTLP──▶ otel-collector (base) +# (this overlay) (unchanged) + +services: + otel-collector-compat: + image: otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} + container_name: otel-collector-compat + pull_policy: always + command: ["--config=/etc/otelcol-config.yml"] + volumes: + - ./compat/collector/config.compat.yaml:/etc/otelcol-config.yml + ports: + # Datadog trace-agent (datadogreceiver) + - "${COMPAT_DATADOG_APM_PORT:-8126}:8126" + # DogStatsD (statsdreceiver, UDP) + - "${COMPAT_DATADOG_STATSD_PORT:-8125}:8125/udp" + # Jaeger native wire protocol. Modern Jaeger apps emit OTLP and should + # send directly to the base collector on 4317/4318 instead of this hop. + - "${COMPAT_JAEGER_GRPC_PORT:-14250}:14250" + - "${COMPAT_JAEGER_THRIFT_HTTP_PORT:-14268}:14268" + # Splunk HEC + - "${COMPAT_SPLUNK_HEC_PORT:-8088}:8088" + networks: + - observability-stack-network + restart: unless-stopped + depends_on: + - otel-collector + deploy: + resources: + limits: + memory: ${COMPAT_COLLECTOR_MEMORY_LIMIT:-256M} + + # Jaeger hotrod demo app. Emits OTLP directly to the base collector, so it + # does not exercise the compat collector. Included as a built-in trace + # generator for verifying the base pipeline end-to-end. + # + # http://localhost:8080 hotrod UI + # http://localhost:5601 OpenSearch Dashboards → Trace Analytics + hotrod: + image: jaegertracing/example-hotrod:1.60 + container_name: hotrod + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + command: ["all"] + networks: + - observability-stack-network + ports: + - "8080:8080" + depends_on: + - otel-collector diff --git a/docker-compose.yml b/docker-compose.yml index 5b347578..98c4d685 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ include: - path: ${INCLUDE_COMPOSE_AGENT_EVAL_LLM:-docker-compose/util/docker-compose.empty.yml} - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH:-docker-compose/util/docker-compose.empty.yml} - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS:-docker-compose/util/docker-compose.empty.yml} + - path: ${INCLUDE_COMPOSE_COMPAT:-docker-compose/util/docker-compose.empty.yml} x-default-logging: &logging driver: "json-file" diff --git a/docs/starlight-docs/astro.config.mjs b/docs/starlight-docs/astro.config.mjs index 65dfc3fc..2d8d950e 100644 --- a/docs/starlight-docs/astro.config.mjs +++ b/docs/starlight-docs/astro.config.mjs @@ -107,6 +107,10 @@ export default defineConfig({ label: 'Infrastructure', autogenerate: { directory: 'send-data/infrastructure' }, }, + { + label: 'From Vendor Agents', + autogenerate: { directory: 'send-data/from-vendor' }, + }, { label: 'Data Pipeline', autogenerate: { directory: 'send-data/data-pipeline' }, diff --git a/docs/starlight-docs/src/content/docs/send-data/from-vendor/datadog.md b/docs/starlight-docs/src/content/docs/send-data/from-vendor/datadog.md new file mode 100644 index 00000000..94e97478 --- /dev/null +++ b/docs/starlight-docs/src/content/docs/send-data/from-vendor/datadog.md @@ -0,0 +1,116 @@ +--- +title: "Datadog" +description: "Accept Datadog agent and SDK telemetry in observability-stack" +--- + +Point Datadog agents, `dd-trace-*` SDKs, and DogStatsD clients at observability-stack by changing an endpoint URL or environment variable. No application code changes required. + +## Which path applies to your setup? + +| If you run... | Use | +|---------------|-----| +| Datadog Agent on your hosts, forwarding app traces | Reconfigure the agent (or route traffic to the compat collector on 8126). | +| Apps instrumented with a `dd-trace-*` SDK sending directly | Set `DD_AGENT_HOST` and related env vars on each app. | +| DogStatsD clients (`statsd`/`dogstatsd` libraries) | Point them at port 8125/udp. | + +All three paths require the compat overlay to be enabled. See the [overview](/docs/send-data/from-vendor/). + +## Protocol support + +| Signal | Default port | Upstream stability | +|--------|--------------|--------------------| +| Traces | 8126/tcp | Alpha | +| Metrics | 8126/tcp | Development | +| Logs | 8126/tcp | Development | +| DogStatsD metrics | 8125/udp | Beta | + +**Stability note:** "Development" is the least mature OpenTelemetry Collector stability tier. APIs and behavior may change without notice. Evaluate carefully before routing production metric or log traffic through the Datadog receiver. Traces are "Alpha" — more mature but still pre-Beta. + +For current per-endpoint behavior, consult the upstream [`datadogreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/datadogreceiver). + +## Configuration + +### `dd-trace-*` SDKs + +Datadog maintains tracer libraries for Python (`dd-trace-py`), Java (`dd-trace-java`), Go (`dd-trace-go`), Ruby (`dd-trace-rb`), JavaScript/Node (`dd-trace-js`), .NET (`dd-trace-dotnet`), and PHP (`dd-trace-php`). All support endpoint override via environment variables, though variable names may differ slightly by library. + +Common environment variables: + +| Variable | Default | Purpose | +|----------|---------|---------| +| `DD_AGENT_HOST` | `localhost` | Agent hostname | +| `DD_TRACE_AGENT_PORT` | `8126` | Agent trace port | +| `DD_TRACE_AGENT_URL` | — | Full URL override (takes precedence) | +| `DD_SERVICE` | — | Service name (maps to OTel `service.name`) | +| `DD_ENV` | — | Environment (maps to OTel `deployment.environment`) | +| `DD_VERSION` | — | Version (maps to OTel `service.version`) | + +```bash +export DD_AGENT_HOST= +export DD_TRACE_AGENT_PORT=8126 +export DD_SERVICE=my-service +export DD_ENV=prod +export DD_VERSION=1.2.3 +``` + +Consult each library's documentation for language-specific configuration. + +### Datadog Agent + +If you run a Datadog Agent on your hosts, update its config to forward to observability-stack's compat collector, or change your app's `DD_AGENT_HOST` to bypass the agent entirely. + +### DogStatsD clients + +DogStatsD clients send UDP to port 8125. Most DogStatsD libraries accept `STATSD_HOST` / `STATSD_PORT` env vars or constructor arguments: + +```bash +export STATSD_HOST= +export STATSD_PORT=8125 +``` + +observability-stack uses the upstream [`statsdreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/statsdreceiver), which supports DogStatsD tag extensions (`#tag1:value1,tag2:value2`). + +## Canonical attribute mapping + +These are the three Datadog fields with well-known, universally-used OTel equivalents — the [Datadog unified service tagging](https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/) fields. + +| Datadog | OTel | +|---------|------| +| `service` | `service.name` | +| `env` | `deployment.environment` | +| `version` | `service.version` | + +The receiver also performs internal translations for HTTP, database, gRPC, and AWS SDK spans that don't require user action. For all other attributes, consult the upstream [`datadogreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/datadogreceiver) and Datadog's [tagging documentation](https://docs.datadoghq.com/getting_started/tagging/). + +## Deployment modes + +1. **Greenfield** — observability-stack binds port 8126; no Datadog Agent on the host. +2. **Side-by-side** — Datadog Agent on 8126, compat collector on a remapped port. Set `COMPAT_DATADOG_APM_PORT=8127` and configure a subset of apps with `DD_TRACE_AGENT_PORT=8127`. Useful for A/B validation during migration. +3. **Full replacement** — Datadog Agent decommissioned, observability-stack on 8126. + +## Verify + +Send a DogStatsD metric and confirm the compat collector accepts it: + +```bash +echo "test.metric:1|c|#env:test" | nc -u -w1 localhost 8125 +``` + +For traces, point a `dd-trace-*` application at observability-stack and dispatch a request. Traces appear under **APM** or **Discover Traces** in OpenSearch Dashboards within seconds. + +## Caveats + +- **128-bit trace IDs** are feature-gated upstream (`receiver.datadogreceiver.Enable128BitTraceID`, off by default). Traces originating in an OpenTelemetry-instrumented service and passing through a Datadog-instrumented service may not correlate without this flag. +- **Metric temporality:** `dd-trace-*` emits delta metrics. Some backends expect cumulative. Add a [`deltatocumulativeprocessor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) to the pipeline if needed. + +## Not covered + +- Live processes, profiling, network monitoring — no open-source OpenTelemetry receivers exist for these Datadog products. +- Synthetic monitoring — not a telemetry-ingest concern. +- Datadog UI features (notebooks, SLOs, monitors) — use OpenSearch Dashboards equivalents. + +## References + +- [`datadogreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/datadogreceiver) +- [`statsdreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/statsdreceiver) +- [Datadog unified service tagging](https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/) diff --git a/docs/starlight-docs/src/content/docs/send-data/from-vendor/index.md b/docs/starlight-docs/src/content/docs/send-data/from-vendor/index.md new file mode 100644 index 00000000..0f7aba3e --- /dev/null +++ b/docs/starlight-docs/src/content/docs/send-data/from-vendor/index.md @@ -0,0 +1,87 @@ +--- +title: "From Vendor Agents" +description: "Accept telemetry from Datadog, Jaeger, and Splunk agents without re-instrumenting your applications" +--- + +The vendor compatibility overlay adds a dedicated OpenTelemetry Collector that accepts Datadog, Jaeger, and Splunk HEC wire protocols, translates them to OTLP, and forwards to the base collector. Existing vendor agents can be reconfigured to send their telemetry to observability-stack with only an endpoint URL change — no application code changes required. + +## Do I need this overlay? + +| If your apps emit... | Need this overlay? | +|----------------------|--------------------| +| OpenTelemetry OTLP (gRPC or HTTP) | No. Send directly to the base collector on 4317 or 4318. | +| Jaeger via modern OpenTelemetry SDK + OTLP | No. | +| Datadog (`dd-trace-*` SDKs, DogStatsD) | Yes. | +| Jaeger native wire protocol (`jaeger-client-*`) | Yes. | +| Splunk HEC | Yes. | + +SignalFx is not supported. The upstream `signalfxreceiver` was deprecated with guidance to migrate to OTLP. + +## Architecture + +```mermaid +flowchart LR + A["Vendor agents
(Datadog, Jaeger legacy, Splunk HEC)"] -->|Vendor wire protocols| B["OTel Collector
(compat overlay)"] + G["OTLP apps
(incl. modern Jaeger)"] -->|OTLP| C + B -->|OTLP| C["OTel Collector
(base)"] + C --> D["Data Prepper"] + C --> E["Prometheus"] + D --> F["OpenSearch"] + E --> F +``` + +OTLP apps bypass the compat hop and send directly to the base collector. All enrichment and downstream routing happens in the base pipeline, as it does for OTLP-native traffic. + +## Enabling the overlay + +Activation uses the standard `INCLUDE_COMPOSE_*` pattern used by observability-stack overlays: + +```bash +echo "INCLUDE_COMPOSE_COMPAT=docker-compose.compat.yml" >> .env +docker compose up -d +``` + +This adds an `otel-collector-compat` service with receivers for each supported vendor, plus the bundled Jaeger `hotrod` demo for validation. + +### Verify + +```bash +docker compose ps otel-collector-compat +curl -sI http://localhost:8126/info # HTTP 200 = Datadog receiver is live +``` + +## Supported vendors + +| Vendor | Default ports | Migration guide | +|--------|---------------|-----------------| +| Datadog | 8126/tcp, 8125/udp | [Datadog](/docs/send-data/from-vendor/datadog/) | +| Jaeger (legacy wire protocol) | 14250/tcp, 14268/tcp | [Jaeger](/docs/send-data/from-vendor/jaeger/) | +| Splunk HEC | 8088/tcp | [Splunk HEC](/docs/send-data/from-vendor/splunk/) | + +## Port customization + +Default ports are remappable via environment variables. Useful when a real vendor agent already occupies the default port on the host. + +| Variable | Default | Maps to | +|----------|---------|---------| +| `COMPAT_DATADOG_APM_PORT` | 8126 | Datadog trace-agent | +| `COMPAT_DATADOG_STATSD_PORT` | 8125 | DogStatsD | +| `COMPAT_JAEGER_GRPC_PORT` | 14250 | Jaeger gRPC | +| `COMPAT_JAEGER_THRIFT_HTTP_PORT` | 14268 | Jaeger Thrift HTTP | +| `COMPAT_SPLUNK_HEC_PORT` | 8088 | Splunk HEC | +| `COMPAT_COLLECTOR_MEMORY_LIMIT` | 256M | Compat collector memory limit | + +## Attribute translation + +Each receiver translates vendor-specific data to the OpenTelemetry data model. Translation behavior is defined by the upstream receivers. See each per-vendor guide for canonical attribute mappings, and: + +- Upstream receiver READMEs under [`opentelemetry-collector-contrib`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver) +- Each vendor's own instrumentation and tagging documentation + +## Viewing ingested data + +Once data reaches OpenSearch: + +- **Traces** — [APM](/docs/apm/) for service maps and RED metrics, or [Discover Traces](/docs/investigate/discover-traces/) for trace-level exploration +- **Logs** — [Discover Logs](/docs/investigate/discover-logs/) (default index pattern: `logs-otel-v1-*`) +- **Metrics** — [Discover Metrics](/docs/investigate/discover-metrics/) (via Prometheus datasource) diff --git a/docs/starlight-docs/src/content/docs/send-data/from-vendor/jaeger.md b/docs/starlight-docs/src/content/docs/send-data/from-vendor/jaeger.md new file mode 100644 index 00000000..ecc23757 --- /dev/null +++ b/docs/starlight-docs/src/content/docs/send-data/from-vendor/jaeger.md @@ -0,0 +1,79 @@ +--- +title: "Jaeger" +description: "Migrate Jaeger-instrumented applications to observability-stack" +--- + +Send traces from Jaeger-instrumented applications to observability-stack by changing the OTLP endpoint (modern apps) or enabling the compat overlay (legacy `jaeger-client-*` apps). No application code changes required. + +:::note +Jaeger is an open-source CNCF project rather than a proprietary vendor. This page lives in the vendor section because users migrating from Jaeger deployments typically look here. +::: + +## Which path applies to your apps? + +| If your apps use... | Use | +|---------------------|-----| +| OpenTelemetry SDKs (`opentelemetry-*`) with an OTLP exporter | The **OTLP path** below. No compat overlay required. | +| Archived `jaeger-client-*` libraries (jaeger-client-go, jaeger-client-java, jaeger-client-python, etc.) | The **legacy wire protocol path** below. Enable the compat overlay. | + +## OTLP path + +Change the OTLP exporter endpoint in your apps to observability-stack's base collector: + +```bash +# gRPC +OTEL_EXPORTER_OTLP_ENDPOINT=http://:4317 + +# HTTP +OTEL_EXPORTER_OTLP_ENDPOINT=http://:4318 +``` + +OTLP traffic arrives at the base collector's `otlp` receiver, which is a pure ingest path — no attribute translation happens here. Your apps' telemetry lands in OpenSearch exactly as your OpenTelemetry SDK produces it. + +### Try it with the bundled hotrod demo + +With the compat overlay enabled (see the [overview](/docs/send-data/from-vendor/)), a pre-configured Jaeger hotrod demo is available on port 8080. It emits OTLP directly to the base collector — the same path your apps will take. + +```bash +curl http://localhost:8080/dispatch?customer=123 +``` + +This produces a multi-service trace (frontend → customer → driver → route → redis → mysql) visible in OpenSearch Dashboards under **Trace Analytics**. + +## Legacy wire protocol path + +Applications using `jaeger-client-*` libraries send Jaeger's native wire protocol. With the compat overlay enabled, observability-stack accepts these via the upstream [`jaegerreceiver`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jaegerreceiver), which translates to the OpenTelemetry data model using [`pkg/translator/jaeger`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/jaeger). + +| Protocol | Default port | Upstream stability | +|----------|--------------|--------------------| +| Thrift HTTP | 14268/tcp | Beta | +| gRPC (Jaeger protobuf) | 14250/tcp | Beta | + +### Configuration + +Example for `jaeger-client-go` using Thrift HTTP: + +```bash +JAEGER_ENDPOINT=http://:14268/api/traces +``` + +Other `jaeger-client-*` libraries expose similar endpoint overrides. Consult each library's documentation for specifics. + +The `jaeger-client-*` libraries are archived upstream and no longer receive patches. The long-term migration path is to OpenTelemetry SDKs with OTLP export. + +### Deployment modes + +1. **Greenfield** — compat collector binds Jaeger ports 14250 and 14268. +2. **Side-by-side** — Jaeger Collector and compat collector run simultaneously via port remapping (`COMPAT_JAEGER_THRIFT_HTTP_PORT`, `COMPAT_JAEGER_GRPC_PORT`). Useful for A/B validation during migration. +3. **Full replacement** — Jaeger Collector decommissioned, compat collector on 14250/14268. + +## Not covered + +- Jaeger query UI — use OpenSearch Dashboards' APM and Discover Traces views. +- Jaeger-specific storage backends (Cassandra, Elasticsearch, badger) — data writes to OpenSearch via Data Prepper. +- Jaeger's dependency graph — OpenSearch Dashboards has a service map. + +## References + +- [`jaegerreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jaegerreceiver) +- [Jaeger hotrod demo](https://github.com/jaegertracing/jaeger/tree/main/examples/hotrod) diff --git a/docs/starlight-docs/src/content/docs/send-data/from-vendor/splunk.md b/docs/starlight-docs/src/content/docs/send-data/from-vendor/splunk.md new file mode 100644 index 00000000..5afb8b59 --- /dev/null +++ b/docs/starlight-docs/src/content/docs/send-data/from-vendor/splunk.md @@ -0,0 +1,99 @@ +--- +title: "Splunk HEC" +description: "Route Splunk HTTP Event Collector traffic into OpenSearch" +--- + +Point Splunk HTTP Event Collector (HEC) clients at observability-stack by changing the collector URL. No application code changes required. + +## Which path applies to your setup? + +| If you use... | Use | +|---------------|-----| +| HEC API clients (curl, scripts, HEC-compatible libraries) | The **HEC endpoint** below. | +| Splunk Universal Forwarders configured with HEC output | The **HEC endpoint** below. | +| Splunk Universal Forwarders in default mode (S2S protocol) | **Not supported.** No open-source OpenTelemetry receiver exists for the Splunk forwarder-to-forwarder protocol. Reconfigure forwarders to use HEC output, or replace them with an OpenTelemetry-native log shipper. | + +All supported paths require the compat overlay to be enabled. See the [overview](/docs/send-data/from-vendor/). + +## Protocol support + +| Signal | Default port | Upstream stability | +|--------|--------------|--------------------| +| Logs (HEC JSON events) | 8088/tcp | Beta | +| Metrics (HEC metric events) | 8088/tcp | Beta | + +Traces are not wired in the default compat configuration. + +For current behavior, consult the upstream [`splunkhecreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/splunkhecreceiver). + +## Configuration + +Point existing HEC clients at observability-stack's compat collector: + +``` +http://:8088/services/collector +``` + +The default compat configuration listens on HTTP (no TLS). Real Splunk HEC deployments typically use HTTPS. For non-localhost deployments, add `tls` receiver config and update client URLs to `https://`. + +Raw log events can be POSTed to `/services/collector/raw` (newline-separated) in addition to JSON events on any collector path. + +## Canonical metadata mapping + +The upstream receiver exposes four HEC metadata fields as OpenTelemetry attribute mappings, configurable via `hec_metadata_to_otel_attrs`: + +| HEC metadata | Default OTel attribute | +|--------------|------------------------| +| `source` | `com.splunk.source` | +| `sourcetype` | `com.splunk.sourcetype` | +| `index` | `com.splunk.index` | +| `host` | `host.name` | + +For event payload details (the `event` field, custom `fields` objects, multi-event batches), consult Splunk's [HEC documentation](https://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector). + +## Deployment modes + +1. **Greenfield** — compat collector binds HEC port 8088. +2. **Side-by-side** — real Splunk HEC and compat collector on different ports. Set `COMPAT_SPLUNK_HEC_PORT=8089` and configure a subset of clients to use the new port. Useful for A/B validation during migration. +3. **Full replacement** — Splunk decommissioned, compat collector on 8088. + +## Verify + +Send a test event to the compat collector: + +```bash +curl -X POST http://localhost:8088/services/collector \ + -H "Authorization: Splunk any-token" \ + -H "Content-Type: application/json" \ + -d '{"event":"hello","sourcetype":"manual","source":"curl"}' +# {"text": "Success", "code": 0} +``` + +## What lands in OpenSearch + +An HEC event with `sourcetype=nginx:access` and `source=/var/log/nginx/access.log` becomes a log record in the `logs-otel-v1-*` index with: + +- `attributes.com.splunk.sourcetype`: `nginx:access` +- `attributes.com.splunk.source`: `/var/log/nginx/access.log` +- `body`: the event payload + +## Caveats + +- **No auth validation.** The default compat configuration accepts any `Authorization: Splunk ` header. Add an auth extension for production deployments. +- **No TLS.** The default compat configuration listens on HTTP. Add `tls` receiver config for non-localhost deployments. +- **`host.name` may be overridden downstream.** The receiver maps the HEC `host` field to OTel `host.name`, but the base collector's resource detection typically overrides `host.name` with the collector's own hostname. If you rely on the HEC `host` field for per-sender identification, route it to a different attribute via `hec_metadata_to_otel_attrs`, or disable resource detection in the base collector. +- **Python `logging` severities do not translate.** HEC does not have a native severity field. The `severityText` and `severityNumber` on the resulting OTel log records will be empty. Extract severity from the message body via a log parser in the pipeline if you need it. +- **HEC client libraries vary in how they ship custom fields.** Only clients that use HEC's `fields` object will produce OTel log attributes in OpenSearch. Clients that serialize structured data into the `event` message body (e.g., `splunk_handler` for Python) will have those fields embedded in `body` rather than broken out as attributes. Verify your client's behavior before relying on field-level queries. + +## Not covered + +- **Splunk forwarder-to-forwarder (S2S) protocol** — no open-source OpenTelemetry receiver exists. +- **SPL queries** — use [PPL (Piped Processing Language)](https://opensearch.org/docs/latest/search-plugins/ppl/index/) in OpenSearch. +- **Splunk apps, dashboards, alerts** — rebuild in OpenSearch Dashboards. +- **SignalFx** — the upstream `signalfxreceiver` is deprecated with explicit guidance to migrate to OTLP. Re-instrument SignalFx-monitored services with OpenTelemetry SDKs. + +## References + +- [`splunkhecreceiver` README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/splunkhecreceiver) +- [Splunk HEC documentation](https://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector) +- [`pkg/translator/splunk`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/splunk) diff --git a/docs/starlight-docs/src/content/docs/send-data/index.md b/docs/starlight-docs/src/content/docs/send-data/index.md index aa67049b..d646c1fc 100644 --- a/docs/starlight-docs/src/content/docs/send-data/index.md +++ b/docs/starlight-docs/src/content/docs/send-data/index.md @@ -56,6 +56,10 @@ Language-specific guides for instrumenting your services. Covers Python, Java, N Collect telemetry from your infrastructure. Covers host metrics, container monitoring, Kubernetes observability, and cloud provider integrations. +### [From Vendor Agents](/docs/send-data/from-vendor/) + +Accept telemetry from existing Datadog, Jaeger, or Splunk agents without re-instrumenting your applications. Point your vendor agents at the observability stack by changing one environment variable. + ### [Data Pipeline](/docs/send-data/data-pipeline/) Configure the backend processing layer. Covers Data Prepper pipelines for trace and log processing, Prometheus for metrics storage, and index management in OpenSearch. From 9e1b4812c46b01041cf6eb7cb985004351976203 Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Wed, 6 May 2026 08:49:51 -0700 Subject: [PATCH 2/7] config: upgrade OpenSearch and Dashboards to 3.7.0 staging (#225) Signed-off-by: ps48 --- .env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 2a504c8f..1f9841eb 100644 --- a/.env +++ b/.env @@ -11,11 +11,11 @@ INCLUDE_COMPOSE_EXAMPLES=docker-compose.examples.yml INCLUDE_COMPOSE_LOCAL_OPENSEARCH=docker-compose.local-opensearch.yml INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS=docker-compose.local-opensearch-dashboards.yml -OPENSEARCH_DOCKER_REPO=opensearchproject +OPENSEARCH_DOCKER_REPO=opensearchstaging # OpenSearch Configuration -OPENSEARCH_VERSION=3.6.0 +OPENSEARCH_VERSION=3.7.0 OPENSEARCH_USER=admin OPENSEARCH_PASSWORD='My_password_123!@#' OPENSEARCH_HOST=opensearch @@ -24,7 +24,7 @@ OPENSEARCH_PROTOCOL=https OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g # OpenSearch Dashboards Configuration -OPENSEARCH_DASHBOARDS_VERSION=3.6.0 +OPENSEARCH_DASHBOARDS_VERSION=3.7.0 OPENSEARCH_DASHBOARDS_HOST=opensearch-dashboards OPENSEARCH_DASHBOARDS_PORT=5601 OPENSEARCH_DASHBOARDS_PROTOCOL=http From 367d855b8fb660d6849a58db0b2bbc5be0a93204 Mon Sep 17 00:00:00 2001 From: Shenoy Pratik Date: Thu, 7 May 2026 09:48:44 -0700 Subject: [PATCH 3/7] config(opensearch-dashboards): enable icon side nav v2 (#227) Signed-off-by: ps48 --- .../opensearch-dashboards/opensearch_dashboards.template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose/opensearch-dashboards/opensearch_dashboards.template.yml b/docker-compose/opensearch-dashboards/opensearch_dashboards.template.yml index 990d2304..2bb5655d 100644 --- a/docker-compose/opensearch-dashboards/opensearch_dashboards.template.yml +++ b/docker-compose/opensearch-dashboards/opensearch_dashboards.template.yml @@ -84,6 +84,7 @@ data_source.ssl.verificationMode: none datasetManagement.enabled: true data.savedQueriesNewUI.enabled: true opensearchDashboards.branding.useExpandedHeader: false +opensearchDashboards.enableIconSideNav: true uiSettings.overrides.home:useNewHomePage: true uiSettings.overrides.query:enhancements:enabled: true From b4163466745ab46d2d29f6a89a5e487f10f19246 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 8 May 2026 15:14:07 -0700 Subject: [PATCH 4/7] poc(splunk): add Splunk OTel Distribution side-by-side overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local POC overlay that inserts Splunk's OTel Collector distribution as a tee in front of the base otel-collector. Demo apps send OTLP to splunk-otel-collector (redirected via OTEL_COLLECTOR_HOST override), which fans out to: - Splunk Observability Cloud (APM via otlphttp, IM via signalfx, Log Observer via splunk_hec) - base otel-collector -> Data Prepper -> OpenSearch (unchanged existing path) Purpose: ground-truth what Splunk Observability Cloud accepts from otel-demo so we can make informed decisions about compat overlay receiver scope (signalfxreceiver, zipkinreceiver) and the 'drop-in for Splunk Observability Cloud' framing. Files: - docker-compose.splunk-demo.yml: compose overlay adding splunk-otel-collector service - docker-compose/splunk-otel-collector/config.yaml: container-friendly Splunk collector config (trimmed from Splunk's agent_config.yaml — host-level receivers and smartagent extensions removed; OTLP-only receivers; tee exporter to base otel-collector:4317) - docker-compose/splunk-otel-collector/README.md: how to run and verify - .env.splunk-poc.example: credential + demo-redirect template - .gitignore: ignore the real .env.splunk-poc Not a production configuration. Local exploration only. Signed-off-by: Kyle Hounslow --- .env.splunk-poc.example | 26 +++++ .gitignore | 4 + docker-compose.splunk-demo.yml | 60 ++++++++++ .../splunk-otel-collector/README.md | 81 +++++++++++++ .../splunk-otel-collector/config.yaml | 110 ++++++++++++++++++ 5 files changed, 281 insertions(+) create mode 100644 .env.splunk-poc.example create mode 100644 docker-compose.splunk-demo.yml create mode 100644 docker-compose/splunk-otel-collector/README.md create mode 100644 docker-compose/splunk-otel-collector/config.yaml diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example new file mode 100644 index 00000000..0c6f17bd --- /dev/null +++ b/.env.splunk-poc.example @@ -0,0 +1,26 @@ +# Splunk Distribution POC — credentials + demo-app redirect +# +# Copy this file to .env.splunk-poc and fill in your Splunk Observability Cloud values. +# The actual .env.splunk-poc is gitignored and must never be committed. +# +# Where to find these in Splunk Observability Cloud: +# Access Token: Organization → Access Tokens → any token with ingest scope +# Realm: Visible in your Splunk Observability Cloud URL +# (e.g. us0, us1, eu0, au0) +# HEC Token: Data Management → Connect Data → HTTP Event Collector +# (if absent, can often fall back to the Access Token) + +# --- Splunk credentials --- +SPLUNK_ACCESS_TOKEN=your-splunk-access-token-here +SPLUNK_REALM=us1 +SPLUNK_HEC_TOKEN=your-hec-token-here-or-reuse-access-token + +# --- Redirect demo apps through Splunk collector (the tee) --- +# Demo apps read OTEL_COLLECTOR_HOST to build OTEL_EXPORTER_OTLP_ENDPOINT. +# Overriding it to the Splunk collector's service name routes all telemetry through +# the Splunk distribution first; Splunk then fans out to Splunk Observability Cloud +# and forwards a copy via OTLP back to the base otel-collector for OpenSearch. +OTEL_COLLECTOR_HOST=splunk-otel-collector + +# --- Splunk collector image pin (optional) --- +SPLUNK_COLLECTOR_VERSION=latest diff --git a/.gitignore b/.gitignore index 7630b3b4..2ab7482f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,7 @@ charts/*/charts/*.tgz # Terraform plan files *.plan + +# Splunk distribution POC +.env.splunk-poc +docker-compose/splunk-otel-collector/*.local.* diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml new file mode 100644 index 00000000..160c9e0b --- /dev/null +++ b/docker-compose.splunk-demo.yml @@ -0,0 +1,60 @@ +# Splunk OTel Distribution — Side-by-side POC Overlay +# +# Inserts Splunk's OTel Collector distribution as a tee in front of the base otel-collector. +# Demo apps send OTLP to splunk-otel-collector (via OTEL_COLLECTOR_HOST override), which fans +# out to: +# - Splunk Observability Cloud (APM + IM + Log Observer) +# - base otel-collector → Data Prepper → OpenSearch (unchanged existing path) +# +# Purpose: see the same demo data in both Splunk's UI and OpenSearch Dashboards at the same +# time, to ground-truth what Splunk Observability Cloud accepts and how it renders. +# +# NOT a production configuration. Local POC only. +# +# Usage: +# 1. Copy .env.splunk-poc.example to .env.splunk-poc and fill in Splunk credentials. +# 2. docker compose \ +# --env-file .env --env-file .env.splunk-poc \ +# -f docker-compose.yml \ +# -f docker-compose.otel-demo.yml \ +# -f docker-compose.splunk-demo.yml \ +# up -d + +services: + splunk-otel-collector: + image: quay.io/signalfx/splunk-otel-collector:${SPLUNK_COLLECTOR_VERSION:-latest} + container_name: splunk-otel-collector + pull_policy: always + command: ["--config=/etc/otelcol-config.yml"] + volumes: + - ./docker-compose/splunk-otel-collector/config.yaml:/etc/otelcol-config.yml + ports: + # Host-exposed on non-default ports to avoid collision with the base collector. + # Demo apps reach splunk-otel-collector over the internal Docker network on 4317/4318. + - "14317:4317" + - "14318:4318" + - "13133:13133" + networks: + - observability-stack-network + restart: unless-stopped + environment: + - SPLUNK_ACCESS_TOKEN + - SPLUNK_REALM + - SPLUNK_API_URL=https://api.${SPLUNK_REALM}.observability.splunkcloud.com + - SPLUNK_INGEST_URL=https://ingest.${SPLUNK_REALM}.observability.splunkcloud.com + - SPLUNK_HEC_TOKEN + - SPLUNK_HEC_URL=https://ingest.${SPLUNK_REALM}.observability.splunkcloud.com/v1/log + - SPLUNK_LISTEN_INTERFACE=0.0.0.0 + - SPLUNK_MEMORY_TOTAL_MIB=512 + deploy: + resources: + limits: + memory: 768M + depends_on: + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:13133"] + interval: 10s + timeout: 5s + retries: 5 diff --git a/docker-compose/splunk-otel-collector/README.md b/docker-compose/splunk-otel-collector/README.md new file mode 100644 index 00000000..d069002d --- /dev/null +++ b/docker-compose/splunk-otel-collector/README.md @@ -0,0 +1,81 @@ +# Splunk OTel Distribution — POC + +Runs Splunk's OpenTelemetry Collector distribution as a tee in front of observability-stack's base collector, so demo telemetry lands in both Splunk Observability Cloud and OpenSearch at the same time. + +Local POC only. Not a production configuration, not part of the default stack. + +## Dataflow + +``` +otel-demo apps + │ OTLP + ▼ +splunk-otel-collector (Splunk distribution) + │ + ├── signalfx → Splunk Infrastructure Monitoring + ├── otlphttp → Splunk APM (trace OTLP endpoint) + ├── splunk_hec → Splunk Log Observer + └── otlp (tee) → otel-collector → Data Prepper → OpenSearch (unchanged) +``` + +Demo apps are redirected via `OTEL_COLLECTOR_HOST=splunk-otel-collector` (set in `.env.splunk-poc`). No demo code changes required. + +## Run + +```bash +# From the repo root +cp .env.splunk-poc.example .env.splunk-poc +# edit .env.splunk-poc with your Splunk Access Token, Realm, HEC Token + +docker compose \ + --env-file .env --env-file .env.splunk-poc \ + -f docker-compose.yml \ + -f docker-compose.otel-demo.yml \ + -f docker-compose.splunk-demo.yml \ + up -d +``` + +## Verify + +Splunk Observability Cloud (URL varies by realm, e.g. `https://app.us1.signalfx.com`): + +- **APM → Services** — look for demo services (frontend, cart, checkout, ad, ...) +- **Infrastructure → Metrics Explorer** — filter on `service.name` +- **Log Observer** — filter on `sourcetype=otel`, `source=otel-demo` + +OpenSearch Dashboards (http://localhost:5601 by default): + +- Trace Analytics — same demo services should appear in the existing OSD trace UI +- Discover on `logs-otel-v1-*` — log records from the demo + +Splunk collector's own health endpoint: `http://localhost:13133`. + +Splunk collector's exposed OTLP ports on the host (non-default to avoid collision with base collector): + +- gRPC: `localhost:14317` +- HTTP: `localhost:14318` + +## Dev commands + +```bash +# Tail Splunk collector logs +docker logs -f splunk-otel-collector + +# Verify collector internal metrics (self-monitoring) +curl -s http://localhost:13133 + +# Tear down just the Splunk overlay +docker compose -f docker-compose.yml -f docker-compose.otel-demo.yml -f docker-compose.splunk-demo.yml rm -sf splunk-otel-collector +``` + +## Scope + +- **Out of scope:** `smartagentreceiver` (proprietary to Splunk's distribution, requires host bundle), `host_metrics` (meaningless in Docker), discovery mode, gateway forwarding. +- **In scope:** OTLP traces/metrics/logs from the otel-demo apps, fanned out to Splunk Observability Cloud + OpenSearch. + +## References + +- [Splunk OpenTelemetry Collector distribution](https://github.com/signalfx/splunk-otel-collector) +- [Splunk agent_config.yaml](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml) — reference for the fuller config this one is trimmed from +- [signalfxexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter) +- [splunkhecexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter) diff --git a/docker-compose/splunk-otel-collector/config.yaml b/docker-compose/splunk-otel-collector/config.yaml new file mode 100644 index 00000000..83192633 --- /dev/null +++ b/docker-compose/splunk-otel-collector/config.yaml @@ -0,0 +1,110 @@ +# Splunk OpenTelemetry Collector — POC Configuration +# +# Acts as a tee: receives OTLP from demo apps, fans out to: +# 1. Splunk Observability Cloud (APM via otlphttp, IM via signalfx, Log Observer via splunk_hec) +# 2. observability-stack's base otel-collector (via OTLP forward), which continues the +# existing Data Prepper / OpenSearch pipeline unchanged. +# +# Trimmed from Splunk's default agent_config.yaml: +# - Host-level receivers (host_metrics, fluent_forward, smartagent/*) removed — not useful in container. +# - Extensions that require host integration (smartagent bundle, opamp, http_forwarder) removed. +# - Prometheus internal / entities pipelines removed — this is app-trace focused. +# +# References: +# Splunk agent config: https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml +# signalfxexporter: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter +# splunkhecexporter: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter + +extensions: + health_check: + endpoint: "0.0.0.0:13133" + +receivers: + # OTLP from demo apps (same shape as base otel-collector so apps' OTEL_EXPORTER_OTLP_ENDPOINT + # only needs the hostname changed via OTEL_COLLECTOR_HOST override in .env.splunk-poc). + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + cors: + allowed_origins: + - "http://*" + - "https://*" + +processors: + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 + + batch: + timeout: 10s + send_batch_size: 1024 + + resourcedetection: + detectors: [env, system] + +exporters: + # ---- Splunk Observability Cloud ---- + # Splunk APM (traces) via OTLP HTTP to Splunk's trace ingest + otlphttp/splunk-apm: + traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp" + headers: + "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}" + + # Splunk Infrastructure Monitoring (metrics + events) + signalfx: + access_token: "${SPLUNK_ACCESS_TOKEN}" + api_url: "${SPLUNK_API_URL}" + ingest_url: "${SPLUNK_INGEST_URL}" + sync_host_metadata: true + + # Splunk Log Observer (logs). If SPLUNK_HEC_TOKEN is unset or invalid, the logs + # pipeline will error but traces/metrics pipelines stay healthy. + splunk_hec: + token: "${SPLUNK_HEC_TOKEN}" + endpoint: "${SPLUNK_HEC_URL}" + source: "otel-demo" + sourcetype: "otel" + profiling_data_enabled: false + + # ---- Tee to observability-stack base collector (OpenSearch path) ---- + otlp/tee: + endpoint: "otel-collector:4317" + tls: + insecure: true + + debug: + verbosity: basic + +service: + extensions: [health_check] + + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, batch] + exporters: [otlphttp/splunk-apm, signalfx, otlp/tee] + + metrics: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, batch] + exporters: [signalfx, otlp/tee] + + logs: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, batch] + exporters: [splunk_hec, otlp/tee] + + telemetry: + logs: + level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 From 22f966e7053f2b8315b3db0d5e939121e2b16e76 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 8 May 2026 15:15:52 -0700 Subject: [PATCH 5/7] poc(splunk): simplify run command via include-directive pattern .env.splunk-poc now sets INCLUDE_COMPOSE_OTEL_DEMO so the otel-demo overlay is pulled in automatically via the base compose's include directive. Run command drops from three -f flags to two. Also switches README example from 'docker compose' to 'finch compose' to match the project's default container runtime. Signed-off-by: Kyle Hounslow --- .env.splunk-poc.example | 3 +++ docker-compose.splunk-demo.yml | 5 +++-- docker-compose/splunk-otel-collector/README.md | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example index 0c6f17bd..e0841eaa 100644 --- a/.env.splunk-poc.example +++ b/.env.splunk-poc.example @@ -22,5 +22,8 @@ SPLUNK_HEC_TOKEN=your-hec-token-here-or-reuse-access-token # and forwards a copy via OTLP back to the base otel-collector for OpenSearch. OTEL_COLLECTOR_HOST=splunk-otel-collector +# --- Activate the otel-demo overlay via the repo's include-directive pattern --- +INCLUDE_COMPOSE_OTEL_DEMO=docker-compose.otel-demo.yml + # --- Splunk collector image pin (optional) --- SPLUNK_COLLECTOR_VERSION=latest diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml index 160c9e0b..2f51efab 100644 --- a/docker-compose.splunk-demo.yml +++ b/docker-compose.splunk-demo.yml @@ -13,12 +13,13 @@ # # Usage: # 1. Copy .env.splunk-poc.example to .env.splunk-poc and fill in Splunk credentials. -# 2. docker compose \ +# 2. finch compose \ # --env-file .env --env-file .env.splunk-poc \ # -f docker-compose.yml \ -# -f docker-compose.otel-demo.yml \ # -f docker-compose.splunk-demo.yml \ # up -d +# (The otel-demo overlay is pulled in automatically via the base compose's include: directive +# because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO.) services: splunk-otel-collector: diff --git a/docker-compose/splunk-otel-collector/README.md b/docker-compose/splunk-otel-collector/README.md index d069002d..fa62dd52 100644 --- a/docker-compose/splunk-otel-collector/README.md +++ b/docker-compose/splunk-otel-collector/README.md @@ -27,10 +27,11 @@ Demo apps are redirected via `OTEL_COLLECTOR_HOST=splunk-otel-collector` (set in cp .env.splunk-poc.example .env.splunk-poc # edit .env.splunk-poc with your Splunk Access Token, Realm, HEC Token -docker compose \ +# Base compose's include: directive pulls in the otel-demo overlay automatically +# because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO. +finch compose \ --env-file .env --env-file .env.splunk-poc \ -f docker-compose.yml \ - -f docker-compose.otel-demo.yml \ -f docker-compose.splunk-demo.yml \ up -d ``` From ebc7adebc4ad99696ecc4f090e476b580172dbe8 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 8 May 2026 15:53:33 -0700 Subject: [PATCH 6/7] poc(splunk): tighten docs and config to match rubric Applies technical-writing.md and otel-contrib-pr-rubric.md: - README: strip narrative prose, us-vs-them framing, and AI-tone defensives; drop docker commands in favor of the finch path users actually run. - README: document finch's --env-file substitution limitation explicitly (cannot feed ${VAR} expansion in compose files) and show the working append-and-restore workflow. - README: add real validation caveat for splunk_hec 404 on /v1/log with the access token reused as HEC token; note logs still land in OpenSearch. - config.yaml: terse header, one-line comments per exporter, no rationale prose. - docker-compose.splunk-demo.yml: two-line header, one-line port comment. - .env.splunk-poc.example: remove explanatory prose, leave only locations and required vars. No behavioral changes. Signed-off-by: Kyle Hounslow --- .env.splunk-poc.example | 31 ++---- docker-compose.splunk-demo.yml | 27 +----- .../splunk-otel-collector/README.md | 95 ++++++++----------- .../splunk-otel-collector/config.yaml | 32 ++----- 4 files changed, 64 insertions(+), 121 deletions(-) diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example index e0841eaa..66fb18a0 100644 --- a/.env.splunk-poc.example +++ b/.env.splunk-poc.example @@ -1,29 +1,18 @@ -# Splunk Distribution POC — credentials + demo-app redirect -# -# Copy this file to .env.splunk-poc and fill in your Splunk Observability Cloud values. -# The actual .env.splunk-poc is gitignored and must never be committed. -# -# Where to find these in Splunk Observability Cloud: -# Access Token: Organization → Access Tokens → any token with ingest scope -# Realm: Visible in your Splunk Observability Cloud URL -# (e.g. us0, us1, eu0, au0) -# HEC Token: Data Management → Connect Data → HTTP Event Collector -# (if absent, can often fall back to the Access Token) +# Splunk Distribution POC credentials. Copy to .env.splunk-poc (gitignored). +# Locations in Splunk Observability Cloud: +# SPLUNK_ACCESS_TOKEN: Organization > Access Tokens (ingest scope) +# SPLUNK_REALM: visible in the Splunk Observability Cloud URL +# SPLUNK_HEC_TOKEN: Data Management > Connect Data > HTTP Event Collector +# (may not be available on all tiers; see POC README caveat) -# --- Splunk credentials --- -SPLUNK_ACCESS_TOKEN=your-splunk-access-token-here +SPLUNK_ACCESS_TOKEN= SPLUNK_REALM=us1 -SPLUNK_HEC_TOKEN=your-hec-token-here-or-reuse-access-token +SPLUNK_HEC_TOKEN= -# --- Redirect demo apps through Splunk collector (the tee) --- -# Demo apps read OTEL_COLLECTOR_HOST to build OTEL_EXPORTER_OTLP_ENDPOINT. -# Overriding it to the Splunk collector's service name routes all telemetry through -# the Splunk distribution first; Splunk then fans out to Splunk Observability Cloud -# and forwards a copy via OTLP back to the base otel-collector for OpenSearch. +# Redirect otel-demo apps through the Splunk collector. OTEL_COLLECTOR_HOST=splunk-otel-collector -# --- Activate the otel-demo overlay via the repo's include-directive pattern --- +# Activate the otel-demo overlay via the base compose's include directive. INCLUDE_COMPOSE_OTEL_DEMO=docker-compose.otel-demo.yml -# --- Splunk collector image pin (optional) --- SPLUNK_COLLECTOR_VERSION=latest diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml index 2f51efab..582b6e98 100644 --- a/docker-compose.splunk-demo.yml +++ b/docker-compose.splunk-demo.yml @@ -1,25 +1,5 @@ -# Splunk OTel Distribution — Side-by-side POC Overlay -# -# Inserts Splunk's OTel Collector distribution as a tee in front of the base otel-collector. -# Demo apps send OTLP to splunk-otel-collector (via OTEL_COLLECTOR_HOST override), which fans -# out to: -# - Splunk Observability Cloud (APM + IM + Log Observer) -# - base otel-collector → Data Prepper → OpenSearch (unchanged existing path) -# -# Purpose: see the same demo data in both Splunk's UI and OpenSearch Dashboards at the same -# time, to ground-truth what Splunk Observability Cloud accepts and how it renders. -# -# NOT a production configuration. Local POC only. -# -# Usage: -# 1. Copy .env.splunk-poc.example to .env.splunk-poc and fill in Splunk credentials. -# 2. finch compose \ -# --env-file .env --env-file .env.splunk-poc \ -# -f docker-compose.yml \ -# -f docker-compose.splunk-demo.yml \ -# up -d -# (The otel-demo overlay is pulled in automatically via the base compose's include: directive -# because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO.) +# Splunk OTel distribution overlay for the side-by-side POC. +# See docker-compose/splunk-otel-collector/README.md to run. services: splunk-otel-collector: @@ -30,8 +10,7 @@ services: volumes: - ./docker-compose/splunk-otel-collector/config.yaml:/etc/otelcol-config.yml ports: - # Host-exposed on non-default ports to avoid collision with the base collector. - # Demo apps reach splunk-otel-collector over the internal Docker network on 4317/4318. + # Non-default host ports to avoid collision with the base collector's 4317/4318. - "14317:4317" - "14318:4318" - "13133:13133" diff --git a/docker-compose/splunk-otel-collector/README.md b/docker-compose/splunk-otel-collector/README.md index fa62dd52..d3d8a044 100644 --- a/docker-compose/splunk-otel-collector/README.md +++ b/docker-compose/splunk-otel-collector/README.md @@ -1,82 +1,69 @@ -# Splunk OTel Distribution — POC +# Splunk OTel Distribution POC -Runs Splunk's OpenTelemetry Collector distribution as a tee in front of observability-stack's base collector, so demo telemetry lands in both Splunk Observability Cloud and OpenSearch at the same time. +Local POC. Inserts Splunk's [OpenTelemetry Collector distribution](https://github.com/signalfx/splunk-otel-collector) between the otel-demo apps and the base `otel-collector`. Demo telemetry fans out to Splunk Observability Cloud and OpenSearch at the same time. -Local POC only. Not a production configuration, not part of the default stack. +Branch: `feat/splunk-distribution-poc`. Not intended for upstream. ## Dataflow ``` otel-demo apps - │ OTLP + │ OTLP (OTEL_COLLECTOR_HOST=splunk-otel-collector) ▼ -splunk-otel-collector (Splunk distribution) - │ - ├── signalfx → Splunk Infrastructure Monitoring - ├── otlphttp → Splunk APM (trace OTLP endpoint) - ├── splunk_hec → Splunk Log Observer - └── otlp (tee) → otel-collector → Data Prepper → OpenSearch (unchanged) +splunk-otel-collector + ├── signalfx exporter → Splunk Infrastructure Monitoring + ├── otlphttp/splunk-apm → Splunk APM (ingest/v2/trace/otlp) + ├── splunk_hec → Splunk Log Observer (see caveats) + └── otlp/tee → base otel-collector (unchanged OpenSearch path) ``` -Demo apps are redirected via `OTEL_COLLECTOR_HOST=splunk-otel-collector` (set in `.env.splunk-poc`). No demo code changes required. - ## Run ```bash -# From the repo root cp .env.splunk-poc.example .env.splunk-poc -# edit .env.splunk-poc with your Splunk Access Token, Realm, HEC Token - -# Base compose's include: directive pulls in the otel-demo overlay automatically -# because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO. -finch compose \ - --env-file .env --env-file .env.splunk-poc \ - -f docker-compose.yml \ - -f docker-compose.splunk-demo.yml \ - up -d -``` - -## Verify +# fill in SPLUNK_ACCESS_TOKEN, SPLUNK_REALM, SPLUNK_HEC_TOKEN -Splunk Observability Cloud (URL varies by realm, e.g. `https://app.us1.signalfx.com`): - -- **APM → Services** — look for demo services (frontend, cart, checkout, ad, ...) -- **Infrastructure → Metrics Explorer** — filter on `service.name` -- **Log Observer** — filter on `sourcetype=otel`, `source=otel-demo` +# finch compose's --env-file doesn't feed ${VAR} substitution in compose files, only +# container env. Append the POC vars into .env before `up`, restore after. +cat .env.splunk-poc >> .env +finch compose -f docker-compose.yml -f docker-compose.splunk-demo.yml up -d +git checkout -- .env +``` -OpenSearch Dashboards (http://localhost:5601 by default): +Teardown: -- Trace Analytics — same demo services should appear in the existing OSD trace UI -- Discover on `logs-otel-v1-*` — log records from the demo +```bash +finch compose -f docker-compose.yml -f docker-compose.splunk-demo.yml down +``` -Splunk collector's own health endpoint: `http://localhost:13133`. +## Verify -Splunk collector's exposed OTLP ports on the host (non-default to avoid collision with base collector): +Counters on the Splunk collector (send success numbers; non-zero = data flowing): -- gRPC: `localhost:14317` -- HTTP: `localhost:14318` +```bash +finch run --rm --network observability-stack-network curlimages/curl:latest \ + -s http://splunk-otel-collector:8888/metrics \ + | grep -E '^otelcol_exporter_sent_(spans|metric_points)_total' +``` -## Dev commands +Host-exposed endpoints on `splunk-otel-collector`: -```bash -# Tail Splunk collector logs -docker logs -f splunk-otel-collector +- `localhost:13133` - health check (200 = ready) +- `localhost:14317` - OTLP gRPC +- `localhost:14318` - OTLP HTTP -# Verify collector internal metrics (self-monitoring) -curl -s http://localhost:13133 +In Splunk Observability Cloud: **APM > Services** shows demo services (frontend, cart, checkout, ad, recommendation, …). **Infrastructure > Metrics Explorer** shows host metadata from `splunk-otel-collector` and app metrics from the demo. -# Tear down just the Splunk overlay -docker compose -f docker-compose.yml -f docker-compose.otel-demo.yml -f docker-compose.splunk-demo.yml rm -sf splunk-otel-collector -``` +OpenSearch side is unaffected: Trace Analytics in OSD renders the same demo services via the tee. -## Scope +## Caveats from validation -- **Out of scope:** `smartagentreceiver` (proprietary to Splunk's distribution, requires host bundle), `host_metrics` (meaningless in Docker), discovery mode, gateway forwarding. -- **In scope:** OTLP traces/metrics/logs from the otel-demo apps, fanned out to Splunk Observability Cloud + OpenSearch. +- **Splunk HEC logs return HTTP 404** on `/v1/log` with the access token reused as HEC token. The `splunk_hec` exporter retries indefinitely without blocking other pipelines. Logs still land in OpenSearch via `otlp/tee`. Cause is likely one of: access token lacks log-ingest scope, trial tier without Log Observer, or a distinct HEC token is required. +- `smartagentreceiver` and `host_metrics` from Splunk's default `agent_config.yaml` are not included. The first is proprietary to Splunk's distribution bundle; the second is meaningless in a container. -## References +## Reference -- [Splunk OpenTelemetry Collector distribution](https://github.com/signalfx/splunk-otel-collector) -- [Splunk agent_config.yaml](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml) — reference for the fuller config this one is trimmed from -- [signalfxexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter) -- [splunkhecexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter) +- [Splunk OTel Collector distribution](https://github.com/signalfx/splunk-otel-collector) +- [Splunk default `agent_config.yaml`](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml) +- [signalfxexporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter) +- [splunkhecexporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter) diff --git a/docker-compose/splunk-otel-collector/config.yaml b/docker-compose/splunk-otel-collector/config.yaml index 83192633..543dfba4 100644 --- a/docker-compose/splunk-otel-collector/config.yaml +++ b/docker-compose/splunk-otel-collector/config.yaml @@ -1,27 +1,17 @@ -# Splunk OpenTelemetry Collector — POC Configuration +# Splunk OpenTelemetry Collector config for the side-by-side POC. +# See docker-compose/splunk-otel-collector/README.md to run. # -# Acts as a tee: receives OTLP from demo apps, fans out to: -# 1. Splunk Observability Cloud (APM via otlphttp, IM via signalfx, Log Observer via splunk_hec) -# 2. observability-stack's base otel-collector (via OTLP forward), which continues the -# existing Data Prepper / OpenSearch pipeline unchanged. +# Based on Splunk's default agent_config.yaml with host-level receivers and +# smartagent/opamp/http_forwarder extensions removed (not viable in-container). # -# Trimmed from Splunk's default agent_config.yaml: -# - Host-level receivers (host_metrics, fluent_forward, smartagent/*) removed — not useful in container. -# - Extensions that require host integration (smartagent bundle, opamp, http_forwarder) removed. -# - Prometheus internal / entities pipelines removed — this is app-trace focused. -# -# References: -# Splunk agent config: https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml -# signalfxexporter: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter -# splunkhecexporter: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter +# Reference: https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml extensions: health_check: endpoint: "0.0.0.0:13133" receivers: - # OTLP from demo apps (same shape as base otel-collector so apps' OTEL_EXPORTER_OTLP_ENDPOINT - # only needs the hostname changed via OTEL_COLLECTOR_HOST override in .env.splunk-poc). + # OTLP on the same ports as the base collector so demo apps only need OTEL_COLLECTOR_HOST changed. otlp: protocols: grpc: @@ -47,22 +37,20 @@ processors: detectors: [env, system] exporters: - # ---- Splunk Observability Cloud ---- - # Splunk APM (traces) via OTLP HTTP to Splunk's trace ingest + # Splunk APM: OTLP traces to Splunk's trace ingest endpoint. otlphttp/splunk-apm: traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp" headers: "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}" - # Splunk Infrastructure Monitoring (metrics + events) + # Splunk Infrastructure Monitoring: metrics + events. signalfx: access_token: "${SPLUNK_ACCESS_TOKEN}" api_url: "${SPLUNK_API_URL}" ingest_url: "${SPLUNK_INGEST_URL}" sync_host_metadata: true - # Splunk Log Observer (logs). If SPLUNK_HEC_TOKEN is unset or invalid, the logs - # pipeline will error but traces/metrics pipelines stay healthy. + # Splunk Log Observer via HEC. Returns 404 in current POC validation; see README caveats. splunk_hec: token: "${SPLUNK_HEC_TOKEN}" endpoint: "${SPLUNK_HEC_URL}" @@ -70,7 +58,7 @@ exporters: sourcetype: "otel" profiling_data_enabled: false - # ---- Tee to observability-stack base collector (OpenSearch path) ---- + # Forward to observability-stack's base otel-collector for the OpenSearch path. otlp/tee: endpoint: "otel-collector:4317" tls: From e8b2286f58d3dd1222bc8c33503214987debf70a Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 18 May 2026 08:41:50 -0700 Subject: [PATCH 7/7] poc(splunk): fix HEC config for Splunk Cloud Platform - Point HEC at Splunk Cloud Platform (port 8088) instead of defunct Observability Cloud /v1/log endpoint - Add insecure_skip_verify for trial cert mismatch - Add SPLUNK_HEC_INDEX env var - Update .env.splunk-poc.example with correct guidance for both Splunk products (Observability Cloud + Cloud Platform) Signed-off-by: Kyle Hounslow --- .env.splunk-poc.example | 19 +++++++++++++++---- docker-compose.splunk-demo.yml | 3 ++- .../splunk-otel-collector/config.yaml | 5 ++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example index 66fb18a0..5ba57429 100644 --- a/.env.splunk-poc.example +++ b/.env.splunk-poc.example @@ -1,13 +1,24 @@ # Splunk Distribution POC credentials. Copy to .env.splunk-poc (gitignored). -# Locations in Splunk Observability Cloud: +# +# This POC sends to TWO Splunk products: +# 1. Splunk Observability Cloud (traces via APM, metrics via IM) +# 2. Splunk Cloud Platform (logs via HEC, searchable in Splunk Search) +# +# Splunk Observability Cloud: # SPLUNK_ACCESS_TOKEN: Organization > Access Tokens (ingest scope) -# SPLUNK_REALM: visible in the Splunk Observability Cloud URL -# SPLUNK_HEC_TOKEN: Data Management > Connect Data > HTTP Event Collector -# (may not be available on all tiers; see POC README caveat) +# SPLUNK_REALM: visible in the Splunk Observability Cloud URL (us0, us1, eu0, au0) +# +# Splunk Cloud Platform: +# SPLUNK_HEC_TOKEN: Settings > Data Inputs > HTTP Event Collector > your token +# SPLUNK_HEC_URL: https://.splunkcloud.com:8088/services/collector +# SPLUNK_HEC_INDEX: index the token is allowed to write to (default: main) SPLUNK_ACCESS_TOKEN= SPLUNK_REALM=us1 + SPLUNK_HEC_TOKEN= +SPLUNK_HEC_URL=https://prd-p-XXXXX.splunkcloud.com:8088/services/collector +SPLUNK_HEC_INDEX=main # Redirect otel-demo apps through the Splunk collector. OTEL_COLLECTOR_HOST=splunk-otel-collector diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml index 582b6e98..abce3285 100644 --- a/docker-compose.splunk-demo.yml +++ b/docker-compose.splunk-demo.yml @@ -23,7 +23,8 @@ services: - SPLUNK_API_URL=https://api.${SPLUNK_REALM}.observability.splunkcloud.com - SPLUNK_INGEST_URL=https://ingest.${SPLUNK_REALM}.observability.splunkcloud.com - SPLUNK_HEC_TOKEN - - SPLUNK_HEC_URL=https://ingest.${SPLUNK_REALM}.observability.splunkcloud.com/v1/log + - SPLUNK_HEC_URL + - SPLUNK_HEC_INDEX - SPLUNK_LISTEN_INTERFACE=0.0.0.0 - SPLUNK_MEMORY_TOTAL_MIB=512 deploy: diff --git a/docker-compose/splunk-otel-collector/config.yaml b/docker-compose/splunk-otel-collector/config.yaml index 543dfba4..49056cff 100644 --- a/docker-compose/splunk-otel-collector/config.yaml +++ b/docker-compose/splunk-otel-collector/config.yaml @@ -50,13 +50,16 @@ exporters: ingest_url: "${SPLUNK_INGEST_URL}" sync_host_metadata: true - # Splunk Log Observer via HEC. Returns 404 in current POC validation; see README caveats. + # Splunk Cloud Platform (logs via HEC). splunk_hec: token: "${SPLUNK_HEC_TOKEN}" endpoint: "${SPLUNK_HEC_URL}" source: "otel-demo" sourcetype: "otel" + index: "${SPLUNK_HEC_INDEX}" profiling_data_enabled: false + tls: + insecure_skip_verify: true # Forward to observability-stack's base otel-collector for the OpenSearch path. otlp/tee: