diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 864cf27eb2c..69f8e0bacb5 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -42,6 +42,14 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { return; } + // LLM Observability spans are backed by tracer spans, so without a tracer there is nothing to + // build them on: AgentTracer.get().buildSpan() returns null on the no-op tracer. Leave the + // no-op SDK implementations in place so the public API stays safe to call. + if (!config.isTraceEnabled()) { + LOGGER.debug("LLM Observability is disabled: tracing is disabled"); + return; + } + sco.createRemaining(config); String mlApp = config.getLlmObsMlApp(); diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java new file mode 100644 index 00000000000..9abde374b2d --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java @@ -0,0 +1,31 @@ +package datadog.trace.llmobs; + +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; +import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; +import static org.junit.jupiter.api.Assertions.assertSame; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.noop.NoOpLLMObsSpan; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.util.DDJavaSpecification; +import org.junit.jupiter.api.Test; + +class LLMObsSystemTest extends DDJavaSpecification { + + /** + * LLM Observability spans are backed by tracer spans, so the subsystem must stay off when tracing + * is disabled. Starting it would install the real SDK implementation on top of the no-op tracer, + * whose {@code buildSpan()} returns null, and every span-starting call would throw. + */ + @WithConfig(key = LLMOBS_ENABLED, value = "true") + @WithConfig(key = TRACE_ENABLED, value = "false") + @Test + void staysNoOpWhenTracingIsDisabled() { + // The null SharedCommunicationObjects doubles as an assertion that start() returns before + // touching it: any use would throw. + LLMObsSystem.start(null, null); + + assertSame( + NoOpLLMObsSpan.INSTANCE, LLMObs.startLLMSpan("span", "model", "provider", null, null)); + } +} diff --git a/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java b/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java index d471fcb630b..6ca074d8e84 100644 --- a/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java +++ b/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java @@ -1,5 +1,7 @@ package datadog.smoketest.apmtracingdisabled; +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.util.GlobalTracer; @@ -81,6 +83,14 @@ public void write( } } + @GetMapping("/llmobs") + public String llmObs() { + LLMObsSpan span = LLMObs.startLLMSpan("llm-call", "gpt-4", "openai", null, null); + span.annotateIO("input", "output"); + span.finish(); + return "llmobs"; + } + @GetMapping("/late-outbound") public String lateOutbound(@RequestParam(name = "url") String url) { final Span span = GlobalTracer.get().activeSpan(); diff --git a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy new file mode 100644 index 00000000000..87cd018f051 --- /dev/null +++ b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy @@ -0,0 +1,59 @@ +package datadog.smoketest.apmtracingdisabled + +import datadog.trace.api.sampling.PrioritySampling +import okhttp3.Request +import spock.util.concurrent.PollingConditions + +/** + * With {@code DD_APM_TRACING_ENABLED=false}, LLM Observability keeps working — its spans go to the + * LLM Observability intake, which is independent of APM sampling — while the APM traces they ride + * on are dropped. + */ +class LlmObsApmTracingDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest { + + @Override + ProcessBuilder createProcessBuilder() { + final String[] processProperties = [ + "-Ddd.apm.tracing.enabled=false", + "-Ddd.llmobs.enabled=true", + "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest", + "-Ddd.service.name=llmobs-apm-tracing-disabled-smoketest-app", + ] + return createProcess(processProperties) + } + + @Override + protected String traceAgentProtocolVersion() { + return '0.4' + } + + @Override + Closure decodedEvpProxyMessageCallback() { + return { String path, request -> + // The payload is msgpack; the path alone tells us the LLM Observability intake was reached. + return path.contains('api/v2/llmobs') ? path : null + } + } + + void 'LLMObs spans reach the LLM Observability intake while the APM trace is dropped'() { + setup: + final url = "http://localhost:${httpPort}/rest-api/llmobs" + final request = new Request.Builder().url(url).get().build() + + when: + final response = client.newCall(request).execute() + + then: + response.successful + waitForTraceCount(1) + + and: 'the APM trace is dropped' + checkRootSpanPrioritySampling(traces[0], PrioritySampling.SAMPLER_DROP) + hasApmDisabledTagOnEverySpan(traces[0]) + + and: 'the LLMObs span still reached the LLM Observability intake' + new PollingConditions(timeout: 30).eventually { + assert !evpProxyMessages.isEmpty() + } + } +} diff --git a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy new file mode 100644 index 00000000000..5a44ebf0e81 --- /dev/null +++ b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy @@ -0,0 +1,38 @@ +package datadog.smoketest.apmtracingdisabled + +import okhttp3.Request + +/** + * {@code DD_TRACE_ENABLED=false} leaves no tracer to back LLM Observability spans, so the SDK must + * stay no-op instead of throwing out of instrumented application code. + */ +class LlmObsTraceDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest { + + @Override + ProcessBuilder createProcessBuilder() { + final String[] processProperties = [ + "-Ddd.trace.enabled=false", + "-Ddd.llmobs.enabled=true", + "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest", + "-Ddd.service.name=llmobs-trace-disabled-smoketest-app", + ] + return createProcess(processProperties) + } + + @Override + protected String traceAgentProtocolVersion() { + return '0.4' + } + + void 'the LLMObs SDK stays safe to call when tracing is disabled'() { + setup: + final url = "http://localhost:${httpPort}/rest-api/llmobs" + final request = new Request.Builder().url(url).get().build() + + when: + final response = client.newCall(request).execute() + + then: + response.successful + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java index 823f0d4959b..6f057782232 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java @@ -36,9 +36,16 @@ final class Builder { public static Sampler forConfig(final Config config, final TraceConfig traceConfig) { Sampler sampler; if (config != null) { - if (!config.isApmTracingEnabled() && isAsmEnabled(config)) { - log.debug("APM is disabled. Only 1 trace per minute will be sent."); - return new AsmStandaloneSampler(Clock.systemUTC()); + if (!config.isApmTracingEnabled()) { + if (isAsmEnabled(config)) { + log.debug( + "APM tracing is disabled, but ASM is enabled. Only 1 APM trace per minute will be sent."); + return new AsmStandaloneSampler(Clock.systemUTC()); + } + // No product needs a continuous APM trace, so drop them all. Products that keep their + // own traces (ASM, AI Guard) force-keep them, which this sampler cannot override. + log.debug("APM tracing is disabled. APM traces will be dropped."); + return new ForcePrioritySampler(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT); } final Map serviceRules = config.getTraceSamplingServiceRules(); final Map operationRules = config.getTraceSamplingOperationRules(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index e4658e7046b..98815701e7c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -4,10 +4,12 @@ import static datadog.trace.api.config.AppSecConfig.APPSEC_SCA_ENABLED; import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING; import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING_FORCE; import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -67,6 +69,23 @@ void asmStandaloneSamplerNotSelectedWhenApmTracingAndAsmNotEnabled() { assertFalse(sampler instanceof AsmStandaloneSampler); } + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @Test + void apmTracesDroppedWhenApmTracingDisabledAndNoOtherProductEnabled() { + assertApmTracesDropped(); + } + + /** + * LLM Observability rides the tracer but ships its spans to the LLM Observability intake, so + * disabling APM tracing must drop the APM traces without disabling the tracer. + */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @WithConfig(key = LLMOBS_ENABLED, value = "true") + @Test + void apmTracesDroppedWhenApmTracingDisabledAndLlmObsEnabled() { + assertApmTracesDropped(); + } + @Test void asmStandaloneSamplerNotSelectedWhenApmTracingEnabledAndAsmNotEnabled() { Config config = Config.get(); @@ -165,4 +184,22 @@ void spansBuiltWithOtlpEnabledAndPrioritySamplingDisabledHaveNonUnsetSamplingPri tracer.close(); } } + + private static void assertApmTracesDropped() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + + assertInstanceOf(ForcePrioritySampler.class, sampler); + + CoreTracer tracer = CoreTracer.builder().writer(new ListWriter()).sampler(sampler).build(); + try { + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "test").start(); + ((PrioritySampler) sampler).setSamplingPriority(span); + + assertEquals(SAMPLER_DROP, (int) span.getSamplingPriority()); + + span.finish(); + } finally { + tracer.close(); + } + } }