Drop APM traces when APM tracing is disabled and keep LLM Observability working

dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java

@@ -42,6 +42,14 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) {
       return;
     }
 
+    // LLM Observability spans are backed by tracer spans, so without a tracer there is nothing to
+    // build them on: AgentTracer.get().buildSpan() returns null on the no-op tracer. Leave the
+    // no-op SDK implementations in place so the public API stays safe to call.
+    if (!config.isTraceEnabled()) {
+      LOGGER.debug("LLM Observability is disabled: tracing is disabled");
+      return;
+    }
+
     sco.createRemaining(config);
 
     String mlApp = config.getLlmObsMlApp();

dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java

@@ -0,0 +1,31 @@
+package datadog.trace.llmobs;
+
+import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED;
+import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+import datadog.trace.api.llmobs.LLMObs;
+import datadog.trace.api.llmobs.noop.NoOpLLMObsSpan;
+import datadog.trace.test.junit.utils.config.WithConfig;
+import datadog.trace.test.util.DDJavaSpecification;
+import org.junit.jupiter.api.Test;
+
+class LLMObsSystemTest extends DDJavaSpecification {
+
+  /**
+   * LLM Observability spans are backed by tracer spans, so the subsystem must stay off when tracing
+   * is disabled. Starting it would install the real SDK implementation on top of the no-op tracer,
+   * whose {@code buildSpan()} returns null, and every span-starting call would throw.
+   */
+  @WithConfig(key = LLMOBS_ENABLED, value = "true")
+  @WithConfig(key = TRACE_ENABLED, value = "false")
+  @Test
+  void staysNoOpWhenTracingIsDisabled() {
+    // The null SharedCommunicationObjects doubles as an assertion that start() returns before
+    // touching it: any use would throw.
+    LLMObsSystem.start(null, null);
+
+    assertSame(
+        NoOpLLMObsSpan.INSTANCE, LLMObs.startLLMSpan("span", "model", "provider", null, null));
+  }
+}

dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java

@@ -1,5 +1,7 @@
 package datadog.smoketest.apmtracingdisabled;
 
+import datadog.trace.api.llmobs.LLMObs;
+import datadog.trace.api.llmobs.LLMObsSpan;
 import io.opentracing.Scope;
 import io.opentracing.Span;
 import io.opentracing.util.GlobalTracer;
@@ -81,6 +83,14 @@ public void write(
     }
   }
 
+  @GetMapping("/llmobs")
+  public String llmObs() {
+    LLMObsSpan span = LLMObs.startLLMSpan("llm-call", "gpt-4", "openai", null, null);
+    span.annotateIO("input", "output");
+    span.finish();
+    return "llmobs";
+  }
+
   @GetMapping("/late-outbound")
   public String lateOutbound(@RequestParam(name = "url") String url) {
     final Span span = GlobalTracer.get().activeSpan();

dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy

@@ -0,0 +1,59 @@
+package datadog.smoketest.apmtracingdisabled
+
+import datadog.trace.api.sampling.PrioritySampling
+import okhttp3.Request
+import spock.util.concurrent.PollingConditions
+
+/**
+ * With {@code DD_APM_TRACING_ENABLED=false}, LLM Observability keeps working — its spans go to the
+ * LLM Observability intake, which is independent of APM sampling — while the APM traces they ride
+ * on are dropped.
+ */
+class LlmObsApmTracingDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest {
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    final String[] processProperties = [
+      "-Ddd.apm.tracing.enabled=false",
+      "-Ddd.llmobs.enabled=true",
+      "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest",
+      "-Ddd.service.name=llmobs-apm-tracing-disabled-smoketest-app",
+    ]
+    return createProcess(processProperties)
+  }
+
+  @Override
+  protected String traceAgentProtocolVersion() {
+    return '0.4'
+  }
+
+  @Override
+  Closure decodedEvpProxyMessageCallback() {
+    return { String path, request ->
+      // The payload is msgpack; the path alone tells us the LLM Observability intake was reached.
+      return path.contains('api/v2/llmobs') ? path : null
+    }
+  }
+
+  void 'LLMObs spans reach the LLM Observability intake while the APM trace is dropped'() {
+    setup:
+    final url = "http://localhost:${httpPort}/rest-api/llmobs"
+    final request = new Request.Builder().url(url).get().build()
+
+    when:
+    final response = client.newCall(request).execute()
+
+    then:
+    response.successful
+    waitForTraceCount(1)
+
+    and: 'the APM trace is dropped'
+    checkRootSpanPrioritySampling(traces[0], PrioritySampling.SAMPLER_DROP)
+    hasApmDisabledTagOnEverySpan(traces[0])
+
+    and: 'the LLMObs span still reached the LLM Observability intake'
+    new PollingConditions(timeout: 30).eventually {
+      assert !evpProxyMessages.isEmpty()
+    }
+  }
+}

dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy

@@ -0,0 +1,38 @@
+package datadog.smoketest.apmtracingdisabled
+
+import okhttp3.Request
+
+/**
+ * {@code DD_TRACE_ENABLED=false} leaves no tracer to back LLM Observability spans, so the SDK must
+ * stay no-op instead of throwing out of instrumented application code.
+ */
+class LlmObsTraceDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest {
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    final String[] processProperties = [
+      "-Ddd.trace.enabled=false",
+      "-Ddd.llmobs.enabled=true",
+      "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest",
+      "-Ddd.service.name=llmobs-trace-disabled-smoketest-app",
+    ]
+    return createProcess(processProperties)
+  }
+
+  @Override
+  protected String traceAgentProtocolVersion() {
+    return '0.4'
+  }
+
+  void 'the LLMObs SDK stays safe to call when tracing is disabled'() {
+    setup:
+    final url = "http://localhost:${httpPort}/rest-api/llmobs"
+    final request = new Request.Builder().url(url).get().build()
+
+    when:
+    final response = client.newCall(request).execute()
+
+    then:
+    response.successful
+  }
+}

dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java

@@ -36,9 +36,16 @@ final class Builder {
     public static Sampler forConfig(final Config config, final TraceConfig traceConfig) {
       Sampler sampler;
       if (config != null) {
-        if (!config.isApmTracingEnabled() && isAsmEnabled(config)) {
-          log.debug("APM is disabled. Only 1 trace per minute will be sent.");
-          return new AsmStandaloneSampler(Clock.systemUTC());
+        if (!config.isApmTracingEnabled()) {
+          if (isAsmEnabled(config)) {
+            log.debug(
+                "APM tracing is disabled, but ASM is enabled. Only 1 APM trace per minute will be sent.");
+            return new AsmStandaloneSampler(Clock.systemUTC());
+          }
+          // No product needs a continuous APM trace, so drop them all. Products that keep their
+          // own traces (ASM, AI Guard) force-keep them, which this sampler cannot override.
+          log.debug("APM tracing is disabled. APM traces will be dropped.");
+          return new ForcePrioritySampler(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT);
         }
         final Map<String, String> serviceRules = config.getTraceSamplingServiceRules();
         final Map<String, String> operationRules = config.getTraceSamplingOperationRules();

dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java

@@ -4,10 +4,12 @@
 import static datadog.trace.api.config.AppSecConfig.APPSEC_SCA_ENABLED;
 import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED;
 import static datadog.trace.api.config.IastConfig.IAST_ENABLED;
+import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED;
 import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER;
 import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING;
 import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING_FORCE;
 import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE;
+import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP;
 import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -67,6 +69,23 @@ void asmStandaloneSamplerNotSelectedWhenApmTracingAndAsmNotEnabled() {
     assertFalse(sampler instanceof AsmStandaloneSampler);
   }
 
+  @WithConfig(key = APM_TRACING_ENABLED, value = "false")
+  @Test
+  void apmTracesDroppedWhenApmTracingDisabledAndNoOtherProductEnabled() {
+    assertApmTracesDropped();
+  }
+
+  /**
+   * LLM Observability rides the tracer but ships its spans to the LLM Observability intake, so
+   * disabling APM tracing must drop the APM traces without disabling the tracer.
+   */
+  @WithConfig(key = APM_TRACING_ENABLED, value = "false")
+  @WithConfig(key = LLMOBS_ENABLED, value = "true")
+  @Test
+  void apmTracesDroppedWhenApmTracingDisabledAndLlmObsEnabled() {
+    assertApmTracesDropped();
+  }
+
   @Test
   void asmStandaloneSamplerNotSelectedWhenApmTracingEnabledAndAsmNotEnabled() {
     Config config = Config.get();
@@ -165,4 +184,22 @@ void spansBuiltWithOtlpEnabledAndPrioritySamplingDisabledHaveNonUnsetSamplingPri
       tracer.close();
     }
   }
+
+  private static void assertApmTracesDropped() {
+    Sampler sampler = Sampler.Builder.forConfig(Config.get(), null);
+
+    assertInstanceOf(ForcePrioritySampler.class, sampler);
+
+    CoreTracer tracer = CoreTracer.builder().writer(new ListWriter()).sampler(sampler).build();
+    try {
+      DDSpan span = (DDSpan) tracer.buildSpan("datadog", "test").start();
+      ((PrioritySampler) sampler).setSamplingPriority(span);
+
+      assertEquals(SAMPLER_DROP, (int) span.getSamplingPriority());
+
+      span.finish();
+    } finally {
+      tracer.close();
+    }
+  }
 }