From b8665597597099ad0cf6bc7e342c233ef9e9871f Mon Sep 17 00:00:00 2001 From: "federico.mon" Date: Fri, 11 Sep 2026 14:25:13 +0000 Subject: [PATCH 1/2] feat(ci_visibility): dynamic atr retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add dynamic Auto Test Retries (ATR) budgets based on test duration, instead of the flat per-test retry limit. When enabled, the number of retries allowed for a test is determined by the duration of its initial attempt, using the same duration buckets as Early Flake Detection (5s / 10s / 30s / 5m / >5m). Two new env vars control the behavior: - DD_CIVISIBILITY_DYNAMIC_ATR_ENABLED — enables duration-based ATR budgets. When unset/false, ATR stays on its existing flat-limit path. - DD_CIVISIBILITY_DYNAMIC_ATR_BUCKETS — optionally overrides the five duration-based retry budgets with five positive comma-separated integers in [1, 20]. When unset/empty, the EFD retry settings from the backend are used. The new handler classifies each test once (by its initial-attempt duration) and caches the resulting max-retries count for the lifetime of that test. The duration bucket index is computed via shared helpers on the EFD settings type, so bucket boundaries stay consistent with EFD (the existing EFD retry handler is refactored to use them too). Telemetry: records a `dynamic_atr_retries.enabled` count metric with a `has_custom_buckets` tag when dynamic ATR is enabled. Feature parity with dd-trace-py PR #20028. --- .../civisibility/CiVisibilitySystem.java | 6 +- .../config/EarlyFlakeDetectionSettings.java | 34 ++ .../civisibility/execution/AutoTestRetry.java | 2 +- .../execution/DynamicAutoTestRetry.java | 49 +++ .../execution/EarlyFlakeDetection.java | 22 +- .../civisibility/test/ExecutionStrategy.java | 74 +++- .../buildsystem/ProxyTestModuleTest.groovy | 3 +- .../headless/HeadlessTestModuleTest.groovy | 3 +- .../headless/HeadlessTestSessionTest.groovy | 3 +- .../execution/EarlyFlakeDetectionTest.groovy | 16 +- .../test/ExecutionStrategyTest.groovy | 4 +- .../execution/DynamicAutoTestRetryTest.java | 362 ++++++++++++++++++ .../trace/api/config/CiVisibilityConfig.java | 4 + .../main/java/datadog/trace/api/Config.java | 20 + .../telemetry/CiVisibilityCountMetric.java | 5 +- .../telemetry/tag/HasCustomBuckets.java | 12 + metadata/supported-configurations.json | 16 + 17 files changed, 604 insertions(+), 31 deletions(-) create mode 100644 dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/DynamicAutoTestRetry.java create mode 100644 dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java create mode 100644 internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/HasCustomBuckets.java diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java index af7f653e921..e43f8812d4a 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java @@ -276,7 +276,8 @@ private static TestFrameworkSession.Factory childTestFrameworkSessionFactory( services.config, executionSettings, repoServices.sourcePathResolver, - services.linesResolver); + services.linesResolver, + services.metricCollector); // only add report upload capability for children sessions, // because report upload is only supported when the build system is instrumented @@ -319,7 +320,8 @@ private static TestFrameworkSession.Factory headlessTestFrameworkSessionFactory( services.config, executionSettings, repoServices.sourcePathResolver, - services.linesResolver); + services.linesResolver, + services.metricCollector); return new HeadlessTestSession( projectName, startTime, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettings.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettings.java index b11e0a4cb53..17d47bb684e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettings.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettings.java @@ -40,6 +40,40 @@ public List getExecutionsByDuration() { return executionsByDuration; } + /** + * Returns the EFD retry-bucket index for an initial test duration. Bucket boundaries match the EFD + * duration buckets: {@code <=5s -> 0, <=10s -> 1, <=30s -> 2, <=5m -> 3, >5m -> 4}. + */ + public int retryBucketIndexForDuration(long durationMillis) { + if (durationMillis <= 5_000) { + return 0; + } + if (durationMillis <= 10_000) { + return 1; + } + if (durationMillis <= 30_000) { + return 2; + } + if (durationMillis <= 300_000) { + return 3; + } + return 4; + } + + /** + * Returns the configured retry budget for an initial test duration, based on the backend-provided + * {@code executionsByDuration} list. Returns 0 if no bucket matches (e.g. test ran longer than the + * longest configured duration). + */ + public int retriesForDuration(long durationMillis) { + for (ExecutionsByDuration e : executionsByDuration) { + if (durationMillis <= e.getDurationMillis()) { + return e.getExecutions(); + } + } + return 0; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/AutoTestRetry.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/AutoTestRetry.java index 9dabc20c7c3..bc980e5afaa 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/AutoTestRetry.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/AutoTestRetry.java @@ -17,7 +17,7 @@ "TestExecutionPolicy instances are confined to a single thread and are not meant to be thread-safe") public class AutoTestRetry implements TestExecutionPolicy { - private final int maxExecutions; + protected int maxExecutions; private final boolean suppressFailures; private int executions; private ExecutionAggregation results; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/DynamicAutoTestRetry.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/DynamicAutoTestRetry.java new file mode 100644 index 00000000000..a6aa5cdd954 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/DynamicAutoTestRetry.java @@ -0,0 +1,49 @@ +package datadog.trace.civisibility.execution; + +import datadog.trace.api.civisibility.execution.TestStatus; +import datadog.trace.api.civisibility.telemetry.tag.RetryReason; +import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Execution policy for dynamic Auto Test Retries (ATR). Instead of a flat per-test retry limit, the + * number of retries is determined by the duration of the initial attempt, using the same duration + * buckets as Early Flake Detection. When custom buckets are provided they override the EFD retry + * settings; otherwise the EFD settings from the backend are used. + */ +public class DynamicAutoTestRetry extends AutoTestRetry { + + private final EarlyFlakeDetectionSettings efdSettings; + private final int[] customBuckets; // null = use EFD settings + private boolean maxExecutionsDetermined = false; + + public DynamicAutoTestRetry( + EarlyFlakeDetectionSettings efdSettings, + int[] customBuckets, + boolean suppressFailures, + AtomicInteger totalRetryCount) { + super(Integer.MAX_VALUE, suppressFailures, totalRetryCount); + this.efdSettings = efdSettings; + this.customBuckets = customBuckets; + } + + @Override + public ExecutionOutcome registerExecution(TestStatus status, long durationMillis) { + if (!maxExecutionsDetermined) { + maxExecutions = computeMaxExecutions(durationMillis); + maxExecutionsDetermined = true; + } + return super.registerExecution(status, durationMillis); + } + + private int computeMaxExecutions(long durationMillis) { + int retries; + if (customBuckets != null) { + int index = efdSettings.retryBucketIndexForDuration(durationMillis); + retries = customBuckets[index]; + } else { + retries = efdSettings.retriesForDuration(durationMillis); + } + return Math.max(1, retries) + 1; // +1 for the initial attempt + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/EarlyFlakeDetection.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/EarlyFlakeDetection.java index 9e0819e0ca0..3c6d375f832 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/EarlyFlakeDetection.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/EarlyFlakeDetection.java @@ -4,8 +4,7 @@ import datadog.trace.api.civisibility.execution.TestExecutionPolicy; import datadog.trace.api.civisibility.execution.TestStatus; import datadog.trace.api.civisibility.telemetry.tag.RetryReason; -import datadog.trace.civisibility.config.ExecutionsByDuration; -import java.util.List; +import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings; /** * Execution policy for Early Flake Detection. Runs a new or modified test case multiple times to @@ -15,18 +14,18 @@ public class EarlyFlakeDetection implements TestExecutionPolicy { private final boolean suppressFailures; - private final List executionsByDuration; + private final EarlyFlakeDetectionSettings efdSettings; private int executions; private int maxExecutions; private ExecutionAggregation results; private TestStatus lastStatus; public EarlyFlakeDetection( - List executionsByDuration, boolean suppressFailures) { + EarlyFlakeDetectionSettings efdSettings, boolean suppressFailures) { this.suppressFailures = suppressFailures; - this.executionsByDuration = executionsByDuration; + this.efdSettings = efdSettings; this.executions = 0; - this.maxExecutions = getExecutions(0); + this.maxExecutions = efdSettings.retriesForDuration(0); this.results = ExecutionAggregation.NONE; } @@ -35,7 +34,7 @@ public ExecutionOutcome registerExecution(TestStatus status, long durationMillis lastStatus = status; ++executions; results = results.withExecution(status); - int maxExecutionsForGivenDuration = getExecutions(durationMillis); + int maxExecutionsForGivenDuration = efdSettings.retriesForDuration(durationMillis); maxExecutions = Math.min(maxExecutions, maxExecutionsForGivenDuration); boolean lastExecution = !retriesLeft(); @@ -80,15 +79,6 @@ public boolean propagateFailure() { return !suppressFailures && results == ExecutionAggregation.MIXED; } - private int getExecutions(long durationMillis) { - for (ExecutionsByDuration e : executionsByDuration) { - if (durationMillis <= e.getDurationMillis()) { - return e.getExecutions(); - } - } - return 0; - } - @Override public boolean failedTestReplayApplicable() { return false; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java index dc89fbe7d97..17fe8e51d2c 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java @@ -6,6 +6,9 @@ import datadog.trace.api.civisibility.config.TestMetadata; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.execution.TestExecutionPolicy; +import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; +import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; +import datadog.trace.api.civisibility.telemetry.tag.HasCustomBuckets; import datadog.trace.api.civisibility.telemetry.tag.SkipReason; import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings; import datadog.trace.civisibility.config.ExecutionSettings; @@ -13,6 +16,7 @@ import datadog.trace.civisibility.config.TestSetting; import datadog.trace.civisibility.execution.AttemptToFix; import datadog.trace.civisibility.execution.AutoTestRetry; +import datadog.trace.civisibility.execution.DynamicAutoTestRetry; import datadog.trace.civisibility.execution.EarlyFlakeDetection; import datadog.trace.civisibility.execution.Quarantine; import datadog.trace.civisibility.execution.Regular; @@ -38,16 +42,29 @@ public class ExecutionStrategy { @Nonnull private final ExecutionSettings executionSettings; @Nonnull private final SourcePathResolver sourcePathResolver; @Nonnull private final LinesResolver linesResolver; + @Nonnull private final CiVisibilityMetricCollector metricCollector; + private final int[] dynamicAtrBuckets; public ExecutionStrategy( @Nonnull Config config, @Nonnull ExecutionSettings executionSettings, @Nonnull SourcePathResolver sourcePathResolver, - @Nonnull LinesResolver linesResolver) { + @Nonnull LinesResolver linesResolver, + @Nonnull CiVisibilityMetricCollector metricCollector) { this.config = config; this.executionSettings = executionSettings; this.sourcePathResolver = sourcePathResolver; this.linesResolver = linesResolver; + this.metricCollector = metricCollector; + this.dynamicAtrBuckets = parseDynamicAtrBuckets(config.getCiVisibilityDynamicAtrBuckets()); + + if (config.isCiVisibilityDynamicAtrEnabled() + && executionSettings.isFlakyTestRetriesEnabled()) { + metricCollector.add( + CiVisibilityCountMetric.DYNAMIC_ATR_RETRIES_ENABLED, + 1, + dynamicAtrBuckets != null ? HasCustomBuckets.TRUE : null); + } } @Nonnull @@ -135,13 +152,20 @@ public TestExecutionPolicy executionPolicy( // but we don't care if we go "a bit" over the limit, it does not have to be precise earlyFlakeDetectionsUsed.incrementAndGet(); return new EarlyFlakeDetection( - executionSettings.getEarlyFlakeDetectionSettings().getExecutionsByDuration(), + executionSettings.getEarlyFlakeDetectionSettings(), isQuarantined(test)); } if (isAutoRetryApplicable(test)) { // check-then-act with "autoRetriesUsed" is not atomic here, // but we don't care if we go "a bit" over the limit, it does not have to be precise + if (config.isCiVisibilityDynamicAtrEnabled()) { + return new DynamicAutoTestRetry( + executionSettings.getEarlyFlakeDetectionSettings(), + dynamicAtrBuckets, + isQuarantined(test), + autoRetriesUsed); + } return new AutoTestRetry( config.getCiVisibilityFlakyRetryCount(), isQuarantined(test), autoRetriesUsed); } @@ -247,4 +271,50 @@ public int executionPriority(@Nullable TestIdentifier test, @Nonnull TestSourceD // then the rest return 0; } + + private static final int RETRY_BUCKET_COUNT = 5; + private static final int MAX_RETRIES_PER_BUCKET = 20; + + /** + * Parses the {@code DD_CIVISIBILITY_DYNAMIC_ATR_BUCKETS} env var into five positive integers in + * [1, 20]. Returns {@code null} if the value is unset/empty or invalid (wrong count, + * non-integer, out of range) — in which case the EFD retry settings are used as fallback. + */ + private static int[] parseDynamicAtrBuckets(String rawBuckets) { + if (rawBuckets == null || rawBuckets.isEmpty()) { + return null; + } + String[] parts = rawBuckets.split(","); + if (parts.length != RETRY_BUCKET_COUNT) { + LOGGER.warn( + "Invalid {} value '{}'; expected five comma-separated integers in [1, {}]", + "DD_CIVISIBILITY_DYNAMIC_ATR_BUCKETS", + rawBuckets, + MAX_RETRIES_PER_BUCKET); + return null; + } + int[] buckets = new int[RETRY_BUCKET_COUNT]; + try { + for (int i = 0; i < RETRY_BUCKET_COUNT; i++) { + int value = Integer.parseInt(parts[i].trim()); + if (value < 1 || value > MAX_RETRIES_PER_BUCKET) { + LOGGER.warn( + "Invalid {} value '{}'; expected five comma-separated integers in [1, {}]", + "DD_CIVISIBILITY_DYNAMIC_ATR_BUCKETS", + rawBuckets, + MAX_RETRIES_PER_BUCKET); + return null; + } + buckets[i] = value; + } + } catch (NumberFormatException e) { + LOGGER.warn( + "Invalid {} value '{}'; expected five comma-separated integers in [1, {}]", + "DD_CIVISIBILITY_DYNAMIC_ATR_BUCKETS", + rawBuckets, + MAX_RETRIES_PER_BUCKET); + return null; + } + return buckets; + } } diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/buildsystem/ProxyTestModuleTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/buildsystem/ProxyTestModuleTest.groovy index b5310c20298..043e54a1f4b 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/buildsystem/ProxyTestModuleTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/buildsystem/ProxyTestModuleTest.groovy @@ -9,6 +9,7 @@ import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.api.civisibility.coverage.CoverageStore import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector +import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector import datadog.trace.civisibility.codeowners.Codeowners import datadog.trace.civisibility.config.ExecutionSettings import datadog.trace.civisibility.coverage.report.child.ChildProcessCoverageReporter @@ -30,7 +31,7 @@ class ProxyTestModuleTest extends DDSpecification { config.getCiVisibilityFlakyRetryCount() >> 2 // this counts all executions of a test case (first attempt is counted too) config.getCiVisibilityTotalFlakyRetryCount() >> 2 // this counts retries across all tests (first attempt is not a retry, so it is not counted) - def executionStrategy = new ExecutionStrategy(config, executionSettings, Stub(SourcePathResolver), Stub(LinesResolver)) + def executionStrategy = new ExecutionStrategy(config, executionSettings, Stub(SourcePathResolver), Stub(LinesResolver), NoOpMetricCollector.INSTANCE) def traceId = Stub(DDTraceId) traceId.toLong() >> 123 diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestModuleTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestModuleTest.groovy index 3cbe1b6c571..9117aeab596 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestModuleTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestModuleTest.groovy @@ -6,6 +6,7 @@ import datadog.trace.api.civisibility.config.TestSourceData import datadog.trace.api.civisibility.coverage.CoverageStore import datadog.trace.api.civisibility.execution.TestStatus import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector +import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext import datadog.trace.civisibility.codeowners.Codeowners import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings @@ -58,7 +59,7 @@ class HeadlessTestModuleTest extends SpanWriterTest { config.getCiVisibilityTotalFlakyRetryCount() >> 2 // this counts retries across all tests (first attempt is not a retry, so it is not counted) - def executionStrategy = new ExecutionStrategy(config, executionSettings, Stub(SourcePathResolver), Stub(LinesResolver)) + def executionStrategy = new ExecutionStrategy(config, executionSettings, Stub(SourcePathResolver), Stub(LinesResolver), NoOpMetricCollector.INSTANCE) new HeadlessTestModule( Stub(AgentSpanContext), diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestSessionTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestSessionTest.groovy index 2b0ddfeeadb..a9b7f06c40b 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestSessionTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestSessionTest.groovy @@ -5,6 +5,7 @@ import datadog.trace.api.Config import datadog.trace.api.DDSpanTypes import datadog.trace.api.civisibility.coverage.CoverageStore import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector +import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector import datadog.trace.api.civisibility.telemetry.tag.Provider import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.civisibility.codeowners.Codeowners @@ -47,7 +48,7 @@ class HeadlessTestSessionTest extends SpanWriterTest { def executionSettings = Stub(ExecutionSettings) executionSettings.getTestManagementSettings() >> new TestManagementSettings(true, 10) - def executionStrategy = new ExecutionStrategy(Stub(Config), executionSettings, Stub(SourcePathResolver), Stub(LinesResolver)) + def executionStrategy = new ExecutionStrategy(Stub(Config), executionSettings, Stub(SourcePathResolver), Stub(LinesResolver), NoOpMetricCollector.INSTANCE) new HeadlessTestSession( "project-name", diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/execution/EarlyFlakeDetectionTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/execution/EarlyFlakeDetectionTest.groovy index 7addf0e724a..d6fe000aa81 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/execution/EarlyFlakeDetectionTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/execution/EarlyFlakeDetectionTest.groovy @@ -3,6 +3,7 @@ package datadog.trace.civisibility.execution import datadog.trace.api.civisibility.execution.ExecutionAggregation import datadog.trace.api.civisibility.execution.TestStatus import datadog.trace.api.civisibility.telemetry.tag.RetryReason +import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings import datadog.trace.civisibility.config.ExecutionsByDuration import spock.lang.Specification @@ -10,7 +11,8 @@ class EarlyFlakeDetectionTest extends Specification { def "test EFD exits on flake"() { setup: - def executionPolicy = new EarlyFlakeDetection([new ExecutionsByDuration(Long.MAX_VALUE, 3)], false) + def efdSettings = new EarlyFlakeDetectionSettings(false, [new ExecutionsByDuration(Long.MAX_VALUE, 3)], -1) + def executionPolicy = new EarlyFlakeDetection(efdSettings, false) when: def outcome = executionPolicy.registerExecution(TestStatus.fail, 0) @@ -35,7 +37,8 @@ class EarlyFlakeDetectionTest extends Specification { def "test EFD failed all executions"() { setup: - def executionPolicy = new EarlyFlakeDetection([new ExecutionsByDuration(Long.MAX_VALUE, 3)], false) + def efdSettings = new EarlyFlakeDetectionSettings(false, [new ExecutionsByDuration(Long.MAX_VALUE, 3)], -1) + def executionPolicy = new EarlyFlakeDetection(efdSettings, false) when: def outcome = executionPolicy.registerExecution(TestStatus.fail, 0) @@ -70,7 +73,8 @@ class EarlyFlakeDetectionTest extends Specification { def "test EFD succeeded all executions"() { setup: - def executionPolicy = new EarlyFlakeDetection([new ExecutionsByDuration(Long.MAX_VALUE, 3)], false) + def efdSettings = new EarlyFlakeDetectionSettings(false, [new ExecutionsByDuration(Long.MAX_VALUE, 3)], -1) + def executionPolicy = new EarlyFlakeDetection(efdSettings, false) when: def outcome = executionPolicy.registerExecution(TestStatus.pass, 0) @@ -105,7 +109,8 @@ class EarlyFlakeDetectionTest extends Specification { def "test EFD adaptive retry count"() { when: - def executionPolicy = new EarlyFlakeDetection([new ExecutionsByDuration(100, 3), new ExecutionsByDuration(Long.MAX_VALUE, 1)], false) + def efdSettings = new EarlyFlakeDetectionSettings(false, [new ExecutionsByDuration(100, 3), new ExecutionsByDuration(Long.MAX_VALUE, 1)], -1) + def executionPolicy = new EarlyFlakeDetection(efdSettings, false) then: !executionPolicy.registerExecution(TestStatus.fail, 0).lastExecution() @@ -113,7 +118,8 @@ class EarlyFlakeDetectionTest extends Specification { executionPolicy.registerExecution(TestStatus.fail, 0).lastExecution() when: - executionPolicy = new EarlyFlakeDetection([new ExecutionsByDuration(100, 3), new ExecutionsByDuration(Long.MAX_VALUE, 1)], false) + efdSettings = new EarlyFlakeDetectionSettings(false, [new ExecutionsByDuration(100, 3), new ExecutionsByDuration(Long.MAX_VALUE, 1)], -1) + executionPolicy = new EarlyFlakeDetection(efdSettings, false) then: !executionPolicy.registerExecution(TestStatus.fail, 0).lastExecution() diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/test/ExecutionStrategyTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/test/ExecutionStrategyTest.groovy index e01e57315fc..cc8f752eee5 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/test/ExecutionStrategyTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/test/ExecutionStrategyTest.groovy @@ -11,6 +11,7 @@ import datadog.trace.api.civisibility.telemetry.tag.SkipReason import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings import datadog.trace.civisibility.config.ExecutionSettings import datadog.trace.civisibility.config.TestManagementSettings +import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector import datadog.trace.civisibility.execution.AttemptToFix import datadog.trace.civisibility.source.LinesResolver import datadog.trace.civisibility.source.SourcePathResolver @@ -126,7 +127,8 @@ class ExecutionStrategyTest extends Specification { config, executionSettings, resolver, - linesResolver + linesResolver, + NoOpMetricCollector.INSTANCE ) } } diff --git a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java new file mode 100644 index 00000000000..001afe2f5d3 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java @@ -0,0 +1,362 @@ +package datadog.trace.civisibility.execution; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.Config; +import datadog.trace.api.civisibility.execution.TestStatus; +import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; +import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric; +import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; +import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricData; +import datadog.trace.api.civisibility.telemetry.TagValue; +import datadog.trace.api.civisibility.telemetry.tag.HasCustomBuckets; +import datadog.trace.civisibility.config.EarlyFlakeDetectionSettings; +import datadog.trace.civisibility.config.ExecutionSettings; +import datadog.trace.civisibility.config.ExecutionsByDuration; +import datadog.trace.civisibility.source.LinesResolver; +import datadog.trace.civisibility.source.SourcePathResolver; +import datadog.trace.civisibility.test.ExecutionStrategy; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class DynamicAutoTestRetryTest { + + private static final int BUCKET_5S_MILLIS = 5_000; + private static final int BUCKET_10S_MILLIS = 10_000; + private static final int BUCKET_30S_MILLIS = 30_000; + private static final int BUCKET_5M_MILLIS = 300_000; + + private static EarlyFlakeDetectionSettings efdSettings( + int retries5s, int retries10s, int retries30s, int retries5m) { + List durations = + Arrays.asList( + new ExecutionsByDuration(BUCKET_5S_MILLIS, retries5s), + new ExecutionsByDuration(BUCKET_10S_MILLIS, retries10s), + new ExecutionsByDuration(BUCKET_30S_MILLIS, retries30s), + new ExecutionsByDuration(BUCKET_5M_MILLIS, retries5m)); + return new EarlyFlakeDetectionSettings(false, durations, -1); + } + + private static ExecutionSettings executionSettings( + boolean autoRetryEnabled, EarlyFlakeDetectionSettings efdSettings) { + return new ExecutionSettings( + false, + false, + false, + autoRetryEnabled, + false, + false, + false, + efdSettings, + datadog.trace.civisibility.config.TestManagementSettings.DEFAULT, + null, + Collections.emptyMap(), + Collections.emptyMap(), + null, + null, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + datadog.trace.civisibility.diff.LineDiff.EMPTY, + datadog.trace.civisibility.config.ConfigurationErrors.NONE); + } + + // ---- DynamicAutoTestRetry unit tests ---- + + @ParameterizedTest(name = "duration={0}ms -> retries={1}") + @MethodSource("efdBucketsProvider") + void testDynamicAtrUsesEfdRetryBudgets(long durationMillis, int expectedRetries) { + EarlyFlakeDetectionSettings settings = efdSettings(10, 2, 3, 4); + AtomicInteger totalRetries = new AtomicInteger(0); + DynamicAutoTestRetry policy = new DynamicAutoTestRetry(settings, null, false, totalRetries); + + int retries = 0; + while (true) { + policy.registerExecution(TestStatus.fail, durationMillis); + if (!policy.applicable()) { + break; + } + retries++; + } + assertEquals(expectedRetries, retries); + } + + static Stream efdBucketsProvider() { + return Stream.of( + arguments(1_000L, 10), // 5s bucket -> 10 retries + arguments(6_000L, 2), // 10s bucket -> 2 retries + arguments(31_000L, 4), // 5m bucket (31s > 30s) -> 4 retries + arguments(301_000L, 1) // >5m bucket -> 0 retries, but max(1,0)=1 + ); + } + + @ParameterizedTest(name = "duration={0}ms -> retries={1}") + @MethodSource("customBucketsProvider") + void testDynamicAtrUsesCustomRetryBudgets(long durationMillis, int expectedRetries) { + EarlyFlakeDetectionSettings settings = efdSettings(10, 2, 3, 4); + AtomicInteger totalRetries = new AtomicInteger(0); + int[] customBuckets = {4, 1, 1, 1, 1}; + DynamicAutoTestRetry policy = + new DynamicAutoTestRetry(settings, customBuckets, false, totalRetries); + + int retries = 0; + while (true) { + policy.registerExecution(TestStatus.fail, durationMillis); + if (!policy.applicable()) { + break; + } + retries++; + } + assertEquals(expectedRetries, retries); + } + + static Stream customBucketsProvider() { + return Stream.of( + arguments(1_000L, 4), // 5s bucket -> 4 retries (custom) + arguments(6_000L, 1), // 10s bucket -> 1 retry (custom) + arguments(31_000L, 1), // 30s bucket -> 1 retry (custom) + arguments(301_000L, 1) // >5m bucket -> 1 retry (custom) + ); + } + + @Test + void testDynamicAtrStopsAfterFirstPass() { + EarlyFlakeDetectionSettings settings = efdSettings(5, 1, 1, 1); + AtomicInteger totalRetries = new AtomicInteger(0); + int[] customBuckets = {5, 1, 1, 1, 1}; + DynamicAutoTestRetry policy = + new DynamicAutoTestRetry(settings, customBuckets, false, totalRetries); + + // first attempt fails + policy.registerExecution(TestStatus.fail, 1_000); + assertTrue(policy.applicable()); + + // second attempt passes -> should stop + policy.registerExecution(TestStatus.pass, 1_000); + assertFalse(policy.applicable()); + assertEquals(1, totalRetries.get()); + } + + @Test + void testDynamicAtrClassificationIsCached() { + EarlyFlakeDetectionSettings settings = efdSettings(10, 2, 3, 4); + AtomicInteger totalRetries = new AtomicInteger(0); + DynamicAutoTestRetry policy = + new DynamicAutoTestRetry(settings, null, false, totalRetries); + + // first attempt: 1s duration -> 5s bucket -> 10 retries + policy.registerExecution(TestStatus.fail, 1_000); + assertTrue(policy.applicable()); + + // second attempt: 600s duration -> should still use 10 retries (initial duration cached) + policy.registerExecution(TestStatus.fail, 600_000); + assertTrue(policy.applicable()); + } + + @Test + void testDynamicAtrMinOneRetry() { + // EFD bucket with 0 retries for >5m -> max(1, 0) = 1 retry + EarlyFlakeDetectionSettings settings = efdSettings(10, 2, 3, 4); + AtomicInteger totalRetries = new AtomicInteger(0); + DynamicAutoTestRetry policy = + new DynamicAutoTestRetry(settings, null, false, totalRetries); + + policy.registerExecution(TestStatus.fail, 600_000); + assertTrue(policy.applicable()); + + policy.registerExecution(TestStatus.fail, 600_000); + assertFalse(policy.applicable()); + assertEquals(1, totalRetries.get()); + } + + // ---- ExecutionStrategy bucket parsing + telemetry tests ---- + + private Config mockConfig( + boolean dynamicAtrEnabled, + String dynamicAtrBuckets, + boolean flakyRetryEnabled) { + Config config = mock(Config.class); + when(config.isCiVisibilityDynamicAtrEnabled()).thenReturn(dynamicAtrEnabled); + when(config.getCiVisibilityDynamicAtrBuckets()).thenReturn(dynamicAtrBuckets); + when(config.isCiVisibilityFlakyRetryEnabled()).thenReturn(flakyRetryEnabled); + when(config.getCiVisibilityFlakyRetryCount()).thenReturn(5); + when(config.getCiVisibilityTotalFlakyRetryCount()).thenReturn(1000); + return config; + } + + private ExecutionStrategy newStrategy( + Config config, ExecutionSettings settings, CiVisibilityMetricCollector collector) { + return new ExecutionStrategy( + config, + settings, + mock(SourcePathResolver.class), + mock(LinesResolver.class), + collector); + } + + @Test + void testExecutionStrategyParsesValidBucketsAndRecordsTelemetryWithCustomBuckets() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "3,1,1,1,1", true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertTrue(collector.recordedDynamicAtrWithCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithoutCustomBuckets); + } + + @Test + void testExecutionStrategyNoBucketsRecordsTelemetryWithoutCustomBuckets() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, null, true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertTrue(collector.recordedDynamicAtrWithoutCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + } + + @Test + void testExecutionStrategyEmptyBucketsRecordsTelemetryWithoutCustomBuckets() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "", true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertTrue(collector.recordedDynamicAtrWithoutCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + } + + @Test + void testExecutionStrategyInvalidBucketsFallsBackToNull() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "not,enough,values", true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertTrue(collector.recordedDynamicAtrWithoutCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + } + + @Test + void testExecutionStrategyOutOfRangeBucketsFallsBackToNull() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "21,4,1,1,1", true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertTrue(collector.recordedDynamicAtrWithoutCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + } + + @Test + void testExecutionStrategyZeroValueBucketsFallsBackToNull() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "10,4,0,1,1", true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertTrue(collector.recordedDynamicAtrWithoutCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + } + + @Test + void testExecutionStrategyDisabledDoesNotRecordTelemetry() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(false, null, true); + newStrategy(config, executionSettings(true, efdSettings(10, 2, 3, 4)), collector); + + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithoutCustomBuckets); + } + + @Test + void testExecutionStrategyAutoRetryDisabledDoesNotRecordTelemetry() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "3,1,1,1,1", true); + // auto retry disabled in backend settings + newStrategy(config, executionSettings(false, efdSettings(10, 2, 3, 4)), collector); + + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithoutCustomBuckets); + } + + // ---- EarlyFlakeDetectionSettings helper tests ---- + + @ParameterizedTest(name = "duration={0}ms -> index={1}") + @MethodSource("bucketIndexProvider") + void testRetryBucketIndexForDuration(long durationMillis, int expectedIndex) { + EarlyFlakeDetectionSettings settings = EarlyFlakeDetectionSettings.DEFAULT; + assertEquals(expectedIndex, settings.retryBucketIndexForDuration(durationMillis)); + } + + static Stream bucketIndexProvider() { + return Stream.of( + arguments(0L, 0), + arguments(5_000L, 0), + arguments(5_001L, 1), + arguments(10_000L, 1), + arguments(10_001L, 2), + arguments(30_000L, 2), + arguments(30_001L, 3), + arguments(300_000L, 3), + arguments(300_001L, 4)); + } + + @Test + void testEfdRetriesForDurationUsesExecutionsByDuration() { + EarlyFlakeDetectionSettings settings = efdSettings(10, 2, 3, 4); + assertEquals(10, settings.retriesForDuration(1_000)); + assertEquals(2, settings.retriesForDuration(6_000)); + assertEquals(3, settings.retriesForDuration(20_000)); + assertEquals(4, settings.retriesForDuration(31_000)); // 31s > 30s, falls into 5m bucket + assertEquals(4, settings.retriesForDuration(300_000)); // exactly 5m + assertEquals(0, settings.retriesForDuration(301_000)); // >5m, no bucket matches + assertEquals(0, settings.retriesForDuration(600_000)); + } + + // ---- Test doubles ---- + + private static class TestMetricCollector implements CiVisibilityMetricCollector { + boolean recordedDynamicAtrWithCustomBuckets = false; + boolean recordedDynamicAtrWithoutCustomBuckets = false; + + @Override + public void add(CiVisibilityCountMetric metric, long value, TagValue... tags) { + if (metric == CiVisibilityCountMetric.DYNAMIC_ATR_RETRIES_ENABLED) { + boolean hasCustomBuckets = false; + for (TagValue tag : tags) { + if (tag == HasCustomBuckets.TRUE) { + hasCustomBuckets = true; + break; + } + } + if (hasCustomBuckets) { + recordedDynamicAtrWithCustomBuckets = true; + } else { + recordedDynamicAtrWithoutCustomBuckets = true; + } + } + } + + @Override + public void add(CiVisibilityDistributionMetric metric, int value, TagValue... tags) {} + + @Override + public void prepareMetrics() {} + + @Override + public Collection drain() { + return Collections.emptyList(); + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java index a42de2ebc98..b5bd8c2318e 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java @@ -69,6 +69,10 @@ public final class CiVisibilityConfig { public static final String CIVISIBILITY_FLAKY_RETRY_COUNT = "civisibility.flaky.retry.count"; public static final String CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT = "civisibility.total.flaky.retry.count"; + public static final String CIVISIBILITY_DYNAMIC_ATR_ENABLED = + "civisibility.dynamic.atr.enabled"; + public static final String CIVISIBILITY_DYNAMIC_ATR_BUCKETS = + "civisibility.dynamic.atr.buckets"; public static final String CIVISIBILITY_MODULE_NAME = "civisibility.module.name"; public static final String CIVISIBILITY_TEST_COMMAND = "civisibility.test.command"; public static final String CIVISIBILITY_TELEMETRY_ENABLED = "civisibility.telemetry.enabled"; diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index a97f7bda2d0..2436209385b 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -265,6 +265,8 @@ import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_LINES_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_REPORT_DUMP_DIR; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_REPORT_UPLOAD_ENABLED; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_DYNAMIC_ATR_BUCKETS; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_DYNAMIC_ATR_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_ROOT_PACKAGES_LIMIT; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_AUTO_CONFIGURATION_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_VERSION; @@ -1214,6 +1216,8 @@ public static String getHostName() { private final boolean ciVisibilityFlakyRetryOnlyKnownFlakes; private final int ciVisibilityFlakyRetryCount; private final int ciVisibilityTotalFlakyRetryCount; + private final boolean ciVisibilityDynamicAtrEnabled; + private final String ciVisibilityDynamicAtrBuckets; private final boolean ciVisibilityEarlyFlakeDetectionEnabled; private final int ciVisibilityEarlyFlakeDetectionLowerLimit; private final String ciVisibilitySessionName; @@ -2866,6 +2870,9 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) ciVisibilityFlakyRetryCount = configProvider.getInteger(CIVISIBILITY_FLAKY_RETRY_COUNT, 5); ciVisibilityTotalFlakyRetryCount = configProvider.getInteger(CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT, 1000); + ciVisibilityDynamicAtrEnabled = + configProvider.getBoolean(CIVISIBILITY_DYNAMIC_ATR_ENABLED, false); + ciVisibilityDynamicAtrBuckets = configProvider.getString(CIVISIBILITY_DYNAMIC_ATR_BUCKETS); ciVisibilitySessionName = configProvider.getString(TEST_SESSION_NAME); ciVisibilityModuleName = configProvider.getString(CIVISIBILITY_MODULE_NAME); ciVisibilityTestCommand = configProvider.getString(CIVISIBILITY_TEST_COMMAND); @@ -4770,6 +4777,14 @@ public int getCiVisibilityTotalFlakyRetryCount() { return ciVisibilityTotalFlakyRetryCount; } + public boolean isCiVisibilityDynamicAtrEnabled() { + return ciVisibilityDynamicAtrEnabled; + } + + public String getCiVisibilityDynamicAtrBuckets() { + return ciVisibilityDynamicAtrBuckets; + } + public String getCiVisibilitySessionName() { return ciVisibilitySessionName; } @@ -7078,6 +7093,11 @@ public String toString() { + otlpTracesTimeout + ", ciVisibilityGradleDependencyVerificationEnabled=" + ciVisibilityGradleDependencyVerificationEnabled + + ", ciVisibilityDynamicAtrEnabled=" + + ciVisibilityDynamicAtrEnabled + + ", ciVisibilityDynamicAtrBuckets='" + + ciVisibilityDynamicAtrBuckets + + '\'' + ", serviceDiscoveryEnabled=" + serviceDiscoveryEnabled + ", sfnInjectDatadogAttributeEnabled=" diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java index 43d98bf6955..f46b14c03d1 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java @@ -19,6 +19,7 @@ import datadog.trace.api.civisibility.telemetry.tag.GitProviderExpected; import datadog.trace.api.civisibility.telemetry.tag.GitShaDiscrepancyType; import datadog.trace.api.civisibility.telemetry.tag.GitShaMatch; +import datadog.trace.api.civisibility.telemetry.tag.HasCustomBuckets; import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner; import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries; import datadog.trace.api.civisibility.telemetry.tag.ImpactedTestsDetectionEnabled; @@ -180,7 +181,9 @@ public enum CiVisibilityCountMetric { COVERAGE_UPLOAD_REQUEST("coverage_upload.request", RequestCompressed.class), /** The number of coverage upload requests that errored */ COVERAGE_UPLOAD_REQUEST_ERRORS( - "coverage_upload.request_errors", ErrorType.class, StatusCode.class); + "coverage_upload.request_errors", ErrorType.class, StatusCode.class), + /** Recorded once per session when dynamic, duration-based ATR retries are enabled */ + DYNAMIC_ATR_RETRIES_ENABLED("dynamic_atr_retries.enabled", HasCustomBuckets.class); // need a "holder" class, as accessing static fields from enum constructors is illegal static class IndexHolder { diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/HasCustomBuckets.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/HasCustomBuckets.java new file mode 100644 index 00000000000..68200870765 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/HasCustomBuckets.java @@ -0,0 +1,12 @@ +package datadog.trace.api.civisibility.telemetry.tag; + +import datadog.trace.api.civisibility.telemetry.TagValue; + +public enum HasCustomBuckets implements TagValue { + TRUE; + + @Override + public String asString() { + return "has_custom_buckets:true"; + } +} diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 1892f63130c..281f0ac436a 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -643,6 +643,22 @@ "aliases": [] } ], + "DD_CIVISIBILITY_DYNAMIC_ATR_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], + "DD_CIVISIBILITY_DYNAMIC_ATR_BUCKETS": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED": [ { "version": "A", From 9d5eaeeea63ec451f200e8cfcef98f1bcdb37527 Mon Sep 17 00:00:00 2001 From: "federico.mon" Date: Fri, 11 Sep 2026 15:03:41 +0000 Subject: [PATCH 2/2] Fix dynamic ATR bucket validation --- .../civisibility/test/ExecutionStrategy.java | 2 +- .../execution/DynamicAutoTestRetryTest.java | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java index 17fe8e51d2c..c175222115d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java @@ -284,7 +284,7 @@ private static int[] parseDynamicAtrBuckets(String rawBuckets) { if (rawBuckets == null || rawBuckets.isEmpty()) { return null; } - String[] parts = rawBuckets.split(","); + String[] parts = rawBuckets.split(",", -1); if (parts.length != RETRY_BUCKET_COUNT) { LOGGER.warn( "Invalid {} value '{}'; expected five comma-separated integers in [1, {}]", diff --git a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java index 001afe2f5d3..2d6c57e1ce6 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java +++ b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/execution/DynamicAutoTestRetryTest.java @@ -10,6 +10,10 @@ import static org.mockito.Mockito.when; import datadog.trace.api.Config; +import datadog.trace.api.civisibility.config.TestFQN; +import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.config.TestSourceData; +import datadog.trace.api.civisibility.execution.TestExecutionPolicy; import datadog.trace.api.civisibility.execution.TestStatus; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric; @@ -249,6 +253,27 @@ void testExecutionStrategyInvalidBucketsFallsBackToNull() { assertFalse(collector.recordedDynamicAtrWithCustomBuckets); } + @Test + void testExecutionStrategyTrailingCommaBucketsFallbackToEfdWithoutCustomBucketTelemetry() { + TestMetricCollector collector = new TestMetricCollector(); + Config config = mockConfig(true, "3,1,1,1,1,", true); + ExecutionStrategy strategy = + newStrategy(config, executionSettings(true, efdSettings(1, 2, 3, 4)), collector); + + TestExecutionPolicy policy = + strategy.executionPolicy( + new TestIdentifier(new TestFQN("suite", "name"), null), + TestSourceData.UNKNOWN, + Collections.emptyList()); + policy.registerExecution(TestStatus.fail, 1_000); + assertTrue(policy.applicable()); + policy.registerExecution(TestStatus.fail, 1_000); + + assertFalse(policy.applicable()); + assertTrue(collector.recordedDynamicAtrWithoutCustomBuckets); + assertFalse(collector.recordedDynamicAtrWithCustomBuckets); + } + @Test void testExecutionStrategyOutOfRangeBucketsFallsBackToNull() { TestMetricCollector collector = new TestMetricCollector();