From 5292d88cacb7e4410ed0ac53b135694d6154d7dd Mon Sep 17 00:00:00 2001 From: Lukas Klingsbo Date: Tue, 22 Sep 2026 09:12:04 +0200 Subject: [PATCH] feat: Lock random seeds in the flutter/tests customer testing run The customer testing script now passes a fixed RANDOM_SEED define to flutter test so that the run is deterministic. To make that work, the testRandom helper offsets a fixed seed by the repeat index instead of reusing it, testWidgetsRandom honors the define too, and the two tests that created an unseeded Random directly now seed it from the environment. --- doc/development/testing_guide.md | 11 ++++++ .../fixed_aspect_ratio_viewport_test.dart | 2 +- .../test/effects/opacity_effect_test.dart | 2 +- packages/flame_test/lib/src/random_test.dart | 30 ++++++++++------ .../flame_test/test/random_test_test.dart | 34 ++++++++++++++++--- scripts/customer_testing.dart | 11 +++++- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/doc/development/testing_guide.md b/doc/development/testing_guide.md index d9b1a292a0d..0f731fe0b00 100644 --- a/doc/development/testing_guide.md +++ b/doc/development/testing_guide.md @@ -174,3 +174,14 @@ If the test breaks at some particular seed, then that seed will be shown in the as the `seed: NNN` parameter to your test, and you'll be able to run it for the same seed as long as you need until the test is fixed. Do not leave the `seed:` parameter when submitting your code, as it defeats the purpose of having the test randomized. + +You can also fix the seed for a whole test run without touching the code, by passing it as a +compile-time define: + +```bash +flutter test --dart-define=RANDOM_SEED=1234 +``` + +Every randomized test then starts from that seed, and each repeat of a test offsets it by the +repeat index so that the repeats stay distinct. This is what the flutter/tests customer testing run +uses to stay deterministic. diff --git a/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart b/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart index 15a6d4bd5fe..8ce0ae3d3bb 100644 --- a/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart +++ b/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart @@ -45,7 +45,7 @@ void main() { game.add(camera); await game.ready(); - final random = Random(); + final random = Random(seedFromEnvironment(null)); for (var i = 0; i < 20; i++) { // keep it as a float32 value final width = prevFloat32(random.nextDouble() * 1000.0 + 10.0); diff --git a/packages/flame/test/effects/opacity_effect_test.dart b/packages/flame/test/effects/opacity_effect_test.dart index 299a708fd58..f880709a893 100644 --- a/packages/flame/test/effects/opacity_effect_test.dart +++ b/packages/flame/test/effects/opacity_effect_test.dart @@ -154,7 +154,7 @@ void main() { testWithFlameGame( 'fade out', (game) async { - final rng = Random(); + final rng = Random(seedFromEnvironment(null)); final component = _PaintComponent(); await game.ensureAdd(component); diff --git a/packages/flame_test/lib/src/random_test.dart b/packages/flame_test/lib/src/random_test.dart index bf4309cbdee..11a028a6239 100644 --- a/packages/flame_test/lib/src/random_test.dart +++ b/packages/flame_test/lib/src/random_test.dart @@ -8,10 +8,13 @@ final _seedGenerator = Random(); const _maxSeed = 1 << 32; /// Get the random seed for a test. If the [seed] parameter is passed in, -/// it takes precedence. Otherwise, if the environment variable -/// `RANDOM_SEED` is set, it is used. If neither is set, returns null. -/// Note: When using this, the `random_test_test` will fail because it will -/// use the same seed for all tests. This is expected. +/// it takes precedence. Otherwise, if the compile-time environment variable +/// `RANDOM_SEED` is set (`flutter test --dart-define=RANDOM_SEED=NNN`), it is +/// used. If neither is set, returns null. +/// +/// When `RANDOM_SEED` is set, every randomized test in the run becomes +/// deterministic: each test starts from that seed and each repeat of a test +/// offsets it by the repeat index. int? seedFromEnvironment(int? seed) { if (seed != null) { return seed; @@ -48,7 +51,10 @@ int? seedFromEnvironment(int? seed) { /// to use that specific seed. /// /// Optional parameter `repeatCount` allows the test to be repeated multiple -/// times, each time with a different seed. +/// times, each time with a different seed. When the seed is fixed, either +/// through the `seed` parameter or the `RANDOM_SEED` environment variable, +/// repeat number `i` uses `seed + i` so that the repeats stay distinct while +/// remaining reproducible. @isTest void testRandom( String name, @@ -65,7 +71,9 @@ void testRandom( assert(repeatCount > 0, 'repeatCount needs to be a positive number'); final resolvedSeed = seedFromEnvironment(seed); for (var i = 0; i < repeatCount; i++) { - final seed0 = resolvedSeed ?? _seedGenerator.nextInt(_maxSeed); + final seed0 = resolvedSeed != null + ? resolvedSeed + i + : _seedGenerator.nextInt(_maxSeed); test( '$name [seed=$seed0]', () => body(Random(seed0)), @@ -107,7 +115,8 @@ typedef TestWidgetsCallback = /// ``` /// Then if the test output shows that the test failed with seed `s`, /// simply adding parameter `seed=s` into the function will force it -/// to run for that specific seed. +/// to run for that specific seed. The `RANDOM_SEED` environment variable is +/// honored in the same way as for [testRandom]. @isTest void testWidgetsRandom( String description, @@ -118,10 +127,11 @@ void testWidgetsRandom( bool semanticsEnabled = true, dynamic tags, }) { - seed ??= _seedGenerator.nextInt(_maxSeed); + final resolvedSeed = + seedFromEnvironment(seed) ?? _seedGenerator.nextInt(_maxSeed); testWidgets( - '$description [seed=$seed]', - (WidgetTester widgetTester) => callback(Random(seed), widgetTester), + '$description [seed=$resolvedSeed]', + (WidgetTester widgetTester) => callback(Random(resolvedSeed), widgetTester), skip: skip, timeout: timeout, semanticsEnabled: semanticsEnabled, diff --git a/packages/flame_test/test/random_test_test.dart b/packages/flame_test/test/random_test_test.dart index 7533af7bf93..2a2cdca6ca0 100644 --- a/packages/flame_test/test/random_test_test.dart +++ b/packages/flame_test/test/random_test_test.dart @@ -15,11 +15,17 @@ void main() { for (var i = 0; i < 50; i++) { testRandom('a', (Random rnd) => seeds.add(rnd.nextInt(1000000))); } - test('verify', () { - final nTotal = seeds.length; - // Allow some seeds to coincide by pure luck - expect(seeds.toSet().length, greaterThanOrEqualTo(nTotal - 2)); - }); + test( + 'verify', + () { + final nTotal = seeds.length; + // Allow some seeds to coincide by pure luck + expect(seeds.toSet().length, greaterThanOrEqualTo(nTotal - 2)); + }, + skip: seedFromEnvironment(null) != null + ? 'RANDOM_SEED is set, so every test uses the same seed' + : null, + ); }); group('Uses specific seed', () { @@ -46,5 +52,23 @@ void main() { expect(seeds.toSet().length, greaterThanOrEqualTo(18)); }); }); + + group('Repeat count offsets a fixed seed', () { + final seeds = []; + testRandom( + 'd', + (Random rnd) => seeds.add(rnd.nextInt(1000000)), + seed: 123456, + repeatCount: 3, + ); + test('verify', () { + expect(seeds, [ + Random(123456).nextInt(1000000), + Random(123457).nextInt(1000000), + Random(123458).nextInt(1000000), + ]); + expect(seeds.toSet().length, 3); + }); + }); }); } diff --git a/scripts/customer_testing.dart b/scripts/customer_testing.dart index bfbf05b1470..f5704b25038 100644 --- a/scripts/customer_testing.dart +++ b/scripts/customer_testing.dart @@ -17,6 +17,12 @@ import 'dart:io'; // of the framework change under test, so it is excluded as well. const _excludedPackages = {'flame_3d', 'flame_forge2d'}; +// The purpose of this run is to catch regressions in flutter/flutter, so the +// randomized tests (see testRandom in flame_test) are pinned to a fixed seed to +// keep the run deterministic. Flame's own CI keeps running them with fresh +// seeds. +const _randomSeed = 20260922; + Future main() async { final packages = Directory('packages') @@ -45,7 +51,10 @@ Future main() async { (directory) => Directory('${directory.path}/test').existsSync(), )) { stdout.writeln('Running tests in ${package.path}'); - await _run('flutter', ['test'], workingDirectory: package.path); + await _run('flutter', [ + 'test', + '--dart-define=RANDOM_SEED=$_randomSeed', + ], workingDirectory: package.path); } }