diff --git a/doc/development/testing_guide.md b/doc/development/testing_guide.md index d9b1a292a0d..0f731fe0b00 100644 --- a/doc/development/testing_guide.md +++ b/doc/development/testing_guide.md @@ -174,3 +174,14 @@ If the test breaks at some particular seed, then that seed will be shown in the as the `seed: NNN` parameter to your test, and you'll be able to run it for the same seed as long as you need until the test is fixed. Do not leave the `seed:` parameter when submitting your code, as it defeats the purpose of having the test randomized. + +You can also fix the seed for a whole test run without touching the code, by passing it as a +compile-time define: + +```bash +flutter test --dart-define=RANDOM_SEED=1234 +``` + +Every randomized test then starts from that seed, and each repeat of a test offsets it by the +repeat index so that the repeats stay distinct. This is what the flutter/tests customer testing run +uses to stay deterministic. diff --git a/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart b/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart index 15a6d4bd5fe..8ce0ae3d3bb 100644 --- a/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart +++ b/packages/flame/test/camera/viewports/fixed_aspect_ratio_viewport_test.dart @@ -45,7 +45,7 @@ void main() { game.add(camera); await game.ready(); - final random = Random(); + final random = Random(seedFromEnvironment(null)); for (var i = 0; i < 20; i++) { // keep it as a float32 value final width = prevFloat32(random.nextDouble() * 1000.0 + 10.0); diff --git a/packages/flame/test/effects/opacity_effect_test.dart b/packages/flame/test/effects/opacity_effect_test.dart index 299a708fd58..f880709a893 100644 --- a/packages/flame/test/effects/opacity_effect_test.dart +++ b/packages/flame/test/effects/opacity_effect_test.dart @@ -154,7 +154,7 @@ void main() { testWithFlameGame( 'fade out', (game) async { - final rng = Random(); + final rng = Random(seedFromEnvironment(null)); final component = _PaintComponent(); await game.ensureAdd(component); diff --git a/packages/flame_test/lib/src/random_test.dart b/packages/flame_test/lib/src/random_test.dart index bf4309cbdee..11a028a6239 100644 --- a/packages/flame_test/lib/src/random_test.dart +++ b/packages/flame_test/lib/src/random_test.dart @@ -8,10 +8,13 @@ final _seedGenerator = Random(); const _maxSeed = 1 << 32; /// Get the random seed for a test. If the [seed] parameter is passed in, -/// it takes precedence. Otherwise, if the environment variable -/// `RANDOM_SEED` is set, it is used. If neither is set, returns null. -/// Note: When using this, the `random_test_test` will fail because it will -/// use the same seed for all tests. This is expected. +/// it takes precedence. Otherwise, if the compile-time environment variable +/// `RANDOM_SEED` is set (`flutter test --dart-define=RANDOM_SEED=NNN`), it is +/// used. If neither is set, returns null. +/// +/// When `RANDOM_SEED` is set, every randomized test in the run becomes +/// deterministic: each test starts from that seed and each repeat of a test +/// offsets it by the repeat index. int? seedFromEnvironment(int? seed) { if (seed != null) { return seed; @@ -48,7 +51,10 @@ int? seedFromEnvironment(int? seed) { /// to use that specific seed. /// /// Optional parameter `repeatCount` allows the test to be repeated multiple -/// times, each time with a different seed. +/// times, each time with a different seed. When the seed is fixed, either +/// through the `seed` parameter or the `RANDOM_SEED` environment variable, +/// repeat number `i` uses `seed + i` so that the repeats stay distinct while +/// remaining reproducible. @isTest void testRandom( String name, @@ -65,7 +71,9 @@ void testRandom( assert(repeatCount > 0, 'repeatCount needs to be a positive number'); final resolvedSeed = seedFromEnvironment(seed); for (var i = 0; i < repeatCount; i++) { - final seed0 = resolvedSeed ?? _seedGenerator.nextInt(_maxSeed); + final seed0 = resolvedSeed != null + ? resolvedSeed + i + : _seedGenerator.nextInt(_maxSeed); test( '$name [seed=$seed0]', () => body(Random(seed0)), @@ -107,7 +115,8 @@ typedef TestWidgetsCallback = /// ``` /// Then if the test output shows that the test failed with seed `s`, /// simply adding parameter `seed=s` into the function will force it -/// to run for that specific seed. +/// to run for that specific seed. The `RANDOM_SEED` environment variable is +/// honored in the same way as for [testRandom]. @isTest void testWidgetsRandom( String description, @@ -118,10 +127,11 @@ void testWidgetsRandom( bool semanticsEnabled = true, dynamic tags, }) { - seed ??= _seedGenerator.nextInt(_maxSeed); + final resolvedSeed = + seedFromEnvironment(seed) ?? _seedGenerator.nextInt(_maxSeed); testWidgets( - '$description [seed=$seed]', - (WidgetTester widgetTester) => callback(Random(seed), widgetTester), + '$description [seed=$resolvedSeed]', + (WidgetTester widgetTester) => callback(Random(resolvedSeed), widgetTester), skip: skip, timeout: timeout, semanticsEnabled: semanticsEnabled, diff --git a/packages/flame_test/test/random_test_test.dart b/packages/flame_test/test/random_test_test.dart index 7533af7bf93..2a2cdca6ca0 100644 --- a/packages/flame_test/test/random_test_test.dart +++ b/packages/flame_test/test/random_test_test.dart @@ -15,11 +15,17 @@ void main() { for (var i = 0; i < 50; i++) { testRandom('a', (Random rnd) => seeds.add(rnd.nextInt(1000000))); } - test('verify', () { - final nTotal = seeds.length; - // Allow some seeds to coincide by pure luck - expect(seeds.toSet().length, greaterThanOrEqualTo(nTotal - 2)); - }); + test( + 'verify', + () { + final nTotal = seeds.length; + // Allow some seeds to coincide by pure luck + expect(seeds.toSet().length, greaterThanOrEqualTo(nTotal - 2)); + }, + skip: seedFromEnvironment(null) != null + ? 'RANDOM_SEED is set, so every test uses the same seed' + : null, + ); }); group('Uses specific seed', () { @@ -46,5 +52,23 @@ void main() { expect(seeds.toSet().length, greaterThanOrEqualTo(18)); }); }); + + group('Repeat count offsets a fixed seed', () { + final seeds = []; + testRandom( + 'd', + (Random rnd) => seeds.add(rnd.nextInt(1000000)), + seed: 123456, + repeatCount: 3, + ); + test('verify', () { + expect(seeds, [ + Random(123456).nextInt(1000000), + Random(123457).nextInt(1000000), + Random(123458).nextInt(1000000), + ]); + expect(seeds.toSet().length, 3); + }); + }); }); } diff --git a/scripts/customer_testing.dart b/scripts/customer_testing.dart index bfbf05b1470..f5704b25038 100644 --- a/scripts/customer_testing.dart +++ b/scripts/customer_testing.dart @@ -17,6 +17,12 @@ import 'dart:io'; // of the framework change under test, so it is excluded as well. const _excludedPackages = {'flame_3d', 'flame_forge2d'}; +// The purpose of this run is to catch regressions in flutter/flutter, so the +// randomized tests (see testRandom in flame_test) are pinned to a fixed seed to +// keep the run deterministic. Flame's own CI keeps running them with fresh +// seeds. +const _randomSeed = 20260922; + Future main() async { final packages = Directory('packages') @@ -45,7 +51,10 @@ Future main() async { (directory) => Directory('${directory.path}/test').existsSync(), )) { stdout.writeln('Running tests in ${package.path}'); - await _run('flutter', ['test'], workingDirectory: package.path); + await _run('flutter', [ + 'test', + '--dart-define=RANDOM_SEED=$_randomSeed', + ], workingDirectory: package.path); } }