From d8ed53f617d0a3367cbb1fa0895844e5faf063f5 Mon Sep 17 00:00:00 2001 From: cogwheel Date: Tue, 29 Sep 2026 00:25:19 +0530 Subject: [PATCH] Run the app test suite through combined entrypoints flutter test compiles test files one at a time and loads a full program for each, so with ~320 files most of a run was per-file overhead rather than tests. tool/run_test_shards.dart groups the files into 8 generated entrypoints (each file's tests run in a group named after it), keeps files that open real sockets standalone, runs them, and deletes the generated files afterwards. The shard count is fixed so CI and every machine group the same files and a cross-file failure reproduces anywhere. CI now runs it (with a 30-minute step timeout) and tracks the latest Flutter stable release instead of a pinned version. docs/BUILDING.md documents the runner and no longer says CI skips the analyzer and tests. --- .github/workflows/ci.yml | 16 +- .gitignore | 2 + docs/BUILDING.md | 11 +- test/shared/widgets/sheet_handle_test.dart | 23 --- tool/run_test_shards.dart | 161 +++++++++++++++++++++ 5 files changed, 175 insertions(+), 38 deletions(-) delete mode 100644 test/shared/widgets/sheet_handle_test.dart create mode 100644 tool/run_test_shards.dart diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b30455fcc..f2a5e697a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,8 @@ concurrency: permissions: contents: read -env: - # Pinned so a Flutter release cannot change what CI means overnight. Keep in - # step with the `environment` block in pubspec.yaml. - FLUTTER_VERSION: '3.47.1' - +# Flutter is the latest stable release (no version pin), so CI keeps up with +# what developers install. pubspec.yaml's `environment` sets the floor. jobs: # The gate that matters most during the extraction: the mobile app must stay # exactly as it was while code moves out from under it. @@ -30,7 +27,6 @@ jobs: submodules: recursive - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 with: - flutter-version: ${{ env.FLUTTER_VERSION }} channel: 'stable' cache: true # One `pub get` at the workspace root resolves every member. @@ -52,7 +48,10 @@ jobs: # the same ones), which want their own clean-up. The packages below are # clean and stay held to --fatal-infos. - run: flutter analyze --no-fatal-warnings --no-fatal-infos - - run: flutter test + # Compiling and loading each test file separately was most of this + # step; the runner groups the files into a few entrypoints. + - run: dart run tool/run_test_shards.dart + timeout-minutes: 30 # Each workspace member analyzes and tests on its own, which is what proves # the pub workspace actually works per package. @@ -73,7 +72,6 @@ jobs: submodules: recursive - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 with: - flutter-version: ${{ env.FLUTTER_VERSION }} channel: 'stable' cache: true - run: flutter pub get @@ -109,7 +107,6 @@ jobs: submodules: recursive - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 with: - flutter-version: ${{ env.FLUTTER_VERSION }} channel: 'stable' cache: true - run: flutter pub get @@ -134,7 +131,6 @@ jobs: submodules: recursive - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2 with: - flutter-version: ${{ env.FLUTTER_VERSION }} channel: 'stable' cache: true - run: flutter pub get diff --git a/.gitignore b/.gitignore index 7e028dab4..81193904c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,8 @@ app-store-screenshots/ # Flutter/Dart/Pub related **/doc/api/ +# Generated by tool/run_test_shards.dart and removed after each run. +/test/.shards_*/ **/ios/Flutter/.last_build_id .dart_tool/ .flutter-plugins diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 71e001c28..6af9545f7 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -96,13 +96,14 @@ iOS audio sessions. flutter pub get dart run build_runner build flutter analyze -flutter test +dart run tool/run_test_shards.dart ``` -`flutter analyze` and `flutter test` are the local gates before handing work -off. GitHub Actions only runs localization validation (`.github/workflows/l10n.yml`) -and releases (`.github/workflows/release.yml`). Nothing checks analyzer or test -health on every push, so run them yourself. +`flutter analyze` and the test suite are the local gates before handing work +off. `tool/run_test_shards.dart` runs every test file through a few combined +entrypoints, which is much faster than plain `flutter test`; use +`flutter test ` for a single file. CI (`.github/workflows/ci.yml`) runs +the analyzer and this test runner on pushes to `main` and on pull requests. Tests use `flutter_test` with `package:checks` for assertions and `mocktail` for mocks. Lints come from `flutter_lints` plus `riverpod_lint`. diff --git a/test/shared/widgets/sheet_handle_test.dart b/test/shared/widgets/sheet_handle_test.dart deleted file mode 100644 index aa055c5c2..000000000 --- a/test/shared/widgets/sheet_handle_test.dart +++ /dev/null @@ -1,23 +0,0 @@ -import 'package:conduit/shared/widgets/sheet_handle.dart'; -import 'package:material_ui/material_ui.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - group('SheetHandle', () { - testWidgets('renders as a centered widget', (tester) async { - await tester.pumpWidget( - const MaterialApp(home: Scaffold(body: SheetHandle())), - ); - - expect(find.byType(Center), findsOneWidget); - }); - - testWidgets('widget can be found by type', (tester) async { - await tester.pumpWidget( - const MaterialApp(home: Scaffold(body: SheetHandle())), - ); - - expect(find.byType(SheetHandle), findsOneWidget); - }); - }); -} diff --git a/tool/run_test_shards.dart b/tool/run_test_shards.dart new file mode 100644 index 000000000..361d9b803 --- /dev/null +++ b/tool/run_test_shards.dart @@ -0,0 +1,161 @@ +// Runs the app's whole test suite through a few combined entrypoints. +// +// dart run tool/run_test_shards.dart [--shards=N] [flutter test options] +// +// `flutter test` compiles test files one at a time and loads a full program +// for each, so with a few hundred files most of the run is per-file overhead +// rather than tests. This groups the files into N generated entrypoints in a +// temporary test/.shards_*/ directory (each file's tests run inside a group +// named after it), runs them, and deletes them afterwards. Options other than --shards go to +// `flutter test` unchanged. +// +// Files that share an entrypoint share an isolate. Files that talk to a real +// socket keep their own: the first widget test installs the test binding, +// whose HttpOverrides answer every request with 400. A test that passes on +// its own but fails here is relying on state an earlier file left behind +// (preferences, statics); make it set up what it relies on. +// +// Run a single file with plain `flutter test ` as usual. +import 'dart:io'; + +final RegExp _realSocketUse = RegExp( + r'HttpServer\.bind|ServerSocket\.bind|SecureServerSocket|HttpClient\(|' + r'Socket\.connect|WebSocket\.connect|RawDatagramSocket|HttpOverrides', +); + +Future main(List args) async { + // Fixed rather than derived from the core count, so CI and every machine + // group the same files and a failure caused by another file's leftover + // state reproduces anywhere. + var shardCount = 8; + final flutterArgs = []; + for (final arg in args) { + if (arg.startsWith('--shards=')) { + shardCount = int.parse(arg.substring('--shards='.length)); + if (shardCount < 1) { + stderr.writeln('--shards must be at least 1.'); + exitCode = 64; + return; + } + } else { + flutterArgs.add(arg); + } + } + + final sources = { + for (final file in Directory('test').listSync(recursive: true)) + if (file is File && file.path.endsWith('_test.dart')) + file.path.replaceAll(r'\', '/'): file.readAsStringSync(), + }; + final plan = planTestShards(sources, shardCount: shardCount); + + // Under test/ so flutter_test_config.dart applies; per run so concurrent + // runs in one checkout keep their own. + final shardDirectory = Directory('test').createTempSync('.shards_'); + final entrypoints = []; + try { + for (var index = 0; index < plan.shards.length; index++) { + final entrypoint = '${shardDirectory.path}/shard_$index.dart'; + File(entrypoint).writeAsStringSync(renderShard(plan.shards[index])); + entrypoints.add(entrypoint); + } + stdout.writeln( + 'Running ${sources.length} test files as ${plan.shards.length} shards ' + 'and ${plan.standalone.length} standalone files.', + ); + final flutter = await Process.start( + Platform.isWindows ? 'flutter.bat' : 'flutter', + ['test', ...entrypoints, ...plan.standalone, ...flutterArgs], + mode: ProcessStartMode.inheritStdio, + ); + exitCode = await flutter.exitCode; + } finally { + if (shardDirectory.existsSync()) shardDirectory.deleteSync(recursive: true); + } +} + +/// Test files grouped into combined entrypoints, plus the files that must +/// keep their own. +class TestShardPlan { + const TestShardPlan(this.shards, this.standalone); + + /// Each shard's test file paths, heaviest shard first. + final List> shards; + + /// Files that run as their own entrypoint. + final List standalone; +} + +/// Splits [sources] (test file path to contents) into at most [shardCount] +/// runs of alphabetically adjacent files with similar total size. Adding a +/// file then moves only files near a shard boundary, so each file keeps +/// nearly the same neighbours from run to run. Every file is scheduled +/// exactly once. +TestShardPlan planTestShards( + Map sources, { + required int shardCount, +}) { + final paths = sources.keys.toList()..sort(); + final standalone = [ + for (final path in paths) + if (_realSocketUse.hasMatch(sources[path]!)) path, + ]; + final shardable = [ + for (final path in paths) + if (!standalone.contains(path)) path, + ]; + + final total = shardable.fold(0, (sum, path) => sum + sources[path]!.length); + final shards = <({int size, List paths})>[]; + var current = []; + var size = 0; + var cumulative = 0; + for (final path in shardable) { + final length = sources[path]!.length; + // Cut where the running total lands nearest the next even split. + final boundary = total * (shards.length + 1) / shardCount; + if (current.isNotEmpty && + cumulative + length - boundary > boundary - cumulative) { + shards.add((size: size, paths: current)); + current = []; + size = 0; + } + current.add(path); + size += length; + cumulative += length; + } + if (current.isNotEmpty) shards.add((size: size, paths: current)); + shards.sort((a, b) => b.size.compareTo(a.size)); + + final plan = TestShardPlan([ + for (final shard in shards) shard.paths, + ], standalone); + final scheduled = [...plan.shards.expand((paths) => paths), ...standalone]; + if (scheduled.length != paths.length || + !scheduled.toSet().containsAll(paths)) { + throw StateError('Shard plan does not schedule every test file once.'); + } + return plan; +} + +/// The source of a shard entrypoint that runs each of [paths] (relative to +/// the package root) in a group named after the file. +String renderShard(List paths) { + final buffer = StringBuffer() + ..writeln('// Generated by tool/run_test_shards.dart. Do not edit.') + ..writeln("import 'package:flutter_test/flutter_test.dart';") + ..writeln(); + for (var index = 0; index < paths.length; index++) { + final relative = paths[index].replaceFirst('test/', '../'); + buffer.writeln("import '$relative' as t$index;"); + } + buffer + ..writeln() + ..writeln('void main() {'); + for (var index = 0; index < paths.length; index++) { + final name = paths[index].replaceFirst('test/', ''); + buffer.writeln(" group('$name', t$index.main);"); + } + buffer.writeln('}'); + return buffer.toString(); +}