[pigeon] Split CI and developer test scripts (#3076)

* Split into two files with no changes

* Adjust to fix compilation

* Duplicate run_tests to test with no changes

* Rework run_tests.dart

* Extract sharable code

* Make test.dart run every supported test

* Update all docs

* Update for changes in base PR

* Analysis, missed tests

* Move custom-test to heavy workload due to OOM

* Re-merge Cirrus configuration

Co-authored-by: Tarrin Neal <tarrinneal@gmail.com>

.cirrus.yml

@@ -183,15 +183,6 @@ task:
       env:
         CIRRUS_CLONE_SUBMODULES: true
       script: ./script/tool_runner.sh update-excerpts --fail-on-change
-    - name: linux-custom_package_tests
-      env:
-        PATH: $PATH:/usr/local/bin
-        matrix:
-          CHANNEL: "master"
-          CHANNEL: "stable"
-      << : *INSTALL_CHROME_LINUX
-      local_tests_script:
-        - ./script/tool_runner.sh custom-test
     ### Web tasks ###
     - name: web-build_all_packages
       env:
@@ -237,6 +228,16 @@ task:
     cpu: 4
     memory: 16G
   matrix:
+    ### Platform-agnostic tasks ###
+    - name: linux-custom_package_tests
+      env:
+        PATH: $PATH:/usr/local/bin
+        matrix:
+          CHANNEL: "master"
+          CHANNEL: "stable"
+      << : *INSTALL_CHROME_LINUX
+      local_tests_script:
+        - ./script/tool_runner.sh custom-test
     ### Android tasks ###
     - name: android-platform_tests
       # Don't run full platform tests on both channels in pre-submit.

packages/pigeon/CONTRIBUTING.md

@@ -35,7 +35,7 @@ generators with that AST.
 ## Testing Overview
 
 Pigeon has 3 types of tests, you'll find them all in
-[run_tests.dart](./tool/run_tests.dart).
+[test.dart](./tool/test.dart).
 
 * Unit tests - These are the fastest tests that are just typical unit tests,
   they may be generating code and checking it against a regular expression to

packages/pigeon/platform_tests/README.md

@@ -1,11 +1,11 @@
 # Native Pigeon Tests
 
 This directory contains native test harnesses for native and end-to-end tests
-of Pigeon-generated code. The [test script](../tool/run_tests.dart) generates
+of Pigeon-generated code. The [test script](../tool/test.dart) generates
 native code from [pigeons/](../pigeons/) into the native test scaffolding, and
 then drives the tests there.
 
-To run these tests, use [`run_tests.dart`](../tool/run_tests.dart).
+To run these tests, use [`test.dart`](../tool/test.dart).
 
 Alternately, if you are running them directly (e.g., from within a platform
 IDE), you can use [`generate.dart`](../tool/generate.dart) to generate the

packages/pigeon/platform_tests/alternate_language_test_plugin/README.md

@@ -3,4 +3,4 @@
 Tests for languages not covered by `test_plugin`.
 See [the `platform_tests` README](../README.md) for details.
 
-To run these tests, use [`run_tests.dart`](../tool/run_tests.dart)
+To run these tests, use [`test.dart`](../tool/test.dart)

packages/pigeon/platform_tests/flutter_null_safe_unit_tests/lib/.gitignore

@@ -1,5 +1,5 @@
 # TODO(stuartmorgan) Remove this file when these are no longer generated;
-# see the TODO in _runFlutterUnitTests in run_tests.dart
+# see the TODO in _runFlutterUnitTests in test_suites.dart
 async_handlers.gen.dart
 host2flutter.gen.dart
 list.gen.dart

packages/pigeon/tool/run_tests.dart

@@ -5,553 +5,95 @@
 // ignore_for_file: avoid_print
 
 ////////////////////////////////////////////////////////////////////////////////
-/// Script for executing the Pigeon tests
+/// CI entrypoint for running Pigeon tests.
 ///
-/// usage: dart run tool/run_tests.dart
+/// For any use other than CI, use test.dart instead.
 ////////////////////////////////////////////////////////////////////////////////
-import 'dart:io' show File, Directory, Platform, exit;
+import 'dart:io' show Platform, exit;
-import 'dart:math';
 
-import 'package:args/args.dart';
+import 'shared/test_runner.dart';
-import 'package:meta/meta.dart';
+import 'shared/test_suites.dart';
-import 'package:path/path.dart' as p;
 
-import 'shared/flutter_utils.dart';
+/// Exits with failure if any tests in [testSuites] are not included in any of
-import 'shared/generation.dart';
+/// the given test [shards].
-import 'shared/native_project_runners.dart';
+void _validateTestCoverage(List<List<String>> shards) {
-import 'shared/process_utils.dart';
+  final Set<String> missing = testSuites.keys.toSet();
+  shards.forEach(missing.removeAll);
 
-const String _testFlag = 'test';
+  if (missing.isNotEmpty) {
-const String _listFlag = 'list';
+    print('The following test suites are not being run on any host:');
-const String _skipGenerationFlag = 'skip-generation';
+    for (final String suite in missing) {
-
+      print('  $suite');
-const int _noDeviceAvailableExitCode = 100;
-
-const String _testPluginRelativePath = 'platform_tests/test_plugin';
-const String _alternateLanguageTestPluginRelativePath =
-    'platform_tests/alternate_language_test_plugin';
-const String _integrationTestFileRelativePath = 'integration_test/test.dart';
-
-@immutable
-class _TestInfo {
-  const _TestInfo({required this.function, this.description});
-  final Future<int> Function() function;
-  final String? description;
-}
-
-// Test suite names.
-const String androidJavaUnitTests = 'android_java_unittests';
-const String androidJavaIntegrationTests = 'android_java_integration_tests';
-const String androidKotlinUnitTests = 'android_kotlin_unittests';
-const String androidKotlinIntegrationTests = 'android_kotlin_integration_tests';
-const String iOSObjCUnitTests = 'ios_objc_unittests';
-const String iOSObjCUnitTestsLegacy = 'ios_objc_legacy_unittests';
-const String iOSObjCIntegrationTests = 'ios_objc_integration_tests';
-const String iOSSwiftUnitTests = 'ios_swift_unittests';
-const String iOSSwiftIntegrationTests = 'ios_swift_integration_tests';
-const String macOSSwiftUnitTests = 'macos_swift_unittests';
-const String macOSSwiftIntegrationTests = 'macos_swift_integration_tests';
-const String windowsUnitTests = 'windows_unittests';
-const String windowsIntegrationTests = 'windows_integration_tests';
-const String dartUnitTests = 'dart_unittests';
-const String flutterUnitTests = 'flutter_unittests';
-const String mockHandlerTests = 'mock_handler_tests';
-const String commandLineTests = 'command_line_tests';
-
-const Map<String, _TestInfo> _tests = <String, _TestInfo>{
-  windowsUnitTests: _TestInfo(
-      function: _runWindowsUnitTests,
-      description: 'Unit tests on generated Windows C++ code.'),
-  windowsIntegrationTests: _TestInfo(
-      function: _runWindowsIntegrationTests,
-      description: 'Integration tests on generated Windows C++ code.'),
-  androidJavaUnitTests: _TestInfo(
-      function: _runAndroidJavaUnitTests,
-      description: 'Unit tests on generated Java code.'),
-  androidJavaIntegrationTests: _TestInfo(
-      function: _runAndroidJavaIntegrationTests,
-      description: 'Integration tests on generated Java code.'),
-  androidKotlinUnitTests: _TestInfo(
-      function: _runAndroidKotlinUnitTests,
-      description: 'Unit tests on generated Kotlin code.'),
-  androidKotlinIntegrationTests: _TestInfo(
-      function: _runAndroidKotlinIntegrationTests,
-      description: 'Integration tests on generated Kotlin code.'),
-  dartUnitTests: _TestInfo(
-      function: _runDartUnitTests,
-      description: "Unit tests on and analysis on Pigeon's implementation."),
-  flutterUnitTests: _TestInfo(
-      function: _runFlutterUnitTests,
-      description: 'Unit tests on generated Dart code.'),
-  iOSObjCUnitTests: _TestInfo(
-      function: _runIOSObjCUnitTests,
-      description: 'Unit tests on generated Objective-C code.'),
-  iOSObjCUnitTestsLegacy: _TestInfo(
-      function: _runIOSObjCLegacyUnitTests,
-      description:
-          'Unit tests on generated Objective-C code (legacy test harness).'),
-  iOSObjCIntegrationTests: _TestInfo(
-      function: _runIOSObjCIntegrationTests,
-      description: 'Integration tests on generated Objective-C code.'),
-  iOSSwiftUnitTests: _TestInfo(
-      function: _runIOSSwiftUnitTests,
-      description: 'Unit tests on generated Swift code.'),
-  iOSSwiftIntegrationTests: _TestInfo(
-      function: _runIOSSwiftIntegrationTests,
-      description: 'Integration tests on generated Swift code.'),
-  macOSSwiftUnitTests: _TestInfo(
-      function: _runMacOSSwiftUnitTests,
-      description: 'Unit tests on generated Swift code on macOS.'),
-  macOSSwiftIntegrationTests: _TestInfo(
-      function: _runMacOSSwiftIntegrationTests,
-      description: 'Integration tests on generated Swift code on macOS.'),
-  mockHandlerTests: _TestInfo(
-      function: _runMockHandlerTests,
-      description: 'Unit tests on generated Dart mock handler code.'),
-  commandLineTests: _TestInfo(
-      function: _runCommandLineTests,
-      description: 'Tests running pigeon with various command-line options.'),
-};
-
-Future<int> _runAndroidJavaUnitTests() async {
-  return _runAndroidUnitTests(_alternateLanguageTestPluginRelativePath);
-}
-
-Future<int> _runAndroidJavaIntegrationTests() async {
-  return _runMobileIntegrationTests(
-      'Android', _alternateLanguageTestPluginRelativePath);
-}
-
-Future<int> _runAndroidKotlinUnitTests() async {
-  return _runAndroidUnitTests(_testPluginRelativePath);
-}
-
-Future<int> _runAndroidUnitTests(String testPluginPath) async {
-  final String examplePath = './$testPluginPath/example';
-  final String androidProjectPath = '$examplePath/android';
-  final File gradleFile = File(p.join(androidProjectPath, 'gradlew'));
-  if (!gradleFile.existsSync()) {
-    final int compileCode = await runFlutterBuild(examplePath, 'apk');
-    if (compileCode != 0) {
-      return compileCode;
     }
+    exit(1);
   }
-
-  return runGradleBuild(androidProjectPath, 'testDebugUnitTest');
-}
-
-Future<int> _runAndroidKotlinIntegrationTests() async {
-  return _runMobileIntegrationTests('Android', _testPluginRelativePath);
-}
-
-Future<int> _runMobileIntegrationTests(
-    String platform, String testPluginPath) async {
-  final String? device = await getDeviceForPlatform(platform.toLowerCase());
-  if (device == null) {
-    print('No $platform device available. Attach an $platform device or start '
-        'an emulator/simulator to run integration tests');
-    return _noDeviceAvailableExitCode;
-  }
-
-  final String examplePath = './$testPluginPath/example';
-  return runFlutterCommand(
-    examplePath,
-    'test',
-    <String>[_integrationTestFileRelativePath, '-d', device],
-  );
-}
-
-Future<int> _runDartUnitTests() async {
-  int exitCode = await runProcess('dart', <String>['analyze', 'bin']);
-  if (exitCode != 0) {
-    return exitCode;
-  }
-  exitCode = await runProcess('dart', <String>['analyze', 'lib']);
-  if (exitCode != 0) {
-    return exitCode;
-  }
-  exitCode = await runProcess('dart', <String>['test']);
-  return exitCode;
-}
-
-/// Generates multiple dart files based on the jobs defined in [jobs] which is
-/// in the format of (key: input pigeon file path, value: output dart file
-/// path).
-Future<int> _generateDart(Map<String, String> jobs) async {
-  for (final MapEntry<String, String> job in jobs.entries) {
-    // TODO(gaaclarke): Make this run the jobs in parallel.  A bug in Dart
-    // blocked this (https://github.com/dart-lang/pub/pull/3285).
-    final int result = await runPigeon(input: job.key, dartOut: job.value);
-    if (result != 0) {
-      return result;
-    }
-  }
-  return 0;
-}
-
-Future<int> _analyzeFlutterUnitTests(String flutterUnitTestsPath) async {
-  final String messagePath = '$flutterUnitTestsPath/lib/message.gen.dart';
-  final String messageTestPath = '$flutterUnitTestsPath/test/message_test.dart';
-  final int generateTestCode = await runPigeon(
-    input: 'pigeons/message.dart',
-    dartOut: messagePath,
-    dartTestOut: messageTestPath,
-  );
-  if (generateTestCode != 0) {
-    return generateTestCode;
-  }
-
-  final int analyzeCode =
-      await runFlutterCommand(flutterUnitTestsPath, 'analyze');
-  if (analyzeCode != 0) {
-    return analyzeCode;
-  }
-
-  // Delete these files that were just generated to help with the analyzer step.
-  File(messagePath).deleteSync();
-  File(messageTestPath).deleteSync();
-  return 0;
-}
-
-Future<int> _runFlutterUnitTests() async {
-  // TODO(stuartmorgan): Migrate Dart unit tests to use the generated output in
-  // shared_test_plugin_code instead of having multiple copies of generation.
-  const String flutterUnitTestsPath =
-      'platform_tests/flutter_null_safe_unit_tests';
-  // Files from the pigeons/ directory to generate output for.
-  const List<String> inputPigeons = <String>[
-    'flutter_unittests',
-    'core_tests',
-    'primitive',
-    'multiple_arity',
-    'non_null_fields',
-    'null_fields',
-    'nullable_returns',
-    // TODO(stuartmorgan): Eliminate these files by ensuring that everything
-    // they are intended to cover is in core_tests.dart (or, if necessary in
-    // the short term due to limitations in non-Dart generators, a single other
-    // file). They aren't being unit tested, only analyzed.
-    'async_handlers',
-    'host2flutter',
-    'list',
-    'message',
-    'void_arg_flutter',
-    'void_arg_host',
-    'voidflutter',
-    'voidhost',
-  ];
-  final int generateCode = await _generateDart(<String, String>{
-    for (final String name in inputPigeons)
-      'pigeons/$name.dart': '$flutterUnitTestsPath/lib/$name.gen.dart'
-  });
-  if (generateCode != 0) {
-    return generateCode;
-  }
-
-  final int analyzeCode = await _analyzeFlutterUnitTests(flutterUnitTestsPath);
-  if (analyzeCode != 0) {
-    return analyzeCode;
-  }
-
-  final int testCode = await runFlutterCommand(flutterUnitTestsPath, 'test');
-  if (testCode != 0) {
-    return testCode;
-  }
-
-  return 0;
-}
-
-Future<int> _runIOSObjCUnitTests() async {
-  return _runIOSPluginUnitTests(_alternateLanguageTestPluginRelativePath);
-}
-
-// TODO(stuartmorgan): Remove this, and the ios_unit_tests directory, once
-// _runIOSObjCUnitTests works in CI; see
-// https://github.com/flutter/packages/pull/2816.
-Future<int> _runIOSObjCLegacyUnitTests() async {
-  return _runIOSProjectUnitTests('platform_tests/ios_unit_tests');
-}
-
-Future<int> _runIOSObjCIntegrationTests() async {
-  final String? device = await getDeviceForPlatform('ios');
-  if (device == null) {
-    print('No iOS device available. Attach an iOS device or start '
-        'a simulator to run integration tests');
-    return _noDeviceAvailableExitCode;
-  }
-
-  const String examplePath =
-      './$_alternateLanguageTestPluginRelativePath/example';
-  return runFlutterCommand(
-    examplePath,
-    'test',
-    <String>[_integrationTestFileRelativePath, '-d', device],
-  );
-}
-
-Future<int> _runMacOSSwiftUnitTests() async {
-  const String examplePath = './$_testPluginRelativePath/example';
-  final int compileCode = await runFlutterBuild(examplePath, 'macos');
-  if (compileCode != 0) {
-    return compileCode;
-  }
-
-  return runXcodeBuild(
-    '$examplePath/macos',
-    extraArguments: <String>['test'],
-  );
-}
-
-Future<int> _runMacOSSwiftIntegrationTests() async {
-  const String examplePath = './$_testPluginRelativePath/example';
-  return runFlutterCommand(
-    examplePath,
-    'test',
-    <String>[_integrationTestFileRelativePath, '-d', 'macos'],
-  );
-}
-
-Future<int> _runIOSSwiftUnitTests() async {
-  return _runIOSPluginUnitTests(_testPluginRelativePath);
-}
-
-Future<int> _runIOSPluginUnitTests(String testPluginPath) async {
-  final String examplePath = './$testPluginPath/example';
-  return _runIOSProjectUnitTests(examplePath);
-}
-
-Future<int> _runIOSProjectUnitTests(String testProjectPath) async {
-  final int compileCode = await runFlutterBuild(
-    testProjectPath,
-    'ios',
-    flags: <String>['--simulator', '--no-codesign'],
-  );
-  if (compileCode != 0) {
-    return compileCode;
-  }
-
-  return runXcodeBuild(
-    '$testProjectPath/ios',
-    sdk: 'iphonesimulator',
-    destination: 'platform=iOS Simulator,name=iPhone 8',
-    extraArguments: <String>['test'],
-  );
-}
-
-Future<int> _runIOSSwiftIntegrationTests() async {
-  return _runMobileIntegrationTests('iOS', _testPluginRelativePath);
-}
-
-Future<int> _runMockHandlerTests() async {
-  const String unitTestsPath = './mock_handler_tester';
-  final int generateCode = await runPigeon(
-    input: './pigeons/message.dart',
-    dartOut: './mock_handler_tester/test/message.dart',
-    dartTestOut: './mock_handler_tester/test/test.dart',
-  );
-  if (generateCode != 0) {
-    return generateCode;
-  }
-
-  final int testCode = await runFlutterCommand(unitTestsPath, 'test');
-  if (testCode != 0) {
-    return testCode;
-  }
-  return 0;
-}
-
-Future<int> _runWindowsUnitTests() async {
-  const String examplePath = './$_testPluginRelativePath/example';
-  final int compileCode = await runFlutterBuild(examplePath, 'windows');
-  if (compileCode != 0) {
-    return compileCode;
-  }
-
-  return runProcess(
-      '$examplePath/build/windows/plugins/test_plugin/Debug/test_plugin_test.exe',
-      <String>[]);
-}
-
-Future<int> _runWindowsIntegrationTests() async {
-  const String examplePath = './$_testPluginRelativePath/example';
-  return runFlutterCommand(
-    examplePath,
-    'test',
-    <String>[_integrationTestFileRelativePath, '-d', 'windows'],
-  );
-}
-
-Future<int> _runCommandLineTests() async {
-  final Directory tempDir = Directory.systemTemp.createTempSync('pigeon');
-  final String tempOutput = p.join(tempDir.path, 'pigeon_output');
-  const String pigeonScript = 'bin/pigeon.dart';
-  final String snapshot = p.join(tempDir.path, 'pigeon.dart.dill');
-
-  // Precompile to make the repeated calls faster.
-  if (await runProcess('dart', <String>[
-        '--snapshot-kind=kernel',
-        '--snapshot=$snapshot',
-        pigeonScript
-      ]) !=
-      0) {
-    print('Unable to generate $snapshot from $pigeonScript');
-    return 1;
-  }
-
-  final List<List<String>> testArguments = <List<String>>[
-    // Test with no arguments.
-    <String>[],
-    // Test one_language flag. With this flag specified, java_out can be
-    // generated without dart_out.
-    <String>[
-      '--input',
-      'pigeons/message.dart',
-      '--one_language',
-      '--java_out',
-      tempOutput
-    ],
-    // Test dartOut in ConfigurePigeon overrides output.
-    <String>['--input', 'pigeons/configure_pigeon_dart_out.dart'],
-    // Make sure AST generation exits correctly.
-    <String>[
-      '--input',
-      'pigeons/message.dart',
-      '--one_language',
-      '--ast_out',
-      tempOutput
-    ],
-  ];
-
-  int exitCode = 0;
-  for (final List<String> arguments in testArguments) {
-    print('Testing dart $pigeonScript ${arguments.join(', ')}');
-    exitCode = await runProcess('dart', <String>[snapshot, ...arguments],
-        streamOutput: false, logFailure: true);
-    if (exitCode != 0) {
-      break;
-    }
-  }
-
-  tempDir.deleteSync(recursive: true);
-  return exitCode;
 }
 
 Future<void> main(List<String> args) async {
-  final ArgParser parser = ArgParser()
+  // Run most tests on Linux, since Linux tends to be the easiest and cheapest.
-    ..addMultiOption(_testFlag, abbr: 't', help: 'Only run specified tests.')
+  const List<String> linuxHostTests = <String>[
-    ..addFlag(_listFlag,
+    dartUnitTests,
-        negatable: false, abbr: 'l', help: 'List available tests.')
+    flutterUnitTests,
-    // Temporarily provide a way for run_test.sh to bypass generation, since
+    mockHandlerTests,
-    // it generates before doing anything else.
+    commandLineTests,
-    // TODO(stuartmorgan): Remove this once run_test.sh is fully migrated to
+    androidJavaUnitTests,
-    // this script.
+    androidKotlinUnitTests,
-    ..addFlag(_skipGenerationFlag, negatable: false, hide: true)
+    // TODO(stuartmorgan): Include these once CI supports running simulator
-    ..addFlag('help',
+    // tests. Currently these tests aren't run in CI.
-        negatable: false, abbr: 'h', help: 'Print this reference.');
+    // See https://github.com/flutter/flutter/issues/111505.
+    // androidJavaIntegrationTests,
+    // androidKotlinIntegrationTests,
+  ];
+  // Run macOS and iOS tests on macOS, since that's the only place they can run.
+  const List<String> macOSHostTests = <String>[
+    // TODO(stuartmorgan): Replace this with iOSObjCUnitTests once the CI
+    // issues are resolved; see https://github.com/flutter/packages/pull/2816.
+    iOSObjCUnitTestsLegacy,
+    // TODO(stuartmorgan): Enable by default once CI issues are solved; see
+    // https://github.com/flutter/packages/pull/2816.
+    // iOSObjCIntegrationTests,
+    iOSSwiftUnitTests,
+    // Currently these are testing exactly the same thing as
+    // macOSSwiftIntegrationTests, so we don't need to run both by default. This
+    // should be enabled if any iOS-only tests are added (e.g., for a feature
+    // not supported by macOS).
+    // iOSSwiftIntegrationTests,
+    macOSSwiftUnitTests,
+    macOSSwiftIntegrationTests,
+  ];
+  // Run Windows tests on Windows, since that's the only place they can run.
+  const List<String> windowsHostTests = <String>[
+    windowsUnitTests,
+    windowsIntegrationTests,
+  ];
 
-  final ArgResults argResults = parser.parse(args);
+  _validateTestCoverage(<List<String>>[
-  List<String> testsToRun = <String>[];
+    linuxHostTests,
-  if (argResults.wasParsed(_listFlag)) {
+    macOSHostTests,
-    print('available tests:');
+    windowsHostTests,
+    // Tests that are deliberately not included in CI:
+    <String>[
+      // See comment in linuxHostTests:
+      androidJavaIntegrationTests,
+      androidKotlinIntegrationTests,
+      // See comments in macOSHostTests:
+      iOSObjCUnitTests,
+      iOSObjCIntegrationTests,
+      iOSSwiftIntegrationTests,
+    ],
+  ]);
 
-    final int columnWidth =
+  final List<String> testsToRun;
-        _tests.keys.map((String key) => key.length).reduce(max) + 4;
+  if (Platform.isMacOS) {
-
+    testsToRun = macOSHostTests;
-    for (final MapEntry<String, _TestInfo> info in _tests.entries) {
+  } else if (Platform.isWindows) {
-      print('${info.key.padRight(columnWidth)}- ${info.value.description}');
+    testsToRun = windowsHostTests;
-    }
+  } else if (Platform.isLinux) {
-    exit(0);
+    testsToRun = linuxHostTests;
-  } else if (argResults.wasParsed('help')) {
+  } else {
-    print('''
+    print('Unsupported host platform.');
-Pigeon run_tests
+    exit(2);
-usage: dart run tool/run_tests.dart [-l | -t <test name>]
-
-${parser.usage}''');
-    exit(0);
-  } else if (argResults.wasParsed(_testFlag)) {
-    testsToRun = argResults[_testFlag];
   }
 
-  if (!argResults.wasParsed(_skipGenerationFlag)) {
+  await runTests(testsToRun);
-    final String baseDir = p.dirname(p.dirname(Platform.script.toFilePath()));
-    print('# Generating platform_test/ output...');
-    final int generateExitCode = await generatePigeons(baseDir: baseDir);
-    if (generateExitCode == 0) {
-      print('Generation complete!');
-    } else {
-      print('Generation failed; see above for errors.');
-    }
-  }
-
-  // If no tests are provided, run a default based on the host platform. This is
-  // the mode used by CI.
-  if (testsToRun.isEmpty) {
-    const List<String> androidTests = <String>[
-      androidJavaUnitTests,
-      androidKotlinUnitTests,
-      // TODO(stuartmorgan): Include these once CI supports running simulator
-      // tests. Currently these tests aren't run in CI.
-      // See https://github.com/flutter/flutter/issues/111505.
-      // androidJavaIntegrationTests,
-      // androidKotlinIntegrationTests,
-    ];
-    const List<String> macOSTests = <String>[
-      macOSSwiftUnitTests,
-      macOSSwiftIntegrationTests
-    ];
-    const List<String> iOSTests = <String>[
-      // TODO(stuartmorgan): Replace this with iOSObjCUnitTests once the CI
-      // issues are resolved; see https://github.com/flutter/packages/pull/2816.
-      iOSObjCUnitTestsLegacy,
-      // TODO(stuartmorgan): Enable by default once CI issues are solved; see
-      // https://github.com/flutter/packages/pull/2816.
-      // iOSObjCIntegrationTests,
-      iOSSwiftUnitTests,
-      // Currently these are testing exactly the same thing as
-      // macos_swift_e2e_tests, so we don't need to run both by default. This
-      // should be enabled if any iOS-only tests are added (e.g., for a feature
-      // not supported by macOS).
-      // iOSSwiftIntegrationTests,
-    ];
-    const List<String> windowsTests = <String>[
-      windowsUnitTests,
-      windowsIntegrationTests,
-    ];
-    const List<String> dartTests = <String>[
-      dartUnitTests,
-      flutterUnitTests,
-      mockHandlerTests,
-      commandLineTests,
-    ];
-
-    if (Platform.isMacOS) {
-      testsToRun = <String>[
-        ...dartTests,
-        ...androidTests,
-        ...iOSTests,
-        ...macOSTests,
-      ];
-    } else if (Platform.isWindows) {
-      testsToRun = windowsTests;
-    } else {
-      // TODO(stuartmorgan): Make a new entrypoint for developers that runs
-      // all tests their host supports by default, and move some of the tests
-      // above here. See https://github.com/flutter/flutter/issues/115393
-    }
-  }
-
-  for (final String test in testsToRun) {
-    final _TestInfo? info = _tests[test];
-    if (info != null) {
-      print('# Running $test');
-      final int testCode = await info.function();
-      if (testCode != 0) {
-        exit(testCode);
-      }
-    } else {
-      print('unknown test: $test');
-      exit(1);
-    }
-  }
-  exit(0);
 }

packages/pigeon/tool/shared/test_runner.dart

@@ -0,0 +1,45 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// ignore_for_file: avoid_print
+
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+
+import 'generation.dart';
+import 'test_suites.dart';
+
+/// Runs the given tests, printing status and exiting with failure if any of
+/// them fails.
+Future<void> runTests(List<String> testsToRun) async {
+  // Pre-generate the necessary common output files.
+  // TODO(stuartmorgan): Consider making this conditional on the specific
+  // tests being run, as not all of them need these files.
+  final String baseDir = p.dirname(p.dirname(Platform.script.toFilePath()));
+  print('# Generating platform_test/ output...');
+  final int generateExitCode = await generatePigeons(baseDir: baseDir);
+  if (generateExitCode == 0) {
+    print('Generation complete!');
+  } else {
+    print('Generation failed; see above for errors.');
+  }
+
+  for (final String test in testsToRun) {
+    final TestInfo? info = testSuites[test];
+    if (info != null) {
+      print('##############################');
+      print('# Running $test');
+      final int testCode = await info.function();
+      if (testCode != 0) {
+        exit(testCode);
+      }
+      print('');
+      print('');
+    } else {
+      print('Unknown test: $test');
+      exit(1);
+    }
+  }
+}

packages/pigeon/tool/shared/test_suites.dart

@@ -0,0 +1,431 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// ignore_for_file: avoid_print
+
+import 'dart:io' show File, Directory;
+
+import 'package:meta/meta.dart';
+import 'package:path/path.dart' as p;
+
+import 'flutter_utils.dart';
+import 'generation.dart';
+import 'native_project_runners.dart';
+import 'process_utils.dart';
+
+const int _noDeviceAvailableExitCode = 100;
+
+const String _testPluginRelativePath = 'platform_tests/test_plugin';
+const String _alternateLanguageTestPluginRelativePath =
+    'platform_tests/alternate_language_test_plugin';
+const String _integrationTestFileRelativePath = 'integration_test/test.dart';
+
+/// Information about a test suite.
+@immutable
+class TestInfo {
+  const TestInfo({required this.function, this.description});
+
+  /// The function to run the test suite.
+  final Future<int> Function() function;
+
+  /// A user-facing description of the test suite.
+  final String? description;
+}
+
+// Test suite names.
+const String androidJavaUnitTests = 'android_java_unittests';
+const String androidJavaIntegrationTests = 'android_java_integration_tests';
+const String androidKotlinUnitTests = 'android_kotlin_unittests';
+const String androidKotlinIntegrationTests = 'android_kotlin_integration_tests';
+const String iOSObjCUnitTests = 'ios_objc_unittests';
+const String iOSObjCUnitTestsLegacy = 'ios_objc_legacy_unittests';
+const String iOSObjCIntegrationTests = 'ios_objc_integration_tests';
+const String iOSSwiftUnitTests = 'ios_swift_unittests';
+const String iOSSwiftIntegrationTests = 'ios_swift_integration_tests';
+const String macOSSwiftUnitTests = 'macos_swift_unittests';
+const String macOSSwiftIntegrationTests = 'macos_swift_integration_tests';
+const String windowsUnitTests = 'windows_unittests';
+const String windowsIntegrationTests = 'windows_integration_tests';
+const String dartUnitTests = 'dart_unittests';
+const String flutterUnitTests = 'flutter_unittests';
+const String mockHandlerTests = 'mock_handler_tests';
+const String commandLineTests = 'command_line_tests';
+
+const Map<String, TestInfo> testSuites = <String, TestInfo>{
+  windowsUnitTests: TestInfo(
+      function: _runWindowsUnitTests,
+      description: 'Unit tests on generated Windows C++ code.'),
+  windowsIntegrationTests: TestInfo(
+      function: _runWindowsIntegrationTests,
+      description: 'Integration tests on generated Windows C++ code.'),
+  androidJavaUnitTests: TestInfo(
+      function: _runAndroidJavaUnitTests,
+      description: 'Unit tests on generated Java code.'),
+  androidJavaIntegrationTests: TestInfo(
+      function: _runAndroidJavaIntegrationTests,
+      description: 'Integration tests on generated Java code.'),
+  androidKotlinUnitTests: TestInfo(
+      function: _runAndroidKotlinUnitTests,
+      description: 'Unit tests on generated Kotlin code.'),
+  androidKotlinIntegrationTests: TestInfo(
+      function: _runAndroidKotlinIntegrationTests,
+      description: 'Integration tests on generated Kotlin code.'),
+  dartUnitTests: TestInfo(
+      function: _runDartUnitTests,
+      description: "Unit tests on and analysis on Pigeon's implementation."),
+  flutterUnitTests: TestInfo(
+      function: _runFlutterUnitTests,
+      description: 'Unit tests on generated Dart code.'),
+  iOSObjCUnitTests: TestInfo(
+      function: _runIOSObjCUnitTests,
+      description: 'Unit tests on generated Objective-C code.'),
+  iOSObjCUnitTestsLegacy: TestInfo(
+      function: _runIOSObjCLegacyUnitTests,
+      description:
+          'Unit tests on generated Objective-C code (legacy test harness).'),
+  iOSObjCIntegrationTests: TestInfo(
+      function: _runIOSObjCIntegrationTests,
+      description: 'Integration tests on generated Objective-C code.'),
+  iOSSwiftUnitTests: TestInfo(
+      function: _runIOSSwiftUnitTests,
+      description: 'Unit tests on generated Swift code.'),
+  iOSSwiftIntegrationTests: TestInfo(
+      function: _runIOSSwiftIntegrationTests,
+      description: 'Integration tests on generated Swift code.'),
+  macOSSwiftUnitTests: TestInfo(
+      function: _runMacOSSwiftUnitTests,
+      description: 'Unit tests on generated Swift code on macOS.'),
+  macOSSwiftIntegrationTests: TestInfo(
+      function: _runMacOSSwiftIntegrationTests,
+      description: 'Integration tests on generated Swift code on macOS.'),
+  mockHandlerTests: TestInfo(
+      function: _runMockHandlerTests,
+      description: 'Unit tests on generated Dart mock handler code.'),
+  commandLineTests: TestInfo(
+      function: _runCommandLineTests,
+      description: 'Tests running pigeon with various command-line options.'),
+};
+
+Future<int> _runAndroidJavaUnitTests() async {
+  return _runAndroidUnitTests(_alternateLanguageTestPluginRelativePath);
+}
+
+Future<int> _runAndroidJavaIntegrationTests() async {
+  return _runMobileIntegrationTests(
+      'Android', _alternateLanguageTestPluginRelativePath);
+}
+
+Future<int> _runAndroidKotlinUnitTests() async {
+  return _runAndroidUnitTests(_testPluginRelativePath);
+}
+
+Future<int> _runAndroidUnitTests(String testPluginPath) async {
+  final String examplePath = './$testPluginPath/example';
+  final String androidProjectPath = '$examplePath/android';
+  final File gradleFile = File(p.join(androidProjectPath, 'gradlew'));
+  if (!gradleFile.existsSync()) {
+    final int compileCode = await runFlutterBuild(examplePath, 'apk');
+    if (compileCode != 0) {
+      return compileCode;
+    }
+  }
+
+  return runGradleBuild(androidProjectPath, 'testDebugUnitTest');
+}
+
+Future<int> _runAndroidKotlinIntegrationTests() async {
+  return _runMobileIntegrationTests('Android', _testPluginRelativePath);
+}
+
+Future<int> _runMobileIntegrationTests(
+    String platform, String testPluginPath) async {
+  final String? device = await getDeviceForPlatform(platform.toLowerCase());
+  if (device == null) {
+    print('No $platform device available. Attach an $platform device or start '
+        'an emulator/simulator to run integration tests');
+    return _noDeviceAvailableExitCode;
+  }
+
+  final String examplePath = './$testPluginPath/example';
+  return runFlutterCommand(
+    examplePath,
+    'test',
+    <String>[_integrationTestFileRelativePath, '-d', device],
+  );
+}
+
+Future<int> _runDartUnitTests() async {
+  int exitCode = await runProcess('dart', <String>['analyze', 'bin']);
+  if (exitCode != 0) {
+    return exitCode;
+  }
+  exitCode = await runProcess('dart', <String>['analyze', 'lib']);
+  if (exitCode != 0) {
+    return exitCode;
+  }
+  exitCode = await runProcess('dart', <String>['test']);
+  return exitCode;
+}
+
+/// Generates multiple dart files based on the jobs defined in [jobs] which is
+/// in the format of (key: input pigeon file path, value: output dart file
+/// path).
+Future<int> _generateDart(Map<String, String> jobs) async {
+  for (final MapEntry<String, String> job in jobs.entries) {
+    // TODO(gaaclarke): Make this run the jobs in parallel.  A bug in Dart
+    // blocked this (https://github.com/dart-lang/pub/pull/3285).
+    final int result = await runPigeon(input: job.key, dartOut: job.value);
+    if (result != 0) {
+      return result;
+    }
+  }
+  return 0;
+}
+
+Future<int> _analyzeFlutterUnitTests(String flutterUnitTestsPath) async {
+  final String messagePath = '$flutterUnitTestsPath/lib/message.gen.dart';
+  final String messageTestPath = '$flutterUnitTestsPath/test/message_test.dart';
+  final int generateTestCode = await runPigeon(
+    input: 'pigeons/message.dart',
+    dartOut: messagePath,
+    dartTestOut: messageTestPath,
+  );
+  if (generateTestCode != 0) {
+    return generateTestCode;
+  }
+
+  final int analyzeCode =
+      await runFlutterCommand(flutterUnitTestsPath, 'analyze');
+  if (analyzeCode != 0) {
+    return analyzeCode;
+  }
+
+  // Delete these files that were just generated to help with the analyzer step.
+  File(messagePath).deleteSync();
+  File(messageTestPath).deleteSync();
+  return 0;
+}
+
+Future<int> _runFlutterUnitTests() async {
+  // TODO(stuartmorgan): Migrate Dart unit tests to use the generated output in
+  // shared_test_plugin_code instead of having multiple copies of generation.
+  const String flutterUnitTestsPath =
+      'platform_tests/flutter_null_safe_unit_tests';
+  // Files from the pigeons/ directory to generate output for.
+  const List<String> inputPigeons = <String>[
+    'flutter_unittests',
+    'core_tests',
+    'primitive',
+    'multiple_arity',
+    'non_null_fields',
+    'null_fields',
+    'nullable_returns',
+    // TODO(stuartmorgan): Eliminate these files by ensuring that everything
+    // they are intended to cover is in core_tests.dart (or, if necessary in
+    // the short term due to limitations in non-Dart generators, a single other
+    // file). They aren't being unit tested, only analyzed.
+    'async_handlers',
+    'host2flutter',
+    'list',
+    'message',
+    'void_arg_flutter',
+    'void_arg_host',
+    'voidflutter',
+    'voidhost',
+  ];
+  final int generateCode = await _generateDart(<String, String>{
+    for (final String name in inputPigeons)
+      'pigeons/$name.dart': '$flutterUnitTestsPath/lib/$name.gen.dart'
+  });
+  if (generateCode != 0) {
+    return generateCode;
+  }
+
+  final int analyzeCode = await _analyzeFlutterUnitTests(flutterUnitTestsPath);
+  if (analyzeCode != 0) {
+    return analyzeCode;
+  }
+
+  final int testCode = await runFlutterCommand(flutterUnitTestsPath, 'test');
+  if (testCode != 0) {
+    return testCode;
+  }
+
+  return 0;
+}
+
+Future<int> _runIOSObjCUnitTests() async {
+  return _runIOSPluginUnitTests(_alternateLanguageTestPluginRelativePath);
+}
+
+// TODO(stuartmorgan): Remove this, and the ios_unit_tests directory, once
+// _runIOSObjCUnitTests works in CI; see
+// https://github.com/flutter/packages/pull/2816.
+Future<int> _runIOSObjCLegacyUnitTests() async {
+  return _runIOSProjectUnitTests('platform_tests/ios_unit_tests');
+}
+
+Future<int> _runIOSObjCIntegrationTests() async {
+  final String? device = await getDeviceForPlatform('ios');
+  if (device == null) {
+    print('No iOS device available. Attach an iOS device or start '
+        'a simulator to run integration tests');
+    return _noDeviceAvailableExitCode;
+  }
+
+  const String examplePath =
+      './$_alternateLanguageTestPluginRelativePath/example';
+  return runFlutterCommand(
+    examplePath,
+    'test',
+    <String>[_integrationTestFileRelativePath, '-d', device],
+  );
+}
+
+Future<int> _runMacOSSwiftUnitTests() async {
+  const String examplePath = './$_testPluginRelativePath/example';
+  final int compileCode = await runFlutterBuild(examplePath, 'macos');
+  if (compileCode != 0) {
+    return compileCode;
+  }
+
+  return runXcodeBuild(
+    '$examplePath/macos',
+    extraArguments: <String>['test'],
+  );
+}
+
+Future<int> _runMacOSSwiftIntegrationTests() async {
+  const String examplePath = './$_testPluginRelativePath/example';
+  return runFlutterCommand(
+    examplePath,
+    'test',
+    <String>[_integrationTestFileRelativePath, '-d', 'macos'],
+  );
+}
+
+Future<int> _runIOSSwiftUnitTests() async {
+  return _runIOSPluginUnitTests(_testPluginRelativePath);
+}
+
+Future<int> _runIOSPluginUnitTests(String testPluginPath) async {
+  final String examplePath = './$testPluginPath/example';
+  return _runIOSProjectUnitTests(examplePath);
+}
+
+Future<int> _runIOSProjectUnitTests(String testProjectPath) async {
+  final int compileCode = await runFlutterBuild(
+    testProjectPath,
+    'ios',
+    flags: <String>['--simulator', '--no-codesign'],
+  );
+  if (compileCode != 0) {
+    return compileCode;
+  }
+
+  return runXcodeBuild(
+    '$testProjectPath/ios',
+    sdk: 'iphonesimulator',
+    destination: 'platform=iOS Simulator,name=iPhone 8',
+    extraArguments: <String>['test'],
+  );
+}
+
+Future<int> _runIOSSwiftIntegrationTests() async {
+  return _runMobileIntegrationTests('iOS', _testPluginRelativePath);
+}
+
+Future<int> _runMockHandlerTests() async {
+  const String unitTestsPath = './mock_handler_tester';
+  final int generateCode = await runPigeon(
+    input: './pigeons/message.dart',
+    dartOut: './mock_handler_tester/test/message.dart',
+    dartTestOut: './mock_handler_tester/test/test.dart',
+  );
+  if (generateCode != 0) {
+    return generateCode;
+  }
+
+  final int testCode = await runFlutterCommand(unitTestsPath, 'test');
+  if (testCode != 0) {
+    return testCode;
+  }
+  return 0;
+}
+
+Future<int> _runWindowsUnitTests() async {
+  const String examplePath = './$_testPluginRelativePath/example';
+  final int compileCode = await runFlutterBuild(examplePath, 'windows');
+  if (compileCode != 0) {
+    return compileCode;
+  }
+
+  return runProcess(
+      '$examplePath/build/windows/plugins/test_plugin/Debug/test_plugin_test.exe',
+      <String>[]);
+}
+
+Future<int> _runWindowsIntegrationTests() async {
+  const String examplePath = './$_testPluginRelativePath/example';
+  return runFlutterCommand(
+    examplePath,
+    'test',
+    <String>[_integrationTestFileRelativePath, '-d', 'windows'],
+  );
+}
+
+Future<int> _runCommandLineTests() async {
+  final Directory tempDir = Directory.systemTemp.createTempSync('pigeon');
+  final String tempOutput = p.join(tempDir.path, 'pigeon_output');
+  const String pigeonScript = 'bin/pigeon.dart';
+  final String snapshot = p.join(tempDir.path, 'pigeon.dart.dill');
+
+  // Precompile to make the repeated calls faster.
+  if (await runProcess('dart', <String>[
+        '--snapshot-kind=kernel',
+        '--snapshot=$snapshot',
+        pigeonScript
+      ]) !=
+      0) {
+    print('Unable to generate $snapshot from $pigeonScript');
+    return 1;
+  }
+
+  final List<List<String>> testArguments = <List<String>>[
+    // Test with no arguments.
+    <String>[],
+    // Test one_language flag. With this flag specified, java_out can be
+    // generated without dart_out.
+    <String>[
+      '--input',
+      'pigeons/message.dart',
+      '--one_language',
+      '--java_out',
+      tempOutput
+    ],
+    // Test dartOut in ConfigurePigeon overrides output.
+    <String>['--input', 'pigeons/configure_pigeon_dart_out.dart'],
+    // Make sure AST generation exits correctly.
+    <String>[
+      '--input',
+      'pigeons/message.dart',
+      '--one_language',
+      '--ast_out',
+      tempOutput
+    ],
+  ];
+
+  int exitCode = 0;
+  for (final List<String> arguments in testArguments) {
+    print('Testing dart $pigeonScript ${arguments.join(', ')}');
+    exitCode = await runProcess('dart', <String>[snapshot, ...arguments],
+        streamOutput: false, logFailure: true);
+    if (exitCode != 0) {
+      break;
+    }
+  }
+
+  tempDir.deleteSync(recursive: true);
+  return exitCode;
+}

packages/pigeon/tool/test.dart

@@ -0,0 +1,108 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// ignore_for_file: avoid_print
+
+////////////////////////////////////////////////////////////////////////////////
+/// Script for executing the Pigeon tests
+///
+/// usage: dart run tool/test.dart
+////////////////////////////////////////////////////////////////////////////////
+import 'dart:io' show Platform, exit;
+import 'dart:math';
+
+import 'package:args/args.dart';
+
+import 'shared/test_runner.dart';
+import 'shared/test_suites.dart';
+
+const String _testFlag = 'test';
+const String _listFlag = 'list';
+
+Future<void> main(List<String> args) async {
+  final ArgParser parser = ArgParser()
+    ..addMultiOption(_testFlag, abbr: 't', help: 'Only run specified tests.')
+    ..addFlag(_listFlag,
+        negatable: false, abbr: 'l', help: 'List available tests.')
+    ..addFlag('help',
+        negatable: false, abbr: 'h', help: 'Print this reference.');
+
+  final ArgResults argResults = parser.parse(args);
+  List<String> testsToRun = <String>[];
+  if (argResults.wasParsed(_listFlag)) {
+    print('available tests:');
+
+    final int columnWidth =
+        testSuites.keys.map((String key) => key.length).reduce(max) + 4;
+
+    for (final MapEntry<String, TestInfo> info in testSuites.entries) {
+      print('${info.key.padRight(columnWidth)}- ${info.value.description}');
+    }
+    exit(0);
+  } else if (argResults.wasParsed('help')) {
+    print('''
+Pigeon run_tests
+usage: dart run tool/test.dart [-l | -t <test name>]
+
+${parser.usage}''');
+    exit(0);
+  } else if (argResults.wasParsed(_testFlag)) {
+    testsToRun = argResults[_testFlag];
+  }
+
+  // If no tests are provided, run everything that is supported on the current
+  // platform.
+  if (testsToRun.isEmpty) {
+    const List<String> dartTests = <String>[
+      dartUnitTests,
+      flutterUnitTests,
+      mockHandlerTests,
+      commandLineTests,
+    ];
+    const List<String> androidTests = <String>[
+      androidJavaUnitTests,
+      androidKotlinUnitTests,
+      androidJavaIntegrationTests,
+      androidKotlinIntegrationTests,
+    ];
+    const List<String> iOSTests = <String>[
+      iOSObjCUnitTests,
+      iOSObjCIntegrationTests,
+      iOSSwiftUnitTests,
+      iOSSwiftIntegrationTests,
+    ];
+    const List<String> macOSTests = <String>[
+      macOSSwiftUnitTests,
+      macOSSwiftIntegrationTests
+    ];
+    const List<String> windowsTests = <String>[
+      windowsUnitTests,
+      windowsIntegrationTests,
+    ];
+
+    if (Platform.isMacOS) {
+      testsToRun = <String>[
+        ...dartTests,
+        ...androidTests,
+        ...iOSTests,
+        ...macOSTests,
+      ];
+    } else if (Platform.isWindows) {
+      testsToRun = <String>[
+        ...dartTests,
+        ...windowsTests,
+      ];
+    } else if (Platform.isLinux) {
+      testsToRun = <String>[
+        ...dartTests,
+        ...androidTests,
+      ];
+    } else {
+      print('Unsupported host platform.');
+      exit(1);
+    }
+  }
+
+  await runTests(testsToRun);
+}