From a4d8c39b1823bd35131e033a420708867bfe8833 Mon Sep 17 00:00:00 2001 From: stuartmorgan Date: Sat, 21 Jan 2023 13:50:04 -0800 Subject: [PATCH] [pigeon] Split CI and developer test scripts (#3076) * Split into two files with no changes * Adjust to fix compilation * Duplicate run_tests to test with no changes * Rework run_tests.dart * Extract sharable code * Make test.dart run every supported test * Update all docs * Update for changes in base PR * Analysis, missed tests * Move custom-test to heavy workload due to OOM * Re-merge Cirrus configuration Co-authored-by: Tarrin Neal --- .cirrus.yml | 19 +- packages/pigeon/CONTRIBUTING.md | 2 +- packages/pigeon/platform_tests/README.md | 4 +- .../alternate_language_test_plugin/README.md | 2 +- .../lib/.gitignore | 2 +- packages/pigeon/tool/run_tests.dart | 612 +++--------------- packages/pigeon/tool/shared/test_runner.dart | 45 ++ packages/pigeon/tool/shared/test_suites.dart | 431 ++++++++++++ packages/pigeon/tool/test.dart | 108 ++++ 9 files changed, 676 insertions(+), 549 deletions(-) create mode 100644 packages/pigeon/tool/shared/test_runner.dart create mode 100644 packages/pigeon/tool/shared/test_suites.dart create mode 100644 packages/pigeon/tool/test.dart diff --git a/.cirrus.yml b/.cirrus.yml index f11a21f8ed..96315cbf68 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -183,15 +183,6 @@ task: env: CIRRUS_CLONE_SUBMODULES: true script: ./script/tool_runner.sh update-excerpts --fail-on-change - - name: linux-custom_package_tests - env: - PATH: $PATH:/usr/local/bin - matrix: - CHANNEL: "master" - CHANNEL: "stable" - << : *INSTALL_CHROME_LINUX - local_tests_script: - - ./script/tool_runner.sh custom-test ### Web tasks ### - name: web-build_all_packages env: @@ -237,6 +228,16 @@ task: cpu: 4 memory: 16G matrix: + ### Platform-agnostic tasks ### + - name: linux-custom_package_tests + env: + PATH: $PATH:/usr/local/bin + matrix: + CHANNEL: "master" + CHANNEL: "stable" + << : *INSTALL_CHROME_LINUX + local_tests_script: + - ./script/tool_runner.sh custom-test ### Android tasks ### - name: android-platform_tests # Don't run full platform tests on both channels in pre-submit. diff --git a/packages/pigeon/CONTRIBUTING.md b/packages/pigeon/CONTRIBUTING.md index ed8541aa2c..0caf41ec42 100644 --- a/packages/pigeon/CONTRIBUTING.md +++ b/packages/pigeon/CONTRIBUTING.md @@ -35,7 +35,7 @@ generators with that AST. ## Testing Overview Pigeon has 3 types of tests, you'll find them all in -[run_tests.dart](./tool/run_tests.dart). +[test.dart](./tool/test.dart). * Unit tests - These are the fastest tests that are just typical unit tests, they may be generating code and checking it against a regular expression to diff --git a/packages/pigeon/platform_tests/README.md b/packages/pigeon/platform_tests/README.md index 5421536a6d..25a05cb3fc 100644 --- a/packages/pigeon/platform_tests/README.md +++ b/packages/pigeon/platform_tests/README.md @@ -1,11 +1,11 @@ # Native Pigeon Tests This directory contains native test harnesses for native and end-to-end tests -of Pigeon-generated code. The [test script](../tool/run_tests.dart) generates +of Pigeon-generated code. The [test script](../tool/test.dart) generates native code from [pigeons/](../pigeons/) into the native test scaffolding, and then drives the tests there. -To run these tests, use [`run_tests.dart`](../tool/run_tests.dart). +To run these tests, use [`test.dart`](../tool/test.dart). Alternately, if you are running them directly (e.g., from within a platform IDE), you can use [`generate.dart`](../tool/generate.dart) to generate the diff --git a/packages/pigeon/platform_tests/alternate_language_test_plugin/README.md b/packages/pigeon/platform_tests/alternate_language_test_plugin/README.md index 5bc262b99d..b507df6f3f 100644 --- a/packages/pigeon/platform_tests/alternate_language_test_plugin/README.md +++ b/packages/pigeon/platform_tests/alternate_language_test_plugin/README.md @@ -3,4 +3,4 @@ Tests for languages not covered by `test_plugin`. See [the `platform_tests` README](../README.md) for details. -To run these tests, use [`run_tests.dart`](../tool/run_tests.dart) +To run these tests, use [`test.dart`](../tool/test.dart) diff --git a/packages/pigeon/platform_tests/flutter_null_safe_unit_tests/lib/.gitignore b/packages/pigeon/platform_tests/flutter_null_safe_unit_tests/lib/.gitignore index 94ed05fd4e..955123f360 100644 --- a/packages/pigeon/platform_tests/flutter_null_safe_unit_tests/lib/.gitignore +++ b/packages/pigeon/platform_tests/flutter_null_safe_unit_tests/lib/.gitignore @@ -1,5 +1,5 @@ # TODO(stuartmorgan) Remove this file when these are no longer generated; -# see the TODO in _runFlutterUnitTests in run_tests.dart +# see the TODO in _runFlutterUnitTests in test_suites.dart async_handlers.gen.dart host2flutter.gen.dart list.gen.dart diff --git a/packages/pigeon/tool/run_tests.dart b/packages/pigeon/tool/run_tests.dart index d5daff68e4..b710048062 100644 --- a/packages/pigeon/tool/run_tests.dart +++ b/packages/pigeon/tool/run_tests.dart @@ -5,553 +5,95 @@ // ignore_for_file: avoid_print //////////////////////////////////////////////////////////////////////////////// -/// Script for executing the Pigeon tests +/// CI entrypoint for running Pigeon tests. /// -/// usage: dart run tool/run_tests.dart +/// For any use other than CI, use test.dart instead. //////////////////////////////////////////////////////////////////////////////// -import 'dart:io' show File, Directory, Platform, exit; -import 'dart:math'; +import 'dart:io' show Platform, exit; -import 'package:args/args.dart'; -import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p; +import 'shared/test_runner.dart'; +import 'shared/test_suites.dart'; -import 'shared/flutter_utils.dart'; -import 'shared/generation.dart'; -import 'shared/native_project_runners.dart'; -import 'shared/process_utils.dart'; +/// Exits with failure if any tests in [testSuites] are not included in any of +/// the given test [shards]. +void _validateTestCoverage(List> shards) { + final Set missing = testSuites.keys.toSet(); + shards.forEach(missing.removeAll); -const String _testFlag = 'test'; -const String _listFlag = 'list'; -const String _skipGenerationFlag = 'skip-generation'; - -const int _noDeviceAvailableExitCode = 100; - -const String _testPluginRelativePath = 'platform_tests/test_plugin'; -const String _alternateLanguageTestPluginRelativePath = - 'platform_tests/alternate_language_test_plugin'; -const String _integrationTestFileRelativePath = 'integration_test/test.dart'; - -@immutable -class _TestInfo { - const _TestInfo({required this.function, this.description}); - final Future Function() function; - final String? description; -} - -// Test suite names. -const String androidJavaUnitTests = 'android_java_unittests'; -const String androidJavaIntegrationTests = 'android_java_integration_tests'; -const String androidKotlinUnitTests = 'android_kotlin_unittests'; -const String androidKotlinIntegrationTests = 'android_kotlin_integration_tests'; -const String iOSObjCUnitTests = 'ios_objc_unittests'; -const String iOSObjCUnitTestsLegacy = 'ios_objc_legacy_unittests'; -const String iOSObjCIntegrationTests = 'ios_objc_integration_tests'; -const String iOSSwiftUnitTests = 'ios_swift_unittests'; -const String iOSSwiftIntegrationTests = 'ios_swift_integration_tests'; -const String macOSSwiftUnitTests = 'macos_swift_unittests'; -const String macOSSwiftIntegrationTests = 'macos_swift_integration_tests'; -const String windowsUnitTests = 'windows_unittests'; -const String windowsIntegrationTests = 'windows_integration_tests'; -const String dartUnitTests = 'dart_unittests'; -const String flutterUnitTests = 'flutter_unittests'; -const String mockHandlerTests = 'mock_handler_tests'; -const String commandLineTests = 'command_line_tests'; - -const Map _tests = { - windowsUnitTests: _TestInfo( - function: _runWindowsUnitTests, - description: 'Unit tests on generated Windows C++ code.'), - windowsIntegrationTests: _TestInfo( - function: _runWindowsIntegrationTests, - description: 'Integration tests on generated Windows C++ code.'), - androidJavaUnitTests: _TestInfo( - function: _runAndroidJavaUnitTests, - description: 'Unit tests on generated Java code.'), - androidJavaIntegrationTests: _TestInfo( - function: _runAndroidJavaIntegrationTests, - description: 'Integration tests on generated Java code.'), - androidKotlinUnitTests: _TestInfo( - function: _runAndroidKotlinUnitTests, - description: 'Unit tests on generated Kotlin code.'), - androidKotlinIntegrationTests: _TestInfo( - function: _runAndroidKotlinIntegrationTests, - description: 'Integration tests on generated Kotlin code.'), - dartUnitTests: _TestInfo( - function: _runDartUnitTests, - description: "Unit tests on and analysis on Pigeon's implementation."), - flutterUnitTests: _TestInfo( - function: _runFlutterUnitTests, - description: 'Unit tests on generated Dart code.'), - iOSObjCUnitTests: _TestInfo( - function: _runIOSObjCUnitTests, - description: 'Unit tests on generated Objective-C code.'), - iOSObjCUnitTestsLegacy: _TestInfo( - function: _runIOSObjCLegacyUnitTests, - description: - 'Unit tests on generated Objective-C code (legacy test harness).'), - iOSObjCIntegrationTests: _TestInfo( - function: _runIOSObjCIntegrationTests, - description: 'Integration tests on generated Objective-C code.'), - iOSSwiftUnitTests: _TestInfo( - function: _runIOSSwiftUnitTests, - description: 'Unit tests on generated Swift code.'), - iOSSwiftIntegrationTests: _TestInfo( - function: _runIOSSwiftIntegrationTests, - description: 'Integration tests on generated Swift code.'), - macOSSwiftUnitTests: _TestInfo( - function: _runMacOSSwiftUnitTests, - description: 'Unit tests on generated Swift code on macOS.'), - macOSSwiftIntegrationTests: _TestInfo( - function: _runMacOSSwiftIntegrationTests, - description: 'Integration tests on generated Swift code on macOS.'), - mockHandlerTests: _TestInfo( - function: _runMockHandlerTests, - description: 'Unit tests on generated Dart mock handler code.'), - commandLineTests: _TestInfo( - function: _runCommandLineTests, - description: 'Tests running pigeon with various command-line options.'), -}; - -Future _runAndroidJavaUnitTests() async { - return _runAndroidUnitTests(_alternateLanguageTestPluginRelativePath); -} - -Future _runAndroidJavaIntegrationTests() async { - return _runMobileIntegrationTests( - 'Android', _alternateLanguageTestPluginRelativePath); -} - -Future _runAndroidKotlinUnitTests() async { - return _runAndroidUnitTests(_testPluginRelativePath); -} - -Future _runAndroidUnitTests(String testPluginPath) async { - final String examplePath = './$testPluginPath/example'; - final String androidProjectPath = '$examplePath/android'; - final File gradleFile = File(p.join(androidProjectPath, 'gradlew')); - if (!gradleFile.existsSync()) { - final int compileCode = await runFlutterBuild(examplePath, 'apk'); - if (compileCode != 0) { - return compileCode; + if (missing.isNotEmpty) { + print('The following test suites are not being run on any host:'); + for (final String suite in missing) { + print(' $suite'); } + exit(1); } - - return runGradleBuild(androidProjectPath, 'testDebugUnitTest'); -} - -Future _runAndroidKotlinIntegrationTests() async { - return _runMobileIntegrationTests('Android', _testPluginRelativePath); -} - -Future _runMobileIntegrationTests( - String platform, String testPluginPath) async { - final String? device = await getDeviceForPlatform(platform.toLowerCase()); - if (device == null) { - print('No $platform device available. Attach an $platform device or start ' - 'an emulator/simulator to run integration tests'); - return _noDeviceAvailableExitCode; - } - - final String examplePath = './$testPluginPath/example'; - return runFlutterCommand( - examplePath, - 'test', - [_integrationTestFileRelativePath, '-d', device], - ); -} - -Future _runDartUnitTests() async { - int exitCode = await runProcess('dart', ['analyze', 'bin']); - if (exitCode != 0) { - return exitCode; - } - exitCode = await runProcess('dart', ['analyze', 'lib']); - if (exitCode != 0) { - return exitCode; - } - exitCode = await runProcess('dart', ['test']); - return exitCode; -} - -/// Generates multiple dart files based on the jobs defined in [jobs] which is -/// in the format of (key: input pigeon file path, value: output dart file -/// path). -Future _generateDart(Map jobs) async { - for (final MapEntry job in jobs.entries) { - // TODO(gaaclarke): Make this run the jobs in parallel. A bug in Dart - // blocked this (https://github.com/dart-lang/pub/pull/3285). - final int result = await runPigeon(input: job.key, dartOut: job.value); - if (result != 0) { - return result; - } - } - return 0; -} - -Future _analyzeFlutterUnitTests(String flutterUnitTestsPath) async { - final String messagePath = '$flutterUnitTestsPath/lib/message.gen.dart'; - final String messageTestPath = '$flutterUnitTestsPath/test/message_test.dart'; - final int generateTestCode = await runPigeon( - input: 'pigeons/message.dart', - dartOut: messagePath, - dartTestOut: messageTestPath, - ); - if (generateTestCode != 0) { - return generateTestCode; - } - - final int analyzeCode = - await runFlutterCommand(flutterUnitTestsPath, 'analyze'); - if (analyzeCode != 0) { - return analyzeCode; - } - - // Delete these files that were just generated to help with the analyzer step. - File(messagePath).deleteSync(); - File(messageTestPath).deleteSync(); - return 0; -} - -Future _runFlutterUnitTests() async { - // TODO(stuartmorgan): Migrate Dart unit tests to use the generated output in - // shared_test_plugin_code instead of having multiple copies of generation. - const String flutterUnitTestsPath = - 'platform_tests/flutter_null_safe_unit_tests'; - // Files from the pigeons/ directory to generate output for. - const List inputPigeons = [ - 'flutter_unittests', - 'core_tests', - 'primitive', - 'multiple_arity', - 'non_null_fields', - 'null_fields', - 'nullable_returns', - // TODO(stuartmorgan): Eliminate these files by ensuring that everything - // they are intended to cover is in core_tests.dart (or, if necessary in - // the short term due to limitations in non-Dart generators, a single other - // file). They aren't being unit tested, only analyzed. - 'async_handlers', - 'host2flutter', - 'list', - 'message', - 'void_arg_flutter', - 'void_arg_host', - 'voidflutter', - 'voidhost', - ]; - final int generateCode = await _generateDart({ - for (final String name in inputPigeons) - 'pigeons/$name.dart': '$flutterUnitTestsPath/lib/$name.gen.dart' - }); - if (generateCode != 0) { - return generateCode; - } - - final int analyzeCode = await _analyzeFlutterUnitTests(flutterUnitTestsPath); - if (analyzeCode != 0) { - return analyzeCode; - } - - final int testCode = await runFlutterCommand(flutterUnitTestsPath, 'test'); - if (testCode != 0) { - return testCode; - } - - return 0; -} - -Future _runIOSObjCUnitTests() async { - return _runIOSPluginUnitTests(_alternateLanguageTestPluginRelativePath); -} - -// TODO(stuartmorgan): Remove this, and the ios_unit_tests directory, once -// _runIOSObjCUnitTests works in CI; see -// https://github.com/flutter/packages/pull/2816. -Future _runIOSObjCLegacyUnitTests() async { - return _runIOSProjectUnitTests('platform_tests/ios_unit_tests'); -} - -Future _runIOSObjCIntegrationTests() async { - final String? device = await getDeviceForPlatform('ios'); - if (device == null) { - print('No iOS device available. Attach an iOS device or start ' - 'a simulator to run integration tests'); - return _noDeviceAvailableExitCode; - } - - const String examplePath = - './$_alternateLanguageTestPluginRelativePath/example'; - return runFlutterCommand( - examplePath, - 'test', - [_integrationTestFileRelativePath, '-d', device], - ); -} - -Future _runMacOSSwiftUnitTests() async { - const String examplePath = './$_testPluginRelativePath/example'; - final int compileCode = await runFlutterBuild(examplePath, 'macos'); - if (compileCode != 0) { - return compileCode; - } - - return runXcodeBuild( - '$examplePath/macos', - extraArguments: ['test'], - ); -} - -Future _runMacOSSwiftIntegrationTests() async { - const String examplePath = './$_testPluginRelativePath/example'; - return runFlutterCommand( - examplePath, - 'test', - [_integrationTestFileRelativePath, '-d', 'macos'], - ); -} - -Future _runIOSSwiftUnitTests() async { - return _runIOSPluginUnitTests(_testPluginRelativePath); -} - -Future _runIOSPluginUnitTests(String testPluginPath) async { - final String examplePath = './$testPluginPath/example'; - return _runIOSProjectUnitTests(examplePath); -} - -Future _runIOSProjectUnitTests(String testProjectPath) async { - final int compileCode = await runFlutterBuild( - testProjectPath, - 'ios', - flags: ['--simulator', '--no-codesign'], - ); - if (compileCode != 0) { - return compileCode; - } - - return runXcodeBuild( - '$testProjectPath/ios', - sdk: 'iphonesimulator', - destination: 'platform=iOS Simulator,name=iPhone 8', - extraArguments: ['test'], - ); -} - -Future _runIOSSwiftIntegrationTests() async { - return _runMobileIntegrationTests('iOS', _testPluginRelativePath); -} - -Future _runMockHandlerTests() async { - const String unitTestsPath = './mock_handler_tester'; - final int generateCode = await runPigeon( - input: './pigeons/message.dart', - dartOut: './mock_handler_tester/test/message.dart', - dartTestOut: './mock_handler_tester/test/test.dart', - ); - if (generateCode != 0) { - return generateCode; - } - - final int testCode = await runFlutterCommand(unitTestsPath, 'test'); - if (testCode != 0) { - return testCode; - } - return 0; -} - -Future _runWindowsUnitTests() async { - const String examplePath = './$_testPluginRelativePath/example'; - final int compileCode = await runFlutterBuild(examplePath, 'windows'); - if (compileCode != 0) { - return compileCode; - } - - return runProcess( - '$examplePath/build/windows/plugins/test_plugin/Debug/test_plugin_test.exe', - []); -} - -Future _runWindowsIntegrationTests() async { - const String examplePath = './$_testPluginRelativePath/example'; - return runFlutterCommand( - examplePath, - 'test', - [_integrationTestFileRelativePath, '-d', 'windows'], - ); -} - -Future _runCommandLineTests() async { - final Directory tempDir = Directory.systemTemp.createTempSync('pigeon'); - final String tempOutput = p.join(tempDir.path, 'pigeon_output'); - const String pigeonScript = 'bin/pigeon.dart'; - final String snapshot = p.join(tempDir.path, 'pigeon.dart.dill'); - - // Precompile to make the repeated calls faster. - if (await runProcess('dart', [ - '--snapshot-kind=kernel', - '--snapshot=$snapshot', - pigeonScript - ]) != - 0) { - print('Unable to generate $snapshot from $pigeonScript'); - return 1; - } - - final List> testArguments = >[ - // Test with no arguments. - [], - // Test one_language flag. With this flag specified, java_out can be - // generated without dart_out. - [ - '--input', - 'pigeons/message.dart', - '--one_language', - '--java_out', - tempOutput - ], - // Test dartOut in ConfigurePigeon overrides output. - ['--input', 'pigeons/configure_pigeon_dart_out.dart'], - // Make sure AST generation exits correctly. - [ - '--input', - 'pigeons/message.dart', - '--one_language', - '--ast_out', - tempOutput - ], - ]; - - int exitCode = 0; - for (final List arguments in testArguments) { - print('Testing dart $pigeonScript ${arguments.join(', ')}'); - exitCode = await runProcess('dart', [snapshot, ...arguments], - streamOutput: false, logFailure: true); - if (exitCode != 0) { - break; - } - } - - tempDir.deleteSync(recursive: true); - return exitCode; } Future main(List args) async { - final ArgParser parser = ArgParser() - ..addMultiOption(_testFlag, abbr: 't', help: 'Only run specified tests.') - ..addFlag(_listFlag, - negatable: false, abbr: 'l', help: 'List available tests.') - // Temporarily provide a way for run_test.sh to bypass generation, since - // it generates before doing anything else. - // TODO(stuartmorgan): Remove this once run_test.sh is fully migrated to - // this script. - ..addFlag(_skipGenerationFlag, negatable: false, hide: true) - ..addFlag('help', - negatable: false, abbr: 'h', help: 'Print this reference.'); + // Run most tests on Linux, since Linux tends to be the easiest and cheapest. + const List linuxHostTests = [ + dartUnitTests, + flutterUnitTests, + mockHandlerTests, + commandLineTests, + androidJavaUnitTests, + androidKotlinUnitTests, + // TODO(stuartmorgan): Include these once CI supports running simulator + // tests. Currently these tests aren't run in CI. + // See https://github.com/flutter/flutter/issues/111505. + // androidJavaIntegrationTests, + // androidKotlinIntegrationTests, + ]; + // Run macOS and iOS tests on macOS, since that's the only place they can run. + const List macOSHostTests = [ + // TODO(stuartmorgan): Replace this with iOSObjCUnitTests once the CI + // issues are resolved; see https://github.com/flutter/packages/pull/2816. + iOSObjCUnitTestsLegacy, + // TODO(stuartmorgan): Enable by default once CI issues are solved; see + // https://github.com/flutter/packages/pull/2816. + // iOSObjCIntegrationTests, + iOSSwiftUnitTests, + // Currently these are testing exactly the same thing as + // macOSSwiftIntegrationTests, so we don't need to run both by default. This + // should be enabled if any iOS-only tests are added (e.g., for a feature + // not supported by macOS). + // iOSSwiftIntegrationTests, + macOSSwiftUnitTests, + macOSSwiftIntegrationTests, + ]; + // Run Windows tests on Windows, since that's the only place they can run. + const List windowsHostTests = [ + windowsUnitTests, + windowsIntegrationTests, + ]; - final ArgResults argResults = parser.parse(args); - List testsToRun = []; - if (argResults.wasParsed(_listFlag)) { - print('available tests:'); + _validateTestCoverage(>[ + linuxHostTests, + macOSHostTests, + windowsHostTests, + // Tests that are deliberately not included in CI: + [ + // See comment in linuxHostTests: + androidJavaIntegrationTests, + androidKotlinIntegrationTests, + // See comments in macOSHostTests: + iOSObjCUnitTests, + iOSObjCIntegrationTests, + iOSSwiftIntegrationTests, + ], + ]); - final int columnWidth = - _tests.keys.map((String key) => key.length).reduce(max) + 4; - - for (final MapEntry info in _tests.entries) { - print('${info.key.padRight(columnWidth)}- ${info.value.description}'); - } - exit(0); - } else if (argResults.wasParsed('help')) { - print(''' -Pigeon run_tests -usage: dart run tool/run_tests.dart [-l | -t ] - -${parser.usage}'''); - exit(0); - } else if (argResults.wasParsed(_testFlag)) { - testsToRun = argResults[_testFlag]; + final List testsToRun; + if (Platform.isMacOS) { + testsToRun = macOSHostTests; + } else if (Platform.isWindows) { + testsToRun = windowsHostTests; + } else if (Platform.isLinux) { + testsToRun = linuxHostTests; + } else { + print('Unsupported host platform.'); + exit(2); } - if (!argResults.wasParsed(_skipGenerationFlag)) { - final String baseDir = p.dirname(p.dirname(Platform.script.toFilePath())); - print('# Generating platform_test/ output...'); - final int generateExitCode = await generatePigeons(baseDir: baseDir); - if (generateExitCode == 0) { - print('Generation complete!'); - } else { - print('Generation failed; see above for errors.'); - } - } - - // If no tests are provided, run a default based on the host platform. This is - // the mode used by CI. - if (testsToRun.isEmpty) { - const List androidTests = [ - androidJavaUnitTests, - androidKotlinUnitTests, - // TODO(stuartmorgan): Include these once CI supports running simulator - // tests. Currently these tests aren't run in CI. - // See https://github.com/flutter/flutter/issues/111505. - // androidJavaIntegrationTests, - // androidKotlinIntegrationTests, - ]; - const List macOSTests = [ - macOSSwiftUnitTests, - macOSSwiftIntegrationTests - ]; - const List iOSTests = [ - // TODO(stuartmorgan): Replace this with iOSObjCUnitTests once the CI - // issues are resolved; see https://github.com/flutter/packages/pull/2816. - iOSObjCUnitTestsLegacy, - // TODO(stuartmorgan): Enable by default once CI issues are solved; see - // https://github.com/flutter/packages/pull/2816. - // iOSObjCIntegrationTests, - iOSSwiftUnitTests, - // Currently these are testing exactly the same thing as - // macos_swift_e2e_tests, so we don't need to run both by default. This - // should be enabled if any iOS-only tests are added (e.g., for a feature - // not supported by macOS). - // iOSSwiftIntegrationTests, - ]; - const List windowsTests = [ - windowsUnitTests, - windowsIntegrationTests, - ]; - const List dartTests = [ - dartUnitTests, - flutterUnitTests, - mockHandlerTests, - commandLineTests, - ]; - - if (Platform.isMacOS) { - testsToRun = [ - ...dartTests, - ...androidTests, - ...iOSTests, - ...macOSTests, - ]; - } else if (Platform.isWindows) { - testsToRun = windowsTests; - } else { - // TODO(stuartmorgan): Make a new entrypoint for developers that runs - // all tests their host supports by default, and move some of the tests - // above here. See https://github.com/flutter/flutter/issues/115393 - } - } - - for (final String test in testsToRun) { - final _TestInfo? info = _tests[test]; - if (info != null) { - print('# Running $test'); - final int testCode = await info.function(); - if (testCode != 0) { - exit(testCode); - } - } else { - print('unknown test: $test'); - exit(1); - } - } - exit(0); + await runTests(testsToRun); } diff --git a/packages/pigeon/tool/shared/test_runner.dart b/packages/pigeon/tool/shared/test_runner.dart new file mode 100644 index 0000000000..abbf5edbd8 --- /dev/null +++ b/packages/pigeon/tool/shared/test_runner.dart @@ -0,0 +1,45 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// ignore_for_file: avoid_print + +import 'dart:io'; + +import 'package:path/path.dart' as p; + +import 'generation.dart'; +import 'test_suites.dart'; + +/// Runs the given tests, printing status and exiting with failure if any of +/// them fails. +Future runTests(List testsToRun) async { + // Pre-generate the necessary common output files. + // TODO(stuartmorgan): Consider making this conditional on the specific + // tests being run, as not all of them need these files. + final String baseDir = p.dirname(p.dirname(Platform.script.toFilePath())); + print('# Generating platform_test/ output...'); + final int generateExitCode = await generatePigeons(baseDir: baseDir); + if (generateExitCode == 0) { + print('Generation complete!'); + } else { + print('Generation failed; see above for errors.'); + } + + for (final String test in testsToRun) { + final TestInfo? info = testSuites[test]; + if (info != null) { + print('##############################'); + print('# Running $test'); + final int testCode = await info.function(); + if (testCode != 0) { + exit(testCode); + } + print(''); + print(''); + } else { + print('Unknown test: $test'); + exit(1); + } + } +} diff --git a/packages/pigeon/tool/shared/test_suites.dart b/packages/pigeon/tool/shared/test_suites.dart new file mode 100644 index 0000000000..809781b5f8 --- /dev/null +++ b/packages/pigeon/tool/shared/test_suites.dart @@ -0,0 +1,431 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// ignore_for_file: avoid_print + +import 'dart:io' show File, Directory; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +import 'flutter_utils.dart'; +import 'generation.dart'; +import 'native_project_runners.dart'; +import 'process_utils.dart'; + +const int _noDeviceAvailableExitCode = 100; + +const String _testPluginRelativePath = 'platform_tests/test_plugin'; +const String _alternateLanguageTestPluginRelativePath = + 'platform_tests/alternate_language_test_plugin'; +const String _integrationTestFileRelativePath = 'integration_test/test.dart'; + +/// Information about a test suite. +@immutable +class TestInfo { + const TestInfo({required this.function, this.description}); + + /// The function to run the test suite. + final Future Function() function; + + /// A user-facing description of the test suite. + final String? description; +} + +// Test suite names. +const String androidJavaUnitTests = 'android_java_unittests'; +const String androidJavaIntegrationTests = 'android_java_integration_tests'; +const String androidKotlinUnitTests = 'android_kotlin_unittests'; +const String androidKotlinIntegrationTests = 'android_kotlin_integration_tests'; +const String iOSObjCUnitTests = 'ios_objc_unittests'; +const String iOSObjCUnitTestsLegacy = 'ios_objc_legacy_unittests'; +const String iOSObjCIntegrationTests = 'ios_objc_integration_tests'; +const String iOSSwiftUnitTests = 'ios_swift_unittests'; +const String iOSSwiftIntegrationTests = 'ios_swift_integration_tests'; +const String macOSSwiftUnitTests = 'macos_swift_unittests'; +const String macOSSwiftIntegrationTests = 'macos_swift_integration_tests'; +const String windowsUnitTests = 'windows_unittests'; +const String windowsIntegrationTests = 'windows_integration_tests'; +const String dartUnitTests = 'dart_unittests'; +const String flutterUnitTests = 'flutter_unittests'; +const String mockHandlerTests = 'mock_handler_tests'; +const String commandLineTests = 'command_line_tests'; + +const Map testSuites = { + windowsUnitTests: TestInfo( + function: _runWindowsUnitTests, + description: 'Unit tests on generated Windows C++ code.'), + windowsIntegrationTests: TestInfo( + function: _runWindowsIntegrationTests, + description: 'Integration tests on generated Windows C++ code.'), + androidJavaUnitTests: TestInfo( + function: _runAndroidJavaUnitTests, + description: 'Unit tests on generated Java code.'), + androidJavaIntegrationTests: TestInfo( + function: _runAndroidJavaIntegrationTests, + description: 'Integration tests on generated Java code.'), + androidKotlinUnitTests: TestInfo( + function: _runAndroidKotlinUnitTests, + description: 'Unit tests on generated Kotlin code.'), + androidKotlinIntegrationTests: TestInfo( + function: _runAndroidKotlinIntegrationTests, + description: 'Integration tests on generated Kotlin code.'), + dartUnitTests: TestInfo( + function: _runDartUnitTests, + description: "Unit tests on and analysis on Pigeon's implementation."), + flutterUnitTests: TestInfo( + function: _runFlutterUnitTests, + description: 'Unit tests on generated Dart code.'), + iOSObjCUnitTests: TestInfo( + function: _runIOSObjCUnitTests, + description: 'Unit tests on generated Objective-C code.'), + iOSObjCUnitTestsLegacy: TestInfo( + function: _runIOSObjCLegacyUnitTests, + description: + 'Unit tests on generated Objective-C code (legacy test harness).'), + iOSObjCIntegrationTests: TestInfo( + function: _runIOSObjCIntegrationTests, + description: 'Integration tests on generated Objective-C code.'), + iOSSwiftUnitTests: TestInfo( + function: _runIOSSwiftUnitTests, + description: 'Unit tests on generated Swift code.'), + iOSSwiftIntegrationTests: TestInfo( + function: _runIOSSwiftIntegrationTests, + description: 'Integration tests on generated Swift code.'), + macOSSwiftUnitTests: TestInfo( + function: _runMacOSSwiftUnitTests, + description: 'Unit tests on generated Swift code on macOS.'), + macOSSwiftIntegrationTests: TestInfo( + function: _runMacOSSwiftIntegrationTests, + description: 'Integration tests on generated Swift code on macOS.'), + mockHandlerTests: TestInfo( + function: _runMockHandlerTests, + description: 'Unit tests on generated Dart mock handler code.'), + commandLineTests: TestInfo( + function: _runCommandLineTests, + description: 'Tests running pigeon with various command-line options.'), +}; + +Future _runAndroidJavaUnitTests() async { + return _runAndroidUnitTests(_alternateLanguageTestPluginRelativePath); +} + +Future _runAndroidJavaIntegrationTests() async { + return _runMobileIntegrationTests( + 'Android', _alternateLanguageTestPluginRelativePath); +} + +Future _runAndroidKotlinUnitTests() async { + return _runAndroidUnitTests(_testPluginRelativePath); +} + +Future _runAndroidUnitTests(String testPluginPath) async { + final String examplePath = './$testPluginPath/example'; + final String androidProjectPath = '$examplePath/android'; + final File gradleFile = File(p.join(androidProjectPath, 'gradlew')); + if (!gradleFile.existsSync()) { + final int compileCode = await runFlutterBuild(examplePath, 'apk'); + if (compileCode != 0) { + return compileCode; + } + } + + return runGradleBuild(androidProjectPath, 'testDebugUnitTest'); +} + +Future _runAndroidKotlinIntegrationTests() async { + return _runMobileIntegrationTests('Android', _testPluginRelativePath); +} + +Future _runMobileIntegrationTests( + String platform, String testPluginPath) async { + final String? device = await getDeviceForPlatform(platform.toLowerCase()); + if (device == null) { + print('No $platform device available. Attach an $platform device or start ' + 'an emulator/simulator to run integration tests'); + return _noDeviceAvailableExitCode; + } + + final String examplePath = './$testPluginPath/example'; + return runFlutterCommand( + examplePath, + 'test', + [_integrationTestFileRelativePath, '-d', device], + ); +} + +Future _runDartUnitTests() async { + int exitCode = await runProcess('dart', ['analyze', 'bin']); + if (exitCode != 0) { + return exitCode; + } + exitCode = await runProcess('dart', ['analyze', 'lib']); + if (exitCode != 0) { + return exitCode; + } + exitCode = await runProcess('dart', ['test']); + return exitCode; +} + +/// Generates multiple dart files based on the jobs defined in [jobs] which is +/// in the format of (key: input pigeon file path, value: output dart file +/// path). +Future _generateDart(Map jobs) async { + for (final MapEntry job in jobs.entries) { + // TODO(gaaclarke): Make this run the jobs in parallel. A bug in Dart + // blocked this (https://github.com/dart-lang/pub/pull/3285). + final int result = await runPigeon(input: job.key, dartOut: job.value); + if (result != 0) { + return result; + } + } + return 0; +} + +Future _analyzeFlutterUnitTests(String flutterUnitTestsPath) async { + final String messagePath = '$flutterUnitTestsPath/lib/message.gen.dart'; + final String messageTestPath = '$flutterUnitTestsPath/test/message_test.dart'; + final int generateTestCode = await runPigeon( + input: 'pigeons/message.dart', + dartOut: messagePath, + dartTestOut: messageTestPath, + ); + if (generateTestCode != 0) { + return generateTestCode; + } + + final int analyzeCode = + await runFlutterCommand(flutterUnitTestsPath, 'analyze'); + if (analyzeCode != 0) { + return analyzeCode; + } + + // Delete these files that were just generated to help with the analyzer step. + File(messagePath).deleteSync(); + File(messageTestPath).deleteSync(); + return 0; +} + +Future _runFlutterUnitTests() async { + // TODO(stuartmorgan): Migrate Dart unit tests to use the generated output in + // shared_test_plugin_code instead of having multiple copies of generation. + const String flutterUnitTestsPath = + 'platform_tests/flutter_null_safe_unit_tests'; + // Files from the pigeons/ directory to generate output for. + const List inputPigeons = [ + 'flutter_unittests', + 'core_tests', + 'primitive', + 'multiple_arity', + 'non_null_fields', + 'null_fields', + 'nullable_returns', + // TODO(stuartmorgan): Eliminate these files by ensuring that everything + // they are intended to cover is in core_tests.dart (or, if necessary in + // the short term due to limitations in non-Dart generators, a single other + // file). They aren't being unit tested, only analyzed. + 'async_handlers', + 'host2flutter', + 'list', + 'message', + 'void_arg_flutter', + 'void_arg_host', + 'voidflutter', + 'voidhost', + ]; + final int generateCode = await _generateDart({ + for (final String name in inputPigeons) + 'pigeons/$name.dart': '$flutterUnitTestsPath/lib/$name.gen.dart' + }); + if (generateCode != 0) { + return generateCode; + } + + final int analyzeCode = await _analyzeFlutterUnitTests(flutterUnitTestsPath); + if (analyzeCode != 0) { + return analyzeCode; + } + + final int testCode = await runFlutterCommand(flutterUnitTestsPath, 'test'); + if (testCode != 0) { + return testCode; + } + + return 0; +} + +Future _runIOSObjCUnitTests() async { + return _runIOSPluginUnitTests(_alternateLanguageTestPluginRelativePath); +} + +// TODO(stuartmorgan): Remove this, and the ios_unit_tests directory, once +// _runIOSObjCUnitTests works in CI; see +// https://github.com/flutter/packages/pull/2816. +Future _runIOSObjCLegacyUnitTests() async { + return _runIOSProjectUnitTests('platform_tests/ios_unit_tests'); +} + +Future _runIOSObjCIntegrationTests() async { + final String? device = await getDeviceForPlatform('ios'); + if (device == null) { + print('No iOS device available. Attach an iOS device or start ' + 'a simulator to run integration tests'); + return _noDeviceAvailableExitCode; + } + + const String examplePath = + './$_alternateLanguageTestPluginRelativePath/example'; + return runFlutterCommand( + examplePath, + 'test', + [_integrationTestFileRelativePath, '-d', device], + ); +} + +Future _runMacOSSwiftUnitTests() async { + const String examplePath = './$_testPluginRelativePath/example'; + final int compileCode = await runFlutterBuild(examplePath, 'macos'); + if (compileCode != 0) { + return compileCode; + } + + return runXcodeBuild( + '$examplePath/macos', + extraArguments: ['test'], + ); +} + +Future _runMacOSSwiftIntegrationTests() async { + const String examplePath = './$_testPluginRelativePath/example'; + return runFlutterCommand( + examplePath, + 'test', + [_integrationTestFileRelativePath, '-d', 'macos'], + ); +} + +Future _runIOSSwiftUnitTests() async { + return _runIOSPluginUnitTests(_testPluginRelativePath); +} + +Future _runIOSPluginUnitTests(String testPluginPath) async { + final String examplePath = './$testPluginPath/example'; + return _runIOSProjectUnitTests(examplePath); +} + +Future _runIOSProjectUnitTests(String testProjectPath) async { + final int compileCode = await runFlutterBuild( + testProjectPath, + 'ios', + flags: ['--simulator', '--no-codesign'], + ); + if (compileCode != 0) { + return compileCode; + } + + return runXcodeBuild( + '$testProjectPath/ios', + sdk: 'iphonesimulator', + destination: 'platform=iOS Simulator,name=iPhone 8', + extraArguments: ['test'], + ); +} + +Future _runIOSSwiftIntegrationTests() async { + return _runMobileIntegrationTests('iOS', _testPluginRelativePath); +} + +Future _runMockHandlerTests() async { + const String unitTestsPath = './mock_handler_tester'; + final int generateCode = await runPigeon( + input: './pigeons/message.dart', + dartOut: './mock_handler_tester/test/message.dart', + dartTestOut: './mock_handler_tester/test/test.dart', + ); + if (generateCode != 0) { + return generateCode; + } + + final int testCode = await runFlutterCommand(unitTestsPath, 'test'); + if (testCode != 0) { + return testCode; + } + return 0; +} + +Future _runWindowsUnitTests() async { + const String examplePath = './$_testPluginRelativePath/example'; + final int compileCode = await runFlutterBuild(examplePath, 'windows'); + if (compileCode != 0) { + return compileCode; + } + + return runProcess( + '$examplePath/build/windows/plugins/test_plugin/Debug/test_plugin_test.exe', + []); +} + +Future _runWindowsIntegrationTests() async { + const String examplePath = './$_testPluginRelativePath/example'; + return runFlutterCommand( + examplePath, + 'test', + [_integrationTestFileRelativePath, '-d', 'windows'], + ); +} + +Future _runCommandLineTests() async { + final Directory tempDir = Directory.systemTemp.createTempSync('pigeon'); + final String tempOutput = p.join(tempDir.path, 'pigeon_output'); + const String pigeonScript = 'bin/pigeon.dart'; + final String snapshot = p.join(tempDir.path, 'pigeon.dart.dill'); + + // Precompile to make the repeated calls faster. + if (await runProcess('dart', [ + '--snapshot-kind=kernel', + '--snapshot=$snapshot', + pigeonScript + ]) != + 0) { + print('Unable to generate $snapshot from $pigeonScript'); + return 1; + } + + final List> testArguments = >[ + // Test with no arguments. + [], + // Test one_language flag. With this flag specified, java_out can be + // generated without dart_out. + [ + '--input', + 'pigeons/message.dart', + '--one_language', + '--java_out', + tempOutput + ], + // Test dartOut in ConfigurePigeon overrides output. + ['--input', 'pigeons/configure_pigeon_dart_out.dart'], + // Make sure AST generation exits correctly. + [ + '--input', + 'pigeons/message.dart', + '--one_language', + '--ast_out', + tempOutput + ], + ]; + + int exitCode = 0; + for (final List arguments in testArguments) { + print('Testing dart $pigeonScript ${arguments.join(', ')}'); + exitCode = await runProcess('dart', [snapshot, ...arguments], + streamOutput: false, logFailure: true); + if (exitCode != 0) { + break; + } + } + + tempDir.deleteSync(recursive: true); + return exitCode; +} diff --git a/packages/pigeon/tool/test.dart b/packages/pigeon/tool/test.dart new file mode 100644 index 0000000000..c94f0570aa --- /dev/null +++ b/packages/pigeon/tool/test.dart @@ -0,0 +1,108 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// ignore_for_file: avoid_print + +//////////////////////////////////////////////////////////////////////////////// +/// Script for executing the Pigeon tests +/// +/// usage: dart run tool/test.dart +//////////////////////////////////////////////////////////////////////////////// +import 'dart:io' show Platform, exit; +import 'dart:math'; + +import 'package:args/args.dart'; + +import 'shared/test_runner.dart'; +import 'shared/test_suites.dart'; + +const String _testFlag = 'test'; +const String _listFlag = 'list'; + +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addMultiOption(_testFlag, abbr: 't', help: 'Only run specified tests.') + ..addFlag(_listFlag, + negatable: false, abbr: 'l', help: 'List available tests.') + ..addFlag('help', + negatable: false, abbr: 'h', help: 'Print this reference.'); + + final ArgResults argResults = parser.parse(args); + List testsToRun = []; + if (argResults.wasParsed(_listFlag)) { + print('available tests:'); + + final int columnWidth = + testSuites.keys.map((String key) => key.length).reduce(max) + 4; + + for (final MapEntry info in testSuites.entries) { + print('${info.key.padRight(columnWidth)}- ${info.value.description}'); + } + exit(0); + } else if (argResults.wasParsed('help')) { + print(''' +Pigeon run_tests +usage: dart run tool/test.dart [-l | -t ] + +${parser.usage}'''); + exit(0); + } else if (argResults.wasParsed(_testFlag)) { + testsToRun = argResults[_testFlag]; + } + + // If no tests are provided, run everything that is supported on the current + // platform. + if (testsToRun.isEmpty) { + const List dartTests = [ + dartUnitTests, + flutterUnitTests, + mockHandlerTests, + commandLineTests, + ]; + const List androidTests = [ + androidJavaUnitTests, + androidKotlinUnitTests, + androidJavaIntegrationTests, + androidKotlinIntegrationTests, + ]; + const List iOSTests = [ + iOSObjCUnitTests, + iOSObjCIntegrationTests, + iOSSwiftUnitTests, + iOSSwiftIntegrationTests, + ]; + const List macOSTests = [ + macOSSwiftUnitTests, + macOSSwiftIntegrationTests + ]; + const List windowsTests = [ + windowsUnitTests, + windowsIntegrationTests, + ]; + + if (Platform.isMacOS) { + testsToRun = [ + ...dartTests, + ...androidTests, + ...iOSTests, + ...macOSTests, + ]; + } else if (Platform.isWindows) { + testsToRun = [ + ...dartTests, + ...windowsTests, + ]; + } else if (Platform.isLinux) { + testsToRun = [ + ...dartTests, + ...androidTests, + ]; + } else { + print('Unsupported host platform.'); + exit(1); + } + } + + await runTests(testsToRun); +}