diff --git a/script/tool/CHANGELOG.md b/script/tool/CHANGELOG.md index d701278ee7..dc30c05f79 100644 --- a/script/tool/CHANGELOG.md +++ b/script/tool/CHANGELOG.md @@ -2,13 +2,21 @@ - Added an `xctest` flag to select specific test targets, to allow running only unit tests or integration tests. -- Split Xcode analysis out of `xctest` and into a new `xcode-analyze` command. +- **Breaking change**: Split Xcode analysis out of `xctest` and into a new + `xcode-analyze` command. - Fixed a bug that caused `firebase-test-lab` to hang if it tried to run more than one plugin's tests in a single run. - **Breaking change**: If `firebase-test-lab` is run on a package that supports Android, but for which no tests are run, it now fails instead of skipping. This matches `drive-examples`, as this command is what is used for driving Android Flutter integration tests on CI. +- **Breaking change**: Replaced `xctest` with a new `native-test` command that + will eventually be able to run native unit and integration tests for all + platforms. + - Adds the ability to disable test types via `--no-unit` or + `--no-integration`. +- **Breaking change**: Replaced `java-test` with Android unit test support for + the new `native-test` command. ## 0.4.1 diff --git a/script/tool/README.md b/script/tool/README.md index 5629dc5064..1a87f09875 100644 --- a/script/tool/README.md +++ b/script/tool/README.md @@ -75,14 +75,28 @@ cd dart run ./script/tool/bin/flutter_plugin_tools.dart test --packages plugin_name ``` -### Run XCTests +### Run Dart Integration Tests ```sh cd -# For iOS: -dart run ./script/tool/bin/flutter_plugin_tools.dart xctest --ios --packages plugin_name -# For macOS: -dart run ./script/tool/bin/flutter_plugin_tools.dart xctest --macos --packages plugin_name +dart run ./script/tool/bin/flutter_plugin_tools.dart build-examples --packages plugin_name +dart run ./script/tool/bin/flutter_plugin_tools.dart drive-examples --packages plugin_name +``` + +### Run Native Tests + +`native-test` takes one or more platform flags to run tests for. By default it +runs both unit tests and (on platforms that support it) integration tests, but +`--no-unit` or `--no-integration` can be used to run just one type. + +Examples: + +```sh +cd +# Run just unit tests for iOS and Android: +dart run ./script/tool/bin/flutter_plugin_tools.dart native-test --ios --android --no-integration --packages plugin_name +# Run all tests for macOS: +dart run ./script/tool/bin/flutter_plugin_tools.dart native-test --macos --packages plugin_name ``` ### Publish a Release diff --git a/script/tool/lib/src/common/package_looping_command.dart b/script/tool/lib/src/common/package_looping_command.dart index 9f4039ec70..0bcde6d296 100644 --- a/script/tool/lib/src/common/package_looping_command.dart +++ b/script/tool/lib/src/common/package_looping_command.dart @@ -165,9 +165,9 @@ abstract class PackageLoopingCommand extends PluginCommand { final List components = p.posix.split(packageName); // For the common federated plugin pattern of `foo/foo_subpackage`, drop // the first part since it's not useful. - if (components.length == 2 && + if (components.length >= 2 && components[1].startsWith('${components[0]}_')) { - packageName = components[1]; + packageName = p.posix.joinAll(components.sublist(1)); } return packageName; } diff --git a/script/tool/lib/src/java_test_command.dart b/script/tool/lib/src/java_test_command.dart deleted file mode 100644 index b36d1102f1..0000000000 --- a/script/tool/lib/src/java_test_command.dart +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:file/file.dart'; -import 'package:platform/platform.dart'; - -import 'common/core.dart'; -import 'common/package_looping_command.dart'; -import 'common/process_runner.dart'; - -/// A command to run the Java tests of Android plugins. -class JavaTestCommand extends PackageLoopingCommand { - /// Creates an instance of the test runner. - JavaTestCommand( - Directory packagesDir, { - ProcessRunner processRunner = const ProcessRunner(), - Platform platform = const LocalPlatform(), - }) : super(packagesDir, processRunner: processRunner, platform: platform); - - static const String _gradleWrapper = 'gradlew'; - - @override - final String name = 'java-test'; - - @override - final String description = 'Runs the Java tests of the example apps.\n\n' - 'Building the apks of the example apps is required before executing this' - 'command.'; - - @override - Future runForPackage(Directory package) async { - final Iterable examplesWithTests = getExamplesForPlugin(package) - .where((Directory d) => - isFlutterPackage(d) && - (d - .childDirectory('android') - .childDirectory('app') - .childDirectory('src') - .childDirectory('test') - .existsSync() || - d.parent - .childDirectory('android') - .childDirectory('src') - .childDirectory('test') - .existsSync())); - - if (examplesWithTests.isEmpty) { - return PackageResult.skip('No Java unit tests.'); - } - - final List errors = []; - for (final Directory example in examplesWithTests) { - final String exampleName = getRelativePosixPath(example, from: package); - print('\nRUNNING JAVA TESTS for $exampleName'); - - final Directory androidDirectory = example.childDirectory('android'); - final File gradleFile = androidDirectory.childFile(_gradleWrapper); - if (!gradleFile.existsSync()) { - printError('ERROR: Run "flutter build apk" on $exampleName, or run ' - 'this tool\'s "build-examples --apk" command, ' - 'before executing tests.'); - errors.add('$exampleName has not been built.'); - continue; - } - - final int exitCode = await processRunner.runAndStream( - gradleFile.path, ['testDebugUnitTest', '--info'], - workingDir: androidDirectory); - if (exitCode != 0) { - errors.add('$exampleName tests failed.'); - } - } - return errors.isEmpty - ? PackageResult.success() - : PackageResult.fail(errors); - } -} diff --git a/script/tool/lib/src/main.dart b/script/tool/lib/src/main.dart index ef1a18ab15..6001c5df7f 100644 --- a/script/tool/lib/src/main.dart +++ b/script/tool/lib/src/main.dart @@ -15,17 +15,16 @@ import 'create_all_plugins_app_command.dart'; import 'drive_examples_command.dart'; import 'firebase_test_lab_command.dart'; import 'format_command.dart'; -import 'java_test_command.dart'; import 'license_check_command.dart'; import 'lint_podspecs_command.dart'; import 'list_command.dart'; +import 'native_test_command.dart'; import 'publish_check_command.dart'; import 'publish_plugin_command.dart'; import 'pubspec_check_command.dart'; import 'test_command.dart'; import 'version_check_command.dart'; import 'xcode_analyze_command.dart'; -import 'xctest_command.dart'; void main(List args) { const FileSystem fileSystem = LocalFileSystem(); @@ -51,17 +50,16 @@ void main(List args) { ..addCommand(DriveExamplesCommand(packagesDir)) ..addCommand(FirebaseTestLabCommand(packagesDir)) ..addCommand(FormatCommand(packagesDir)) - ..addCommand(JavaTestCommand(packagesDir)) ..addCommand(LicenseCheckCommand(packagesDir)) ..addCommand(LintPodspecsCommand(packagesDir)) ..addCommand(ListCommand(packagesDir)) + ..addCommand(NativeTestCommand(packagesDir)) ..addCommand(PublishCheckCommand(packagesDir)) ..addCommand(PublishPluginCommand(packagesDir)) ..addCommand(PubspecCheckCommand(packagesDir)) ..addCommand(TestCommand(packagesDir)) ..addCommand(VersionCheckCommand(packagesDir)) - ..addCommand(XcodeAnalyzeCommand(packagesDir)) - ..addCommand(XCTestCommand(packagesDir)); + ..addCommand(XcodeAnalyzeCommand(packagesDir)); commandRunner.run(args).catchError((Object e) { final ToolExit toolExit = e as ToolExit; diff --git a/script/tool/lib/src/native_test_command.dart b/script/tool/lib/src/native_test_command.dart new file mode 100644 index 0000000000..73a435d83e --- /dev/null +++ b/script/tool/lib/src/native_test_command.dart @@ -0,0 +1,377 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/file.dart'; +import 'package:platform/platform.dart'; + +import 'common/core.dart'; +import 'common/package_looping_command.dart'; +import 'common/plugin_utils.dart'; +import 'common/process_runner.dart'; +import 'common/xcode.dart'; + +const String _unitTestFlag = 'unit'; +const String _integrationTestFlag = 'integration'; + +const String _iosDestinationFlag = 'ios-destination'; + +const int _exitNoIosSimulators = 3; + +/// The command to run native tests for plugins: +/// - iOS and macOS: XCTests (XCUnitTest and XCUITest) in plugins. +class NativeTestCommand extends PackageLoopingCommand { + /// Creates an instance of the test command. + NativeTestCommand( + Directory packagesDir, { + ProcessRunner processRunner = const ProcessRunner(), + Platform platform = const LocalPlatform(), + }) : _xcode = Xcode(processRunner: processRunner, log: true), + super(packagesDir, processRunner: processRunner, platform: platform) { + argParser.addOption( + _iosDestinationFlag, + help: 'Specify the destination when running iOS tests.\n' + 'This is passed to the `-destination` argument in the xcodebuild command.\n' + 'See https://developer.apple.com/library/archive/technotes/tn2339/_index.html#//apple_ref/doc/uid/DTS40014588-CH1-UNIT ' + 'for details on how to specify the destination.', + ); + argParser.addFlag(kPlatformAndroid, help: 'Runs Android tests'); + argParser.addFlag(kPlatformIos, help: 'Runs iOS tests'); + argParser.addFlag(kPlatformMacos, help: 'Runs macOS tests'); + + // By default, both unit tests and integration tests are run, but provide + // flags to disable one or the other. + argParser.addFlag(_unitTestFlag, + help: 'Runs native unit tests', defaultsTo: true); + argParser.addFlag(_integrationTestFlag, + help: 'Runs native integration (UI) tests', defaultsTo: true); + } + + static const String _gradleWrapper = 'gradlew'; + + // The device destination flags for iOS tests. + List _iosDestinationFlags = []; + + final Xcode _xcode; + + @override + final String name = 'native-test'; + + @override + final String description = ''' +Runs native unit tests and native integration tests. + +Currently supported platforms: +- Android (unit tests only) +- iOS: requires 'xcrun' to be in your path. +- macOS: requires 'xcrun' to be in your path. + +The example app(s) must be built for all targeted platforms before running +this command. +'''; + + Map _platforms = {}; + + List _requestedPlatforms = []; + + @override + Future initializeRun() async { + _platforms = { + kPlatformAndroid: _PlatformDetails('Android', _testAndroid), + kPlatformIos: _PlatformDetails('iOS', _testIos), + kPlatformMacos: _PlatformDetails('macOS', _testMacOS), + }; + _requestedPlatforms = _platforms.keys + .where((String platform) => getBoolArg(platform)) + .toList(); + _requestedPlatforms.sort(); + + if (_requestedPlatforms.isEmpty) { + printError('At least one platform flag must be provided.'); + throw ToolExit(exitInvalidArguments); + } + + if (!(getBoolArg(_unitTestFlag) || getBoolArg(_integrationTestFlag))) { + printError('At least one test type must be enabled.'); + throw ToolExit(exitInvalidArguments); + } + + if (getBoolArg(kPlatformAndroid) && getBoolArg(_integrationTestFlag)) { + logWarning('This command currently only supports unit tests for Android. ' + 'See https://github.com/flutter/flutter/issues/86490.'); + } + + // iOS-specific run-level state. + if (_requestedPlatforms.contains('ios')) { + String destination = getStringArg(_iosDestinationFlag); + if (destination.isEmpty) { + final String? simulatorId = + await _xcode.findBestAvailableIphoneSimulator(); + if (simulatorId == null) { + printError('Cannot find any available iOS simulators.'); + throw ToolExit(_exitNoIosSimulators); + } + destination = 'id=$simulatorId'; + } + _iosDestinationFlags = [ + '-destination', + destination, + ]; + } + } + + @override + Future runForPackage(Directory package) async { + final List testPlatforms = []; + for (final String platform in _requestedPlatforms) { + if (pluginSupportsPlatform(platform, package, + requiredMode: PlatformSupport.inline)) { + testPlatforms.add(platform); + } else { + print('No implementation for ${_platforms[platform]!.label}.'); + } + } + + if (testPlatforms.isEmpty) { + return PackageResult.skip('Not implemented for target platform(s).'); + } + + final _TestMode mode = _TestMode( + unit: getBoolArg(_unitTestFlag), + integration: getBoolArg(_integrationTestFlag), + ); + + bool ranTests = false; + bool failed = false; + final List failureMessages = []; + for (final String platform in testPlatforms) { + final _PlatformDetails platformInfo = _platforms[platform]!; + print('Running tests for ${platformInfo.label}...'); + print('----------------------------------------'); + final _PlatformResult result = + await platformInfo.testFunction(package, mode); + ranTests |= result.state != RunState.skipped; + if (result.state == RunState.failed) { + failed = true; + + final String? error = result.error; + // Only provide the failing platforms in the failure details if testing + // multiple platforms, otherwise it's just noise. + if (_requestedPlatforms.length > 1) { + failureMessages.add(error != null + ? '${platformInfo.label}: $error' + : platformInfo.label); + } else if (error != null) { + // If there's only one platform, only provide error details in the + // summary if the platform returned a message. + failureMessages.add(error); + } + } + } + + if (!ranTests) { + return PackageResult.skip('No tests found.'); + } + return failed + ? PackageResult.fail(failureMessages) + : PackageResult.success(); + } + + Future<_PlatformResult> _testAndroid(Directory plugin, _TestMode mode) async { + final List examplesWithTests = []; + for (final Directory example in getExamplesForPlugin(plugin)) { + if (!isFlutterPackage(example)) { + continue; + } + if (example + .childDirectory('android') + .childDirectory('app') + .childDirectory('src') + .childDirectory('test') + .existsSync() || + example.parent + .childDirectory('android') + .childDirectory('src') + .childDirectory('test') + .existsSync()) { + examplesWithTests.add(example); + } else { + _printNoExampleTestsMessage(example, 'Android'); + } + } + + if (examplesWithTests.isEmpty) { + return _PlatformResult(RunState.skipped); + } + + bool failed = false; + bool hasMissingBuild = false; + for (final Directory example in examplesWithTests) { + final String exampleName = getPackageDescription(example); + _printRunningExampleTestsMessage(example, 'Android'); + + final Directory androidDirectory = example.childDirectory('android'); + final File gradleFile = androidDirectory.childFile(_gradleWrapper); + if (!gradleFile.existsSync()) { + printError('ERROR: Run "flutter build apk" on $exampleName, or run ' + 'this tool\'s "build-examples --apk" command, ' + 'before executing tests.'); + failed = true; + hasMissingBuild = true; + continue; + } + + final int exitCode = await processRunner.runAndStream( + gradleFile.path, ['testDebugUnitTest', '--info'], + workingDir: androidDirectory); + if (exitCode != 0) { + printError('$exampleName tests failed.'); + failed = true; + } + } + return _PlatformResult(failed ? RunState.failed : RunState.succeeded, + error: + hasMissingBuild ? 'Examples must be built before testing.' : null); + } + + Future<_PlatformResult> _testIos(Directory plugin, _TestMode mode) { + return _runXcodeTests(plugin, 'iOS', mode, + extraFlags: _iosDestinationFlags); + } + + Future<_PlatformResult> _testMacOS(Directory plugin, _TestMode mode) { + return _runXcodeTests(plugin, 'macOS', mode); + } + + /// Runs all applicable tests for [plugin], printing status and returning + /// the test result. + /// + /// The tests targets must be added to the Xcode project of the example app, + /// usually at "example/{ios,macos}/Runner.xcworkspace". + Future<_PlatformResult> _runXcodeTests( + Directory plugin, + String platform, + _TestMode mode, { + List extraFlags = const [], + }) async { + String? testTarget; + if (mode.unitOnly) { + testTarget = 'RunnerTests'; + } else if (mode.integrationOnly) { + testTarget = 'RunnerUITests'; + } + + // Assume skipped until at least one test has run. + RunState overallResult = RunState.skipped; + for (final Directory example in getExamplesForPlugin(plugin)) { + final String exampleName = getPackageDescription(example); + + if (testTarget != null) { + final Directory project = example + .childDirectory(platform.toLowerCase()) + .childDirectory('Runner.xcodeproj'); + final bool? hasTarget = + await _xcode.projectHasTarget(project, testTarget); + if (hasTarget == null) { + printError('Unable to check targets for $exampleName.'); + overallResult = RunState.failed; + continue; + } else if (!hasTarget) { + print('No "$testTarget" target in $exampleName; skipping.'); + continue; + } + } + + _printRunningExampleTestsMessage(example, platform); + final int exitCode = await _xcode.runXcodeBuild( + example, + actions: ['test'], + workspace: '${platform.toLowerCase()}/Runner.xcworkspace', + scheme: 'Runner', + configuration: 'Debug', + extraFlags: [ + if (testTarget != null) '-only-testing:$testTarget', + ...extraFlags, + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + ); + + // The exit code from 'xcodebuild test' when there are no tests. + const int _xcodebuildNoTestExitCode = 66; + switch (exitCode) { + case _xcodebuildNoTestExitCode: + _printNoExampleTestsMessage(example, platform); + continue; + case 0: + printSuccess('Successfully ran $platform xctest for $exampleName'); + // If this is the first test, assume success until something fails. + if (overallResult == RunState.skipped) { + overallResult = RunState.succeeded; + } + break; + default: + // Any failure means a failure overall. + overallResult = RunState.failed; + break; + } + } + return _PlatformResult(overallResult); + } + + /// Prints a standard format message indicating that [platform] tests for + /// [plugin]'s [example] are about to be run. + void _printRunningExampleTestsMessage(Directory example, String platform) { + print('Running $platform tests for ${getPackageDescription(example)}...'); + } + + /// Prints a standard format message indicating that no tests were found for + /// [plugin]'s [example] for [platform]. + void _printNoExampleTestsMessage(Directory example, String platform) { + print('No $platform tests found for ${getPackageDescription(example)}'); + } +} + +// The type for a function that takes a plugin directory and runs its native +// tests for a specific platform. +typedef _TestFunction = Future<_PlatformResult> Function(Directory, _TestMode); + +/// A collection of information related to a specific platform. +class _PlatformDetails { + const _PlatformDetails( + this.label, + this.testFunction, + ); + + /// The name to use in output. + final String label; + + /// The function to call to run tests. + final _TestFunction testFunction; +} + +/// Enabled state for different test types. +class _TestMode { + const _TestMode({required this.unit, required this.integration}); + + final bool unit; + final bool integration; + + bool get integrationOnly => integration && !unit; + bool get unitOnly => unit && !integration; +} + +/// The result of running a single platform's tests. +class _PlatformResult { + _PlatformResult(this.state, {this.error}); + + /// The overall state of the platform's tests. This should be: + /// - failed if any tests failed. + /// - succeeded if at least one test ran, and all tests passed. + /// - skipped if no tests ran. + final RunState state; + + /// An optional error string to include in the summary for this platform. + /// + /// Ignored unless [state] is `failed`. + final String? error; +} diff --git a/script/tool/lib/src/xctest_command.dart b/script/tool/lib/src/xctest_command.dart deleted file mode 100644 index 44fc3a87d5..0000000000 --- a/script/tool/lib/src/xctest_command.dart +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:file/file.dart'; -import 'package:platform/platform.dart'; - -import 'common/core.dart'; -import 'common/package_looping_command.dart'; -import 'common/plugin_utils.dart'; -import 'common/process_runner.dart'; -import 'common/xcode.dart'; - -const String _iosDestinationFlag = 'ios-destination'; -const String _testTargetFlag = 'test-target'; - -// The exit code from 'xcodebuild test' when there are no tests. -const int _xcodebuildNoTestExitCode = 66; - -const int _exitNoSimulators = 3; - -/// The command to run XCTests (XCUnitTest and XCUITest) in plugins. -/// The tests target have to be added to the Xcode project of the example app, -/// usually at "example/{ios,macos}/Runner.xcworkspace". -class XCTestCommand extends PackageLoopingCommand { - /// Creates an instance of the test command. - XCTestCommand( - Directory packagesDir, { - ProcessRunner processRunner = const ProcessRunner(), - Platform platform = const LocalPlatform(), - }) : _xcode = Xcode(processRunner: processRunner, log: true), - super(packagesDir, processRunner: processRunner, platform: platform) { - argParser.addOption( - _iosDestinationFlag, - help: - 'Specify the destination when running the test, used for -destination flag for xcodebuild command.\n' - 'this is passed to the `-destination` argument in xcodebuild command.\n' - 'See https://developer.apple.com/library/archive/technotes/tn2339/_index.html#//apple_ref/doc/uid/DTS40014588-CH1-UNIT for details on how to specify the destination.', - ); - argParser.addOption( - _testTargetFlag, - help: - 'Limits the tests to a specific target (e.g., RunnerTests or RunnerUITests)', - ); - argParser.addFlag(kPlatformIos, help: 'Runs the iOS tests'); - argParser.addFlag(kPlatformMacos, help: 'Runs the macOS tests'); - } - - // The device destination flags for iOS tests. - List _iosDestinationFlags = []; - - final Xcode _xcode; - - @override - final String name = 'xctest'; - - @override - final String description = - 'Runs the xctests in the iOS and/or macOS example apps.\n\n' - 'This command requires "flutter" and "xcrun" to be in your path.'; - - @override - Future initializeRun() async { - final bool shouldTestIos = getBoolArg(kPlatformIos); - final bool shouldTestMacos = getBoolArg(kPlatformMacos); - - if (!(shouldTestIos || shouldTestMacos)) { - printError('At least one platform flag must be provided.'); - throw ToolExit(exitInvalidArguments); - } - - if (shouldTestIos) { - String destination = getStringArg(_iosDestinationFlag); - if (destination.isEmpty) { - final String? simulatorId = - await _xcode.findBestAvailableIphoneSimulator(); - if (simulatorId == null) { - printError('Cannot find any available simulators, tests failed'); - throw ToolExit(_exitNoSimulators); - } - destination = 'id=$simulatorId'; - } - _iosDestinationFlags = [ - '-destination', - destination, - ]; - } - } - - @override - Future runForPackage(Directory package) async { - final bool testIos = getBoolArg(kPlatformIos) && - pluginSupportsPlatform(kPlatformIos, package, - requiredMode: PlatformSupport.inline); - final bool testMacos = getBoolArg(kPlatformMacos) && - pluginSupportsPlatform(kPlatformMacos, package, - requiredMode: PlatformSupport.inline); - - final bool multiplePlatformsRequested = - getBoolArg(kPlatformIos) && getBoolArg(kPlatformMacos); - if (!(testIos || testMacos)) { - String description; - if (multiplePlatformsRequested) { - description = 'Neither iOS nor macOS is'; - } else if (getBoolArg(kPlatformIos)) { - description = 'iOS is not'; - } else { - description = 'macOS is not'; - } - return PackageResult.skip( - '$description implemented by this plugin package.'); - } - - if (multiplePlatformsRequested && (!testIos || !testMacos)) { - print('Only running for ${testIos ? 'iOS' : 'macOS'}\n'); - } - - final List failures = []; - bool ranTests = false; - if (testIos) { - final RunState result = await _testPlugin(package, 'iOS', - extraXcrunFlags: _iosDestinationFlags); - ranTests |= result != RunState.skipped; - if (result == RunState.failed) { - failures.add('iOS'); - } - } - if (testMacos) { - final RunState result = await _testPlugin(package, 'macOS'); - ranTests |= result != RunState.skipped; - if (result == RunState.failed) { - failures.add('macOS'); - } - } - - if (!ranTests) { - return PackageResult.skip('No tests found.'); - } - // Only provide the failing platform in the failure details if testing - // multiple platforms, otherwise it's just noise. - return failures.isEmpty - ? PackageResult.success() - : PackageResult.fail( - multiplePlatformsRequested ? failures : []); - } - - /// Runs all applicable tests for [plugin], printing status and returning - /// the test result. - Future _testPlugin( - Directory plugin, - String platform, { - List extraXcrunFlags = const [], - }) async { - final String testTarget = getStringArg(_testTargetFlag); - - // Assume skipped until at least one test has run. - RunState overallResult = RunState.skipped; - for (final Directory example in getExamplesForPlugin(plugin)) { - final String examplePath = - getRelativePosixPath(example, from: plugin.parent); - - if (testTarget.isNotEmpty) { - final Directory project = example - .childDirectory(platform.toLowerCase()) - .childDirectory('Runner.xcodeproj'); - final bool? hasTarget = - await _xcode.projectHasTarget(project, testTarget); - if (hasTarget == null) { - printError('Unable to check targets for $examplePath.'); - overallResult = RunState.failed; - continue; - } else if (!hasTarget) { - print('No "$testTarget" target in $examplePath; skipping.'); - continue; - } - } - - print('Running $platform tests for $examplePath...'); - final int exitCode = await _xcode.runXcodeBuild( - example, - actions: ['test'], - workspace: '${platform.toLowerCase()}/Runner.xcworkspace', - scheme: 'Runner', - configuration: 'Debug', - extraFlags: [ - if (testTarget.isNotEmpty) '-only-testing:$testTarget', - ...extraXcrunFlags, - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - ); - - switch (exitCode) { - case _xcodebuildNoTestExitCode: - print('No tests found for $examplePath'); - continue; - case 0: - printSuccess('Successfully ran $platform xctest for $examplePath'); - // If this is the first test, assume success until something fails. - if (overallResult == RunState.skipped) { - overallResult = RunState.succeeded; - } - break; - default: - // Any failure means a failure overall. - overallResult = RunState.failed; - break; - } - } - return overallResult; - } -} diff --git a/script/tool/test/java_test_command_test.dart b/script/tool/test/java_test_command_test.dart deleted file mode 100644 index 13e0e7fc0f..0000000000 --- a/script/tool/test/java_test_command_test.dart +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io' as io; - -import 'package:args/command_runner.dart'; -import 'package:file/file.dart'; -import 'package:file/memory.dart'; -import 'package:flutter_plugin_tools/src/common/core.dart'; -import 'package:flutter_plugin_tools/src/common/plugin_utils.dart'; -import 'package:flutter_plugin_tools/src/java_test_command.dart'; -import 'package:test/test.dart'; - -import 'mocks.dart'; -import 'util.dart'; - -void main() { - group('$JavaTestCommand', () { - late FileSystem fileSystem; - late MockPlatform mockPlatform; - late Directory packagesDir; - late CommandRunner runner; - late RecordingProcessRunner processRunner; - - setUp(() { - fileSystem = MemoryFileSystem(); - mockPlatform = MockPlatform(); - packagesDir = createPackagesDirectory(fileSystem: fileSystem); - processRunner = RecordingProcessRunner(); - final JavaTestCommand command = JavaTestCommand( - packagesDir, - processRunner: processRunner, - platform: mockPlatform, - ); - - runner = - CommandRunner('java_test_test', 'Test for $JavaTestCommand'); - runner.addCommand(command); - }); - - test('Should run Java tests in Android implementation folder', () async { - final Directory plugin = createFakePlugin( - 'plugin1', - packagesDir, - platformSupport: { - kPlatformAndroid: PlatformSupport.inline - }, - extraFiles: [ - 'example/android/gradlew', - 'android/src/test/example_test.java', - ], - ); - - await runCapturingPrint(runner, ['java-test']); - - final Directory androidFolder = - plugin.childDirectory('example').childDirectory('android'); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - androidFolder.childFile('gradlew').path, - const ['testDebugUnitTest', '--info'], - androidFolder.path, - ), - ]), - ); - }); - - test('Should run Java tests in example folder', () async { - final Directory plugin = createFakePlugin( - 'plugin1', - packagesDir, - platformSupport: { - kPlatformAndroid: PlatformSupport.inline - }, - extraFiles: [ - 'example/android/gradlew', - 'example/android/app/src/test/example_test.java', - ], - ); - - await runCapturingPrint(runner, ['java-test']); - - final Directory androidFolder = - plugin.childDirectory('example').childDirectory('android'); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - androidFolder.childFile('gradlew').path, - const ['testDebugUnitTest', '--info'], - androidFolder.path, - ), - ]), - ); - }); - - test('fails when the app needs to be built', () async { - createFakePlugin( - 'plugin1', - packagesDir, - platformSupport: { - kPlatformAndroid: PlatformSupport.inline - }, - extraFiles: [ - 'example/android/app/src/test/example_test.java', - ], - ); - - Error? commandError; - final List output = await runCapturingPrint( - runner, ['java-test'], errorHandler: (Error e) { - commandError = e; - }); - - expect(commandError, isA()); - - expect( - output, - containsAllInOrder([ - contains('ERROR: Run "flutter build apk" on example'), - contains('plugin1:\n' - ' example has not been built.') - ]), - ); - }); - - test('fails when a test fails', () async { - final Directory pluginDir = createFakePlugin( - 'plugin1', - packagesDir, - platformSupport: { - kPlatformAndroid: PlatformSupport.inline - }, - extraFiles: [ - 'example/android/gradlew', - 'example/android/app/src/test/example_test.java', - ], - ); - - final String gradlewPath = pluginDir - .childDirectory('example') - .childDirectory('android') - .childFile('gradlew') - .path; - processRunner.mockProcessesForExecutable[gradlewPath] = [ - MockProcess.failing() - ]; - - Error? commandError; - final List output = await runCapturingPrint( - runner, ['java-test'], errorHandler: (Error e) { - commandError = e; - }); - - expect(commandError, isA()); - - expect( - output, - containsAllInOrder([ - contains('plugin1:\n' - ' example tests failed.') - ]), - ); - }); - - test('Skips when running no tests', () async { - createFakePlugin( - 'plugin1', - packagesDir, - ); - - final List output = - await runCapturingPrint(runner, ['java-test']); - - expect( - output, - containsAllInOrder( - [contains('SKIPPING: No Java unit tests.')]), - ); - }); - }); -} diff --git a/script/tool/test/native_test_command_test.dart b/script/tool/test/native_test_command_test.dart new file mode 100644 index 0000000000..ca28a6cff0 --- /dev/null +++ b/script/tool/test/native_test_command_test.dart @@ -0,0 +1,1071 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:args/command_runner.dart'; +import 'package:file/file.dart'; +import 'package:file/memory.dart'; +import 'package:flutter_plugin_tools/src/common/core.dart'; +import 'package:flutter_plugin_tools/src/common/plugin_utils.dart'; +import 'package:flutter_plugin_tools/src/native_test_command.dart'; +import 'package:test/test.dart'; + +import 'mocks.dart'; +import 'util.dart'; + +final Map _kDeviceListMap = { + 'runtimes': >[ + { + 'bundlePath': + '/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS 13.4.simruntime', + 'buildversion': '17L255', + 'runtimeRoot': + '/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS 13.4.simruntime/Contents/Resources/RuntimeRoot', + 'identifier': 'com.apple.CoreSimulator.SimRuntime.iOS-13-4', + 'version': '13.4', + 'isAvailable': true, + 'name': 'iOS 13.4' + }, + ], + 'devices': { + 'com.apple.CoreSimulator.SimRuntime.iOS-13-4': >[ + { + 'dataPath': + '/Users/xxx/Library/Developer/CoreSimulator/Devices/1E76A0FD-38AC-4537-A989-EA639D7D012A/data', + 'logPath': + '/Users/xxx/Library/Logs/CoreSimulator/1E76A0FD-38AC-4537-A989-EA639D7D012A', + 'udid': '1E76A0FD-38AC-4537-A989-EA639D7D012A', + 'isAvailable': true, + 'deviceTypeIdentifier': + 'com.apple.CoreSimulator.SimDeviceType.iPhone-8-Plus', + 'state': 'Shutdown', + 'name': 'iPhone 8 Plus' + } + ] + } +}; + +// TODO(stuartmorgan): Rework these tests to use a mock Xcode instead of +// doing all the process mocking and validation. +void main() { + const String _kDestination = '--ios-destination'; + + group('test native_test_command', () { + late FileSystem fileSystem; + late MockPlatform mockPlatform; + late Directory packagesDir; + late CommandRunner runner; + late RecordingProcessRunner processRunner; + + setUp(() { + fileSystem = MemoryFileSystem(); + mockPlatform = MockPlatform(isMacOS: true); + packagesDir = createPackagesDirectory(fileSystem: fileSystem); + processRunner = RecordingProcessRunner(); + final NativeTestCommand command = NativeTestCommand(packagesDir, + processRunner: processRunner, platform: mockPlatform); + + runner = CommandRunner( + 'native_test_command', 'Test for native_test_command'); + runner.addCommand(command); + }); + + test('fails if no platforms are provided', () async { + Error? commandError; + final List output = await runCapturingPrint( + runner, ['native-test'], errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('At least one platform flag must be provided.'), + ]), + ); + }); + + test('fails if all test types are disabled', () async { + Error? commandError; + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--macos', + '--no-unit', + '--no-integration', + ], errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('At least one test type must be enabled.'), + ]), + ); + }); + + test('reports skips with no tests', () async { + final Directory pluginDirectory1 = createFakePlugin('plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + // Exit code 66 from testing indicates no tests. + final MockProcess noTestsProcessResult = MockProcess(); + noTestsProcessResult.exitCodeCompleter.complete(66); + processRunner.mockProcessesForExecutable['xcrun'] = [ + noTestsProcessResult, + ]; + final List output = + await runCapturingPrint(runner, ['native-test', '--macos']); + + expect(output, contains(contains('No tests found.'))); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'macos/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + group('iOS', () { + test('skip if iOS is not supported', () async { + createFakePlugin('plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final List output = await runCapturingPrint(runner, + ['native-test', '--ios', _kDestination, 'foo_destination']); + expect( + output, + containsAllInOrder([ + contains('No implementation for iOS.'), + contains('SKIPPING: Not implemented for target platform(s).'), + ])); + expect(processRunner.recordedCalls, orderedEquals([])); + }); + + test('skip if iOS is implemented in a federated package', () async { + createFakePlugin('plugin', packagesDir, + platformSupport: { + kPlatformIos: PlatformSupport.federated + }); + + final List output = await runCapturingPrint(runner, + ['native-test', '--ios', _kDestination, 'foo_destination']); + expect( + output, + containsAllInOrder([ + contains('No implementation for iOS.'), + contains('SKIPPING: Not implemented for target platform(s).'), + ])); + expect(processRunner.recordedCalls, orderedEquals([])); + }); + + test('running with correct destination', () async { + final Directory pluginDirectory = createFakePlugin( + 'plugin', packagesDir, platformSupport: { + kPlatformIos: PlatformSupport.inline + }); + + final Directory pluginExampleDirectory = + pluginDirectory.childDirectory('example'); + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--ios', + _kDestination, + 'foo_destination', + ]); + + expect( + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('Successfully ran iOS xctest for plugin/example') + ])); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'ios/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + '-destination', + 'foo_destination', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + test('Not specifying --ios-destination assigns an available simulator', + () async { + final Directory pluginDirectory = createFakePlugin( + 'plugin', packagesDir, platformSupport: { + kPlatformIos: PlatformSupport.inline + }); + + final Directory pluginExampleDirectory = + pluginDirectory.childDirectory('example'); + + processRunner.processToReturn = MockProcess.succeeding(); + processRunner.resultStdout = jsonEncode(_kDeviceListMap); + await runCapturingPrint(runner, ['native-test', '--ios']); + + expect( + processRunner.recordedCalls, + orderedEquals([ + const ProcessCall( + 'xcrun', + [ + 'simctl', + 'list', + 'devices', + 'runtimes', + 'available', + '--json', + ], + null), + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'ios/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + '-destination', + 'id=1E76A0FD-38AC-4537-A989-EA639D7D012A', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + }); + + group('macOS', () { + test('skip if macOS is not supported', () async { + createFakePlugin('plugin', packagesDir); + + final List output = + await runCapturingPrint(runner, ['native-test', '--macos']); + + expect( + output, + containsAllInOrder([ + contains('No implementation for macOS.'), + contains('SKIPPING: Not implemented for target platform(s).'), + ])); + expect(processRunner.recordedCalls, orderedEquals([])); + }); + + test('skip if macOS is implemented in a federated package', () async { + createFakePlugin('plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.federated, + }); + + final List output = + await runCapturingPrint(runner, ['native-test', '--macos']); + + expect( + output, + containsAllInOrder([ + contains('No implementation for macOS.'), + contains('SKIPPING: Not implemented for target platform(s).'), + ])); + expect(processRunner.recordedCalls, orderedEquals([])); + }); + + test('runs for macOS plugin', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--macos', + ]); + + expect( + output, + contains( + contains('Successfully ran macOS xctest for plugin/example'))); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'macos/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + }); + + group('Android', () { + test('runs Java tests in Android implementation folder', () async { + final Directory plugin = createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline + }, + extraFiles: [ + 'example/android/gradlew', + 'android/src/test/example_test.java', + ], + ); + + await runCapturingPrint(runner, ['native-test', '--android']); + + final Directory androidFolder = + plugin.childDirectory('example').childDirectory('android'); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + androidFolder.childFile('gradlew').path, + const ['testDebugUnitTest', '--info'], + androidFolder.path, + ), + ]), + ); + }); + + test('runs Java tests in example folder', () async { + final Directory plugin = createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline + }, + extraFiles: [ + 'example/android/gradlew', + 'example/android/app/src/test/example_test.java', + ], + ); + + await runCapturingPrint(runner, ['native-test', '--android']); + + final Directory androidFolder = + plugin.childDirectory('example').childDirectory('android'); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + androidFolder.childFile('gradlew').path, + const ['testDebugUnitTest', '--info'], + androidFolder.path, + ), + ]), + ); + }); + + test('fails when the app needs to be built', () async { + createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline + }, + extraFiles: [ + 'example/android/app/src/test/example_test.java', + ], + ); + + Error? commandError; + final List output = await runCapturingPrint( + runner, ['native-test', '--android'], + errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + + expect( + output, + containsAllInOrder([ + contains('ERROR: Run "flutter build apk" on plugin/example'), + contains('plugin:\n' + ' Examples must be built before testing.') + ]), + ); + }); + + test('fails when a test fails', () async { + final Directory pluginDir = createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline + }, + extraFiles: [ + 'example/android/gradlew', + 'example/android/app/src/test/example_test.java', + ], + ); + + final String gradlewPath = pluginDir + .childDirectory('example') + .childDirectory('android') + .childFile('gradlew') + .path; + processRunner.mockProcessesForExecutable[gradlewPath] = [ + MockProcess.failing() + ]; + + Error? commandError; + final List output = await runCapturingPrint( + runner, ['native-test', '--android'], + errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + + expect( + output, + containsAllInOrder([ + contains('plugin/example tests failed.'), + contains('The following packages had errors:'), + contains('plugin') + ]), + ); + }); + + test('skips if Android is not supported', () async { + createFakePlugin( + 'plugin', + packagesDir, + ); + + final List output = await runCapturingPrint( + runner, ['native-test', '--android']); + + expect( + output, + containsAllInOrder([ + contains('No implementation for Android.'), + contains('SKIPPING: Not implemented for target platform(s).'), + ]), + ); + }); + + test('skips when running no tests', () async { + createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline + }, + ); + + final List output = await runCapturingPrint( + runner, ['native-test', '--android']); + + expect( + output, + containsAllInOrder([ + contains('No Android tests found for plugin/example'), + contains('SKIPPING: No tests found.'), + ]), + ); + }); + }); + + // Tests behaviors of implementation that is shared between iOS and macOS. + group('iOS/macOS', () { + test('fails if xcrun fails', () async { + createFakePlugin('plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + processRunner.mockProcessesForExecutable['xcrun'] = [ + MockProcess.failing() + ]; + + Error? commandError; + final List output = + await runCapturingPrint(runner, ['native-test', '--macos'], + errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('The following packages had errors:'), + contains(' plugin'), + ]), + ); + }); + + test('honors unit-only', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + processRunner.processToReturn = MockProcess.succeeding(); + processRunner.resultStdout = + '{"project":{"targets":["RunnerTests", "RunnerUITests"]}}'; + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--macos', + '--no-integration', + ]); + + expect( + output, + contains( + contains('Successfully ran macOS xctest for plugin/example'))); + + // --no-integration should translate to '-only-testing:RunnerTests'. + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + [ + 'xcodebuild', + '-list', + '-json', + '-project', + pluginExampleDirectory + .childDirectory('macos') + .childDirectory('Runner.xcodeproj') + .path, + ], + null), + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'macos/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + '-only-testing:RunnerTests', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + test('honors integration-only', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + processRunner.processToReturn = MockProcess.succeeding(); + processRunner.resultStdout = + '{"project":{"targets":["RunnerTests", "RunnerUITests"]}}'; + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--macos', + '--no-unit', + ]); + + expect( + output, + contains( + contains('Successfully ran macOS xctest for plugin/example'))); + + // --no-unit should translate to '-only-testing:RunnerUITests'. + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + [ + 'xcodebuild', + '-list', + '-json', + '-project', + pluginExampleDirectory + .childDirectory('macos') + .childDirectory('Runner.xcodeproj') + .path, + ], + null), + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'macos/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + '-only-testing:RunnerUITests', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + test('skips when the requested target is not present', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + processRunner.processToReturn = MockProcess.succeeding(); + // Simulate a project with unit tests but no integration tests... + processRunner.resultStdout = '{"project":{"targets":["RunnerTests"]}}'; + // ... then try to run only integration tests. + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--macos', + '--no-unit', + ]); + + expect( + output, + containsAllInOrder([ + contains( + 'No "RunnerUITests" target in plugin/example; skipping.'), + ])); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + [ + 'xcodebuild', + '-list', + '-json', + '-project', + pluginExampleDirectory + .childDirectory('macos') + .childDirectory('Runner.xcodeproj') + .path, + ], + null), + ])); + }); + + test('fails if unable to check for requested target', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + processRunner.processToReturn = MockProcess.failing(); + + Error? commandError; + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--macos', + '--no-integration', + ], errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Unable to check targets for plugin/example.'), + ]), + ); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + [ + 'xcodebuild', + '-list', + '-json', + '-project', + pluginExampleDirectory + .childDirectory('macos') + .childDirectory('Runner.xcodeproj') + .path, + ], + null), + ])); + }); + }); + + group('multiplatform', () { + test('runs all platfroms when supported', () async { + final Directory pluginDirectory = createFakePlugin( + 'plugin', + packagesDir, + extraFiles: [ + 'example/android/gradlew', + 'android/src/test/example_test.java', + ], + platformSupport: { + kPlatformAndroid: PlatformSupport.inline, + kPlatformIos: PlatformSupport.inline, + kPlatformMacos: PlatformSupport.inline, + }, + ); + + final Directory pluginExampleDirectory = + pluginDirectory.childDirectory('example'); + final Directory androidFolder = + pluginExampleDirectory.childDirectory('android'); + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--android', + '--ios', + '--macos', + _kDestination, + 'foo_destination', + ]); + + expect( + output, + containsAll([ + contains('Running Android tests for plugin/example'), + contains('Successfully ran iOS xctest for plugin/example'), + contains('Successfully ran macOS xctest for plugin/example'), + ])); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + androidFolder.childFile('gradlew').path, + const ['testDebugUnitTest', '--info'], + androidFolder.path), + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'ios/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + '-destination', + 'foo_destination', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'macos/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + test('runs only macOS for a macOS plugin', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: PlatformSupport.inline, + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--ios', + '--macos', + _kDestination, + 'foo_destination', + ]); + + expect( + output, + containsAllInOrder([ + contains('No implementation for iOS.'), + contains('Successfully ran macOS xctest for plugin/example'), + ])); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'macos/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + test('runs only iOS for a iOS plugin', () async { + final Directory pluginDirectory = createFakePlugin( + 'plugin', packagesDir, platformSupport: { + kPlatformIos: PlatformSupport.inline + }); + + final Directory pluginExampleDirectory = + pluginDirectory.childDirectory('example'); + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--ios', + '--macos', + _kDestination, + 'foo_destination', + ]); + + expect( + output, + containsAllInOrder([ + contains('No implementation for macOS.'), + contains('Successfully ran iOS xctest for plugin/example') + ])); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + 'xcrun', + const [ + 'xcodebuild', + 'test', + '-workspace', + 'ios/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + '-destination', + 'foo_destination', + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + pluginExampleDirectory.path), + ])); + }); + + test('skips when nothing is supported', () async { + createFakePlugin('plugin', packagesDir); + + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--android', + '--ios', + '--macos', + _kDestination, + 'foo_destination', + ]); + + expect( + output, + containsAllInOrder([ + contains('No implementation for Android.'), + contains('No implementation for iOS.'), + contains('No implementation for macOS.'), + contains('SKIPPING: Not implemented for target platform(s).'), + ])); + + expect(processRunner.recordedCalls, orderedEquals([])); + }); + + test('failing one platform does not stop the tests', () async { + final Directory pluginDir = createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline, + kPlatformIos: PlatformSupport.inline, + }, + extraFiles: [ + 'example/android/gradlew', + 'example/android/app/src/test/example_test.java', + ], + ); + + // Simulate failing Android, but not iOS. + final String gradlewPath = pluginDir + .childDirectory('example') + .childDirectory('android') + .childFile('gradlew') + .path; + processRunner.mockProcessesForExecutable[gradlewPath] = [ + MockProcess.failing() + ]; + + Error? commandError; + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--android', + '--ios', + '--ios-destination', + 'foo_destination', + ], errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + + expect( + output, + containsAllInOrder([ + contains('Running tests for Android...'), + contains('plugin/example tests failed.'), + contains('Running tests for iOS...'), + contains('Successfully ran iOS xctest for plugin/example'), + contains('The following packages had errors:'), + contains('plugin:\n' + ' Android') + ]), + ); + }); + + test('failing multiple platforms reports multiple failures', () async { + final Directory pluginDir = createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + kPlatformAndroid: PlatformSupport.inline, + kPlatformIos: PlatformSupport.inline, + }, + extraFiles: [ + 'example/android/gradlew', + 'example/android/app/src/test/example_test.java', + ], + ); + + // Simulate failing Android. + final String gradlewPath = pluginDir + .childDirectory('example') + .childDirectory('android') + .childFile('gradlew') + .path; + processRunner.mockProcessesForExecutable[gradlewPath] = [ + MockProcess.failing() + ]; + // Simulate failing Android. + processRunner.mockProcessesForExecutable['xcrun'] = [ + MockProcess.failing() + ]; + + Error? commandError; + final List output = await runCapturingPrint(runner, [ + 'native-test', + '--android', + '--ios', + '--ios-destination', + 'foo_destination', + ], errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + + expect( + output, + containsAllInOrder([ + contains('Running tests for Android...'), + contains('Running tests for iOS...'), + contains('The following packages had errors:'), + contains('plugin:\n' + ' Android\n' + ' iOS') + ]), + ); + }); + }); + }); +} diff --git a/script/tool/test/xctest_command_test.dart b/script/tool/test/xctest_command_test.dart deleted file mode 100644 index 324dea0e71..0000000000 --- a/script/tool/test/xctest_command_test.dart +++ /dev/null @@ -1,705 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:convert'; -import 'dart:io' as io; - -import 'package:args/command_runner.dart'; -import 'package:file/file.dart'; -import 'package:file/memory.dart'; -import 'package:flutter_plugin_tools/src/common/core.dart'; -import 'package:flutter_plugin_tools/src/common/plugin_utils.dart'; -import 'package:flutter_plugin_tools/src/xctest_command.dart'; -import 'package:test/test.dart'; - -import 'mocks.dart'; -import 'util.dart'; - -final Map _kDeviceListMap = { - 'runtimes': >[ - { - 'bundlePath': - '/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS 13.4.simruntime', - 'buildversion': '17L255', - 'runtimeRoot': - '/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS 13.4.simruntime/Contents/Resources/RuntimeRoot', - 'identifier': 'com.apple.CoreSimulator.SimRuntime.iOS-13-4', - 'version': '13.4', - 'isAvailable': true, - 'name': 'iOS 13.4' - }, - ], - 'devices': { - 'com.apple.CoreSimulator.SimRuntime.iOS-13-4': >[ - { - 'dataPath': - '/Users/xxx/Library/Developer/CoreSimulator/Devices/1E76A0FD-38AC-4537-A989-EA639D7D012A/data', - 'logPath': - '/Users/xxx/Library/Logs/CoreSimulator/1E76A0FD-38AC-4537-A989-EA639D7D012A', - 'udid': '1E76A0FD-38AC-4537-A989-EA639D7D012A', - 'isAvailable': true, - 'deviceTypeIdentifier': - 'com.apple.CoreSimulator.SimDeviceType.iPhone-8-Plus', - 'state': 'Shutdown', - 'name': 'iPhone 8 Plus' - } - ] - } -}; - -// TODO(stuartmorgan): Rework these tests to use a mock Xcode instead of -// doing all the process mocking and validation. -void main() { - const String _kDestination = '--ios-destination'; - - group('test xctest_command', () { - late FileSystem fileSystem; - late MockPlatform mockPlatform; - late Directory packagesDir; - late CommandRunner runner; - late RecordingProcessRunner processRunner; - - setUp(() { - fileSystem = MemoryFileSystem(); - mockPlatform = MockPlatform(isMacOS: true); - packagesDir = createPackagesDirectory(fileSystem: fileSystem); - processRunner = RecordingProcessRunner(); - final XCTestCommand command = XCTestCommand(packagesDir, - processRunner: processRunner, platform: mockPlatform); - - runner = CommandRunner('xctest_command', 'Test for xctest_command'); - runner.addCommand(command); - }); - - test('Fails if no platforms are provided', () async { - Error? commandError; - final List output = await runCapturingPrint( - runner, ['xctest'], errorHandler: (Error e) { - commandError = e; - }); - - expect(commandError, isA()); - expect( - output, - containsAllInOrder([ - contains('At least one platform flag must be provided'), - ]), - ); - }); - - test('allows target filtering', () async { - final Directory pluginDirectory1 = createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - processRunner.processToReturn = MockProcess.succeeding(); - processRunner.resultStdout = '{"project":{"targets":["RunnerTests"]}}'; - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--macos', - '--test-target=RunnerTests', - ]); - - expect( - output, - contains( - contains('Successfully ran macOS xctest for plugin/example'))); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-only-testing:RunnerTests', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('skips when the requested target is not present', () async { - final Directory pluginDirectory1 = createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - processRunner.processToReturn = MockProcess.succeeding(); - processRunner.resultStdout = '{"project":{"targets":["Runner"]}}'; - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--macos', - '--test-target=RunnerTests', - ]); - - expect( - output, - containsAllInOrder([ - contains('No "RunnerTests" target in plugin/example; skipping.'), - ])); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), - ])); - }); - - test('fails if unable to check for requested target', () async { - final Directory pluginDirectory1 = createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - processRunner.processToReturn = MockProcess.failing(); - - Error? commandError; - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--macos', - '--test-target=RunnerTests', - ], errorHandler: (Error e) { - commandError = e; - }); - - expect(commandError, isA()); - expect( - output, - containsAllInOrder([ - contains('Unable to check targets for plugin/example.'), - ]), - ); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), - ])); - }); - - test('reports skips with no tests', () async { - final Directory pluginDirectory1 = createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - // Exit code 66 from testing indicates no tests. - final MockProcess noTestsProcessResult = MockProcess(); - noTestsProcessResult.exitCodeCompleter.complete(66); - processRunner.mockProcessesForExecutable['xcrun'] = [ - noTestsProcessResult, - ]; - final List output = - await runCapturingPrint(runner, ['xctest', '--macos']); - - expect(output, contains(contains('No tests found.'))); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - group('iOS', () { - test('skip if iOS is not supported', () async { - createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final List output = await runCapturingPrint(runner, - ['xctest', '--ios', _kDestination, 'foo_destination']); - expect( - output, - contains( - contains('iOS is not implemented by this plugin package.'))); - expect(processRunner.recordedCalls, orderedEquals([])); - }); - - test('skip if iOS is implemented in a federated package', () async { - createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformIos: PlatformSupport.federated - }); - - final List output = await runCapturingPrint(runner, - ['xctest', '--ios', _kDestination, 'foo_destination']); - expect( - output, - contains( - contains('iOS is not implemented by this plugin package.'))); - expect(processRunner.recordedCalls, orderedEquals([])); - }); - - test('running with correct destination', () async { - final Directory pluginDirectory = createFakePlugin( - 'plugin', packagesDir, platformSupport: { - kPlatformIos: PlatformSupport.inline - }); - - final Directory pluginExampleDirectory = - pluginDirectory.childDirectory('example'); - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--ios', - _kDestination, - 'foo_destination', - ]); - - expect( - output, - containsAllInOrder([ - contains('Running for plugin'), - contains('Successfully ran iOS xctest for plugin/example') - ])); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'foo_destination', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('Not specifying --ios-destination assigns an available simulator', - () async { - final Directory pluginDirectory = createFakePlugin( - 'plugin', packagesDir, platformSupport: { - kPlatformIos: PlatformSupport.inline - }); - - final Directory pluginExampleDirectory = - pluginDirectory.childDirectory('example'); - - processRunner.processToReturn = MockProcess.succeeding(); - processRunner.resultStdout = jsonEncode(_kDeviceListMap); - await runCapturingPrint(runner, ['xctest', '--ios']); - - expect( - processRunner.recordedCalls, - orderedEquals([ - const ProcessCall( - 'xcrun', - [ - 'simctl', - 'list', - 'devices', - 'runtimes', - 'available', - '--json', - ], - null), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'id=1E76A0FD-38AC-4537-A989-EA639D7D012A', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('fails if xcrun fails', () async { - createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformIos: PlatformSupport.inline - }); - - processRunner.mockProcessesForExecutable['xcrun'] = [ - MockProcess.failing() - ]; - - Error? commandError; - final List output = await runCapturingPrint( - runner, - [ - 'xctest', - '--ios', - _kDestination, - 'foo_destination', - ], - errorHandler: (Error e) { - commandError = e; - }, - ); - - expect(commandError, isA()); - expect( - output, - containsAllInOrder([ - contains('The following packages had errors:'), - contains(' plugin'), - ])); - }); - }); - - group('macOS', () { - test('skip if macOS is not supported', () async { - createFakePlugin('plugin', packagesDir); - - final List output = - await runCapturingPrint(runner, ['xctest', '--macos']); - expect( - output, - contains( - contains('macOS is not implemented by this plugin package.'))); - expect(processRunner.recordedCalls, orderedEquals([])); - }); - - test('skip if macOS is implemented in a federated package', () async { - createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.federated, - }); - - final List output = - await runCapturingPrint(runner, ['xctest', '--macos']); - expect( - output, - contains( - contains('macOS is not implemented by this plugin package.'))); - expect(processRunner.recordedCalls, orderedEquals([])); - }); - - test('runs for macOS plugin', () async { - final Directory pluginDirectory1 = createFakePlugin( - 'plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--macos', - ]); - - expect( - output, - contains( - contains('Successfully ran macOS xctest for plugin/example'))); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('fails if xcrun fails', () async { - createFakePlugin('plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - processRunner.mockProcessesForExecutable['xcrun'] = [ - MockProcess.failing() - ]; - - Error? commandError; - final List output = await runCapturingPrint( - runner, ['xctest', '--macos'], errorHandler: (Error e) { - commandError = e; - }); - - expect(commandError, isA()); - expect( - output, - containsAllInOrder([ - contains('The following packages had errors:'), - contains(' plugin'), - ]), - ); - }); - }); - - group('combined', () { - test('runs both iOS and macOS when supported', () async { - final Directory pluginDirectory1 = createFakePlugin( - 'plugin', packagesDir, - platformSupport: { - kPlatformIos: PlatformSupport.inline, - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--ios', - '--macos', - _kDestination, - 'foo_destination', - ]); - - expect( - output, - containsAll([ - contains('Successfully ran iOS xctest for plugin/example'), - contains('Successfully ran macOS xctest for plugin/example'), - ])); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'foo_destination', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('runs only macOS for a macOS plugin', () async { - final Directory pluginDirectory1 = createFakePlugin( - 'plugin', packagesDir, - platformSupport: { - kPlatformMacos: PlatformSupport.inline, - }); - - final Directory pluginExampleDirectory = - pluginDirectory1.childDirectory('example'); - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--ios', - '--macos', - _kDestination, - 'foo_destination', - ]); - - expect( - output, - containsAllInOrder([ - contains('Only running for macOS'), - contains('Successfully ran macOS xctest for plugin/example'), - ])); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('runs only iOS for a iOS plugin', () async { - final Directory pluginDirectory = createFakePlugin( - 'plugin', packagesDir, platformSupport: { - kPlatformIos: PlatformSupport.inline - }); - - final Directory pluginExampleDirectory = - pluginDirectory.childDirectory('example'); - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--ios', - '--macos', - _kDestination, - 'foo_destination', - ]); - - expect( - output, - containsAllInOrder([ - contains('Only running for iOS'), - contains('Successfully ran iOS xctest for plugin/example') - ])); - - expect( - processRunner.recordedCalls, - orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'foo_destination', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ])); - }); - - test('skips when neither are supported', () async { - createFakePlugin('plugin', packagesDir); - - final List output = await runCapturingPrint(runner, [ - 'xctest', - '--ios', - '--macos', - _kDestination, - 'foo_destination', - ]); - - expect( - output, - containsAllInOrder([ - contains( - 'SKIPPING: Neither iOS nor macOS is implemented by this plugin package.'), - ])); - - expect(processRunner.recordedCalls, orderedEquals([])); - }); - }); - }); -}