diff --git a/script/tool/CHANGELOG.md b/script/tool/CHANGELOG.md index c585bee472..7df6913db7 100644 --- a/script/tool/CHANGELOG.md +++ b/script/tool/CHANGELOG.md @@ -1,7 +1,7 @@ ## NEXT -- `native-test --android` now fails plugins that don't have unit tests, - rather than skipping them. +- `native-test --android`, `--ios`, and `--macos` now fail plugins that don't + have unit tests, rather than skipping them. ## 0.7.1 diff --git a/script/tool/lib/src/native_test_command.dart b/script/tool/lib/src/native_test_command.dart index 78a82afc57..4911b4aeb1 100644 --- a/script/tool/lib/src/native_test_command.dart +++ b/script/tool/lib/src/native_test_command.dart @@ -251,8 +251,6 @@ this command. final bool hasIntegrationTests = exampleHasNativeIntegrationTests(example); - // TODO(stuartmorgan): Make !hasUnitTests fatal. See - // https://github.com/flutter/flutter/issues/85469 if (mode.unit && !hasUnitTests) { _printNoExampleTestsMessage(example, 'Android unit'); } @@ -355,33 +353,40 @@ this command. List extraFlags = const [], }) async { String? testTarget; + const String unitTestTarget = 'RunnerTests'; if (mode.unitOnly) { - testTarget = 'RunnerTests'; + testTarget = unitTestTarget; } else if (mode.integrationOnly) { testTarget = 'RunnerUITests'; } + bool ranUnitTests = false; // Assume skipped until at least one test has run. RunState overallResult = RunState.skipped; for (final RepositoryPackage example in plugin.getExamples()) { final String exampleName = example.displayName; - // TODO(stuartmorgan): Always check for RunnerTests, and make it fatal if - // no examples have it. See - // https://github.com/flutter/flutter/issues/85469 - if (testTarget != null) { - final Directory project = example.directory - .childDirectory(platform.toLowerCase()) - .childDirectory('Runner.xcodeproj'); + // If running a specific target, check that. Otherwise, check if there + // are unit tests, since having no unit tests for a plugin is fatal + // (by repo policy) even if there are integration tests. + bool exampleHasUnitTests = false; + final String? targetToCheck = + testTarget ?? (mode.unit ? unitTestTarget : null); + final Directory xcodeProject = example.directory + .childDirectory(platform.toLowerCase()) + .childDirectory('Runner.xcodeproj'); + if (targetToCheck != null) { final bool? hasTarget = - await _xcode.projectHasTarget(project, testTarget); + await _xcode.projectHasTarget(xcodeProject, targetToCheck); if (hasTarget == null) { printError('Unable to check targets for $exampleName.'); overallResult = RunState.failed; continue; } else if (!hasTarget) { - print('No "$testTarget" target in $exampleName; skipping.'); + print('No "$targetToCheck" target in $exampleName; skipping.'); continue; + } else if (targetToCheck == unitTestTarget) { + exampleHasUnitTests = true; } } @@ -404,20 +409,39 @@ this command. switch (exitCode) { case _xcodebuildNoTestExitCode: _printNoExampleTestsMessage(example, platform); - continue; + break; case 0: printSuccess('Successfully ran $platform xctest for $exampleName'); // If this is the first test, assume success until something fails. if (overallResult == RunState.skipped) { overallResult = RunState.succeeded; } + if (exampleHasUnitTests) { + ranUnitTests = true; + } break; default: // Any failure means a failure overall. overallResult = RunState.failed; + // If unit tests ran, note that even if they failed. + if (exampleHasUnitTests) { + ranUnitTests = true; + } break; } } + + if (!mode.integrationOnly && !ranUnitTests) { + printError('No unit tests ran. Plugins are required to have unit tests.'); + // Only return a specific summary error message about the missing unit + // tests if there weren't also failures, to avoid having a misleadingly + // specific message. + if (overallResult != RunState.failed) { + return _PlatformResult(RunState.failed, + error: 'No unit tests ran (use --exclude if this is intentional).'); + } + } + return _PlatformResult(overallResult); } diff --git a/script/tool/test/native_test_command_test.dart b/script/tool/test/native_test_command_test.dart index f7b2ea5c0d..ba93efcb3a 100644 --- a/script/tool/test/native_test_command_test.dart +++ b/script/tool/test/native_test_command_test.dart @@ -78,6 +78,61 @@ void main() { runner.addCommand(command); }); + // Returns a MockProcess to provide for "xcrun xcodebuild -list" for a + // project that contains [targets]. + MockProcess _getMockXcodebuildListProcess(List targets) { + final Map projects = { + 'project': { + 'targets': targets, + } + }; + return MockProcess(stdout: jsonEncode(projects)); + } + + // Returns the ProcessCall to expect for checking the targets present in + // the [package]'s [platform]/Runner.xcodeproj. + ProcessCall _getTargetCheckCall(Directory package, String platform) { + return ProcessCall( + 'xcrun', + [ + 'xcodebuild', + '-list', + '-json', + '-project', + package + .childDirectory(platform) + .childDirectory('Runner.xcodeproj') + .path, + ], + null); + } + + // Returns the ProcessCall to expect for running the tests in the + // workspace [platform]/Runner.xcworkspace, with the given extra flags. + ProcessCall _getRunTestCall( + Directory package, + String platform, { + String? destination, + List extraFlags = const [], + }) { + return ProcessCall( + 'xcrun', + [ + 'xcodebuild', + 'test', + '-workspace', + '$platform/Runner.xcworkspace', + '-scheme', + 'Runner', + '-configuration', + 'Debug', + if (destination != null) ...['-destination', destination], + ...extraFlags, + 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', + ], + package.path); + } + test('fails if no platforms are provided', () async { Error? commandError; final List output = await runCapturingPrint( @@ -124,31 +179,26 @@ void main() { pluginDirectory1.childDirectory('example'); processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess(['RunnerTests', 'RunnerUITests']), // Exit code 66 from testing indicates no tests. MockProcess(exitCode: 66), ]; - final List output = - await runCapturingPrint(runner, ['native-test', '--macos']); + final List output = await runCapturingPrint( + runner, ['native-test', '--macos', '--no-unit']); - expect(output, contains(contains('No tests found.'))); + expect( + output, + containsAllInOrder([ + contains('No tests found.'), + contains('Skipped 1 package(s)'), + ])); expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + _getRunTestCall(pluginExampleDirectory, 'macos', + extraFlags: ['-only-testing:RunnerUITests']), ])); }); @@ -196,6 +246,11 @@ void main() { final Directory pluginExampleDirectory = pluginDirectory.childDirectory('example'); + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), + ]; + final List output = await runCapturingPrint(runner, [ 'native-test', '--ios', @@ -213,22 +268,9 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'foo_destination', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'ios'), + _getRunTestCall(pluginExampleDirectory, 'ios', + destination: 'foo_destination'), ])); }); @@ -243,6 +285,8 @@ void main() { processRunner.mockProcessesForExecutable['xcrun'] = [ MockProcess(stdout: jsonEncode(_kDeviceListMap)), // simctl + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), ]; await runCapturingPrint(runner, ['native-test', '--ios']); @@ -261,22 +305,9 @@ void main() { '--json', ], null), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'id=1E76A0FD-38AC-4537-A989-EA639D7D012A', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'ios'), + _getRunTestCall(pluginExampleDirectory, 'ios', + destination: 'id=1E76A0FD-38AC-4537-A989-EA639D7D012A'), ])); }); }); @@ -325,6 +356,11 @@ void main() { final Directory pluginExampleDirectory = pluginDirectory1.childDirectory('example'); + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), + ]; + final List output = await runCapturingPrint(runner, [ 'native-test', '--macos', @@ -338,20 +374,8 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + _getRunTestCall(pluginExampleDirectory, 'macos'), ])); }); }); @@ -999,13 +1023,9 @@ void main() { final Directory pluginExampleDirectory = pluginDirectory1.childDirectory('example'); - const Map projects = { - 'project': { - 'targets': ['RunnerTests', 'RunnerUITests'] - } - }; processRunner.mockProcessesForExecutable['xcrun'] = [ - MockProcess(stdout: jsonEncode(projects)), // xcodebuild -list + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), ]; final List output = await runCapturingPrint(runner, [ @@ -1023,34 +1043,9 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-only-testing:RunnerTests', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + _getRunTestCall(pluginExampleDirectory, 'macos', + extraFlags: ['-only-testing:RunnerTests']), ])); }); @@ -1064,13 +1059,9 @@ void main() { final Directory pluginExampleDirectory = pluginDirectory1.childDirectory('example'); - const Map projects = { - 'project': { - 'targets': ['RunnerTests', 'RunnerUITests'] - } - }; processRunner.mockProcessesForExecutable['xcrun'] = [ - MockProcess(stdout: jsonEncode(projects)), // xcodebuild -list + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), ]; final List output = await runCapturingPrint(runner, [ @@ -1088,34 +1079,9 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-only-testing:RunnerUITests', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + _getRunTestCall(pluginExampleDirectory, 'macos', + extraFlags: ['-only-testing:RunnerUITests']), ])); }); @@ -1130,13 +1096,8 @@ void main() { pluginDirectory1.childDirectory('example'); // Simulate a project with unit tests but no integration tests... - const Map projects = { - 'project': { - 'targets': ['RunnerTests'] - } - }; processRunner.mockProcessesForExecutable['xcrun'] = [ - MockProcess(stdout: jsonEncode(projects)), // xcodebuild -list + _getMockXcodebuildListProcess(['RunnerTests']), ]; // ... then try to run only integration tests. @@ -1156,19 +1117,47 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + ])); + }); + + test('fails if there are no unit tests', () async { + final Directory pluginDirectory1 = createFakePlugin( + 'plugin', packagesDir, + platformSupport: { + kPlatformMacos: const PlatformDetails(PlatformSupport.inline), + }); + + final Directory pluginExampleDirectory = + pluginDirectory1.childDirectory('example'); + + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess(['RunnerUITests']), + ]; + + Error? commandError; + final List output = + await runCapturingPrint(runner, ['native-test', '--macos'], + errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('No "RunnerTests" target in plugin/example; skipping.'), + contains( + 'No unit tests ran. Plugins are required to have unit tests.'), + contains('The following packages had errors:'), + contains('plugin:\n' + ' No unit tests ran (use --exclude if this is intentional).'), + ])); + + expect( + processRunner.recordedCalls, + orderedEquals([ + _getTargetCheckCall(pluginExampleDirectory, 'macos'), ])); }); @@ -1206,19 +1195,7 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - [ - 'xcodebuild', - '-list', - '-json', - '-project', - pluginExampleDirectory - .childDirectory('macos') - .childDirectory('Runner.xcodeproj') - .path, - ], - null), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), ])); }); }); @@ -1244,6 +1221,15 @@ void main() { final Directory androidFolder = pluginExampleDirectory.childDirectory('android'); + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), // iOS list + MockProcess(), // iOS run + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), // macOS list + MockProcess(), // macOS run + ]; + final List output = await runCapturingPrint(runner, [ 'native-test', '--android', @@ -1266,36 +1252,11 @@ void main() { orderedEquals([ ProcessCall(androidFolder.childFile('gradlew').path, const ['testDebugUnitTest'], androidFolder.path), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'foo_destination', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'ios'), + _getRunTestCall(pluginExampleDirectory, 'ios', + destination: 'foo_destination'), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + _getRunTestCall(pluginExampleDirectory, 'macos'), ])); }); @@ -1309,6 +1270,11 @@ void main() { final Directory pluginExampleDirectory = pluginDirectory1.childDirectory('example'); + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), + ]; + final List output = await runCapturingPrint(runner, [ 'native-test', '--ios', @@ -1327,20 +1293,8 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'macos/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'macos'), + _getRunTestCall(pluginExampleDirectory, 'macos'), ])); }); @@ -1353,6 +1307,11 @@ void main() { final Directory pluginExampleDirectory = pluginDirectory.childDirectory('example'); + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), + ]; + final List output = await runCapturingPrint(runner, [ 'native-test', '--ios', @@ -1371,22 +1330,9 @@ void main() { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'xcrun', - const [ - 'xcodebuild', - 'test', - '-workspace', - 'ios/Runner.xcworkspace', - '-scheme', - 'Runner', - '-configuration', - 'Debug', - '-destination', - 'foo_destination', - 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', - ], - pluginExampleDirectory.path), + _getTargetCheckCall(pluginExampleDirectory, 'ios'), + _getRunTestCall(pluginExampleDirectory, 'ios', + destination: 'foo_destination'), ])); }); @@ -1460,6 +1406,11 @@ void main() { ], ); + processRunner.mockProcessesForExecutable['xcrun'] = [ + _getMockXcodebuildListProcess( + ['RunnerTests', 'RunnerUITests']), + ]; + // Simulate failing Android, but not iOS. final String gradlewPath = pluginDir .childDirectory('example')