[flutter_plugin_tools] Add a summary for successful runs (#4118)

Add a summary to the end of successful runs for everything using the new looping base command, similar to what we do for summarizing failures. This will make it easy to manually check results for PRs that we know should be changing the set of run packages (adding a new package, adding a new test type to a package, adding a new test type to the tool), as well as spot-checking when we see unexpected results (e.g., looking back and why a PR didn't fail CI when we discover that it should have).

To support better surfacing skips, this restructures the return value of `runForPackage` to have "skip" as one of the options. As a result of it being a return value, packages that used `printSkip` to indicate that *parts* of the command were being skipped have been changed to no longer do that.

Fixes https://github.com/flutter/flutter/issues/85626
This commit is contained in:
stuartmorgan
2021-07-01 16:25:21 -07:00
committed by GitHub
parent 92d6214984
commit edeb10a752
19 changed files with 817 additions and 274 deletions

View File

@ -44,21 +44,16 @@ void main() {
test('building for iOS when plugin is not set up for iOS results in no-op',
() async {
final Directory pluginDirectory = createFakePlugin('plugin', packagesDir,
createFakePlugin('plugin', packagesDir,
extraFiles: <String>['example/test']);
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final List<String> output =
await runCapturingPrint(runner, <String>['build-examples', '--ios']);
final String packageName =
p.relative(pluginExampleDirectory.path, from: packagesDir.path);
expect(
output,
containsAllInOrder(<Matcher>[
contains('BUILDING $packageName for iOS'),
contains('Running for plugin'),
contains('iOS is not supported by this plugin'),
]),
);
@ -113,23 +108,17 @@ void main() {
test(
'building for Linux when plugin is not set up for Linux results in no-op',
() async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
]);
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final List<String> output = await runCapturingPrint(
runner, <String>['build-examples', '--linux']);
final String packageName =
p.relative(pluginExampleDirectory.path, from: packagesDir.path);
expect(
output,
containsAllInOrder(<Matcher>[
contains('BUILDING $packageName for Linux'),
contains('Running for plugin'),
contains('Linux is not supported by this plugin'),
]),
);
@ -176,23 +165,17 @@ void main() {
test('building for macos with no implementation results in no-op',
() async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
]);
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final List<String> output = await runCapturingPrint(
runner, <String>['build-examples', '--macos']);
final String packageName =
p.relative(pluginExampleDirectory.path, from: packagesDir.path);
expect(
output,
containsAllInOrder(<Matcher>[
contains('BUILDING $packageName for macOS'),
contains('Running for plugin'),
contains('macOS is not supported by this plugin'),
]),
);
@ -239,24 +222,18 @@ void main() {
});
test('building for web with no implementation results in no-op', () async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
]);
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final List<String> output =
await runCapturingPrint(runner, <String>['build-examples', '--web']);
final String packageName =
p.relative(pluginExampleDirectory.path, from: packagesDir.path);
expect(
output,
containsAllInOrder(<Matcher>[
contains('BUILDING $packageName for web'),
contains('Web is not supported by this plugin'),
contains('Running for plugin'),
contains('web is not supported by this plugin'),
]),
);
@ -304,29 +281,23 @@ void main() {
test(
'building for Windows when plugin is not set up for Windows results in no-op',
() async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
]);
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final List<String> output = await runCapturingPrint(
runner, <String>['build-examples', '--windows']);
final String packageName =
p.relative(pluginExampleDirectory.path, from: packagesDir.path);
expect(
output,
containsAllInOrder(<Matcher>[
contains('BUILDING $packageName for Windows'),
contains('Running for plugin'),
contains('Windows is not supported by this plugin'),
]),
);
// Output should be empty since running build-examples --macos with no macos
// implementation is a no-op.
// Output should be empty since running build-examples --windows with no
// Windows implementation is a no-op.
expect(processRunner.recordedCalls, orderedEquals(<ProcessCall>[]));
});
@ -368,23 +339,17 @@ void main() {
test(
'building for Android when plugin is not set up for Android results in no-op',
() async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
]);
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final List<String> output =
await runCapturingPrint(runner, <String>['build-examples', '--apk']);
final String packageName =
p.relative(pluginExampleDirectory.path, from: packagesDir.path);
expect(
output,
containsAllInOrder(<Matcher>[
contains('\nBUILDING APK for $packageName'),
contains('Running for plugin'),
contains('Android is not supported by this plugin'),
]),
);
@ -419,7 +384,7 @@ void main() {
expect(
output,
containsAllInOrder(<String>[
'\nBUILDING APK for $packageName',
'\nBUILDING $packageName for Android (apk)',
]),
);

View File

@ -19,14 +19,20 @@ import '../util.dart';
import 'plugin_command_test.mocks.dart';
// Constants for colorized output start and end.
const String _startErrorColor = '\x1B[31m';
const String _startHeadingColor = '\x1B[36m';
const String _startSkipColor = '\x1B[90m';
const String _startSkipWithWarningColor = '\x1B[93m';
const String _startSuccessColor = '\x1B[32m';
const String _startErrorColor = '\x1B[31m';
const String _startWarningColor = '\x1B[33m';
const String _endColor = '\x1B[0m';
// The filename within a package containing errors to return from runForPackage.
const String _errorFile = 'errors';
// The filename within a package indicating that it should be skipped.
const String _skipFile = 'skip';
// The filename within a package containing warnings to log during runForPackage.
const String _warningFile = 'warnings';
void main() {
late FileSystem fileSystem;
@ -48,6 +54,8 @@ void main() {
bool hasLongOutput = true,
bool includeSubpackages = false,
bool failsDuringInit = false,
bool warnsDuringInit = false,
bool warnsDuringCleanup = false,
String? customFailureListHeader,
String? customFailureListFooter,
}) {
@ -68,6 +76,8 @@ void main() {
hasLongOutput: hasLongOutput,
includeSubpackages: includeSubpackages,
failsDuringInit: failsDuringInit,
warnsDuringInit: warnsDuringInit,
warnsDuringCleanup: warnsDuringCleanup,
customFailureListHeader: customFailureListHeader,
customFailureListFooter: customFailureListFooter,
gitDir: gitDir,
@ -216,7 +226,8 @@ void main() {
expect(
output,
containsAllInOrder(<String>[
'$_startSuccessColor\n\nNo issues found!$_endColor',
'\n',
'${_startSuccessColor}No issues found!$_endColor',
]));
});
@ -314,24 +325,153 @@ void main() {
'${_startErrorColor}See above for full details.$_endColor',
]));
});
test('logs skips', () async {
createFakePackage('package_a', packagesDir);
final Directory skipPackage = createFakePackage('package_b', packagesDir);
skipPackage.childFile(_skipFile).writeAsStringSync('For a reason');
final TestPackageLoopingCommand command =
createTestCommand(hasLongOutput: false);
final List<String> output = await runCommand(command);
expect(
output,
containsAllInOrder(<String>[
'${_startHeadingColor}Running for package_a...$_endColor',
'${_startHeadingColor}Running for package_b...$_endColor',
'$_startSkipColor SKIPPING: For a reason$_endColor',
]));
});
test('logs warnings', () async {
final Directory warnPackage = createFakePackage('package_a', packagesDir);
warnPackage
.childFile(_warningFile)
.writeAsStringSync('Warning 1\nWarning 2');
createFakePackage('package_b', packagesDir);
final TestPackageLoopingCommand command =
createTestCommand(hasLongOutput: false);
final List<String> output = await runCommand(command);
expect(
output,
containsAllInOrder(<String>[
'${_startHeadingColor}Running for package_a...$_endColor',
'${_startWarningColor}Warning 1$_endColor',
'${_startWarningColor}Warning 2$_endColor',
'${_startHeadingColor}Running for package_b...$_endColor',
]));
});
test('prints run summary on success', () async {
final Directory warnPackage1 =
createFakePackage('package_a', packagesDir);
warnPackage1
.childFile(_warningFile)
.writeAsStringSync('Warning 1\nWarning 2');
createFakePackage('package_b', packagesDir);
final Directory skipPackage = createFakePackage('package_c', packagesDir);
skipPackage.childFile(_skipFile).writeAsStringSync('For a reason');
final Directory skipAndWarnPackage =
createFakePackage('package_d', packagesDir);
skipAndWarnPackage.childFile(_warningFile).writeAsStringSync('Warning');
skipAndWarnPackage.childFile(_skipFile).writeAsStringSync('See warning');
final Directory warnPackage2 =
createFakePackage('package_e', packagesDir);
warnPackage2
.childFile(_warningFile)
.writeAsStringSync('Warning 1\nWarning 2');
createFakePackage('package_f', packagesDir);
final TestPackageLoopingCommand command =
createTestCommand(hasLongOutput: false);
final List<String> output = await runCommand(command);
expect(
output,
containsAllInOrder(<String>[
'------------------------------------------------------------',
'Ran for 4 package(s) (2 with warnings)',
'Skipped 2 package(s) (1 with warnings)',
'\n',
'${_startSuccessColor}No issues found!$_endColor',
]));
// The long-form summary should not be printed for short-form commands.
expect(output, isNot(contains('Run summary:')));
expect(output, isNot(contains(contains('package a - ran'))));
});
test('prints long-form run summary for long-output commands', () async {
final Directory warnPackage1 =
createFakePackage('package_a', packagesDir);
warnPackage1
.childFile(_warningFile)
.writeAsStringSync('Warning 1\nWarning 2');
createFakePackage('package_b', packagesDir);
final Directory skipPackage = createFakePackage('package_c', packagesDir);
skipPackage.childFile(_skipFile).writeAsStringSync('For a reason');
final Directory skipAndWarnPackage =
createFakePackage('package_d', packagesDir);
skipAndWarnPackage.childFile(_warningFile).writeAsStringSync('Warning');
skipAndWarnPackage.childFile(_skipFile).writeAsStringSync('See warning');
final Directory warnPackage2 =
createFakePackage('package_e', packagesDir);
warnPackage2
.childFile(_warningFile)
.writeAsStringSync('Warning 1\nWarning 2');
createFakePackage('package_f', packagesDir);
final TestPackageLoopingCommand command =
createTestCommand(hasLongOutput: true);
final List<String> output = await runCommand(command);
expect(
output,
containsAllInOrder(<String>[
'------------------------------------------------------------',
'Run overview:',
' package_a - ${_startWarningColor}ran (with warning)$_endColor',
' package_b - ${_startSuccessColor}ran$_endColor',
' package_c - ${_startSkipColor}skipped$_endColor',
' package_d - ${_startSkipWithWarningColor}skipped (with warning)$_endColor',
' package_e - ${_startWarningColor}ran (with warning)$_endColor',
' package_f - ${_startSuccessColor}ran$_endColor',
'',
'Ran for 4 package(s) (2 with warnings)',
'Skipped 2 package(s) (1 with warnings)',
'\n',
'${_startSuccessColor}No issues found!$_endColor',
]));
});
test('handles warnings outside of runForPackage', () async {
createFakePackage('package_a', packagesDir);
final TestPackageLoopingCommand command = createTestCommand(
hasLongOutput: false,
warnsDuringCleanup: true,
warnsDuringInit: true,
);
final List<String> output = await runCommand(command);
expect(
output,
containsAllInOrder(<String>[
'${_startWarningColor}Warning during initializeRun$_endColor',
'${_startHeadingColor}Running for package_a...$_endColor',
'${_startWarningColor}Warning during completeRun$_endColor',
'------------------------------------------------------------',
'Ran for 1 package(s)',
'2 warnings not associated with a package',
'\n',
'${_startSuccessColor}No issues found!$_endColor',
]));
});
});
group('utility', () {
test('printSkip has expected output', () async {
final TestPackageLoopingCommand command =
TestPackageLoopingCommand(packagesDir);
final List<String> printBuffer = <String>[];
Zone.current.fork(specification: ZoneSpecification(
print: (_, __, ___, String message) {
printBuffer.add(message);
},
)).run<void>(() => command.printSkip('For a reason'));
expect(printBuffer.first,
'${_startSkipColor}SKIPPING: For a reason$_endColor');
});
test('getPackageDescription prints packageDir-relative paths by default',
() async {
final TestPackageLoopingCommand command =
@ -380,6 +520,8 @@ class TestPackageLoopingCommand extends PackageLoopingCommand {
this.customFailureListHeader,
this.customFailureListFooter,
this.failsDuringInit = false,
this.warnsDuringInit = false,
this.warnsDuringCleanup = false,
ProcessRunner processRunner = const ProcessRunner(),
GitDir? gitDir,
}) : super(packagesDir, processRunner: processRunner, gitDir: gitDir);
@ -390,6 +532,8 @@ class TestPackageLoopingCommand extends PackageLoopingCommand {
final String? customFailureListFooter;
final bool failsDuringInit;
final bool warnsDuringInit;
final bool warnsDuringCleanup;
@override
bool hasLongOutput;
@ -413,20 +557,38 @@ class TestPackageLoopingCommand extends PackageLoopingCommand {
@override
Future<void> initializeRun() async {
if (warnsDuringInit) {
logWarning('Warning during initializeRun');
}
if (failsDuringInit) {
throw ToolExit(2);
}
}
@override
Future<List<String>> runForPackage(Directory package) async {
Future<PackageResult> runForPackage(Directory package) async {
checkedPackages.add(package.path);
final File warningFile = package.childFile(_warningFile);
if (warningFile.existsSync()) {
final List<String> warnings = warningFile.readAsLinesSync();
warnings.forEach(logWarning);
}
final File skipFile = package.childFile(_skipFile);
if (skipFile.existsSync()) {
return PackageResult.skip(skipFile.readAsStringSync());
}
final File errorFile = package.childFile(_errorFile);
if (errorFile.existsSync()) {
final List<String> errors = errorFile.readAsLinesSync();
return errors.isNotEmpty ? errors : PackageLoopingCommand.failure;
return PackageResult.fail(errorFile.readAsLinesSync());
}
return PackageResult.success();
}
@override
Future<void> completeRun() async {
if (warnsDuringInit) {
logWarning('Warning during completeRun');
}
return PackageLoopingCommand.success;
}
}

View File

@ -740,7 +740,7 @@ void main() {
containsAllInOrder(<Matcher>[
contains('Running for aplugin_platform_interface'),
contains(
'SKIPPING: Platform interfaces are not expected to have integratino tests.'),
'SKIPPING: Platform interfaces are not expected to have integration tests.'),
contains('No issues found!'),
]),
);

View File

@ -38,11 +38,8 @@ void main() {
mockProcess.exitCodeCompleter.complete(1);
processRunner.processToReturn = mockProcess;
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'lib/test/should_not_run_e2e.dart',
'example/test_driver/plugin_e2e.dart',
'example/test_driver/plugin_e2e_test.dart',
'example/integration_test/foo_test.dart',
'example/android/gradlew',
'example/should_not_run_e2e.dart',
'example/android/app/src/androidTest/MainActivityTest.java',
]);
@ -55,8 +52,10 @@ void main() {
expect(commandError, isA<ToolExit>());
expect(
output,
contains(
'\nWarning: gcloud config set returned a non-zero exit code. Continuing anyway.'));
containsAllInOrder(<Matcher>[
contains(
'Warning: gcloud config set returned a non-zero exit code. Continuing anyway.'),
]));
});
test('runs integration tests', () async {

View File

@ -151,5 +151,21 @@ void main() {
]),
);
});
test('Skips when running no tests', () async {
createFakePlugin(
'plugin1',
packagesDir,
);
final List<String> output =
await runCapturingPrint(runner, <String>['java-test']);
expect(
output,
containsAllInOrder(
<Matcher>[contains('SKIPPING: No Java unit tests.')]),
);
});
});
}

View File

@ -187,5 +187,18 @@ void main() {
],
));
});
test('skips when there are no podspecs', () async {
createFakePlugin('plugin1', packagesDir);
final List<String> output =
await runCapturingPrint(runner, <String>['podspecs']);
expect(
output,
containsAllInOrder(
<Matcher>[contains('SKIPPING: No podspecs.')],
));
});
});
}

View File

@ -150,21 +150,16 @@ void main() {
expect(processRunner.recordedCalls, orderedEquals(<ProcessCall>[]));
});
test('running with correct destination, exclude 1 plugin', () async {
createFakePlugin('plugin1', packagesDir, extraFiles: <String>[
'example/test',
], platformSupport: <String, PlatformSupport>{
kPlatformIos: PlatformSupport.inline
});
final Directory pluginDirectory2 =
createFakePlugin('plugin2', packagesDir, extraFiles: <String>[
test('running with correct destination', () async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
], platformSupport: <String, PlatformSupport>{
kPlatformIos: PlatformSupport.inline
});
final Directory pluginExampleDirectory2 =
pluginDirectory2.childDirectory('example');
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final MockProcess mockProcess = MockProcess();
mockProcess.exitCodeCompleter.complete(0);
@ -176,16 +171,14 @@ void main() {
'--ios',
_kDestination,
'foo_destination',
'--exclude',
'plugin1'
]);
expect(output, isNot(contains(contains('Running for plugin1'))));
expect(output, contains(contains('Running for plugin2')));
expect(
output,
contains(
contains('Successfully ran iOS xctest for plugin2/example')));
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Successfully ran iOS xctest for plugin/example')
]));
expect(
processRunner.recordedCalls,
@ -206,7 +199,7 @@ void main() {
'foo_destination',
'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
],
pluginExampleDirectory2.path),
pluginExampleDirectory.path),
]));
});
@ -350,5 +343,211 @@ void main() {
]));
});
});
group('combined', () {
test('runs both iOS and macOS when supported', () async {
final Directory pluginDirectory1 =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
], platformSupport: <String, PlatformSupport>{
kPlatformIos: PlatformSupport.inline,
kPlatformMacos: PlatformSupport.inline,
});
final Directory pluginExampleDirectory =
pluginDirectory1.childDirectory('example');
final MockProcess mockProcess = MockProcess();
mockProcess.exitCodeCompleter.complete(0);
processRunner.processToReturn = mockProcess;
processRunner.resultStdout =
'{"project":{"targets":["bar_scheme", "foo_scheme"]}}';
final List<String> output = await runCapturingPrint(runner, <String>[
'xctest',
'--ios',
'--macos',
_kDestination,
'foo_destination',
]);
expect(
output,
containsAll(<Matcher>[
contains('Successfully ran iOS xctest for plugin/example'),
contains('Successfully ran macOS xctest for plugin/example'),
]));
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'xcrun',
const <String>[
'xcodebuild',
'test',
'analyze',
'-workspace',
'ios/Runner.xcworkspace',
'-configuration',
'Debug',
'-scheme',
'Runner',
'-destination',
'foo_destination',
'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
],
pluginExampleDirectory.path),
ProcessCall(
'xcrun',
const <String>[
'xcodebuild',
'test',
'analyze',
'-workspace',
'macos/Runner.xcworkspace',
'-configuration',
'Debug',
'-scheme',
'Runner',
'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
],
pluginExampleDirectory.path),
]));
});
test('runs only macOS for a macOS plugin', () async {
final Directory pluginDirectory1 =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
], platformSupport: <String, PlatformSupport>{
kPlatformMacos: PlatformSupport.inline,
});
final Directory pluginExampleDirectory =
pluginDirectory1.childDirectory('example');
final MockProcess mockProcess = MockProcess();
mockProcess.exitCodeCompleter.complete(0);
processRunner.processToReturn = mockProcess;
processRunner.resultStdout =
'{"project":{"targets":["bar_scheme", "foo_scheme"]}}';
final List<String> output = await runCapturingPrint(runner, <String>[
'xctest',
'--ios',
'--macos',
_kDestination,
'foo_destination',
]);
expect(
output,
containsAllInOrder(<Matcher>[
contains('Only running for macOS'),
contains('Successfully ran macOS xctest for plugin/example'),
]));
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'xcrun',
const <String>[
'xcodebuild',
'test',
'analyze',
'-workspace',
'macos/Runner.xcworkspace',
'-configuration',
'Debug',
'-scheme',
'Runner',
'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
],
pluginExampleDirectory.path),
]));
});
test('runs only iOS for a iOS plugin', () async {
final Directory pluginDirectory =
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
], platformSupport: <String, PlatformSupport>{
kPlatformIos: PlatformSupport.inline
});
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
final MockProcess mockProcess = MockProcess();
mockProcess.exitCodeCompleter.complete(0);
processRunner.processToReturn = mockProcess;
processRunner.resultStdout =
'{"project":{"targets":["bar_scheme", "foo_scheme"]}}';
final List<String> output = await runCapturingPrint(runner, <String>[
'xctest',
'--ios',
'--macos',
_kDestination,
'foo_destination',
]);
expect(
output,
containsAllInOrder(<Matcher>[
contains('Only running for iOS'),
contains('Successfully ran iOS xctest for plugin/example')
]));
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'xcrun',
const <String>[
'xcodebuild',
'test',
'analyze',
'-workspace',
'ios/Runner.xcworkspace',
'-configuration',
'Debug',
'-scheme',
'Runner',
'-destination',
'foo_destination',
'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
],
pluginExampleDirectory.path),
]));
});
test('skips when neither are supported', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test',
]);
final MockProcess mockProcess = MockProcess();
mockProcess.exitCodeCompleter.complete(0);
processRunner.processToReturn = mockProcess;
processRunner.resultStdout =
'{"project":{"targets":["bar_scheme", "foo_scheme"]}}';
final List<String> output = await runCapturingPrint(runner, <String>[
'xctest',
'--ios',
'--macos',
_kDestination,
'foo_destination',
]);
expect(
output,
containsAllInOrder(<Matcher>[
contains(
'SKIPPING: Neither iOS nor macOS is implemented by this plugin package.'),
]));
expect(processRunner.recordedCalls, orderedEquals(<ProcessCall>[]));
});
});
});
}