flutter/packages
1
0
Fork 0
mirror of https://github.com/flutter/packages.git synced 2025-05-23 19:56:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tools] Switch to flutter test (#4348)

Browse Source 
For non-web platforms, `flutter drive` is deprecated. This switches those platforms from `flutter drive` to `flutter test`. This makes the logic to check for test driver files web-specific, since `flutter test` doesn't require a driver.

Removes support for the legacy test-in-test_driver-directory structure, which is no longer used anywhere in the repository.

Also includes a minor drive-by fix to the way we do process output, noticed while manually testing. Instead of adding all stdout, and only then adding all stderr, this adds both then waits for both, which should allow interleaving of stdout and stderr in the terminal.

Fixes https://github.com/flutter/flutter/issues/105634
This commit is contained in:
stuartmorgan
2023-07-05 11:47:57 -04:00
committed by GitHub
parent d472347aaf
commit 38745bffef
3 changed files with 189 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
script/tool/lib/src/common/process_runner.dart
View File

@ -36,8 +36,10 @@ class ProcessRunner {
'Running command: "$executable ${args.join(' ')}" in ${workingDir?.path ?? io.Directory.current.path}');
final io.Process process = await io.Process.start(executable, args,
workingDirectory: workingDir?.path);
await io.stdout.addStream(process.stdout);
await io.stderr.addStream(process.stderr);
await Future.wait(<Future<dynamic>>[
io.stdout.addStream(process.stdout),
io.stderr.addStream(process.stderr),
]);
if (exitOnError && await process.exitCode != 0) {
final String error =
_getErrorString(executable, args, workingDir: workingDir);

105
script/tool/lib/src/drive_examples_command.dart
View File

@ -17,7 +17,7 @@ import 'common/repository_package.dart';
const int _exitNoPlatformFlags = 2;
const int _exitNoAvailableDevice = 3;
/// A command to run the example applications for packages via Flutter driver.
/// A command to run the integration tests for a package's example applications.
class DriveExamplesCommand extends PackageLoopingCommand {
/// Creates an instance of the drive command.
DriveExamplesCommand(
@ -50,11 +50,9 @@ class DriveExamplesCommand extends PackageLoopingCommand {
final String name = 'drive-examples';
@override
final String description = 'Runs driver tests for package example apps.\n\n'
'For each *_test.dart in test_driver/ it drives an application with '
'either the corresponding test in test_driver (for example, '
'test_driver/app_test.dart would match test_driver/app.dart), or the '
'*_test.dart files in integration_test/.\n\n'
final String description = 'Runs Dart integration tests for example apps.\n\n'
"This runs all tests in each example's integration_test directory, "
'via "flutter test" on most platforms, and "flutter drive" on web.\n\n'
'This command requires "flutter" to be in your path.';
Map<String, List<String>> _targetDeviceFlags = const <String, List<String>>{};
@ -164,47 +162,42 @@ class DriveExamplesCommand extends PackageLoopingCommand {
'Skipping $exampleName; does not support any requested platforms.');
continue;
}
++supportedExamplesFound;
final List<File> drivers = await _getDrivers(example);
if (drivers.isEmpty) {
print('No driver tests found for $exampleName');
final List<File> testTargets = await _getIntegrationTests(example);
if (testTargets.isEmpty) {
print('No integration_test/*.dart files found for $exampleName.');
continue;
}
for (final File driver in drivers) {
final List<File> testTargets = <File>[];
// Try to find a matching app to drive without the _test.dart
// TODO(stuartmorgan): Migrate all remaining uses of this legacy
// approach (currently only video_player) and remove support for it:
// https://github.com/flutter/flutter/issues/85224.
final File? legacyTestFile = _getLegacyTestFileForTestDriver(driver);
if (legacyTestFile != null) {
testTargets.add(legacyTestFile);
} else {
for (final File testFile in await _getIntegrationTests(example)) {
// Check files for known problematic patterns.
final bool passesValidation = _validateIntegrationTest(testFile);
if (!passesValidation) {
testTargets
.where((File file) => !_validateIntegrationTest(file))
.forEach((File file) {
// Report the issue, but continue with the test as the validation
// errors don't prevent running.
errors.add('${testFile.basename} failed validation');
}
testTargets.add(testFile);
}
}
errors.add('${file.basename} failed validation');
});
if (testTargets.isEmpty) {
final String driverRelativePath =
getRelativePosixPath(driver, from: package.directory);
printError(
'Found $driverRelativePath, but no integration_test/*_test.dart files.');
errors.add('No test files for $driverRelativePath');
// `flutter test` doesn't yet support web integration tests, so fall back
// to `flutter drive`.
final bool useFlutterDrive = getBoolArg(platformWeb);
final List<File> drivers;
if (useFlutterDrive) {
drivers = await _getDrivers(example);
if (drivers.isEmpty) {
print('No driver found for $exampleName');
continue;
}
} else {
drivers = <File>[];
}
testsRan = true;
if (useFlutterDrive) {
for (final File driver in drivers) {
final List<File> failingTargets = await _driveTests(
example, driver, testTargets,
deviceFlags: deviceFlags);
@ -213,6 +206,11 @@ class DriveExamplesCommand extends PackageLoopingCommand {
getRelativePosixPath(failingTarget, from: package.directory));
}
}
} else {
if (!await _runTests(example, deviceFlags: deviceFlags)) {
errors.add('Integration tests failed.');
}
}
}
if (!testsRan) {
// It is an error for a plugin not to have integration tests, because that
@ -224,7 +222,7 @@ class DriveExamplesCommand extends PackageLoopingCommand {
} else {
return PackageResult.skip(supportedExamplesFound == 0
? 'No example supports requested platform(s).'
: 'No example is configured for driver tests.');
: 'No example is configured for integration tests.');
}
}
return errors.isEmpty
@ -295,16 +293,6 @@ class DriveExamplesCommand extends PackageLoopingCommand {
return drivers;
}
File? _getLegacyTestFileForTestDriver(File testDriver) {
final String testName = testDriver.basename.replaceAll(
RegExp(r'_test.dart$'),
'.dart',
);
final File testFile = testDriver.parent.childFile(testName);
return testFile.existsSync() ? testFile : null;
}
Future<List<File>> _getIntegrationTests(RepositoryPackage example) async {
final List<File> tests = <File>[];
final Directory integrationTestDir =
@ -378,4 +366,31 @@ class DriveExamplesCommand extends PackageLoopingCommand {
}
return failures;
}
/// Uses `flutter test integration_test` to run [example], returning the
/// success of the test run.
///
/// [deviceFlags] should contain the flags to run the test on a specific
/// target device (plus any supporting device-specific flags). E.g.:
/// - `['-d', 'macos']` for driving for macOS.
/// - `['-d', 'web-server', '--web-port=<port>', '--browser-name=<browser>]`
/// for web
Future<bool> _runTests(
RepositoryPackage example, {
required List<String> deviceFlags,
}) async {
final String enableExperiment = getStringArg(kEnableExperiment);
final int exitCode = await processRunner.runAndStream(
flutterCommand,
<String>[
'test',
...deviceFlags,
if (enableExperiment.isNotEmpty)
'--enable-experiment=$enableExperiment',
'integration_test',
],
workingDir: example.directory);
return exitCode == 0;
}
}

311
script/tool/test/drive_examples_command_test.dart
View File

@ -190,91 +190,6 @@ void main() {
);
});
test('driving under folder "test_driver"', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/android/android.java',
'example/ios/ios.m',
],
platformSupport: <String, PlatformDetails>{
platformAndroid: const PlatformDetails(PlatformSupport.inline),
platformIOS: const PlatformDetails(PlatformSupport.inline),
},
);
final Directory pluginExampleDirectory = getExampleDir(plugin);
setMockFlutterDevicesOutput();
final List<String> output =
await runCapturingPrint(runner, <String>['drive-examples', '--ios']);
expect(
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('No issues found!'),
]),
);
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(getFlutterCommand(mockPlatform),
const <String>['devices', '--machine'], null),
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'-d',
_fakeIOSDevice,
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
],
pluginExampleDirectory.path),
]));
});
test('driving under folder "test_driver" when test files are missing"',
() async {
setMockFlutterDevicesOutput();
createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/android/android.java',
'example/ios/ios.m',
],
platformSupport: <String, PlatformDetails>{
platformAndroid: const PlatformDetails(PlatformSupport.inline),
platformIOS: const PlatformDetails(PlatformSupport.inline),
},
);
Error? commandError;
final List<String> output = await runCapturingPrint(
runner, <String>['drive-examples', '--android'],
errorHandler: (Error e) {
commandError = e;
});
expect(commandError, isA<ToolExit>());
expect(
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('No driver tests were run (1 example(s) found).'),
contains('No test files for example/test_driver/plugin_test.dart'),
]),
);
});
test('a plugin without any integration test files is reported as an error',
() async {
setMockFlutterDevicesOutput();
@ -351,14 +266,11 @@ void main() {
);
});
test(
'driving under folder "test_driver" when targets are under "integration_test"',
() async {
test('tests an iOS plugin', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/integration_test.dart',
'example/integration_test/bar_test.dart',
'example/integration_test/foo_test.dart',
'example/integration_test/ignore_me.dart',
@ -393,25 +305,10 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
_fakeIOSDevice,
'--driver',
'test_driver/integration_test.dart',
'--target',
'integration_test/bar_test.dart',
],
pluginExampleDirectory.path),
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'-d',
_fakeIOSDevice,
'--driver',
'test_driver/integration_test.dart',
'--target',
'integration_test/foo_test.dart',
'integration_test',
],
pluginExampleDirectory.path),
]));
@ -419,8 +316,7 @@ void main() {
test('driving when plugin does not support Linux is a no-op', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
]);
final List<String> output = await runCapturingPrint(runner, <String>[
@ -442,13 +338,12 @@ void main() {
expect(processRunner.recordedCalls, <ProcessCall>[]);
});
test('driving on a Linux plugin', () async {
test('tests a Linux plugin', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/linux/linux.cc',
],
platformSupport: <String, PlatformDetails>{
@ -477,13 +372,10 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
'linux',
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test',
],
pluginExampleDirectory.path),
]));
@ -491,8 +383,7 @@ void main() {
test('driving when plugin does not suppport macOS is a no-op', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
]);
final List<String> output = await runCapturingPrint(runner, <String>[
@ -514,13 +405,12 @@ void main() {
expect(processRunner.recordedCalls, <ProcessCall>[]);
});
test('driving on a macOS plugin', () async {
test('tests a macOS plugin', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/macos/macos.swift',
],
platformSupport: <String, PlatformDetails>{
@ -549,13 +439,10 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
'macos',
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test',
],
pluginExampleDirectory.path),
]));
@ -563,8 +450,7 @@ void main() {
test('driving when plugin does not suppport web is a no-op', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
]);
final List<String> output = await runCapturingPrint(runner, <String>[
@ -585,13 +471,13 @@ void main() {
expect(processRunner.recordedCalls, <ProcessCall>[]);
});
test('driving a web plugin', () async {
test('drives a web plugin', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/test_driver/integration_test.dart',
'example/web/index.html',
],
platformSupport: <String, PlatformDetails>{
@ -626,21 +512,21 @@ void main() {
'--web-port=7357',
'--browser-name=chrome',
'--driver',
'test_driver/plugin_test.dart',
'test_driver/integration_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test/plugin_test.dart',
],
pluginExampleDirectory.path),
]));
});
test('driving a web plugin with CHROME_EXECUTABLE', () async {
test('drives a web plugin with CHROME_EXECUTABLE', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/test_driver/integration_test.dart',
'example/web/index.html',
],
platformSupport: <String, PlatformDetails>{
@ -678,9 +564,9 @@ void main() {
'--browser-name=chrome',
'--chrome-binary=/path/to/chrome',
'--driver',
'test_driver/plugin_test.dart',
'test_driver/integration_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test/plugin_test.dart',
],
pluginExampleDirectory.path),
]));
@ -688,8 +574,7 @@ void main() {
test('driving when plugin does not suppport Windows is a no-op', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
]);
final List<String> output = await runCapturingPrint(runner, <String>[
@ -711,13 +596,12 @@ void main() {
expect(processRunner.recordedCalls, <ProcessCall>[]);
});
test('driving on a Windows plugin', () async {
test('tests a Windows plugin', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/windows/windows.cpp',
],
platformSupport: <String, PlatformDetails>{
@ -746,25 +630,21 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
'windows',
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test',
],
pluginExampleDirectory.path),
]));
});
test('driving on an Android plugin', () async {
test('tests an Android plugin', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/android/android.java',
],
platformSupport: <String, PlatformDetails>{
@ -796,25 +676,21 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
_fakeAndroidDevice,
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test',
],
pluginExampleDirectory.path),
]));
});
test('driving on an Android plugin with alias', () async {
test('tests an Android plugin with "apk" alias', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/android/android.java',
],
platformSupport: <String, PlatformDetails>{
@ -846,13 +722,10 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
_fakeAndroidDevice,
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test',
],
pluginExampleDirectory.path),
]));
@ -863,8 +736,7 @@ void main() {
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
],
platformSupport: <String, PlatformDetails>{
platformMacOS: const PlatformDetails(PlatformSupport.inline),
@ -896,8 +768,7 @@ void main() {
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
],
platformSupport: <String, PlatformDetails>{
platformMacOS: const PlatformDetails(PlatformSupport.inline),
@ -951,8 +822,7 @@ void main() {
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
'example/test_driver/plugin.dart',
'example/integration_test/plugin_test.dart',
'example/android/android.java',
'example/ios/ios.m',
],
@ -979,14 +849,11 @@ void main() {
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'drive',
'test',
'-d',
_fakeIOSDevice,
'--enable-experiment=exp1',
'--driver',
'test_driver/plugin_test.dart',
'--target',
'test_driver/plugin.dart'
'integration_test',
],
pluginExampleDirectory.path),
]));
@ -1021,7 +888,7 @@ void main() {
);
});
test('fails when no driver is present', () async {
test('web fails when no driver is present', () async {
createFakePlugin(
'plugin',
packagesDir,
@ -1046,7 +913,7 @@ void main() {
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('No driver tests found for plugin/example'),
contains('No driver found for plugin/example'),
contains('No driver tests were run (1 example(s) found).'),
contains('The following packages had errors:'),
contains(' plugin:\n'
@ -1055,7 +922,7 @@ void main() {
);
});
test('fails when no integration tests are present', () async {
test('web fails when no integration tests are present', () async {
createFakePlugin(
'plugin',
packagesDir,
@ -1079,18 +946,15 @@ void main() {
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Found example/test_driver/integration_test.dart, but no '
'integration_test/*_test.dart files.'),
contains('No driver tests were run (1 example(s) found).'),
contains('The following packages had errors:'),
contains(' plugin:\n'
' No test files for example/test_driver/integration_test.dart\n'
' No tests ran (use --exclude if this is intentional)'),
]),
);
});
test('reports test failures', () async {
test('"flutter drive" reports test failures', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
@ -1098,10 +962,10 @@ void main() {
'example/test_driver/integration_test.dart',
'example/integration_test/bar_test.dart',
'example/integration_test/foo_test.dart',
'example/macos/macos.swift',
'example/web/index.html',
],
platformSupport: <String, PlatformDetails>{
platformMacOS: const PlatformDetails(PlatformSupport.inline),
platformWeb: const PlatformDetails(PlatformSupport.inline),
},
);
@ -1109,17 +973,14 @@ void main() {
processRunner
.mockProcessesForExecutable[getFlutterCommand(mockPlatform)] =
<FakeProcessInfo>[
// No mock for 'devices', since it's running for macOS.
FakeProcessInfo(
MockProcess(exitCode: 1), <String>['drive']), // 'drive' #1
FakeProcessInfo(
MockProcess(exitCode: 1), <String>['drive']), // 'drive' #2
// Fail both bar_test.dart and foo_test.dart.
FakeProcessInfo(MockProcess(exitCode: 1), <String>['drive']),
FakeProcessInfo(MockProcess(exitCode: 1), <String>['drive']),
];
Error? commandError;
final List<String> output =
await runCapturingPrint(runner, <String>['drive-examples', '--macos'],
errorHandler: (Error e) {
final List<String> output = await runCapturingPrint(
runner, <String>['drive-examples', '--web'], errorHandler: (Error e) {
commandError = e;
});
@ -1144,7 +1005,9 @@ void main() {
const <String>[
'drive',
'-d',
'macos',
'web-server',
'--web-port=7357',
'--browser-name=chrome',
'--driver',
'test_driver/integration_test.dart',
'--target',
@ -1156,7 +1019,9 @@ void main() {
const <String>[
'drive',
'-d',
'macos',
'web-server',
'--web-port=7357',
'--browser-name=chrome',
'--driver',
'test_driver/integration_test.dart',
'--target',
@ -1166,6 +1031,61 @@ void main() {
]));
});
test('"flutter test" reports test failures', () async {
final RepositoryPackage plugin = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/integration_test/bar_test.dart',
'example/integration_test/foo_test.dart',
'example/macos/macos.swift',
],
platformSupport: <String, PlatformDetails>{
platformMacOS: const PlatformDetails(PlatformSupport.inline),
},
);
// Simulate failure from `flutter test`.
processRunner
.mockProcessesForExecutable[getFlutterCommand(mockPlatform)] =
<FakeProcessInfo>[
FakeProcessInfo(MockProcess(exitCode: 1), <String>['test']),
];
Error? commandError;
final List<String> output =
await runCapturingPrint(runner, <String>['drive-examples', '--macos'],
errorHandler: (Error e) {
commandError = e;
});
expect(commandError, isA<ToolExit>());
expect(
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('The following packages had errors:'),
contains(' plugin:\n'
' Integration tests failed.'),
]),
);
final Directory pluginExampleDirectory = getExampleDir(plugin);
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
getFlutterCommand(mockPlatform),
const <String>[
'test',
'-d',
'macos',
'integration_test',
],
pluginExampleDirectory.path),
]));
});
group('packages', () {
test('can be driven', () async {
final RepositoryPackage package =
@ -1301,7 +1221,8 @@ void main() {
output,
containsAllInOrder(<Matcher>[
contains('Running for a_package'),
contains('SKIPPING: No example is configured for driver tests.'),
contains(
'SKIPPING: No example is configured for integration tests.'),
]),
);