[flutter_plugin_tools] Migrate firebase-test-lab to new base command (#4116)

Migrates firebase-test-lab to use the new package-looping base command.

Other changes:
- Extracts several helpers to make the main flow easier to follow
- Removes support for finding and running `*_e2e.dart` files, since we no longer use that file structure for integration tests.

Part of https://github.com/flutter/flutter/issues/83413
This commit is contained in:
stuartmorgan
2021-06-30 11:43:41 -07:00
committed by GitHub
parent ea72f74d0b
commit ec9233eb41
3 changed files with 249 additions and 229 deletions

View File

@ -1,3 +1,9 @@
## NEXT
- Modified the output format of many commands
- **Breaking change**: `firebase-test-lab` no longer supports `*_e2e.dart`
files, only `integration_test/*_test.dart`.
## 0.3.0
- Add a --build-id flag to `firebase-test-lab` instead of hard-coding the use of

View File

@ -10,18 +10,16 @@ import 'package:path/path.dart' as p;
import 'package:uuid/uuid.dart';
import 'common/core.dart';
import 'common/plugin_command.dart';
import 'common/package_looping_command.dart';
import 'common/process_runner.dart';
/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PluginCommand {
class FirebaseTestLabCommand extends PackageLoopingCommand {
/// Creates an instance of the test runner command.
FirebaseTestLabCommand(
Directory packagesDir, {
ProcessRunner processRunner = const ProcessRunner(),
Print print = print,
}) : _print = print,
super(packagesDir, processRunner: processRunner) {
}) : super(packagesDir, processRunner: processRunner) {
argParser.addOption(
'project',
defaultsTo: 'flutter-infra',
@ -74,8 +72,6 @@ class FirebaseTestLabCommand extends PluginCommand {
static const String _gradleWrapper = 'gradlew';
final Print _print;
Completer<void>? _firebaseProjectConfigured;
Future<void> _configureFirebaseProject() async {
@ -86,7 +82,7 @@ class FirebaseTestLabCommand extends PluginCommand {
final String serviceKey = getStringArg('service-key');
if (serviceKey.isEmpty) {
_print('No --service-key provided; skipping gcloud authorization');
print('No --service-key provided; skipping gcloud authorization');
} else {
await processRunner.run(
'gcloud',
@ -105,10 +101,10 @@ class FirebaseTestLabCommand extends PluginCommand {
getStringArg('project'),
]);
if (exitCode == 0) {
_print('\nFirebase project configured.');
print('\nFirebase project configured.');
return;
} else {
_print(
print(
'\nWarning: gcloud config set returned a non-zero exit code. Continuing anyway.');
}
}
@ -116,121 +112,53 @@ class FirebaseTestLabCommand extends PluginCommand {
}
@override
Future<void> run() async {
final Stream<Directory> packagesWithTests = getPackages().where(
(Directory d) =>
isFlutterPackage(d) &&
d
Future<List<String>> runForPackage(Directory package) async {
if (!package
.childDirectory('example')
.childDirectory('android')
.childDirectory('app')
.childDirectory('src')
.childDirectory('androidTest')
.existsSync());
.existsSync()) {
printSkip('No example with androidTest directory');
return PackageLoopingCommand.success;
}
final List<String> failingPackages = <String>[];
final List<String> missingFlutterBuild = <String>[];
int resultsCounter =
0; // We use a unique GCS bucket for each Firebase Test Lab run
await for (final Directory package in packagesWithTests) {
// See https://github.com/flutter/flutter/issues/38983
final List<String> errors = <String>[];
final Directory exampleDirectory = package.childDirectory('example');
final String packageName =
p.relative(package.path, from: packagesDir.path);
_print('\nRUNNING FIREBASE TEST LAB TESTS for $packageName');
final Directory androidDirectory =
exampleDirectory.childDirectory('android');
final String enableExperiment = getStringArg(kEnableExperiment);
final String encodedEnableExperiment =
Uri.encodeComponent('--enable-experiment=$enableExperiment');
// Ensures that gradle wrapper exists
if (!androidDirectory.childFile(_gradleWrapper).existsSync()) {
final int exitCode = await processRunner.runAndStream(
'flutter',
<String>[
'build',
'apk',
if (enableExperiment.isNotEmpty)
'--enable-experiment=$enableExperiment',
],
workingDir: androidDirectory);
if (exitCode != 0) {
failingPackages.add(packageName);
continue;
}
continue;
if (!await _ensureGradleWrapperExists(androidDirectory)) {
errors.add('Unable to build example apk');
return errors;
}
await _configureFirebaseProject();
int exitCode = await processRunner.runAndStream(
p.join(androidDirectory.path, _gradleWrapper),
<String>[
'app:assembleAndroidTest',
'-Pverbose=true',
if (enableExperiment.isNotEmpty)
'-Pextra-front-end-options=$encodedEnableExperiment',
if (enableExperiment.isNotEmpty)
'-Pextra-gen-snapshot-options=$encodedEnableExperiment',
],
workingDir: androidDirectory);
if (exitCode != 0) {
failingPackages.add(packageName);
continue;
if (!await _runGradle(androidDirectory, 'app:assembleAndroidTest')) {
errors.add('Unable to assemble androidTest');
return errors;
}
// Look for tests recursively in folders that start with 'test' and that
// live in the root or example folders.
bool isTestDir(FileSystemEntity dir) {
return dir is Directory &&
(p.basename(dir.path).startsWith('test') ||
p.basename(dir.path) == 'integration_test');
}
final List<Directory> testDirs =
package.listSync().where(isTestDir).cast<Directory>().toList();
final Directory example = package.childDirectory('example');
testDirs.addAll(
example.listSync().where(isTestDir).cast<Directory>().toList());
for (final Directory testDir in testDirs) {
bool isE2ETest(FileSystemEntity file) {
return file.path.endsWith('_e2e.dart') ||
(file.parent.basename == 'integration_test' &&
file.path.endsWith('_test.dart'));
}
final List<FileSystemEntity> testFiles = testDir
.listSync(recursive: true, followLinks: true)
.where(isE2ETest)
.toList();
for (final FileSystemEntity test in testFiles) {
exitCode = await processRunner.runAndStream(
p.join(androidDirectory.path, _gradleWrapper),
<String>[
'app:assembleDebug',
'-Pverbose=true',
'-Ptarget=${test.path}',
if (enableExperiment.isNotEmpty)
'-Pextra-front-end-options=$encodedEnableExperiment',
if (enableExperiment.isNotEmpty)
'-Pextra-gen-snapshot-options=$encodedEnableExperiment',
],
workingDir: androidDirectory);
if (exitCode != 0) {
failingPackages.add(packageName);
// Used within the loop to ensure a unique GCS output location for each
// test file's run.
int resultsCounter = 0;
for (final File test in _findIntegrationTestFiles(package)) {
final String testName = p.relative(test.path, from: package.path);
print('Testing $testName...');
if (!await _runGradle(androidDirectory, 'app:assembleDebug',
testFile: test)) {
printError('Could not build $testName');
errors.add('$testName failed to build');
continue;
}
final String buildId = getStringArg('build-id');
final String testRunId = getStringArg('test-run-id');
final String resultsDir =
'plugins_android_test/$packageName/$buildId/$testRunId/${resultsCounter++}/';
'plugins_android_test/${getPackageDescription(package)}/$buildId/$testRunId/${resultsCounter++}/';
final List<String> args = <String>[
'firebase',
'test',
@ -250,38 +178,89 @@ class FirebaseTestLabCommand extends PluginCommand {
for (final String device in getStringListArg('device')) {
args.addAll(<String>['--device', device]);
}
exitCode = await processRunner.runAndStream('gcloud', args,
final int exitCode = await processRunner.runAndStream('gcloud', args,
workingDir: exampleDirectory);
if (exitCode != 0) {
failingPackages.add(packageName);
continue;
}
printError('Test failure for $testName');
errors.add('$testName failed tests');
}
}
return errors;
}
_print('\n\n');
if (failingPackages.isNotEmpty) {
_print(
'The instrumentation tests for the following packages are failing (see above for'
'details):');
for (final String package in failingPackages) {
_print(' * $package');
/// Checks that 'gradlew' exists in [androidDirectory], and if not runs a
/// Flutter build to generate it.
///
/// Returns true if either gradlew was already present, or the build succeeds.
Future<bool> _ensureGradleWrapperExists(Directory androidDirectory) async {
if (!androidDirectory.childFile(_gradleWrapper).existsSync()) {
print('Running flutter build apk...');
final String experiment = getStringArg(kEnableExperiment);
final int exitCode = await processRunner.runAndStream(
'flutter',
<String>[
'build',
'apk',
if (experiment.isNotEmpty) '--enable-experiment=$experiment',
],
workingDir: androidDirectory);
if (exitCode != 0) {
return false;
}
}
if (missingFlutterBuild.isNotEmpty) {
_print('Run "pub global run flutter_plugin_tools build-examples --apk" on'
'the following packages before executing tests again:');
for (final String package in missingFlutterBuild) {
_print(' * $package');
}
return true;
}
if (failingPackages.isNotEmpty || missingFlutterBuild.isNotEmpty) {
throw ToolExit(1);
/// Builds [target] using 'gradlew' in the given [directory]. Assumes
/// 'gradlew' already exists.
///
/// [testFile] optionally does the Flutter build with the given test file as
/// the build target.
///
/// Returns true if the command succeeds.
Future<bool> _runGradle(
Directory directory,
String target, {
File? testFile,
}) async {
final String experiment = getStringArg(kEnableExperiment);
final String? extraOptions = experiment.isNotEmpty
? Uri.encodeComponent('--enable-experiment=$experiment')
: null;
final int exitCode = await processRunner.runAndStream(
p.join(directory.path, _gradleWrapper),
<String>[
target,
'-Pverbose=true',
if (testFile != null) '-Ptarget=${testFile.path}',
if (extraOptions != null) '-Pextra-front-end-options=$extraOptions',
if (extraOptions != null)
'-Pextra-gen-snapshot-options=$extraOptions',
],
workingDir: directory);
if (exitCode != 0) {
return false;
}
return true;
}
_print('All Firebase Test Lab tests successful!');
/// Finds and returns all integration test files for [package].
Iterable<File> _findIntegrationTestFiles(Directory package) sync* {
final Directory integrationTestDir =
package.childDirectory('example').childDirectory('integration_test');
if (!integrationTestDir.existsSync()) {
return;
}
yield* integrationTestDir
.listSync(recursive: true, followLinks: true)
.where((FileSystemEntity file) =>
file is File && file.basename.endsWith('_test.dart'))
.cast<File>();
}
}

View File

@ -18,18 +18,15 @@ void main() {
group('$FirebaseTestLabCommand', () {
FileSystem fileSystem;
late Directory packagesDir;
late List<String> printedMessages;
late CommandRunner<void> runner;
late RecordingProcessRunner processRunner;
setUp(() {
fileSystem = MemoryFileSystem();
packagesDir = createPackagesDirectory(fileSystem: fileSystem);
printedMessages = <String>[];
processRunner = RecordingProcessRunner();
final FirebaseTestLabCommand command = FirebaseTestLabCommand(packagesDir,
processRunner: processRunner,
print: (Object? message) => printedMessages.add(message.toString()));
final FirebaseTestLabCommand command =
FirebaseTestLabCommand(packagesDir, processRunner: processRunner);
runner = CommandRunner<void>(
'firebase_test_lab_command', 'Test for $FirebaseTestLabCommand');
@ -48,32 +45,31 @@ void main() {
'example/should_not_run_e2e.dart',
'example/android/app/src/androidTest/MainActivityTest.java',
]);
await expectLater(
() => runCapturingPrint(runner, <String>['firebase-test-lab']),
throwsA(const TypeMatcher<ToolExit>()));
Error? commandError;
final List<String> output = await runCapturingPrint(
runner, <String>['firebase-test-lab'], errorHandler: (Error e) {
commandError = e;
});
expect(commandError, isA<ToolExit>());
expect(
printedMessages,
output,
contains(
'\nWarning: gcloud config set returned a non-zero exit code. Continuing anyway.'));
});
test('runs e2e tests', () async {
test('runs integration tests', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'test/plugin_test.dart',
'test/plugin_e2e.dart',
'should_not_run_e2e.dart',
'lib/test/should_not_run_e2e.dart',
'example/test/plugin_e2e.dart',
'example/test_driver/plugin_e2e.dart',
'example/test_driver/plugin_e2e_test.dart',
'example/integration_test/bar_test.dart',
'example/integration_test/foo_test.dart',
'example/integration_test/should_not_run.dart',
'example/android/gradlew',
'example/should_not_run_e2e.dart',
'example/android/app/src/androidTest/MainActivityTest.java',
]);
await runCapturingPrint(runner, <String>[
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--device',
'model=flame,version=29',
@ -86,14 +82,17 @@ void main() {
]);
expect(
printedMessages,
orderedEquals(<String>[
'\nRUNNING FIREBASE TEST LAB TESTS for plugin',
'\nFirebase project configured.',
'\n\n',
'All Firebase Test Lab tests successful!',
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Firebase project configured.'),
contains('Testing example/integration_test/bar_test.dart...'),
contains('Testing example/integration_test/foo_test.dart...'),
]),
);
expect(output, isNot(contains('test/plugin_test.dart')));
expect(output,
isNot(contains('example/integration_test/should_not_run.dart')));
expect(
processRunner.recordedCalls,
@ -111,7 +110,7 @@ void main() {
'/packages/plugin/example/android'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/test/plugin_e2e.dart'
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/bar_test.dart'
.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
@ -121,7 +120,7 @@ void main() {
'/packages/plugin/example'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test/plugin_e2e.dart'
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart'
.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
@ -129,16 +128,91 @@ void main() {
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/1/ --device model=flame,version=29 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example'),
]),
);
});
test('skips packages with no androidTest directory', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/integration_test/foo_test.dart',
'example/android/gradlew',
]);
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--device',
'model=flame,version=29',
'--device',
'model=seoul,version=26',
'--test-run-id',
'testRunId',
'--build-id',
'buildId',
]);
expect(
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('No example with androidTest directory'),
]),
);
expect(output,
isNot(contains('Testing example/integration_test/foo_test.dart...')));
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[]),
);
});
test('builds if gradlew is missing', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'example/integration_test/foo_test.dart',
'example/android/app/src/androidTest/MainActivityTest.java',
]);
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--device',
'model=flame,version=29',
'--device',
'model=seoul,version=26',
'--test-run-id',
'testRunId',
'--build-id',
'buildId',
]);
expect(
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Running flutter build apk...'),
contains('Firebase project configured.'),
contains('Testing example/integration_test/foo_test.dart...'),
]),
);
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test_driver/plugin_e2e.dart'
.split(' '),
'/packages/plugin/example/android'),
'flutter',
'build apk'.split(' '),
'/packages/plugin/example/android',
),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/2/ --device model=flame,version=29 --device model=seoul,version=26'
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
'/packages/plugin/example'),
null),
ProcessCall(
'gcloud', 'config set project flutter-infra'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart'
@ -146,7 +220,7 @@ void main() {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/3/ --device model=flame,version=29 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example'),
]),
@ -155,17 +229,8 @@ void main() {
test('experimental flag', () async {
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
'test/plugin_test.dart',
'test/plugin_e2e.dart',
'should_not_run_e2e.dart',
'lib/test/should_not_run_e2e.dart',
'example/test/plugin_e2e.dart',
'example/test_driver/plugin_e2e.dart',
'example/test_driver/plugin_e2e_test.dart',
'example/integration_test/foo_test.dart',
'example/integration_test/should_not_run.dart',
'example/android/gradlew',
'example/should_not_run_e2e.dart',
'example/android/app/src/androidTest/MainActivityTest.java',
]);
@ -197,7 +262,7 @@ void main() {
'/packages/plugin/example/android'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/test/plugin_e2e.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
@ -205,36 +270,6 @@ void main() {
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29'
.split(' '),
'/packages/plugin/example'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test/plugin_e2e.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/1/ --device model=flame,version=29'
.split(' '),
'/packages/plugin/example'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test_driver/plugin_e2e.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/2/ --device model=flame,version=29'
.split(' '),
'/packages/plugin/example'),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
.split(' '),
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/3/ --device model=flame,version=29'
.split(' '),
'/packages/plugin/example'),
]),
);
});