diff --git a/script/tool/CHANGELOG.md b/script/tool/CHANGELOG.md index a2716cb53d..2b15ccdd2a 100644 --- a/script/tool/CHANGELOG.md +++ b/script/tool/CHANGELOG.md @@ -1,3 +1,9 @@ +## NEXT + +- Modified the output format of many commands +- **Breaking change**: `firebase-test-lab` no longer supports `*_e2e.dart` + files, only `integration_test/*_test.dart`. + ## 0.3.0 - Add a --build-id flag to `firebase-test-lab` instead of hard-coding the use of diff --git a/script/tool/lib/src/firebase_test_lab_command.dart b/script/tool/lib/src/firebase_test_lab_command.dart index b4f5e92933..9f4982b278 100644 --- a/script/tool/lib/src/firebase_test_lab_command.dart +++ b/script/tool/lib/src/firebase_test_lab_command.dart @@ -10,18 +10,16 @@ import 'package:path/path.dart' as p; import 'package:uuid/uuid.dart'; import 'common/core.dart'; -import 'common/plugin_command.dart'; +import 'common/package_looping_command.dart'; import 'common/process_runner.dart'; /// A command to run tests via Firebase test lab. -class FirebaseTestLabCommand extends PluginCommand { +class FirebaseTestLabCommand extends PackageLoopingCommand { /// Creates an instance of the test runner command. FirebaseTestLabCommand( Directory packagesDir, { ProcessRunner processRunner = const ProcessRunner(), - Print print = print, - }) : _print = print, - super(packagesDir, processRunner: processRunner) { + }) : super(packagesDir, processRunner: processRunner) { argParser.addOption( 'project', defaultsTo: 'flutter-infra', @@ -74,8 +72,6 @@ class FirebaseTestLabCommand extends PluginCommand { static const String _gradleWrapper = 'gradlew'; - final Print _print; - Completer? _firebaseProjectConfigured; Future _configureFirebaseProject() async { @@ -86,7 +82,7 @@ class FirebaseTestLabCommand extends PluginCommand { final String serviceKey = getStringArg('service-key'); if (serviceKey.isEmpty) { - _print('No --service-key provided; skipping gcloud authorization'); + print('No --service-key provided; skipping gcloud authorization'); } else { await processRunner.run( 'gcloud', @@ -105,10 +101,10 @@ class FirebaseTestLabCommand extends PluginCommand { getStringArg('project'), ]); if (exitCode == 0) { - _print('\nFirebase project configured.'); + print('\nFirebase project configured.'); return; } else { - _print( + print( '\nWarning: gcloud config set returned a non-zero exit code. Continuing anyway.'); } } @@ -116,172 +112,155 @@ class FirebaseTestLabCommand extends PluginCommand { } @override - Future run() async { - final Stream packagesWithTests = getPackages().where( - (Directory d) => - isFlutterPackage(d) && - d - .childDirectory('example') - .childDirectory('android') - .childDirectory('app') - .childDirectory('src') - .childDirectory('androidTest') - .existsSync()); + Future> runForPackage(Directory package) async { + if (!package + .childDirectory('example') + .childDirectory('android') + .childDirectory('app') + .childDirectory('src') + .childDirectory('androidTest') + .existsSync()) { + printSkip('No example with androidTest directory'); + return PackageLoopingCommand.success; + } - final List failingPackages = []; - final List missingFlutterBuild = []; - int resultsCounter = - 0; // We use a unique GCS bucket for each Firebase Test Lab run - await for (final Directory package in packagesWithTests) { - // See https://github.com/flutter/flutter/issues/38983 + final List errors = []; - final Directory exampleDirectory = package.childDirectory('example'); - final String packageName = - p.relative(package.path, from: packagesDir.path); - _print('\nRUNNING FIREBASE TEST LAB TESTS for $packageName'); + final Directory exampleDirectory = package.childDirectory('example'); + final Directory androidDirectory = + exampleDirectory.childDirectory('android'); - final Directory androidDirectory = - exampleDirectory.childDirectory('android'); + // Ensures that gradle wrapper exists + if (!await _ensureGradleWrapperExists(androidDirectory)) { + errors.add('Unable to build example apk'); + return errors; + } - final String enableExperiment = getStringArg(kEnableExperiment); - final String encodedEnableExperiment = - Uri.encodeComponent('--enable-experiment=$enableExperiment'); + await _configureFirebaseProject(); - // Ensures that gradle wrapper exists - if (!androidDirectory.childFile(_gradleWrapper).existsSync()) { - final int exitCode = await processRunner.runAndStream( - 'flutter', - [ - 'build', - 'apk', - if (enableExperiment.isNotEmpty) - '--enable-experiment=$enableExperiment', - ], - workingDir: androidDirectory); + if (!await _runGradle(androidDirectory, 'app:assembleAndroidTest')) { + errors.add('Unable to assemble androidTest'); + return errors; + } - if (exitCode != 0) { - failingPackages.add(packageName); - continue; - } + // Used within the loop to ensure a unique GCS output location for each + // test file's run. + int resultsCounter = 0; + for (final File test in _findIntegrationTestFiles(package)) { + final String testName = p.relative(test.path, from: package.path); + print('Testing $testName...'); + if (!await _runGradle(androidDirectory, 'app:assembleDebug', + testFile: test)) { + printError('Could not build $testName'); + errors.add('$testName failed to build'); continue; } + final String buildId = getStringArg('build-id'); + final String testRunId = getStringArg('test-run-id'); + final String resultsDir = + 'plugins_android_test/${getPackageDescription(package)}/$buildId/$testRunId/${resultsCounter++}/'; + final List args = [ + 'firebase', + 'test', + 'android', + 'run', + '--type', + 'instrumentation', + '--app', + 'build/app/outputs/apk/debug/app-debug.apk', + '--test', + 'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk', + '--timeout', + '5m', + '--results-bucket=${getStringArg('results-bucket')}', + '--results-dir=$resultsDir', + ]; + for (final String device in getStringListArg('device')) { + args.addAll(['--device', device]); + } + final int exitCode = await processRunner.runAndStream('gcloud', args, + workingDir: exampleDirectory); - await _configureFirebaseProject(); + if (exitCode != 0) { + printError('Test failure for $testName'); + errors.add('$testName failed tests'); + } + } + return errors; + } - int exitCode = await processRunner.runAndStream( - p.join(androidDirectory.path, _gradleWrapper), + /// Checks that 'gradlew' exists in [androidDirectory], and if not runs a + /// Flutter build to generate it. + /// + /// Returns true if either gradlew was already present, or the build succeeds. + Future _ensureGradleWrapperExists(Directory androidDirectory) async { + if (!androidDirectory.childFile(_gradleWrapper).existsSync()) { + print('Running flutter build apk...'); + final String experiment = getStringArg(kEnableExperiment); + final int exitCode = await processRunner.runAndStream( + 'flutter', [ - 'app:assembleAndroidTest', - '-Pverbose=true', - if (enableExperiment.isNotEmpty) - '-Pextra-front-end-options=$encodedEnableExperiment', - if (enableExperiment.isNotEmpty) - '-Pextra-gen-snapshot-options=$encodedEnableExperiment', + 'build', + 'apk', + if (experiment.isNotEmpty) '--enable-experiment=$experiment', ], workingDir: androidDirectory); if (exitCode != 0) { - failingPackages.add(packageName); - continue; - } - - // Look for tests recursively in folders that start with 'test' and that - // live in the root or example folders. - bool isTestDir(FileSystemEntity dir) { - return dir is Directory && - (p.basename(dir.path).startsWith('test') || - p.basename(dir.path) == 'integration_test'); - } - - final List testDirs = - package.listSync().where(isTestDir).cast().toList(); - final Directory example = package.childDirectory('example'); - testDirs.addAll( - example.listSync().where(isTestDir).cast().toList()); - for (final Directory testDir in testDirs) { - bool isE2ETest(FileSystemEntity file) { - return file.path.endsWith('_e2e.dart') || - (file.parent.basename == 'integration_test' && - file.path.endsWith('_test.dart')); - } - - final List testFiles = testDir - .listSync(recursive: true, followLinks: true) - .where(isE2ETest) - .toList(); - for (final FileSystemEntity test in testFiles) { - exitCode = await processRunner.runAndStream( - p.join(androidDirectory.path, _gradleWrapper), - [ - 'app:assembleDebug', - '-Pverbose=true', - '-Ptarget=${test.path}', - if (enableExperiment.isNotEmpty) - '-Pextra-front-end-options=$encodedEnableExperiment', - if (enableExperiment.isNotEmpty) - '-Pextra-gen-snapshot-options=$encodedEnableExperiment', - ], - workingDir: androidDirectory); - - if (exitCode != 0) { - failingPackages.add(packageName); - continue; - } - final String buildId = getStringArg('build-id'); - final String testRunId = getStringArg('test-run-id'); - final String resultsDir = - 'plugins_android_test/$packageName/$buildId/$testRunId/${resultsCounter++}/'; - final List args = [ - 'firebase', - 'test', - 'android', - 'run', - '--type', - 'instrumentation', - '--app', - 'build/app/outputs/apk/debug/app-debug.apk', - '--test', - 'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk', - '--timeout', - '5m', - '--results-bucket=${getStringArg('results-bucket')}', - '--results-dir=$resultsDir', - ]; - for (final String device in getStringListArg('device')) { - args.addAll(['--device', device]); - } - exitCode = await processRunner.runAndStream('gcloud', args, - workingDir: exampleDirectory); - - if (exitCode != 0) { - failingPackages.add(packageName); - continue; - } - } + return false; } } + return true; + } - _print('\n\n'); - if (failingPackages.isNotEmpty) { - _print( - 'The instrumentation tests for the following packages are failing (see above for' - 'details):'); - for (final String package in failingPackages) { - _print(' * $package'); - } + /// Builds [target] using 'gradlew' in the given [directory]. Assumes + /// 'gradlew' already exists. + /// + /// [testFile] optionally does the Flutter build with the given test file as + /// the build target. + /// + /// Returns true if the command succeeds. + Future _runGradle( + Directory directory, + String target, { + File? testFile, + }) async { + final String experiment = getStringArg(kEnableExperiment); + final String? extraOptions = experiment.isNotEmpty + ? Uri.encodeComponent('--enable-experiment=$experiment') + : null; + + final int exitCode = await processRunner.runAndStream( + p.join(directory.path, _gradleWrapper), + [ + target, + '-Pverbose=true', + if (testFile != null) '-Ptarget=${testFile.path}', + if (extraOptions != null) '-Pextra-front-end-options=$extraOptions', + if (extraOptions != null) + '-Pextra-gen-snapshot-options=$extraOptions', + ], + workingDir: directory); + + if (exitCode != 0) { + return false; } - if (missingFlutterBuild.isNotEmpty) { - _print('Run "pub global run flutter_plugin_tools build-examples --apk" on' - 'the following packages before executing tests again:'); - for (final String package in missingFlutterBuild) { - _print(' * $package'); - } + return true; + } + + /// Finds and returns all integration test files for [package]. + Iterable _findIntegrationTestFiles(Directory package) sync* { + final Directory integrationTestDir = + package.childDirectory('example').childDirectory('integration_test'); + + if (!integrationTestDir.existsSync()) { + return; } - if (failingPackages.isNotEmpty || missingFlutterBuild.isNotEmpty) { - throw ToolExit(1); - } - - _print('All Firebase Test Lab tests successful!'); + yield* integrationTestDir + .listSync(recursive: true, followLinks: true) + .where((FileSystemEntity file) => + file is File && file.basename.endsWith('_test.dart')) + .cast(); } } diff --git a/script/tool/test/firebase_test_lab_test.dart b/script/tool/test/firebase_test_lab_command_test.dart similarity index 62% rename from script/tool/test/firebase_test_lab_test.dart rename to script/tool/test/firebase_test_lab_command_test.dart index 32867c949b..e317ba924b 100644 --- a/script/tool/test/firebase_test_lab_test.dart +++ b/script/tool/test/firebase_test_lab_command_test.dart @@ -18,18 +18,15 @@ void main() { group('$FirebaseTestLabCommand', () { FileSystem fileSystem; late Directory packagesDir; - late List printedMessages; late CommandRunner runner; late RecordingProcessRunner processRunner; setUp(() { fileSystem = MemoryFileSystem(); packagesDir = createPackagesDirectory(fileSystem: fileSystem); - printedMessages = []; processRunner = RecordingProcessRunner(); - final FirebaseTestLabCommand command = FirebaseTestLabCommand(packagesDir, - processRunner: processRunner, - print: (Object? message) => printedMessages.add(message.toString())); + final FirebaseTestLabCommand command = + FirebaseTestLabCommand(packagesDir, processRunner: processRunner); runner = CommandRunner( 'firebase_test_lab_command', 'Test for $FirebaseTestLabCommand'); @@ -48,32 +45,31 @@ void main() { 'example/should_not_run_e2e.dart', 'example/android/app/src/androidTest/MainActivityTest.java', ]); - await expectLater( - () => runCapturingPrint(runner, ['firebase-test-lab']), - throwsA(const TypeMatcher())); + + Error? commandError; + final List output = await runCapturingPrint( + runner, ['firebase-test-lab'], errorHandler: (Error e) { + commandError = e; + }); + + expect(commandError, isA()); expect( - printedMessages, + output, contains( '\nWarning: gcloud config set returned a non-zero exit code. Continuing anyway.')); }); - test('runs e2e tests', () async { + test('runs integration tests', () async { createFakePlugin('plugin', packagesDir, extraFiles: [ 'test/plugin_test.dart', - 'test/plugin_e2e.dart', - 'should_not_run_e2e.dart', - 'lib/test/should_not_run_e2e.dart', - 'example/test/plugin_e2e.dart', - 'example/test_driver/plugin_e2e.dart', - 'example/test_driver/plugin_e2e_test.dart', + 'example/integration_test/bar_test.dart', 'example/integration_test/foo_test.dart', 'example/integration_test/should_not_run.dart', 'example/android/gradlew', - 'example/should_not_run_e2e.dart', 'example/android/app/src/androidTest/MainActivityTest.java', ]); - await runCapturingPrint(runner, [ + final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', '--device', 'model=flame,version=29', @@ -86,14 +82,17 @@ void main() { ]); expect( - printedMessages, - orderedEquals([ - '\nRUNNING FIREBASE TEST LAB TESTS for plugin', - '\nFirebase project configured.', - '\n\n', - 'All Firebase Test Lab tests successful!', + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('Firebase project configured.'), + contains('Testing example/integration_test/bar_test.dart...'), + contains('Testing example/integration_test/foo_test.dart...'), ]), ); + expect(output, isNot(contains('test/plugin_test.dart'))); + expect(output, + isNot(contains('example/integration_test/should_not_run.dart'))); expect( processRunner.recordedCalls, @@ -111,7 +110,7 @@ void main() { '/packages/plugin/example/android'), ProcessCall( '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/test/plugin_e2e.dart' + 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/bar_test.dart' .split(' '), '/packages/plugin/example/android'), ProcessCall( @@ -121,7 +120,7 @@ void main() { '/packages/plugin/example'), ProcessCall( '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test/plugin_e2e.dart' + 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart' .split(' '), '/packages/plugin/example/android'), ProcessCall( @@ -129,16 +128,91 @@ void main() { 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/1/ --device model=flame,version=29 --device model=seoul,version=26' .split(' '), '/packages/plugin/example'), + ]), + ); + }); + + test('skips packages with no androidTest directory', () async { + createFakePlugin('plugin', packagesDir, extraFiles: [ + 'example/integration_test/foo_test.dart', + 'example/android/gradlew', + ]); + + final List output = await runCapturingPrint(runner, [ + 'firebase-test-lab', + '--device', + 'model=flame,version=29', + '--device', + 'model=seoul,version=26', + '--test-run-id', + 'testRunId', + '--build-id', + 'buildId', + ]); + + expect( + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('No example with androidTest directory'), + ]), + ); + expect(output, + isNot(contains('Testing example/integration_test/foo_test.dart...'))); + + expect( + processRunner.recordedCalls, + orderedEquals([]), + ); + }); + + test('builds if gradlew is missing', () async { + createFakePlugin('plugin', packagesDir, extraFiles: [ + 'example/integration_test/foo_test.dart', + 'example/android/app/src/androidTest/MainActivityTest.java', + ]); + + final List output = await runCapturingPrint(runner, [ + 'firebase-test-lab', + '--device', + 'model=flame,version=29', + '--device', + 'model=seoul,version=26', + '--test-run-id', + 'testRunId', + '--build-id', + 'buildId', + ]); + + expect( + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('Running flutter build apk...'), + contains('Firebase project configured.'), + contains('Testing example/integration_test/foo_test.dart...'), + ]), + ); + + expect( + processRunner.recordedCalls, + orderedEquals([ ProcessCall( - '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test_driver/plugin_e2e.dart' - .split(' '), - '/packages/plugin/example/android'), + 'flutter', + 'build apk'.split(' '), + '/packages/plugin/example/android', + ), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/2/ --device model=flame,version=29 --device model=seoul,version=26' + 'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json' .split(' '), - '/packages/plugin/example'), + null), + ProcessCall( + 'gcloud', 'config set project flutter-infra'.split(' '), null), + ProcessCall( + '/packages/plugin/example/android/gradlew', + 'app:assembleAndroidTest -Pverbose=true'.split(' '), + '/packages/plugin/example/android'), ProcessCall( '/packages/plugin/example/android/gradlew', 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart' @@ -146,7 +220,7 @@ void main() { '/packages/plugin/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/3/ --device model=flame,version=29 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29 --device model=seoul,version=26' .split(' '), '/packages/plugin/example'), ]), @@ -155,17 +229,8 @@ void main() { test('experimental flag', () async { createFakePlugin('plugin', packagesDir, extraFiles: [ - 'test/plugin_test.dart', - 'test/plugin_e2e.dart', - 'should_not_run_e2e.dart', - 'lib/test/should_not_run_e2e.dart', - 'example/test/plugin_e2e.dart', - 'example/test_driver/plugin_e2e.dart', - 'example/test_driver/plugin_e2e_test.dart', 'example/integration_test/foo_test.dart', - 'example/integration_test/should_not_run.dart', 'example/android/gradlew', - 'example/should_not_run_e2e.dart', 'example/android/app/src/androidTest/MainActivityTest.java', ]); @@ -197,7 +262,7 @@ void main() { '/packages/plugin/example/android'), ProcessCall( '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/test/plugin_e2e.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1' + 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1' .split(' '), '/packages/plugin/example/android'), ProcessCall( @@ -205,36 +270,6 @@ void main() { 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29' .split(' '), '/packages/plugin/example'), - ProcessCall( - '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test/plugin_e2e.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1' - .split(' '), - '/packages/plugin/example/android'), - ProcessCall( - 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/1/ --device model=flame,version=29' - .split(' '), - '/packages/plugin/example'), - ProcessCall( - '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/test_driver/plugin_e2e.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1' - .split(' '), - '/packages/plugin/example/android'), - ProcessCall( - 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/2/ --device model=flame,version=29' - .split(' '), - '/packages/plugin/example'), - ProcessCall( - '/packages/plugin/example/android/gradlew', - 'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1' - .split(' '), - '/packages/plugin/example/android'), - ProcessCall( - 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/3/ --device model=flame,version=29' - .split(' '), - '/packages/plugin/example'), ]), ); });