diff --git a/.ci.yaml b/.ci.yaml index 10500e82f9..faed3c6010 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -464,6 +464,71 @@ targets: {"dependency": "android_virtual_device", "version": "33"} ] + # Device versions of Android integration tests, run via FTL. + # TODO(stuartmorgan): Revisit whether physical device tests are redundant once + # we have more data about emulator tests; see + # https://github.com/flutter/flutter/issues/131429. + + - name: Linux_android android_device_tests_shard_1 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_device_tests.yaml + channel: master + version_file: flutter_master.version + package_sharding: "--shardIndex 0 --shardCount 6" + + - name: Linux_android android_device_tests_shard_2 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_device_tests.yaml + channel: master + version_file: flutter_master.version + package_sharding: "--shardIndex 1 --shardCount 6" + + - name: Linux_android android_device_tests_shard_3 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_device_tests.yaml + channel: master + version_file: flutter_master.version + package_sharding: "--shardIndex 2 --shardCount 6" + + - name: Linux_android android_device_tests_shard_4 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_device_tests.yaml + channel: master + version_file: flutter_master.version + package_sharding: "--shardIndex 3 --shardCount 6" + + - name: Linux_android android_device_tests_shard_5 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_device_tests.yaml + channel: master + version_file: flutter_master.version + package_sharding: "--shardIndex 4 --shardCount 6" + + - name: Linux_android android_device_tests_shard_6 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_device_tests.yaml + channel: master + version_file: flutter_master.version + package_sharding: "--shardIndex 5 --shardCount 6" + ### Web tasks ### - name: Linux_web web_build_all_packages master recipe: packages/packages diff --git a/.ci/targets/android_device_tests.yaml b/.ci/targets/android_device_tests.yaml new file mode 100644 index 0000000000..c152cb07d4 --- /dev/null +++ b/.ci/targets/android_device_tests.yaml @@ -0,0 +1,17 @@ +tasks: + - name: prepare tool + script: .ci/scripts/prepare_tool.sh + infra_step: true # Note infra steps failing prevents "always" from running. + - name: download Dart and Android deps + script: script/tool_runner.sh + infra_step: true + args: ["fetch-deps", "--android", "--supporting-target-platforms-only"] + - name: Firebase Test Lab + script: script/tool_runner.sh + args: + - "firebase-test-lab" + - "--device" + - "model=redfin,version=30" + - "--exclude=script/configs/exclude_integration_android.yaml" + - "--project=flutter-infra-staging" + - "--results-bucket=flutter_firebase_testlab_staging" diff --git a/.cirrus.yml b/.cirrus.yml index 46abccf48b..a964609817 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -81,12 +81,13 @@ task: matrix: CHANNEL: "master" CHANNEL: "stable" + GCLOUD_KEY_PATH: $HOME/gcloud-service-key.json MAPS_API_KEY: ENCRYPTED[d6583b08f79f91ea4844c77460f04539965e46ad2fd97fb7c062b4dfe88016228b86ebe8c220ab4187e0c4bd773dc1e7] GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1a2eebf9367197bbe812d9a0ea83a53a05aeba4bb5e4964fe6a69727883cd87e51238d39237b1f80b0894c48419ac268] firebase_test_lab_script: - if [[ -n "$GCLOUD_FIREBASE_TESTLAB_KEY" ]]; then - - echo $GCLOUD_FIREBASE_TESTLAB_KEY > ${HOME}/gcloud-service-key.json - - ./script/tool_runner.sh firebase-test-lab --device model=redfin,version=30 --exclude=script/configs/exclude_integration_android.yaml + - echo $GCLOUD_FIREBASE_TESTLAB_KEY > "${GCLOUD_KEY_PATH}" + - ./script/tool_runner.sh firebase-test-lab --device model=redfin,version=30 --exclude=script/configs/exclude_integration_android.yaml --project=flutter-cirrus --results-bucket=flutter_cirrus_testlab --service-key="${GCLOUD_KEY_PATH}" - else - echo "This user does not have permission to run Firebase Test Lab tests." - fi diff --git a/script/tool/lib/src/firebase_test_lab_command.dart b/script/tool/lib/src/firebase_test_lab_command.dart index f4d96b5abd..fa2d1e525f 100644 --- a/script/tool/lib/src/firebase_test_lab_command.dart +++ b/script/tool/lib/src/firebase_test_lab_command.dart @@ -14,7 +14,7 @@ import 'common/package_looping_command.dart'; import 'common/plugin_utils.dart'; import 'common/repository_package.dart'; -const int _exitGcloudAuthFailed = 2; +const int _exitGcloudAuthFailed = 3; /// A command to run tests via Firebase test lab. class FirebaseTestLabCommand extends PackageLoopingCommand { @@ -25,18 +25,13 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { super.platform, }) { argParser.addOption( - 'project', - defaultsTo: 'flutter-cirrus', + _gCloudProjectArg, help: 'The Firebase project name.', ); - final String? homeDir = io.Platform.environment['HOME']; - argParser.addOption('service-key', - defaultsTo: homeDir == null - ? null - : path.join(homeDir, 'gcloud-service-key.json'), + argParser.addOption(_gCloudServiceKeyArg, help: 'The path to the service key for gcloud authentication.\n' - r'If not provided, \$HOME/gcloud-service-key.json will be ' - r'assumed if $HOME is set.'); + 'If not provided, setup will be skipped, so testing will fail ' + 'unless gcloud is already configured.'); argParser.addOption('test-run-id', defaultsTo: const Uuid().v4(), help: @@ -57,8 +52,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { ], help: 'Device model(s) to test. See https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run for more info'); - argParser.addOption('results-bucket', - defaultsTo: 'gs://flutter_cirrus_testlab'); + argParser.addOption(_gCloudResultsBucketArg, mandatory: true); argParser.addOption( kEnableExperiment, defaultsTo: '', @@ -66,6 +60,10 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { ); } + static const String _gCloudServiceKeyArg = 'service-key'; + static const String _gCloudProjectArg = 'project'; + static const String _gCloudResultsBucketArg = 'results-bucket'; + @override final String name = 'firebase-test-lab'; @@ -82,9 +80,10 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { return; } - final String serviceKey = getStringArg('service-key'); + final String serviceKey = getStringArg(_gCloudServiceKeyArg); if (serviceKey.isEmpty) { - print('No --service-key provided; skipping gcloud authorization'); + print( + 'No --$_gCloudServiceKeyArg provided; skipping gcloud authorization'); } else { final io.ProcessResult result = await processRunner.run( 'gcloud', @@ -99,11 +98,16 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { printError('Unable to activate gcloud account.'); throw ToolExit(_exitGcloudAuthFailed); } + } + final String project = getStringArg(_gCloudProjectArg); + if (project.isEmpty) { + print('No --$_gCloudProjectArg provided; skipping gcloud config'); + } else { final int exitCode = await processRunner.runAndStream('gcloud', [ 'config', 'set', 'project', - getStringArg('project'), + project, ]); print(''); if (exitCode == 0) { @@ -284,7 +288,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { 'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk', '--timeout', '7m', - '--results-bucket=${getStringArg('results-bucket')}', + '--results-bucket=gs://${getStringArg(_gCloudResultsBucketArg)}', '--results-dir=$resultsDir', for (final String device in getStringListArg('device')) ...[ '--device', diff --git a/script/tool/test/firebase_test_lab_command_test.dart b/script/tool/test/firebase_test_lab_command_test.dart index a1c09dc433..2bed0db0f0 100644 --- a/script/tool/test/firebase_test_lab_command_test.dart +++ b/script/tool/test/firebase_test_lab_command_test.dart @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:args/command_runner.dart'; import 'package:file/file.dart'; import 'package:file/memory.dart'; @@ -70,8 +68,11 @@ public class MainActivityTest { writeJavaTestFile(plugin, javaTestFileRelativePath); Error? commandError; - final List output = await runCapturingPrint( - runner, ['firebase-test-lab'], errorHandler: (Error e) { + final List output = await runCapturingPrint(runner, [ + 'firebase-test-lab', + '--results-bucket=a_bucket', + '--service-key=/path/to/key', + ], errorHandler: (Error e) { commandError = e; }); @@ -99,8 +100,12 @@ public class MainActivityTest { ]); writeJavaTestFile(plugin, javaTestFileRelativePath); - final List output = - await runCapturingPrint(runner, ['firebase-test-lab']); + final List output = await runCapturingPrint(runner, [ + 'firebase-test-lab', + '--results-bucket=a_bucket', + '--service-key=/path/to/key', + '--project=a-project' + ]); expect( output, @@ -132,6 +137,9 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', + '--project=a-project', + '--service-key=/path/to/key', '--device', 'model=redfin,version=30', '--device', @@ -158,11 +166,11 @@ public class MainActivityTest { orderedEquals([ ProcessCall( 'gcloud', - 'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json' + 'auth activate-service-account --key-file=/path/to/key' .split(' '), null), ProcessCall( - 'gcloud', 'config set project flutter-cirrus'.split(' '), null), + 'gcloud', 'config set project a-project'.split(' '), null), ProcessCall( '/packages/plugin1/example/android/gradlew', 'app:assembleAndroidTest -Pverbose=true'.split(' '), @@ -174,7 +182,7 @@ public class MainActivityTest { '/packages/plugin1/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin1/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin1/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26' .split(' '), '/packages/plugin1/example'), ProcessCall( @@ -188,7 +196,7 @@ public class MainActivityTest { '/packages/plugin2/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin2/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin2/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26' .split(' '), '/packages/plugin2/example'), ]), @@ -211,6 +219,7 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', '--device', @@ -225,7 +234,6 @@ public class MainActivityTest { output, containsAllInOrder([ contains('Running for plugin'), - contains('Firebase project configured.'), contains('Testing example/integration_test/bar_test.dart...'), contains('Testing example/integration_test/foo_test.dart...'), ]), @@ -237,13 +245,6 @@ public class MainActivityTest { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'gcloud', - 'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json' - .split(' '), - null), - ProcessCall( - 'gcloud', 'config set project flutter-cirrus'.split(' '), null), ProcessCall( '/packages/plugin/example/android/gradlew', 'app:assembleAndroidTest -Pverbose=true'.split(' '), @@ -255,7 +256,7 @@ public class MainActivityTest { '/packages/plugin/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26' .split(' '), '/packages/plugin/example'), ProcessCall( @@ -265,7 +266,7 @@ public class MainActivityTest { '/packages/plugin/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/1/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/1/ --device model=redfin,version=30 --device model=seoul,version=26' .split(' '), '/packages/plugin/example'), ]), @@ -292,6 +293,7 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', '--device', @@ -320,7 +322,7 @@ public class MainActivityTest { '/packages/plugin/example/example1/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example1/0/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example1/0/ --device model=redfin,version=30 --device model=seoul,version=26' .split(' '), '/packages/plugin/example/example1'), ProcessCall( @@ -330,7 +332,7 @@ public class MainActivityTest { '/packages/plugin/example/example2/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example2/0/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example2/0/ --device model=redfin,version=30 --device model=seoul,version=26' .split(' '), '/packages/plugin/example/example2'), ]), @@ -350,8 +352,6 @@ public class MainActivityTest { writeJavaTestFile(plugin, javaTestFileRelativePath); processRunner.mockProcessesForExecutable['gcloud'] = [ - FakeProcessInfo(MockProcess(), ['auth']), - FakeProcessInfo(MockProcess(), ['config']), FakeProcessInfo(MockProcess(exitCode: 1), ['firebase', 'test']), // integration test #1 FakeProcessInfo(MockProcess(exitCode: 1), @@ -365,6 +365,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -399,8 +400,6 @@ public class MainActivityTest { writeJavaTestFile(plugin, javaTestFileRelativePath); processRunner.mockProcessesForExecutable['gcloud'] = [ - FakeProcessInfo(MockProcess(), ['auth']), - FakeProcessInfo(MockProcess(), ['config']), FakeProcessInfo(MockProcess(exitCode: 1), ['firebase', 'test']), // integration test #1 FakeProcessInfo(MockProcess(), @@ -411,6 +410,7 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ]); @@ -437,6 +437,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -467,6 +468,7 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ]); @@ -497,6 +499,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -539,6 +542,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -567,6 +571,7 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ]); @@ -599,6 +604,7 @@ public class MainActivityTest { final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', '--test-run-id', @@ -612,7 +618,6 @@ public class MainActivityTest { containsAllInOrder([ contains('Running for plugin'), contains('Running flutter build apk...'), - contains('Firebase project configured.'), contains('Testing example/integration_test/foo_test.dart...'), ]), ); @@ -625,13 +630,6 @@ public class MainActivityTest { 'build apk --config-only'.split(' '), '/packages/plugin/example/android', ), - ProcessCall( - 'gcloud', - 'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json' - .split(' '), - null), - ProcessCall( - 'gcloud', 'config set project flutter-cirrus'.split(' '), null), ProcessCall( '/packages/plugin/example/android/gradlew', 'app:assembleAndroidTest -Pverbose=true'.split(' '), @@ -643,7 +641,7 @@ public class MainActivityTest { '/packages/plugin/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30' .split(' '), '/packages/plugin/example'), ]), @@ -669,6 +667,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -711,6 +710,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -753,6 +753,7 @@ public class MainActivityTest { runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', ], @@ -785,6 +786,7 @@ public class MainActivityTest { await runCapturingPrint(runner, [ 'firebase-test-lab', + '--results-bucket=a_bucket', '--device', 'model=redfin,version=30', '--test-run-id', @@ -797,13 +799,6 @@ public class MainActivityTest { expect( processRunner.recordedCalls, orderedEquals([ - ProcessCall( - 'gcloud', - 'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json' - .split(' '), - null), - ProcessCall( - 'gcloud', 'config set project flutter-cirrus'.split(' '), null), ProcessCall( '/packages/plugin/example/android/gradlew', 'app:assembleAndroidTest -Pverbose=true -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1' @@ -816,7 +811,7 @@ public class MainActivityTest { '/packages/plugin/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30' .split(' '), '/packages/plugin/example'), ]),