flutter/packages
1
0
Fork 0
mirror of https://github.com/flutter/packages.git synced 2025-06-30 23:03:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

[ci] Add LUCI version of Android FTL tests in bringup mode (#4571)

Browse Source 
Adds new LUCI versions of the Android FTL tests. These are new tasks, rather than being part of android_platform_tests, since those are already running the emulator version of the tests, and the combination is far too slow (unless we roughly double the number of shards anyway), and this lets us control the behavior separately (e.g., not running them on `stable`, and potentially making them post-submit only in the future).

Adjusts the repo tooling to make the `gcloud auth` step optional, since that's handled automatically for us in the LUCI environment, and to make it less tightly coupled to the Cirrus configuration by removing those defaults from the tool.

Part of https://github.com/flutter/flutter/issues/114373
See also https://github.com/flutter/flutter/issues/131429
This commit is contained in:
stuartmorgan
2023-07-28 08:01:10 -07:00
committed by GitHub
parent 10aab445d9
commit 066e47a5f2
5 changed files with 143 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
script/tool/test/firebase_test_lab_command_test.dart
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:file/file.dart';
import 'package:file/memory.dart';
@ -70,8 +68,11 @@ public class MainActivityTest {
writeJavaTestFile(plugin, javaTestFileRelativePath);
Error? commandError;
final List<String> output = await runCapturingPrint(
runner, <String>['firebase-test-lab'], errorHandler: (Error e) {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--service-key=/path/to/key',
], errorHandler: (Error e) {
commandError = e;
});
@ -99,8 +100,12 @@ public class MainActivityTest {
]);
writeJavaTestFile(plugin, javaTestFileRelativePath);
final List<String> output =
await runCapturingPrint(runner, <String>['firebase-test-lab']);
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--service-key=/path/to/key',
'--project=a-project'
]);
expect(
output,
@ -132,6 +137,9 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--project=a-project',
'--service-key=/path/to/key',
'--device',
'model=redfin,version=30',
'--device',
@ -158,11 +166,11 @@ public class MainActivityTest {
orderedEquals(<ProcessCall>[
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
'auth activate-service-account --key-file=/path/to/key'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
'gcloud', 'config set project a-project'.split(' '), null),
ProcessCall(
'/packages/plugin1/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
@ -174,7 +182,7 @@ public class MainActivityTest {
'/packages/plugin1/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin1/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin1/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin1/example'),
ProcessCall(
@ -188,7 +196,7 @@ public class MainActivityTest {
'/packages/plugin2/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin2/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin2/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin2/example'),
]),
@ -211,6 +219,7 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--device',
@ -225,7 +234,6 @@ public class MainActivityTest {
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Firebase project configured.'),
contains('Testing example/integration_test/bar_test.dart...'),
contains('Testing example/integration_test/foo_test.dart...'),
]),
@ -237,13 +245,6 @@ public class MainActivityTest {
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
@ -255,7 +256,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example'),
ProcessCall(
@ -265,7 +266,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/1/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/1/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example'),
]),
@ -292,6 +293,7 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--device',
@ -320,7 +322,7 @@ public class MainActivityTest {
'/packages/plugin/example/example1/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example1/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example1/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example/example1'),
ProcessCall(
@ -330,7 +332,7 @@ public class MainActivityTest {
'/packages/plugin/example/example2/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example2/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example2/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example/example2'),
]),
@ -350,8 +352,6 @@ public class MainActivityTest {
writeJavaTestFile(plugin, javaTestFileRelativePath);
processRunner.mockProcessesForExecutable['gcloud'] = <FakeProcessInfo>[
FakeProcessInfo(MockProcess(), <String>['auth']),
FakeProcessInfo(MockProcess(), <String>['config']),
FakeProcessInfo(MockProcess(exitCode: 1),
<String>['firebase', 'test']), // integration test #1
FakeProcessInfo(MockProcess(exitCode: 1),
@ -365,6 +365,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -399,8 +400,6 @@ public class MainActivityTest {
writeJavaTestFile(plugin, javaTestFileRelativePath);
processRunner.mockProcessesForExecutable['gcloud'] = <FakeProcessInfo>[
FakeProcessInfo(MockProcess(), <String>['auth']),
FakeProcessInfo(MockProcess(), <String>['config']),
FakeProcessInfo(MockProcess(exitCode: 1),
<String>['firebase', 'test']), // integration test #1
FakeProcessInfo(MockProcess(),
@ -411,6 +410,7 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
]);
@ -437,6 +437,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -467,6 +468,7 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
]);
@ -497,6 +499,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -539,6 +542,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -567,6 +571,7 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
]);
@ -599,6 +604,7 @@ public class MainActivityTest {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--test-run-id',
@ -612,7 +618,6 @@ public class MainActivityTest {
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Running flutter build apk...'),
contains('Firebase project configured.'),
contains('Testing example/integration_test/foo_test.dart...'),
]),
);
@ -625,13 +630,6 @@ public class MainActivityTest {
'build apk --config-only'.split(' '),
'/packages/plugin/example/android',
),
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
@ -643,7 +641,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
.split(' '),
'/packages/plugin/example'),
]),
@ -669,6 +667,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -711,6 +710,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -753,6 +753,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
@ -785,6 +786,7 @@ public class MainActivityTest {
await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--test-run-id',
@ -797,13 +799,6 @@ public class MainActivityTest {
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
@ -816,7 +811,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
.split(' '),
'/packages/plugin/example'),
]),