mirror of
https://github.com/flutter/packages.git
synced 2025-05-23 03:36:45 +08:00
b9f601cc7b
The intent of package-looping commands is that we always want to run them for all the targeted packages, accumulating failures, and then report them all at the end, so we don't end up with the problem of only finding errors one at a time. However, some of them were using `exitOnError: true` for the underlying commands, causing the first error to be fatal to the test. This PR: - Removes all use of `exitOnError: true` from the package-looping commands. (It's still used in `format`, but fixing that is a larger issue, and less important due to the way `format` is currently structured.) - Fixes the mock process runner used in our tests to correctly simulate `exitOrError: true`; the fact that it didn't was hiding this problem in test (e.g., `drive-examples` had a test that asserted that all failures were summarized correctly, which passed because in tests it was behaving as if `exitOnError` were false) - Adjusts the mock process runner to allow setting a list of mock result for a specific executable instead of one result for all calls to anything, in order to fix some tests that were broken by the two changes above and unfixable without this (e.g., a test of a command that had been calling one executable with `exitOnError: true` then another with ``exitOnError: false`, where the test was asserting things about the second call failing, which only worked because the first call's failure wasn't actually checked). To limit the scope of the PR, the old method of setting a single result for all calls is still supported for now as a fallback. - Fixes the fact that the mock `run` and `runAndStream` had opposite default exit code behavior when no mock process was set (since that caused me a lot of confusion while fixing the above until I figured it out).
304 lines
12 KiB
Dart
304 lines
12 KiB
Dart
// Copyright 2013 The Flutter Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
import 'dart:io';
|
|
|
|
import 'package:args/command_runner.dart';
|
|
import 'package:file/file.dart';
|
|
import 'package:file/memory.dart';
|
|
import 'package:flutter_plugin_tools/src/common/core.dart';
|
|
import 'package:flutter_plugin_tools/src/firebase_test_lab_command.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
import 'mocks.dart';
|
|
import 'util.dart';
|
|
|
|
void main() {
|
|
group('$FirebaseTestLabCommand', () {
|
|
FileSystem fileSystem;
|
|
late Directory packagesDir;
|
|
late CommandRunner<void> runner;
|
|
late RecordingProcessRunner processRunner;
|
|
|
|
setUp(() {
|
|
fileSystem = MemoryFileSystem();
|
|
packagesDir = createPackagesDirectory(fileSystem: fileSystem);
|
|
processRunner = RecordingProcessRunner();
|
|
final FirebaseTestLabCommand command =
|
|
FirebaseTestLabCommand(packagesDir, processRunner: processRunner);
|
|
|
|
runner = CommandRunner<void>(
|
|
'firebase_test_lab_command', 'Test for $FirebaseTestLabCommand');
|
|
runner.addCommand(command);
|
|
});
|
|
|
|
test('fails if gcloud auth fails', () async {
|
|
final MockProcess mockProcess = MockProcess();
|
|
mockProcess.exitCodeCompleter.complete(1);
|
|
processRunner.mockProcessesForExecutable['gcloud'] = <Process>[
|
|
mockProcess
|
|
];
|
|
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
|
|
'example/integration_test/foo_test.dart',
|
|
'example/android/gradlew',
|
|
'example/android/app/src/androidTest/MainActivityTest.java',
|
|
]);
|
|
|
|
Error? commandError;
|
|
final List<String> output = await runCapturingPrint(
|
|
runner, <String>['firebase-test-lab'], errorHandler: (Error e) {
|
|
commandError = e;
|
|
});
|
|
|
|
expect(commandError, isA<ToolExit>());
|
|
expect(
|
|
output,
|
|
containsAllInOrder(<Matcher>[
|
|
contains('Unable to activate gcloud account.'),
|
|
]));
|
|
});
|
|
|
|
test('retries gcloud set', () async {
|
|
final MockProcess mockAuthProcess = MockProcess();
|
|
mockAuthProcess.exitCodeCompleter.complete(0);
|
|
final MockProcess mockConfigProcess = MockProcess();
|
|
mockConfigProcess.exitCodeCompleter.complete(1);
|
|
processRunner.mockProcessesForExecutable['gcloud'] = <Process>[
|
|
mockAuthProcess,
|
|
mockConfigProcess,
|
|
];
|
|
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
|
|
'example/integration_test/foo_test.dart',
|
|
'example/android/gradlew',
|
|
'example/android/app/src/androidTest/MainActivityTest.java',
|
|
]);
|
|
|
|
final List<String> output =
|
|
await runCapturingPrint(runner, <String>['firebase-test-lab']);
|
|
|
|
expect(
|
|
output,
|
|
containsAllInOrder(<Matcher>[
|
|
contains(
|
|
'Warning: gcloud config set returned a non-zero exit code. Continuing anyway.'),
|
|
]));
|
|
});
|
|
|
|
test('runs integration tests', () async {
|
|
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
|
|
'test/plugin_test.dart',
|
|
'example/integration_test/bar_test.dart',
|
|
'example/integration_test/foo_test.dart',
|
|
'example/integration_test/should_not_run.dart',
|
|
'example/android/gradlew',
|
|
'example/android/app/src/androidTest/MainActivityTest.java',
|
|
]);
|
|
|
|
final List<String> output = await runCapturingPrint(runner, <String>[
|
|
'firebase-test-lab',
|
|
'--device',
|
|
'model=flame,version=29',
|
|
'--device',
|
|
'model=seoul,version=26',
|
|
'--test-run-id',
|
|
'testRunId',
|
|
'--build-id',
|
|
'buildId',
|
|
]);
|
|
|
|
expect(
|
|
output,
|
|
containsAllInOrder(<Matcher>[
|
|
contains('Running for plugin'),
|
|
contains('Firebase project configured.'),
|
|
contains('Testing example/integration_test/bar_test.dart...'),
|
|
contains('Testing example/integration_test/foo_test.dart...'),
|
|
]),
|
|
);
|
|
expect(output, isNot(contains('test/plugin_test.dart')));
|
|
expect(output,
|
|
isNot(contains('example/integration_test/should_not_run.dart')));
|
|
|
|
expect(
|
|
processRunner.recordedCalls,
|
|
orderedEquals(<ProcessCall>[
|
|
ProcessCall(
|
|
'gcloud',
|
|
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
|
|
.split(' '),
|
|
null),
|
|
ProcessCall(
|
|
'gcloud', 'config set project flutter-infra'.split(' '), null),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleAndroidTest -Pverbose=true'.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/bar_test.dart'
|
|
.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'gcloud',
|
|
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29 --device model=seoul,version=26'
|
|
.split(' '),
|
|
'/packages/plugin/example'),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart'
|
|
.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'gcloud',
|
|
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/1/ --device model=flame,version=29 --device model=seoul,version=26'
|
|
.split(' '),
|
|
'/packages/plugin/example'),
|
|
]),
|
|
);
|
|
});
|
|
|
|
test('skips packages with no androidTest directory', () async {
|
|
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
|
|
'example/integration_test/foo_test.dart',
|
|
'example/android/gradlew',
|
|
]);
|
|
|
|
final List<String> output = await runCapturingPrint(runner, <String>[
|
|
'firebase-test-lab',
|
|
'--device',
|
|
'model=flame,version=29',
|
|
'--device',
|
|
'model=seoul,version=26',
|
|
'--test-run-id',
|
|
'testRunId',
|
|
'--build-id',
|
|
'buildId',
|
|
]);
|
|
|
|
expect(
|
|
output,
|
|
containsAllInOrder(<Matcher>[
|
|
contains('Running for plugin'),
|
|
contains('No example with androidTest directory'),
|
|
]),
|
|
);
|
|
expect(output,
|
|
isNot(contains('Testing example/integration_test/foo_test.dart...')));
|
|
|
|
expect(
|
|
processRunner.recordedCalls,
|
|
orderedEquals(<ProcessCall>[]),
|
|
);
|
|
});
|
|
|
|
test('builds if gradlew is missing', () async {
|
|
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
|
|
'example/integration_test/foo_test.dart',
|
|
'example/android/app/src/androidTest/MainActivityTest.java',
|
|
]);
|
|
|
|
final List<String> output = await runCapturingPrint(runner, <String>[
|
|
'firebase-test-lab',
|
|
'--device',
|
|
'model=flame,version=29',
|
|
'--device',
|
|
'model=seoul,version=26',
|
|
'--test-run-id',
|
|
'testRunId',
|
|
'--build-id',
|
|
'buildId',
|
|
]);
|
|
|
|
expect(
|
|
output,
|
|
containsAllInOrder(<Matcher>[
|
|
contains('Running for plugin'),
|
|
contains('Running flutter build apk...'),
|
|
contains('Firebase project configured.'),
|
|
contains('Testing example/integration_test/foo_test.dart...'),
|
|
]),
|
|
);
|
|
|
|
expect(
|
|
processRunner.recordedCalls,
|
|
orderedEquals(<ProcessCall>[
|
|
ProcessCall(
|
|
'flutter',
|
|
'build apk'.split(' '),
|
|
'/packages/plugin/example/android',
|
|
),
|
|
ProcessCall(
|
|
'gcloud',
|
|
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
|
|
.split(' '),
|
|
null),
|
|
ProcessCall(
|
|
'gcloud', 'config set project flutter-infra'.split(' '), null),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleAndroidTest -Pverbose=true'.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart'
|
|
.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'gcloud',
|
|
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29 --device model=seoul,version=26'
|
|
.split(' '),
|
|
'/packages/plugin/example'),
|
|
]),
|
|
);
|
|
});
|
|
|
|
test('experimental flag', () async {
|
|
createFakePlugin('plugin', packagesDir, extraFiles: <String>[
|
|
'example/integration_test/foo_test.dart',
|
|
'example/android/gradlew',
|
|
'example/android/app/src/androidTest/MainActivityTest.java',
|
|
]);
|
|
|
|
await runCapturingPrint(runner, <String>[
|
|
'firebase-test-lab',
|
|
'--device',
|
|
'model=flame,version=29',
|
|
'--test-run-id',
|
|
'testRunId',
|
|
'--build-id',
|
|
'buildId',
|
|
'--enable-experiment=exp1',
|
|
]);
|
|
|
|
expect(
|
|
processRunner.recordedCalls,
|
|
orderedEquals(<ProcessCall>[
|
|
ProcessCall(
|
|
'gcloud',
|
|
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
|
|
.split(' '),
|
|
null),
|
|
ProcessCall(
|
|
'gcloud', 'config set project flutter-infra'.split(' '), null),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleAndroidTest -Pverbose=true -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
|
|
.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'/packages/plugin/example/android/gradlew',
|
|
'app:assembleDebug -Pverbose=true -Ptarget=/packages/plugin/example/integration_test/foo_test.dart -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
|
|
.split(' '),
|
|
'/packages/plugin/example/android'),
|
|
ProcessCall(
|
|
'gcloud',
|
|
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 5m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=flame,version=29'
|
|
.split(' '),
|
|
'/packages/plugin/example'),
|
|
]),
|
|
);
|
|
});
|
|
});
|
|
}
|