flutter/packages
1
0
Fork 0
mirror of https://github.com/flutter/packages.git synced 2025-06-08 21:09:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

[flutter_plugin_tools] Auto-retry failed FTL tests (#4610)

Browse Source 
Currently the flake situation for Firebase Test Lab tests is very bad,
and the task running those tests are some of the slowest tasks in the
CI. Re-running failed tests is slow, manual work that is signficantly
affecting productivity.

There are plans to actually address the flake in the short-to-medium
term, but in the immediate term this will improve the CI situation, as
well as reducing the drag on engineering time that could be spent on the
root causes.

Part of https://github.com/flutter/flutter/issues/95063
This commit is contained in:
stuartmorgan
2021-12-15 11:22:52 -05:00
committed by GitHub
parent e45a509b35
commit 7855a20369
3 changed files with 93 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
script/tool/lib/src/firebase_test_lab_command.dart
View File

@ -176,30 +176,22 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
final String testRunId = getStringArg('test-run-id');
final String resultsDir =
'plugins_android_test/${package.displayName}/$buildId/$testRunId/${resultsCounter++}/';
final List<String> args = <String>[
'firebase',
'test',
'android',
'run',
'--type',
'instrumentation',
'--app',
'build/app/outputs/apk/debug/app-debug.apk',
'--test',
'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
'--timeout',
'7m',
'--results-bucket=${getStringArg('results-bucket')}',
'--results-dir=$resultsDir',
];
for (final String device in getStringListArg('device')) {
args.addAll(<String>['--device', device]);
}
final int exitCode = await processRunner.runAndStream('gcloud', args,
workingDir: example.directory);
if (exitCode != 0) {
printError('Test failure for $testName');
// Automatically retry failures; there is significant flake with these
// tests whose cause isn't yet understood, and having to re-run the
// entire shard for a flake in any one test is extremely slow. This should
// be removed once the root cause of the flake is understood.
// See https://github.com/flutter/flutter/issues/95063
const int maxRetries = 2;
bool passing = false;
for (int i = 1; i <= maxRetries && !passing; ++i) {
if (i > 1) {
logWarning('$testName failed on attempt ${i - 1}. Retrying...');
}
passing = await _runFirebaseTest(example, test, resultsDir: resultsDir);
}
if (!passing) {
printError('Test failure for $testName after $maxRetries attempts');
errors.add('$testName failed tests');
}
}
@ -238,6 +230,42 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
return true;
}
/// Runs [test] from [example] as a Firebase Test Lab test, returning true if
/// the test passed.
///
/// [resultsDir] should be a unique-to-the-test-run directory to store the
/// results on the server.
Future<bool> _runFirebaseTest(
RepositoryPackage example,
File test, {
required String resultsDir,
}) async {
final List<String> args = <String>[
'firebase',
'test',
'android',
'run',
'--type',
'instrumentation',
'--app',
'build/app/outputs/apk/debug/app-debug.apk',
'--test',
'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
'--timeout',
'7m',
'--results-bucket=${getStringArg('results-bucket')}',
'--results-dir=$resultsDir',
for (final String device in getStringListArg('device')) ...<String>[
'--device',
device
],
];
final int exitCode = await processRunner.runAndStream('gcloud', args,
workingDir: example.directory);
return exitCode == 0;
}
/// Builds [target] using Gradle in the given [project]. Assumes Gradle is
/// already configured.
///