From ff00264ec543b69022e23e0d012088b7ebf77d2e Mon Sep 17 00:00:00 2001 From: stuartmorgan Date: Thu, 6 Jul 2023 18:48:01 -0400 Subject: [PATCH] [ci] Add partial LUCI Android platform tests (#4381) Adds an initial Android platform tests LUCI script, and initial targets in bringup mode (using 6 shards instead of the 8 we have in Cirrus since the added shards were to try to address what in retrospect was likely a device availability issue, and since for now this will be running fewer tests; once everything is migrated we can evaluate whether we need more shards). Rather than wait for emulator and/or FTL support in LUCI to do the migration of this target, this will partially migrate; the script currently does only the parts that don't require any kind of device. That will let us set up a baseline of LUCI Android platform tests bots to easily expand from as we figure out those pieces, and we can turn down these parts of the tests in Cirrus once these come out of bringup mode to minimize duplication. To avoid having to run a full `flutter build` for both versions, this also updates the repo tooling to use the new `flutter build apk --config-only` option to create `gradlew` without doing a full build. Part of https://github.com/flutter/flutter/issues/114373 --- .ci.yaml | 138 ++++++++++++++++++ .ci/targets/android_platform_tests.yaml | 25 ++++ .../lib/src/firebase_test_lab_command.dart | 1 + script/tool/lib/src/lint_android_command.dart | 12 +- script/tool/lib/src/native_test_command.dart | 22 +-- .../test/firebase_test_lab_command_test.dart | 2 +- .../tool/test/lint_android_command_test.dart | 46 +++++- .../tool/test/native_test_command_test.dart | 47 +++++- 8 files changed, 272 insertions(+), 21 deletions(-) create mode 100644 .ci/targets/android_platform_tests.yaml diff --git a/.ci.yaml b/.ci.yaml index cdd848ddc9..8fa65ec06b 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -262,6 +262,144 @@ targets: target_file: android_build_all_packages.yaml channel: stable + - name: Linux_android android_platform_tests_shard_1 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: master + version_file: flutter_master.version + cores: "32" + package_sharding: "--shardIndex 0 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_2 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: master + version_file: flutter_master.version + cores: "32" + package_sharding: "--shardIndex 1 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_3 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: master + version_file: flutter_master.version + cores: "32" + package_sharding: "--shardIndex 2 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_4 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: master + version_file: flutter_master.version + cores: "32" + package_sharding: "--shardIndex 3 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_5 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: master + version_file: flutter_master.version + cores: "32" + package_sharding: "--shardIndex 4 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_6 master + bringup: true # New target + recipe: packages/packages + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: master + version_file: flutter_master.version + cores: "32" + package_sharding: "--shardIndex 6 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_1 stable + bringup: true # New target + recipe: packages/packages + presubmit: false + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: stable + version_file: flutter_stable.version + cores: "32" + package_sharding: "--shardIndex 0 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_2 stable + bringup: true # New target + recipe: packages/packages + presubmit: false + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: stable + version_file: flutter_stable.version + cores: "32" + package_sharding: "--shardIndex 1 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_3 stable + bringup: true # New target + recipe: packages/packages + presubmit: false + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: stable + version_file: flutter_stable.version + cores: "32" + package_sharding: "--shardIndex 2 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_4 stable + bringup: true # New target + recipe: packages/packages + presubmit: false + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: stable + version_file: flutter_stable.version + cores: "32" + package_sharding: "--shardIndex 3 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_5 stable + bringup: true # New target + recipe: packages/packages + presubmit: false + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: stable + version_file: flutter_stable.version + cores: "32" + package_sharding: "--shardIndex 4 --shardCount 6" + + - name: Linux_android android_platform_tests_shard_6 stable + bringup: true # New target + recipe: packages/packages + presubmit: false + timeout: 60 + properties: + target_file: android_platform_tests.yaml + channel: stable + version_file: flutter_stable.version + cores: "32" + package_sharding: "--shardIndex 6 --shardCount 6" + ### Web tasks ### - name: Linux_web web_build_all_packages master recipe: packages/packages diff --git a/.ci/targets/android_platform_tests.yaml b/.ci/targets/android_platform_tests.yaml new file mode 100644 index 0000000000..1944cdecc9 --- /dev/null +++ b/.ci/targets/android_platform_tests.yaml @@ -0,0 +1,25 @@ +tasks: + - name: prepare tool + script: .ci/scripts/prepare_tool.sh + - name: build examples + script: script/tool_runner.sh + args: ["build-examples", "--apk"] + - name: lint + script: script/tool_runner.sh + args: ["lint-android"] + # Native unit and native integration are split into two steps to allow for + # different exclusions. + # TODO(stuartmorgan): Eliminate the native unit test exclusion, and combine + # these steps. + - name: native unit tests + script: script/tool_runner.sh + args: ["native-test", "--android", "--no-integration", "--exclude=script/configs/exclude_native_unit_android.yaml"] + # TODO(stuartmorgan): Enable these once + # https://github.com/flutter/flutter/issues/120736 is implemented. + # See also https://github.com/flutter/flutter/issues/114373 + #- name: native integration tests + # script: script/tool_runner.sh + # args: ["native-test", "--android", "--no-unit"] + #- name: drive examples + # script: script/tool_runner.sh + # args: ["drive-examples", "--android", "--exclude=script/configs/exclude_integration_android.yaml"] diff --git a/script/tool/lib/src/firebase_test_lab_command.dart b/script/tool/lib/src/firebase_test_lab_command.dart index 5aa319c6b2..b679c1a27c 100644 --- a/script/tool/lib/src/firebase_test_lab_command.dart +++ b/script/tool/lib/src/firebase_test_lab_command.dart @@ -251,6 +251,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { [ 'build', 'apk', + '--config-only', if (experiment.isNotEmpty) '--enable-experiment=$experiment', ], workingDir: project.androidDirectory); diff --git a/script/tool/lib/src/lint_android_command.dart b/script/tool/lib/src/lint_android_command.dart index f229fd7469..9b875e7856 100644 --- a/script/tool/lib/src/lint_android_command.dart +++ b/script/tool/lib/src/lint_android_command.dart @@ -43,9 +43,15 @@ class LintAndroidCommand extends PackageLoopingCommand { processRunner: processRunner, platform: platform); if (!project.isConfigured()) { - // TODO(stuartmorgan): Replace this with a --config-only build once - // that's available on stable. - return PackageResult.fail(['Build examples before linting']); + final int exitCode = await processRunner.runAndStream( + flutterCommand, + ['build', 'apk', '--config-only'], + workingDir: example.directory, + ); + if (exitCode != 0) { + printError('Unable to configure Gradle project.'); + return PackageResult.fail(['Unable to configure Gradle.']); + } } final String packageName = package.directory.basename; diff --git a/script/tool/lib/src/native_test_command.dart b/script/tool/lib/src/native_test_command.dart index d5f720a02e..978c41bdf8 100644 --- a/script/tool/lib/src/native_test_command.dart +++ b/script/tool/lib/src/native_test_command.dart @@ -276,7 +276,6 @@ this command. bool ranUnitTests = false; bool ranAnyTests = false; bool failed = false; - bool hasMissingBuild = false; bool hasMisconfiguredIntegrationTest = false; // Iterate through all examples (in the rare case that there is more than // one example); running any tests found for each one. Requirements on what @@ -311,12 +310,16 @@ this command. platform: platform, ); if (!project.isConfigured()) { - printError('ERROR: Run "flutter build apk" on $exampleName, or run ' - 'this tool\'s "build-examples --apk" command, ' - 'before executing tests.'); - failed = true; - hasMissingBuild = true; - continue; + final int exitCode = await processRunner.runAndStream( + flutterCommand, + ['build', 'apk', '--config-only'], + workingDir: example.directory, + ); + if (exitCode != 0) { + printError('Unable to configure Gradle project.'); + failed = true; + continue; + } } if (runUnitTests) { @@ -379,10 +382,7 @@ this command. } if (failed) { - return _PlatformResult(RunState.failed, - error: hasMissingBuild - ? 'Examples must be built before testing.' - : null); + return _PlatformResult(RunState.failed); } if (hasMisconfiguredIntegrationTest) { return _PlatformResult(RunState.failed, diff --git a/script/tool/test/firebase_test_lab_command_test.dart b/script/tool/test/firebase_test_lab_command_test.dart index 70aabf7611..a1c09dc433 100644 --- a/script/tool/test/firebase_test_lab_command_test.dart +++ b/script/tool/test/firebase_test_lab_command_test.dart @@ -622,7 +622,7 @@ public class MainActivityTest { orderedEquals([ ProcessCall( 'flutter', - 'build apk'.split(' '), + 'build apk --config-only'.split(' '), '/packages/plugin/example/android', ), ProcessCall( diff --git a/script/tool/test/lint_android_command_test.dart b/script/tool/test/lint_android_command_test.dart index fda3ee41ce..3c739af7e3 100644 --- a/script/tool/test/lint_android_command_test.dart +++ b/script/tool/test/lint_android_command_test.dart @@ -109,12 +109,54 @@ void main() { ])); }); - test('fails if gradlew is missing', () async { + test('runs --config-only build if gradlew is missing', () async { + final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, + platformSupport: { + platformAndroid: const PlatformDetails(PlatformSupport.inline) + }); + + final Directory androidDir = + plugin.getExamples().first.platformDirectory(FlutterPlatform.android); + + final List output = + await runCapturingPrint(runner, ['lint-android']); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + getFlutterCommand(mockPlatform), + const ['build', 'apk', '--config-only'], + plugin.getExamples().first.directory.path, + ), + ProcessCall( + androidDir.childFile('gradlew').path, + const ['plugin1:lintDebug'], + androidDir.path, + ), + ]), + ); + + expect( + output, + containsAllInOrder([ + contains('Running for plugin1'), + contains('No issues found!'), + ])); + }); + + test('fails if gradlew generation fails', () async { createFakePlugin('plugin1', packagesDir, platformSupport: { platformAndroid: const PlatformDetails(PlatformSupport.inline) }); + processRunner + .mockProcessesForExecutable[getFlutterCommand(mockPlatform)] = + [ + FakeProcessInfo(MockProcess(exitCode: 1)), + ]; + Error? commandError; final List output = await runCapturingPrint( runner, ['lint-android'], errorHandler: (Error e) { @@ -126,7 +168,7 @@ void main() { output, containsAllInOrder( [ - contains('Build examples before linting'), + contains('Unable to configure Gradle project'), ], )); }); diff --git a/script/tool/test/native_test_command_test.dart b/script/tool/test/native_test_command_test.dart index b4e06b9b7e..a180c3cb09 100644 --- a/script/tool/test/native_test_command_test.dart +++ b/script/tool/test/native_test_command_test.dart @@ -773,7 +773,44 @@ public class FlutterActivityTest { ); }); - test('fails when the app needs to be built', () async { + test('runs a config-only build when the app needs to be built', () async { + final RepositoryPackage package = createFakePlugin( + 'plugin', + packagesDir, + platformSupport: { + platformAndroid: const PlatformDetails(PlatformSupport.inline) + }, + extraFiles: [ + 'example/android/app/src/test/example_test.java', + ], + ); + final RepositoryPackage example = package.getExamples().first; + final Directory androidFolder = + example.platformDirectory(FlutterPlatform.android); + + await runCapturingPrint(runner, ['native-test', '--android']); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + getFlutterCommand(mockPlatform), + const ['build', 'apk', '--config-only'], + example.path, + ), + ProcessCall( + androidFolder.childFile('gradlew').path, + const [ + 'app:testDebugUnitTest', + 'plugin:testDebugUnitTest', + ], + androidFolder.path, + ), + ]), + ); + }); + + test('fails when the gradlew generation fails', () async { createFakePlugin( 'plugin', packagesDir, @@ -785,6 +822,10 @@ public class FlutterActivityTest { ], ); + processRunner + .mockProcessesForExecutable[getFlutterCommand(mockPlatform)] = + [FakeProcessInfo(MockProcess(exitCode: 1))]; + Error? commandError; final List output = await runCapturingPrint( runner, ['native-test', '--android'], @@ -797,9 +838,7 @@ public class FlutterActivityTest { expect( output, containsAllInOrder([ - contains('ERROR: Run "flutter build apk" on plugin/example'), - contains('plugin:\n' - ' Examples must be built before testing.') + contains('Unable to configure Gradle project'), ]), ); });