diff --git a/script/tool/CHANGELOG.md b/script/tool/CHANGELOG.md index ee23428e66..a2a2ed8242 100644 --- a/script/tool/CHANGELOG.md +++ b/script/tool/CHANGELOG.md @@ -1,3 +1,7 @@ +## NEXT + +- Ensures that `firebase-test-lab` runs include an `integration_test` runner. + ## 0.7.3 - `native-test` now builds unit tests before running them on Windows and Linux, diff --git a/script/tool/lib/src/firebase_test_lab_command.dart b/script/tool/lib/src/firebase_test_lab_command.dart index 28afc63820..4e53ee8fba 100644 --- a/script/tool/lib/src/firebase_test_lab_command.dart +++ b/script/tool/lib/src/firebase_test_lab_command.dart @@ -127,16 +127,24 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { '${example.displayName} does not support Android.'); } - if (!androidDirectory + final Directory uiTestDirectory = androidDirectory .childDirectory('app') .childDirectory('src') - .childDirectory('androidTest') - .existsSync()) { + .childDirectory('androidTest'); + if (!uiTestDirectory.existsSync()) { printError('No androidTest directory found.'); return PackageResult.fail( ['No tests ran (use --exclude if this is intentional).']); } + // Ensure that the Dart integration tests will be run, not just native UI + // tests. + if (!await _testsContainDartIntegrationTestRunner(uiTestDirectory)) { + printError('No integration_test runner found. ' + 'See the integration_test package README for setup instructions.'); + return PackageResult.fail(['No integration_test runner.']); + } + // Ensures that gradle wrapper exists final GradleProject project = GradleProject(example.directory, processRunner: processRunner, platform: platform); @@ -280,4 +288,19 @@ class FirebaseTestLabCommand extends PackageLoopingCommand { file is File && file.basename.endsWith('_test.dart')) .cast(); } + + /// Returns true if any of the test files in [uiTestDirectory] contain the + /// annotation that means that the test will reports the results of running + /// the Dart integration tests. + Future _testsContainDartIntegrationTestRunner( + Directory uiTestDirectory) async { + return uiTestDirectory + .list(recursive: true, followLinks: false) + .where((FileSystemEntity entity) => entity is File) + .cast() + .any((File file) { + return file.basename.endsWith('.java') && + file.readAsStringSync().contains('@RunWith(FlutterTestRunner.class)'); + }); + } } diff --git a/script/tool/test/firebase_test_lab_command_test.dart b/script/tool/test/firebase_test_lab_command_test.dart index 268210d004..65f398b32c 100644 --- a/script/tool/test/firebase_test_lab_command_test.dart +++ b/script/tool/test/firebase_test_lab_command_test.dart @@ -8,7 +8,9 @@ import 'package:args/command_runner.dart'; import 'package:file/file.dart'; import 'package:file/memory.dart'; import 'package:flutter_plugin_tools/src/common/core.dart'; +import 'package:flutter_plugin_tools/src/common/file_utils.dart'; import 'package:flutter_plugin_tools/src/firebase_test_lab_command.dart'; +import 'package:path/path.dart' as p; import 'package:test/test.dart'; import 'mocks.dart'; @@ -38,15 +40,33 @@ void main() { runner.addCommand(command); }); + void _writeJavaTestFile(Directory pluginDir, String relativeFilePath, + {String runnerClass = 'FlutterTestRunner'}) { + childFileWithSubcomponents(pluginDir, p.posix.split(relativeFilePath)) + .writeAsStringSync(''' +@DartIntegrationTest +@RunWith($runnerClass.class) +public class MainActivityTest { + @Rule + public ActivityTestRule rule = new ActivityTestRule<>(FlutterActivity.class); +} +'''); + } + test('fails if gcloud auth fails', () async { processRunner.mockProcessesForExecutable['gcloud'] = [ MockProcess(exitCode: 1) ]; - createFakePlugin('plugin', packagesDir, extraFiles: [ + + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); Error? commandError; final List output = await runCapturingPrint( @@ -67,11 +87,16 @@ void main() { MockProcess(), // auth MockProcess(exitCode: 1), // config ]; - createFakePlugin('plugin', packagesDir, extraFiles: [ + + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); final List output = await runCapturingPrint(runner, ['firebase-test-lab']); @@ -85,18 +110,24 @@ void main() { }); test('only runs gcloud configuration once', () async { - createFakePlugin('plugin1', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory plugin1Dir = + createFakePlugin('plugin1', packagesDir, extraFiles: [ 'test/plugin_test.dart', 'example/integration_test/foo_test.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); - createFakePlugin('plugin2', packagesDir, extraFiles: [ + _writeJavaTestFile(plugin1Dir, javaTestFileRelativePath); + final Directory plugin2Dir = + createFakePlugin('plugin2', packagesDir, extraFiles: [ 'test/plugin_test.dart', 'example/integration_test/bar_test.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(plugin2Dir, javaTestFileRelativePath); final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', @@ -164,14 +195,18 @@ void main() { }); test('runs integration tests', () async { - createFakePlugin('plugin', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'test/plugin_test.dart', 'example/integration_test/bar_test.dart', 'example/integration_test/foo_test.dart', 'example/integration_test/should_not_run.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', @@ -237,12 +272,16 @@ void main() { }); test('fails if a test fails', () async { - createFakePlugin('plugin', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/bar_test.dart', 'example/integration_test/foo_test.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); processRunner.mockProcessesForExecutable['gcloud'] = [ MockProcess(), // auth @@ -258,12 +297,6 @@ void main() { 'firebase-test-lab', '--device', 'model=redfin,version=30', - '--device', - 'model=seoul,version=26', - '--test-run-id', - 'testRunId', - '--build-id', - 'buildId', ], errorHandler: (Error e) { commandError = e; @@ -295,12 +328,6 @@ void main() { 'firebase-test-lab', '--device', 'model=redfin,version=30', - '--device', - 'model=seoul,version=26', - '--test-run-id', - 'testRunId', - '--build-id', - 'buildId', ], errorHandler: (Error e) { commandError = e; @@ -321,10 +348,14 @@ void main() { }); test('fails for packages with no integration test files', () async { - createFakePlugin('plugin', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); Error? commandError; final List output = await runCapturingPrint( @@ -333,12 +364,6 @@ void main() { 'firebase-test-lab', '--device', 'model=redfin,version=30', - '--device', - 'model=seoul,version=26', - '--test-run-id', - 'testRunId', - '--build-id', - 'buildId', ], errorHandler: (Error e) { commandError = e; @@ -358,6 +383,48 @@ void main() { ); }); + test('fails for packages with no integration_test runner', () async { + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ + 'test/plugin_test.dart', + 'example/integration_test/bar_test.dart', + 'example/integration_test/foo_test.dart', + 'example/integration_test/should_not_run.dart', + 'example/android/gradlew', + javaTestFileRelativePath, + ]); + // Use the wrong @RunWith annotation. + _writeJavaTestFile(pluginDir, javaTestFileRelativePath, + runnerClass: 'AndroidJUnit4.class'); + + Error? commandError; + final List output = await runCapturingPrint( + runner, + [ + 'firebase-test-lab', + '--device', + 'model=redfin,version=30', + ], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('No integration_test runner found. ' + 'See the integration_test package README for setup instructions.'), + contains('plugin:\n' + ' No integration_test runner.'), + ]), + ); + }); + test('skips packages with no android directory', () async { createFakePackage('package', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', @@ -367,12 +434,6 @@ void main() { 'firebase-test-lab', '--device', 'model=redfin,version=30', - '--device', - 'model=seoul,version=26', - '--test-run-id', - 'testRunId', - '--build-id', - 'buildId', ]); expect( @@ -392,17 +453,19 @@ void main() { }); test('builds if gradlew is missing', () async { - createFakePlugin('plugin', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); final List output = await runCapturingPrint(runner, [ 'firebase-test-lab', '--device', 'model=redfin,version=30', - '--device', - 'model=seoul,version=26', '--test-run-id', 'testRunId', '--build-id', @@ -445,7 +508,7 @@ void main() { '/packages/plugin/example/android'), ProcessCall( 'gcloud', - 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=redfin,version=30 --device model=seoul,version=26' + 'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=redfin,version=30' .split(' '), '/packages/plugin/example'), ]), @@ -453,10 +516,14 @@ void main() { }); test('fails if building to generate gradlew fails', () async { - createFakePlugin('plugin', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); processRunner.mockProcessesForExecutable['flutter'] = [ MockProcess(exitCode: 1) // flutter build @@ -484,11 +551,14 @@ void main() { }); test('fails if assembleAndroidTest fails', () async { + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; final Directory pluginDir = createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); final String gradlewPath = pluginDir .childDirectory('example') @@ -521,11 +591,14 @@ void main() { }); test('fails if assembleDebug fails', () async { + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; final Directory pluginDir = createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); final String gradlewPath = pluginDir .childDirectory('example') @@ -562,11 +635,15 @@ void main() { }); test('experimental flag', () async { - createFakePlugin('plugin', packagesDir, extraFiles: [ + const String javaTestFileRelativePath = + 'example/android/app/src/androidTest/MainActivityTest.java'; + final Directory pluginDir = + createFakePlugin('plugin', packagesDir, extraFiles: [ 'example/integration_test/foo_test.dart', 'example/android/gradlew', - 'example/android/app/src/androidTest/MainActivityTest.java', + javaTestFileRelativePath, ]); + _writeJavaTestFile(pluginDir, javaTestFileRelativePath); await runCapturingPrint(runner, [ 'firebase-test-lab',