[flutter_plugin_tools] Check for FlutterTestRunner in FTL tests (#4531)

When running via Firebase Test Lab, ensure that there is a test using `FlutterTestRunner`. This ensures that a plugin can't silently not run any of the Dart integration test on Android by having some other native integration test. Fixes https://github.com/flutter/flutter/issues/93952
2025-06-03 01:30:43 +08:00 · 2021-11-23 21:52:01 -05:00
parent 7b6ac13139
commit b25b31427a
3 changed files with 156 additions and 52 deletions
--- a/script/tool/CHANGELOG.md
+++ b/script/tool/CHANGELOG.md
@ -1,3 +1,7 @@
+## NEXT
+
+- Ensures that `firebase-test-lab` runs include an `integration_test` runner.
+
 ## 0.7.3

 - `native-test` now builds unit tests before running them on Windows and Linux,
--- a/script/tool/lib/src/firebase_test_lab_command.dart
+++ b/script/tool/lib/src/firebase_test_lab_command.dart
@ -127,16 +127,24 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
          '${example.displayName} does not support Android.');
    }

-    if (!androidDirectory
+    final Directory uiTestDirectory = androidDirectory
        .childDirectory('app')
        .childDirectory('src')
-        .childDirectory('androidTest')
-        .existsSync()) {
+        .childDirectory('androidTest');
+    if (!uiTestDirectory.existsSync()) {
      printError('No androidTest directory found.');
      return PackageResult.fail(
          <String>['No tests ran (use --exclude if this is intentional).']);
    }

+    // Ensure that the Dart integration tests will be run, not just native UI
+    // tests.
+    if (!await _testsContainDartIntegrationTestRunner(uiTestDirectory)) {
+      printError('No integration_test runner found. '
+          'See the integration_test package README for setup instructions.');
+      return PackageResult.fail(<String>['No integration_test runner.']);
+    }
+
    // Ensures that gradle wrapper exists
    final GradleProject project = GradleProject(example.directory,
        processRunner: processRunner, platform: platform);
@ -280,4 +288,19 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
            file is File && file.basename.endsWith('_test.dart'))
        .cast<File>();
  }
+
+  /// Returns true if any of the test files in [uiTestDirectory] contain the
+  /// annotation that means that the test will reports the results of running
+  /// the Dart integration tests.
+  Future<bool> _testsContainDartIntegrationTestRunner(
+      Directory uiTestDirectory) async {
+    return uiTestDirectory
+        .list(recursive: true, followLinks: false)
+        .where((FileSystemEntity entity) => entity is File)
+        .cast<File>()
+        .any((File file) {
+      return file.basename.endsWith('.java') &&
+          file.readAsStringSync().contains('@RunWith(FlutterTestRunner.class)');
+    });
+  }
 }
--- a/script/tool/test/firebase_test_lab_command_test.dart
+++ b/script/tool/test/firebase_test_lab_command_test.dart
@ -8,7 +8,9 @@ import 'package:args/command_runner.dart';
 import 'package:file/file.dart';
 import 'package:file/memory.dart';
 import 'package:flutter_plugin_tools/src/common/core.dart';
+import 'package:flutter_plugin_tools/src/common/file_utils.dart';
 import 'package:flutter_plugin_tools/src/firebase_test_lab_command.dart';
+import 'package:path/path.dart' as p;
 import 'package:test/test.dart';

 import 'mocks.dart';
@ -38,15 +40,33 @@ void main() {
      runner.addCommand(command);
    });

+    void _writeJavaTestFile(Directory pluginDir, String relativeFilePath,
+        {String runnerClass = 'FlutterTestRunner'}) {
+      childFileWithSubcomponents(pluginDir, p.posix.split(relativeFilePath))
+          .writeAsStringSync('''
+@DartIntegrationTest
+@RunWith($runnerClass.class)
+public class MainActivityTest {
+  @Rule
+  public ActivityTestRule<FlutterActivity> rule = new ActivityTestRule<>(FlutterActivity.class);
+}
+''');
+    }
+
    test('fails if gcloud auth fails', () async {
      processRunner.mockProcessesForExecutable['gcloud'] = <Process>[
        MockProcess(exitCode: 1)
      ];
+
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      Error? commandError;
      final List<String> output = await runCapturingPrint(
@ -67,11 +87,16 @@ void main() {
        MockProcess(), // auth
        MockProcess(exitCode: 1), // config
      ];
+
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      final List<String> output =
          await runCapturingPrint(runner, <String>['firebase-test-lab']);
@ -85,18 +110,24 @@ void main() {
    });

    test('only runs gcloud configuration once', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory plugin1Dir =
          createFakePlugin('plugin1', packagesDir, extraFiles: <String>[
        'test/plugin_test.dart',
        'example/integration_test/foo_test.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(plugin1Dir, javaTestFileRelativePath);
+      final Directory plugin2Dir =
          createFakePlugin('plugin2', packagesDir, extraFiles: <String>[
        'test/plugin_test.dart',
        'example/integration_test/bar_test.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(plugin2Dir, javaTestFileRelativePath);

      final List<String> output = await runCapturingPrint(runner, <String>[
        'firebase-test-lab',
@ -164,14 +195,18 @@ void main() {
    });

    test('runs integration tests', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'test/plugin_test.dart',
        'example/integration_test/bar_test.dart',
        'example/integration_test/foo_test.dart',
        'example/integration_test/should_not_run.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      final List<String> output = await runCapturingPrint(runner, <String>[
        'firebase-test-lab',
@ -237,12 +272,16 @@ void main() {
    });

    test('fails if a test fails', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/bar_test.dart',
        'example/integration_test/foo_test.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      processRunner.mockProcessesForExecutable['gcloud'] = <Process>[
        MockProcess(), // auth
@ -258,12 +297,6 @@ void main() {
          'firebase-test-lab',
          '--device',
          'model=redfin,version=30',
-          '--device',
-          'model=seoul,version=26',
-          '--test-run-id',
-          'testRunId',
-          '--build-id',
-          'buildId',
        ],
        errorHandler: (Error e) {
          commandError = e;
@ -295,12 +328,6 @@ void main() {
          'firebase-test-lab',
          '--device',
          'model=redfin,version=30',
-          '--device',
-          'model=seoul,version=26',
-          '--test-run-id',
-          'testRunId',
-          '--build-id',
-          'buildId',
        ],
        errorHandler: (Error e) {
          commandError = e;
@ -321,10 +348,14 @@ void main() {
    });

    test('fails for packages with no integration test files', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      Error? commandError;
      final List<String> output = await runCapturingPrint(
@ -333,12 +364,6 @@ void main() {
          'firebase-test-lab',
          '--device',
          'model=redfin,version=30',
-          '--device',
-          'model=seoul,version=26',
-          '--test-run-id',
-          'testRunId',
-          '--build-id',
-          'buildId',
        ],
        errorHandler: (Error e) {
          commandError = e;
@ -358,6 +383,48 @@ void main() {
      );
    });

+    test('fails for packages with no integration_test runner', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
+          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
+        'test/plugin_test.dart',
+        'example/integration_test/bar_test.dart',
+        'example/integration_test/foo_test.dart',
+        'example/integration_test/should_not_run.dart',
+        'example/android/gradlew',
+        javaTestFileRelativePath,
+      ]);
+      // Use the wrong @RunWith annotation.
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath,
+          runnerClass: 'AndroidJUnit4.class');
+
+      Error? commandError;
+      final List<String> output = await runCapturingPrint(
+        runner,
+        <String>[
+          'firebase-test-lab',
+          '--device',
+          'model=redfin,version=30',
+        ],
+        errorHandler: (Error e) {
+          commandError = e;
+        },
+      );
+
+      expect(commandError, isA<ToolExit>());
+      expect(
+        output,
+        containsAllInOrder(<Matcher>[
+          contains('Running for plugin'),
+          contains('No integration_test runner found. '
+              'See the integration_test package README for setup instructions.'),
+          contains('plugin:\n'
+              '    No integration_test runner.'),
+        ]),
+      );
+    });
+
    test('skips packages with no android directory', () async {
      createFakePackage('package', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
@ -367,12 +434,6 @@ void main() {
        'firebase-test-lab',
        '--device',
        'model=redfin,version=30',
-        '--device',
-        'model=seoul,version=26',
-        '--test-run-id',
-        'testRunId',
-        '--build-id',
-        'buildId',
      ]);

      expect(
@ -392,17 +453,19 @@ void main() {
    });

    test('builds if gradlew is missing', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      final List<String> output = await runCapturingPrint(runner, <String>[
        'firebase-test-lab',
        '--device',
        'model=redfin,version=30',
-        '--device',
-        'model=seoul,version=26',
        '--test-run-id',
        'testRunId',
        '--build-id',
@ -445,7 +508,7 @@ void main() {
              '/packages/plugin/example/android'),
          ProcessCall(
              'gcloud',
-              'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=redfin,version=30 --device model=seoul,version=26'
+              'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_firebase_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/0/ --device model=redfin,version=30'
                  .split(' '),
              '/packages/plugin/example'),
        ]),
@ -453,10 +516,14 @@ void main() {
    });

    test('fails if building to generate gradlew fails', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      processRunner.mockProcessesForExecutable['flutter'] = <Process>[
        MockProcess(exitCode: 1) // flutter build
@ -484,11 +551,14 @@ void main() {
    });

    test('fails if assembleAndroidTest fails', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      final String gradlewPath = pluginDir
          .childDirectory('example')
@ -521,11 +591,14 @@ void main() {
    });

    test('fails if assembleDebug fails', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      final String gradlewPath = pluginDir
          .childDirectory('example')
@ -562,11 +635,15 @@ void main() {
    });

    test('experimental flag', () async {
+      const String javaTestFileRelativePath =
+          'example/android/app/src/androidTest/MainActivityTest.java';
+      final Directory pluginDir =
          createFakePlugin('plugin', packagesDir, extraFiles: <String>[
        'example/integration_test/foo_test.dart',
        'example/android/gradlew',
-        'example/android/app/src/androidTest/MainActivityTest.java',
+        javaTestFileRelativePath,
      ]);
+      _writeJavaTestFile(pluginDir, javaTestFileRelativePath);

      await runCapturingPrint(runner, <String>[
        'firebase-test-lab',