[flutter_plugin_tools] Build gtest unit tests (#4492)

script/tool/CHANGELOG.md

@@ -1,5 +1,7 @@
 ## NEXT
 
+- `native-test` now builds unit tests before running them on Windows and Linux,
+  matching the behavior of other platforms.
 - Added `--log-timing` to add timing information to package headers in looping
   commands.
 

script/tool/lib/src/common/cmake.dart

@@ -0,0 +1,118 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:file/file.dart';
+import 'package:flutter_plugin_tools/src/common/core.dart';
+import 'package:platform/platform.dart';
+
+import 'process_runner.dart';
+
+const String _cacheCommandKey = 'CMAKE_COMMAND:INTERNAL';
+
+/// A utility class for interacting with CMake projects.
+class CMakeProject {
+  /// Creates an instance that runs commands for [project] with the given
+  /// [processRunner].
+  CMakeProject(
+    this.flutterProject, {
+    required this.buildMode,
+    this.processRunner = const ProcessRunner(),
+    this.platform = const LocalPlatform(),
+  });
+
+  /// The directory of a Flutter project to run Gradle commands in.
+  final Directory flutterProject;
+
+  /// The [ProcessRunner] used to run commands. Overridable for testing.
+  final ProcessRunner processRunner;
+
+  /// The platform that commands are being run on.
+  final Platform platform;
+
+  /// The build mode (e.g., Debug, Release).
+  ///
+  /// This is a constructor paramater because on Linux many properties depend
+  /// on the build mode since it uses a single-configuration generator.
+  final String buildMode;
+
+  late final String _cmakeCommand = _determineCmakeCommand();
+
+  /// The project's platform directory name.
+  String get _platformDirName => platform.isWindows ? 'windows' : 'linux';
+
+  /// The project's 'example' build directory for this instance's platform.
+  Directory get buildDirectory {
+    Directory buildDir =
+        flutterProject.childDirectory('build').childDirectory(_platformDirName);
+    if (platform.isLinux) {
+      buildDir = buildDir
+          // TODO(stuartmorgan): Support arm64 if that ever becomes a supported
+          // CI configuration for the repository.
+          .childDirectory('x64')
+          // Linux uses a single-config generator, so the base build directory
+          // includes the configuration.
+          .childDirectory(buildMode.toLowerCase());
+    }
+    return buildDir;
+  }
+
+  File get _cacheFile => buildDirectory.childFile('CMakeCache.txt');
+
+  /// Returns the CMake command to run build commands for this project.
+  ///
+  /// Assumes the project has been built at least once, such that the CMake
+  /// generation step has run.
+  String getCmakeCommand() {
+    return _cmakeCommand;
+  }
+
+  /// Returns the CMake command to run build commands for this project. This is
+  /// used to initialize _cmakeCommand, and should not be called directly.
+  ///
+  /// Assumes the project has been built at least once, such that the CMake
+  /// generation step has run.
+  String _determineCmakeCommand() {
+    // On Linux 'cmake' is expected to be in the path, so doesn't need to
+    // be lookup up and cached.
+    if (platform.isLinux) {
+      return 'cmake';
+    }
+    final File cacheFile = _cacheFile;
+    String? command;
+    for (String line in cacheFile.readAsLinesSync()) {
+      line = line.trim();
+      if (line.startsWith(_cacheCommandKey)) {
+        command = line.substring(line.indexOf('=') + 1).trim();
+        break;
+      }
+    }
+    if (command == null) {
+      printError('Unable to find CMake command in ${cacheFile.path}');
+      throw ToolExit(100);
+    }
+    return command;
+  }
+
+  /// Whether or not the project is ready to have CMake commands run on it
+  /// (i.e., whether the `flutter` tool has generated the necessary files).
+  bool isConfigured() => _cacheFile.existsSync();
+
+  /// Runs a `cmake` command with the given parameters.
+  Future<int> runBuild(
+    String target, {
+    List<String> arguments = const <String>[],
+  }) {
+    return processRunner.runAndStream(
+      getCmakeCommand(),
+      <String>[
+        '--build',
+        buildDirectory.path,
+        '--target',
+        target,
+        if (platform.isWindows) ...<String>['--config', buildMode],
+        ...arguments,
+      ],
+    );
+  }
+}

script/tool/lib/src/common/gradle.dart

@@ -14,9 +14,6 @@ const String _gradleWrapperNonWindows = 'gradlew';
 class GradleProject {
   /// Creates an instance that runs commands for [project] with the given
   /// [processRunner].
-  ///
-  /// If [log] is true, commands run by this instance will long various status
-  /// messages.
   GradleProject(
     this.flutterProject, {
     this.processRunner = const ProcessRunner(),

script/tool/lib/src/native_test_command.dart

@@ -5,6 +5,7 @@
 import 'package:file/file.dart';
 import 'package:platform/platform.dart';
 
+import 'common/cmake.dart';
 import 'common/core.dart';
 import 'common/gradle.dart';
 import 'common/package_looping_command.dart';
@@ -456,8 +457,8 @@ this command.
           file.basename.endsWith('_tests.exe');
     }
 
-    return _runGoogleTestTests(plugin,
+    return _runGoogleTestTests(plugin, 'Windows', 'Debug',
-        buildDirectoryName: 'windows', isTestBinary: isTestBinary);
+        isTestBinary: isTestBinary);
   }
 
   Future<_PlatformResult> _testLinux(
@@ -471,8 +472,16 @@ this command.
           file.basename.endsWith('_tests');
     }
 
-    return _runGoogleTestTests(plugin,
+    // Since Linux uses a single-config generator, building-examples only
-        buildDirectoryName: 'linux', isTestBinary: isTestBinary);
+    // generates the build files for release, so the tests have to be run in
+    // release mode as well.
+    //
+    // TODO(stuartmorgan): Consider adding a command to `flutter` that would
+    // generate build files without doing a build, and using that instead of
+    // relying on running build-examples. See
+    // https://github.com/flutter/flutter/issues/93407.
+    return _runGoogleTestTests(plugin, 'Linux', 'Release',
+        isTestBinary: isTestBinary);
   }
 
   /// Finds every file in the [buildDirectoryName] subdirectory of [plugin]'s
@@ -482,38 +491,66 @@ this command.
   /// The binaries are assumed to be Google Test test binaries, thus returning
   /// zero for success and non-zero for failure.
   Future<_PlatformResult> _runGoogleTestTests(
-    RepositoryPackage plugin, {
+    RepositoryPackage plugin,
-    required String buildDirectoryName,
+    String platformName,
+    String buildMode, {
     required bool Function(File) isTestBinary,
   }) async {
     final List<File> testBinaries = <File>[];
+    bool hasMissingBuild = false;
+    bool buildFailed = false;
     for (final RepositoryPackage example in plugin.getExamples()) {
-      final Directory buildDir = example.directory
+      final CMakeProject project = CMakeProject(example.directory,
-          .childDirectory('build')
+          buildMode: buildMode,
-          .childDirectory(buildDirectoryName);
+          processRunner: processRunner,
-      if (!buildDir.existsSync()) {
+          platform: platform);
+      if (!project.isConfigured()) {
+        printError('ERROR: Run "flutter build" on ${example.displayName}, '
+            'or run this tool\'s "build-examples" command, for the target '
+            'platform before executing tests.');
+        hasMissingBuild = true;
         continue;
       }
-      testBinaries.addAll(buildDir
+
+      // By repository convention, example projects create an aggregate target
+      // called 'unit_tests' that builds all unit tests (usually just an alias
+      // for a specific test target).
+      final int exitCode = await project.runBuild('unit_tests');
+      if (exitCode != 0) {
+        printError('${example.displayName} unit tests failed to build.');
+        buildFailed = true;
+      }
+
+      testBinaries.addAll(project.buildDirectory
           .listSync(recursive: true)
           .whereType<File>()
           .where(isTestBinary)
           .where((File file) {
-        // Only run the release build of the unit tests, to avoid running the
+        // Only run the `buildMode` build of the unit tests, to avoid running
-        // same tests multiple times. Release is used rather than debug since
+        // the same tests multiple times.
-        // `build-examples` builds release versions.
         final List<String> components = path.split(file.path);
-        return components.contains('release') || components.contains('Release');
+        return components.contains(buildMode) ||
+            components.contains(buildMode.toLowerCase());
       }));
     }
 
+    if (hasMissingBuild) {
+      return _PlatformResult(RunState.failed,
+          error: 'Examples must be built before testing.');
+    }
+
+    if (buildFailed) {
+      return _PlatformResult(RunState.failed,
+          error: 'Failed to build $platformName unit tests.');
+    }
+
     if (testBinaries.isEmpty) {
       final String binaryExtension = platform.isWindows ? '.exe' : '';
       printError(
           'No test binaries found. At least one *_test(s)$binaryExtension '
           'binary should be built by the example(s)');
       return _PlatformResult(RunState.failed,
-          error: 'No $buildDirectoryName unit tests found');
+          error: 'No $platformName unit tests found');
     }
 
     bool passing = true;

script/tool/test/native_test_command_test.dart

@@ -8,10 +8,12 @@ import 'dart:io' as io;
 import 'package:args/command_runner.dart';
 import 'package:file/file.dart';
 import 'package:file/memory.dart';
+import 'package:flutter_plugin_tools/src/common/cmake.dart';
 import 'package:flutter_plugin_tools/src/common/core.dart';
 import 'package:flutter_plugin_tools/src/common/file_utils.dart';
 import 'package:flutter_plugin_tools/src/common/plugin_utils.dart';
 import 'package:flutter_plugin_tools/src/native_test_command.dart';
+import 'package:platform/platform.dart';
 import 'package:test/test.dart';
 
 import 'mocks.dart';
@@ -53,6 +55,16 @@ final Map<String, dynamic> _kDeviceListMap = <String, dynamic>{
   }
 };
 
+const String _fakeCmakeCommand = 'path/to/cmake';
+
+void _createFakeCMakeCache(Directory pluginDir, Platform platform) {
+  final CMakeProject project = CMakeProject(pluginDir.childDirectory('example'),
+      platform: platform, buildMode: 'Release');
+  final File cache = project.buildDirectory.childFile('CMakeCache.txt');
+  cache.createSync(recursive: true);
+  cache.writeAsStringSync('CMAKE_COMMAND:INTERNAL=$_fakeCmakeCommand');
+}
+
 // TODO(stuartmorgan): Rework these tests to use a mock Xcode instead of
 // doing all the process mocking and validation.
 void main() {
@@ -67,7 +79,10 @@ void main() {
 
     setUp(() {
       fileSystem = MemoryFileSystem();
-      mockPlatform = MockPlatform(isMacOS: true);
+      // iOS and macOS tests expect macOS, Linux tests expect Linux; nothing
+      // needs to distinguish between Linux and macOS, so set both to true to
+      // allow them to share a setup group.
+      mockPlatform = MockPlatform(isMacOS: true, isLinux: true);
       packagesDir = createPackagesDirectory(fileSystem: fileSystem);
       processRunner = RecordingProcessRunner();
       final NativeTestCommand command = NativeTestCommand(packagesDir,
@@ -133,6 +148,26 @@ void main() {
           package.path);
     }
 
+    // Returns the ProcessCall to expect for build the Linux unit tests for the
+    // given plugin.
+    ProcessCall _getLinuxBuildCall(Directory pluginDir) {
+      return ProcessCall(
+          'cmake',
+          <String>[
+            '--build',
+            pluginDir
+                .childDirectory('example')
+                .childDirectory('build')
+                .childDirectory('linux')
+                .childDirectory('x64')
+                .childDirectory('release')
+                .path,
+            '--target',
+            'unit_tests'
+          ],
+          null);
+    }
+
     test('fails if no platforms are provided', () async {
       Error? commandError;
       final List<String> output = await runCapturingPrint(
@@ -844,15 +879,16 @@ void main() {
     });
 
     group('Linux', () {
-      test('runs unit tests', () async {
+      test('builds and runs unit tests', () async {
         const String testBinaryRelativePath =
-            'build/linux/foo/release/bar/plugin_test';
+            'build/linux/x64/release/bar/plugin_test';
         final Directory pluginDirectory =
             createFakePlugin('plugin', packagesDir, extraFiles: <String>[
           'example/$testBinaryRelativePath'
         ], platformSupport: <String, PlatformDetails>{
           kPlatformLinux: const PlatformDetails(PlatformSupport.inline),
         });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
 
         final File testBinary = childFileWithSubcomponents(pluginDirectory,
             <String>['example', ...testBinaryRelativePath.split('/')]);
@@ -874,15 +910,16 @@ void main() {
         expect(
             processRunner.recordedCalls,
             orderedEquals(<ProcessCall>[
+              _getLinuxBuildCall(pluginDirectory),
               ProcessCall(testBinary.path, const <String>[], null),
             ]));
       });
 
       test('only runs release unit tests', () async {
         const String debugTestBinaryRelativePath =
-            'build/linux/foo/debug/bar/plugin_test';
+            'build/linux/x64/debug/bar/plugin_test';
         const String releaseTestBinaryRelativePath =
-            'build/linux/foo/release/bar/plugin_test';
+            'build/linux/x64/release/bar/plugin_test';
         final Directory pluginDirectory =
             createFakePlugin('plugin', packagesDir, extraFiles: <String>[
           'example/$debugTestBinaryRelativePath',
@@ -890,6 +927,7 @@ void main() {
         ], platformSupport: <String, PlatformDetails>{
           kPlatformLinux: const PlatformDetails(PlatformSupport.inline),
         });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
 
         final File releaseTestBinary = childFileWithSubcomponents(
             pluginDirectory,
@@ -909,15 +947,15 @@ void main() {
           ]),
         );
 
-        // Only the release version should be run.
         expect(
             processRunner.recordedCalls,
             orderedEquals(<ProcessCall>[
+              _getLinuxBuildCall(pluginDirectory),
               ProcessCall(releaseTestBinary.path, const <String>[], null),
             ]));
       });
 
-      test('fails if there are no unit tests', () async {
+      test('fails if CMake has not been configured', () async {
         createFakePlugin('plugin', packagesDir,
             platformSupport: <String, PlatformDetails>{
               kPlatformLinux: const PlatformDetails(PlatformSupport.inline),
@@ -936,22 +974,56 @@ void main() {
         expect(
           output,
           containsAllInOrder(<Matcher>[
-            contains('No test binaries found.'),
+            contains('plugin:\n'
+                '    Examples must be built before testing.')
           ]),
         );
 
         expect(processRunner.recordedCalls, orderedEquals(<ProcessCall>[]));
       });
 
+      test('fails if there are no unit tests', () async {
+        final Directory pluginDirectory = createFakePlugin(
+            'plugin', packagesDir,
+            platformSupport: <String, PlatformDetails>{
+              kPlatformLinux: const PlatformDetails(PlatformSupport.inline),
+            });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
+
+        Error? commandError;
+        final List<String> output = await runCapturingPrint(runner, <String>[
+          'native-test',
+          '--linux',
+          '--no-integration',
+        ], errorHandler: (Error e) {
+          commandError = e;
+        });
+
+        expect(commandError, isA<ToolExit>());
+        expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('No test binaries found.'),
+          ]),
+        );
+
+        expect(
+            processRunner.recordedCalls,
+            orderedEquals(<ProcessCall>[
+              _getLinuxBuildCall(pluginDirectory),
+            ]));
+      });
+
       test('fails if a unit test fails', () async {
         const String testBinaryRelativePath =
-            'build/linux/foo/release/bar/plugin_test';
+            'build/linux/x64/release/bar/plugin_test';
         final Directory pluginDirectory =
             createFakePlugin('plugin', packagesDir, extraFiles: <String>[
           'example/$testBinaryRelativePath'
         ], platformSupport: <String, PlatformDetails>{
           kPlatformLinux: const PlatformDetails(PlatformSupport.inline),
         });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
 
         final File testBinary = childFileWithSubcomponents(pluginDirectory,
             <String>['example', ...testBinaryRelativePath.split('/')]);
@@ -979,6 +1051,7 @@ void main() {
         expect(
             processRunner.recordedCalls,
             orderedEquals(<ProcessCall>[
+              _getLinuxBuildCall(pluginDirectory),
               ProcessCall(testBinary.path, const <String>[], null),
             ]));
       });
@@ -1524,16 +1597,37 @@ void main() {
       runner.addCommand(command);
     });
 
+    // Returns the ProcessCall to expect for build the Windows unit tests for
+    // the given plugin.
+    ProcessCall _getWindowsBuildCall(Directory pluginDir) {
+      return ProcessCall(
+          _fakeCmakeCommand,
+          <String>[
+            '--build',
+            pluginDir
+                .childDirectory('example')
+                .childDirectory('build')
+                .childDirectory('windows')
+                .path,
+            '--target',
+            'unit_tests',
+            '--config',
+            'Debug'
+          ],
+          null);
+    }
+
     group('Windows', () {
       test('runs unit tests', () async {
         const String testBinaryRelativePath =
-            'build/windows/foo/Release/bar/plugin_test.exe';
+            'build/windows/Debug/bar/plugin_test.exe';
         final Directory pluginDirectory =
             createFakePlugin('plugin', packagesDir, extraFiles: <String>[
           'example/$testBinaryRelativePath'
         ], platformSupport: <String, PlatformDetails>{
           kPlatformWindows: const PlatformDetails(PlatformSupport.inline),
         });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
 
         final File testBinary = childFileWithSubcomponents(pluginDirectory,
             <String>['example', ...testBinaryRelativePath.split('/')]);
@@ -1555,15 +1649,16 @@ void main() {
         expect(
             processRunner.recordedCalls,
             orderedEquals(<ProcessCall>[
+              _getWindowsBuildCall(pluginDirectory),
               ProcessCall(testBinary.path, const <String>[], null),
             ]));
       });
 
-      test('only runs release unit tests', () async {
+      test('only runs debug unit tests', () async {
         const String debugTestBinaryRelativePath =
-            'build/windows/foo/Debug/bar/plugin_test.exe';
+            'build/windows/Debug/bar/plugin_test.exe';
         const String releaseTestBinaryRelativePath =
-            'build/windows/foo/Release/bar/plugin_test.exe';
+            'build/windows/Release/bar/plugin_test.exe';
         final Directory pluginDirectory =
             createFakePlugin('plugin', packagesDir, extraFiles: <String>[
           'example/$debugTestBinaryRelativePath',
@@ -1571,10 +1666,10 @@ void main() {
         ], platformSupport: <String, PlatformDetails>{
           kPlatformWindows: const PlatformDetails(PlatformSupport.inline),
         });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
 
-        final File releaseTestBinary = childFileWithSubcomponents(
+        final File debugTestBinary = childFileWithSubcomponents(pluginDirectory,
-            pluginDirectory,
+            <String>['example', ...debugTestBinaryRelativePath.split('/')]);
-            <String>['example', ...releaseTestBinaryRelativePath.split('/')]);
 
         final List<String> output = await runCapturingPrint(runner, <String>[
           'native-test',
@@ -1590,15 +1685,15 @@ void main() {
           ]),
         );
 
-        // Only the release version should be run.
         expect(
             processRunner.recordedCalls,
             orderedEquals(<ProcessCall>[
-              ProcessCall(releaseTestBinary.path, const <String>[], null),
+              _getWindowsBuildCall(pluginDirectory),
+              ProcessCall(debugTestBinary.path, const <String>[], null),
             ]));
       });
 
-      test('fails if there are no unit tests', () async {
+      test('fails if CMake has not been configured', () async {
         createFakePlugin('plugin', packagesDir,
             platformSupport: <String, PlatformDetails>{
               kPlatformWindows: const PlatformDetails(PlatformSupport.inline),
@@ -1617,22 +1712,56 @@ void main() {
         expect(
           output,
           containsAllInOrder(<Matcher>[
-            contains('No test binaries found.'),
+            contains('plugin:\n'
+                '    Examples must be built before testing.')
           ]),
         );
 
         expect(processRunner.recordedCalls, orderedEquals(<ProcessCall>[]));
       });
 
+      test('fails if there are no unit tests', () async {
+        final Directory pluginDirectory = createFakePlugin(
+            'plugin', packagesDir,
+            platformSupport: <String, PlatformDetails>{
+              kPlatformWindows: const PlatformDetails(PlatformSupport.inline),
+            });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
+
+        Error? commandError;
+        final List<String> output = await runCapturingPrint(runner, <String>[
+          'native-test',
+          '--windows',
+          '--no-integration',
+        ], errorHandler: (Error e) {
+          commandError = e;
+        });
+
+        expect(commandError, isA<ToolExit>());
+        expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('No test binaries found.'),
+          ]),
+        );
+
+        expect(
+            processRunner.recordedCalls,
+            orderedEquals(<ProcessCall>[
+              _getWindowsBuildCall(pluginDirectory),
+            ]));
+      });
+
       test('fails if a unit test fails', () async {
         const String testBinaryRelativePath =
-            'build/windows/foo/Release/bar/plugin_test.exe';
+            'build/windows/Debug/bar/plugin_test.exe';
         final Directory pluginDirectory =
             createFakePlugin('plugin', packagesDir, extraFiles: <String>[
           'example/$testBinaryRelativePath'
         ], platformSupport: <String, PlatformDetails>{
           kPlatformWindows: const PlatformDetails(PlatformSupport.inline),
         });
+        _createFakeCMakeCache(pluginDirectory, mockPlatform);
 
         final File testBinary = childFileWithSubcomponents(pluginDirectory,
             <String>['example', ...testBinaryRelativePath.split('/')]);
@@ -1660,6 +1789,7 @@ void main() {
         expect(
             processRunner.recordedCalls,
             orderedEquals(<ProcessCall>[
+              _getWindowsBuildCall(pluginDirectory),
               ProcessCall(testBinary.path, const <String>[], null),
             ]));
       });