// Copyright 2013 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:convert'; import 'dart:io' as io; import 'dart:math'; import 'package:args/command_runner.dart'; import 'package:colorize/colorize.dart'; import 'package:file/file.dart'; import 'package:git/git.dart'; import 'package:http/http.dart' as http; import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; import 'package:yaml/yaml.dart'; /// The signature for a print handler for commands that allow overriding the /// print destination. typedef Print = void Function(Object? object); /// Key for windows platform. const String kWindows = 'windows'; /// Key for macos platform. const String kMacos = 'macos'; /// Key for linux platform. const String kLinux = 'linux'; /// Key for IPA (iOS) platform. const String kIos = 'ios'; /// Key for APK (Android) platform. const String kAndroid = 'android'; /// Key for Web platform. const String kWeb = 'web'; /// Key for IPA. const String kIpa = 'ipa'; /// Key for APK. const String kApk = 'apk'; /// Key for enable experiment. const String kEnableExperiment = 'enable-experiment'; /// Returns whether the given directory contains a Flutter package. bool isFlutterPackage(FileSystemEntity entity, FileSystem fileSystem) { if (entity is! Directory) { return false; } try { final File pubspecFile = fileSystem.file(p.join(entity.path, 'pubspec.yaml')); final YamlMap pubspecYaml = loadYaml(pubspecFile.readAsStringSync()) as YamlMap; final YamlMap? dependencies = pubspecYaml['dependencies'] as YamlMap?; if (dependencies == null) { return false; } return dependencies.containsKey('flutter'); } on FileSystemException { return false; } on YamlException { return false; } } /// Returns whether the given directory contains a Flutter [platform] plugin. /// /// It checks this by looking for the following pattern in the pubspec: /// /// flutter: /// plugin: /// platforms: /// [platform]: bool pluginSupportsPlatform( String platform, FileSystemEntity entity, FileSystem fileSystem) { assert(platform == kIos || platform == kAndroid || platform == kWeb || platform == kMacos || platform == kWindows || platform == kLinux); if (entity is! Directory) { return false; } try { final File pubspecFile = fileSystem.file(p.join(entity.path, 'pubspec.yaml')); final YamlMap pubspecYaml = loadYaml(pubspecFile.readAsStringSync()) as YamlMap; final YamlMap? flutterSection = pubspecYaml['flutter'] as YamlMap?; if (flutterSection == null) { return false; } final YamlMap? pluginSection = flutterSection['plugin'] as YamlMap?; if (pluginSection == null) { return false; } final YamlMap? platforms = pluginSection['platforms'] as YamlMap?; if (platforms == null) { // Legacy plugin specs are assumed to support iOS and Android. if (!pluginSection.containsKey('platforms')) { return platform == kIos || platform == kAndroid; } return false; } return platforms.containsKey(platform); } on FileSystemException { return false; } on YamlException { return false; } } /// Returns whether the given directory contains a Flutter Android plugin. bool isAndroidPlugin(FileSystemEntity entity, FileSystem fileSystem) { return pluginSupportsPlatform(kAndroid, entity, fileSystem); } /// Returns whether the given directory contains a Flutter iOS plugin. bool isIosPlugin(FileSystemEntity entity, FileSystem fileSystem) { return pluginSupportsPlatform(kIos, entity, fileSystem); } /// Returns whether the given directory contains a Flutter web plugin. bool isWebPlugin(FileSystemEntity entity, FileSystem fileSystem) { return pluginSupportsPlatform(kWeb, entity, fileSystem); } /// Returns whether the given directory contains a Flutter Windows plugin. bool isWindowsPlugin(FileSystemEntity entity, FileSystem fileSystem) { return pluginSupportsPlatform(kWindows, entity, fileSystem); } /// Returns whether the given directory contains a Flutter macOS plugin. bool isMacOsPlugin(FileSystemEntity entity, FileSystem fileSystem) { return pluginSupportsPlatform(kMacos, entity, fileSystem); } /// Returns whether the given directory contains a Flutter linux plugin. bool isLinuxPlugin(FileSystemEntity entity, FileSystem fileSystem) { return pluginSupportsPlatform(kLinux, entity, fileSystem); } /// Throws a [ToolExit] with `exitCode` and log the `errorMessage` in red. void printErrorAndExit({required String errorMessage, int exitCode = 1}) { final Colorize redError = Colorize(errorMessage)..red(); print(redError); throw ToolExit(exitCode); } /// Error thrown when a command needs to exit with a non-zero exit code. class ToolExit extends Error { /// Creates a tool exit with the given [exitCode]. ToolExit(this.exitCode); /// The code that the process should exit with. final int exitCode; } /// Interface definition for all commands in this tool. abstract class PluginCommand extends Command { /// Creates a command to operate on [packagesDir] with the given environment. PluginCommand( this.packagesDir, this.fileSystem, { this.processRunner = const ProcessRunner(), this.gitDir, }) { argParser.addMultiOption( _pluginsArg, splitCommas: true, help: 'Specifies which plugins the command should run on (before sharding).', valueHelp: 'plugin1,plugin2,...', ); argParser.addOption( _shardIndexArg, help: 'Specifies the zero-based index of the shard to ' 'which the command applies.', valueHelp: 'i', defaultsTo: '0', ); argParser.addOption( _shardCountArg, help: 'Specifies the number of shards into which plugins are divided.', valueHelp: 'n', defaultsTo: '1', ); argParser.addMultiOption( _excludeArg, abbr: 'e', help: 'Exclude packages from this command.', defaultsTo: [], ); argParser.addFlag(_runOnChangedPackagesArg, help: 'Run the command on changed packages/plugins.\n' 'If the $_pluginsArg is specified, this flag is ignored.\n' 'If no plugins have changed, the command runs on all plugins.\n' 'The packages excluded with $_excludeArg is also excluded even if changed.\n' 'See $_kBaseSha if a custom base is needed to determine the diff.'); argParser.addOption(_kBaseSha, help: 'The base sha used to determine git diff. \n' 'This is useful when $_runOnChangedPackagesArg is specified.\n' 'If not specified, merge-base is used as base sha.'); } static const String _pluginsArg = 'plugins'; static const String _shardIndexArg = 'shardIndex'; static const String _shardCountArg = 'shardCount'; static const String _excludeArg = 'exclude'; static const String _runOnChangedPackagesArg = 'run-on-changed-packages'; static const String _kBaseSha = 'base-sha'; /// The directory containing the plugin packages. final Directory packagesDir; /// The file system. /// /// This can be overridden for testing. final FileSystem fileSystem; /// The process runner. /// /// This can be overridden for testing. final ProcessRunner processRunner; /// The git directory to use. By default it uses the parent directory. /// /// This can be mocked for testing. final GitDir? gitDir; int? _shardIndex; int? _shardCount; /// The shard of the overall command execution that this instance should run. int get shardIndex { if (_shardIndex == null) { _checkSharding(); } return _shardIndex!; } /// The number of shards this command is divided into. int get shardCount { if (_shardCount == null) { _checkSharding(); } return _shardCount!; } /// Convenience accessor for boolean arguments. bool getBoolArg(String key) { return (argResults![key] as bool?) ?? false; } /// Convenience accessor for String arguments. String getStringArg(String key) { return (argResults![key] as String?) ?? ''; } /// Convenience accessor for List arguments. List getStringListArg(String key) { return (argResults![key] as List?) ?? []; } void _checkSharding() { final int? shardIndex = int.tryParse(getStringArg(_shardIndexArg)); final int? shardCount = int.tryParse(getStringArg(_shardCountArg)); if (shardIndex == null) { usageException('$_shardIndexArg must be an integer'); } if (shardCount == null) { usageException('$_shardCountArg must be an integer'); } if (shardCount < 1) { usageException('$_shardCountArg must be positive'); } if (shardIndex < 0 || shardCount <= shardIndex) { usageException( '$_shardIndexArg must be in the half-open range [0..$shardCount['); } _shardIndex = shardIndex; _shardCount = shardCount; } /// Returns the root Dart package folders of the plugins involved in this /// command execution. Stream getPlugins() async* { // To avoid assuming consistency of `Directory.list` across command // invocations, we collect and sort the plugin folders before sharding. // This is considered an implementation detail which is why the API still // uses streams. final List allPlugins = await _getAllPlugins().toList(); allPlugins.sort((Directory d1, Directory d2) => d1.path.compareTo(d2.path)); // Sharding 10 elements into 3 shards should yield shard sizes 4, 4, 2. // Sharding 9 elements into 3 shards should yield shard sizes 3, 3, 3. // Sharding 2 elements into 3 shards should yield shard sizes 1, 1, 0. final int shardSize = allPlugins.length ~/ shardCount + (allPlugins.length % shardCount == 0 ? 0 : 1); final int start = min(shardIndex * shardSize, allPlugins.length); final int end = min(start + shardSize, allPlugins.length); for (final Directory plugin in allPlugins.sublist(start, end)) { yield plugin; } } /// Returns the root Dart package folders of the plugins involved in this /// command execution, assuming there is only one shard. /// /// Plugin packages can exist in the following places relative to the packages /// directory: /// /// 1. As a Dart package in a directory which is a direct child of the /// packages directory. This is a plugin where all of the implementations /// exist in a single Dart package. /// 2. Several plugin packages may live in a directory which is a direct /// child of the packages directory. This directory groups several Dart /// packages which implement a single plugin. This directory contains a /// "client library" package, which declares the API for the plugin, as /// well as one or more platform-specific implementations. /// 3./4. Either of the above, but in a third_party/packages/ directory that /// is a sibling of the packages directory. This is used for a small number /// of packages in the flutter/packages repository. Stream _getAllPlugins() async* { Set plugins = Set.from(getStringListArg(_pluginsArg)); final Set excludedPlugins = Set.from(getStringListArg(_excludeArg)); final bool runOnChangedPackages = getBoolArg(_runOnChangedPackagesArg); if (plugins.isEmpty && runOnChangedPackages) { plugins = await _getChangedPackages(); } final Directory thirdPartyPackagesDirectory = packagesDir.parent .childDirectory('third_party') .childDirectory('packages'); for (final Directory dir in [ packagesDir, if (thirdPartyPackagesDirectory.existsSync()) thirdPartyPackagesDirectory, ]) { await for (final FileSystemEntity entity in dir.list(followLinks: false)) { // A top-level Dart package is a plugin package. if (_isDartPackage(entity)) { if (!excludedPlugins.contains(entity.basename) && (plugins.isEmpty || plugins.contains(p.basename(entity.path)))) { yield entity as Directory; } } else if (entity is Directory) { // Look for Dart packages under this top-level directory. await for (final FileSystemEntity subdir in entity.list(followLinks: false)) { if (_isDartPackage(subdir)) { // If --plugin=my_plugin is passed, then match all federated // plugins under 'my_plugin'. Also match if the exact plugin is // passed. final String relativePath = p.relative(subdir.path, from: dir.path); final String packageName = p.basename(subdir.path); final String basenamePath = p.basename(entity.path); if (!excludedPlugins.contains(basenamePath) && !excludedPlugins.contains(packageName) && !excludedPlugins.contains(relativePath) && (plugins.isEmpty || plugins.contains(relativePath) || plugins.contains(basenamePath))) { yield subdir as Directory; } } } } } } } /// Returns the example Dart package folders of the plugins involved in this /// command execution. Stream getExamples() => getPlugins().expand(getExamplesForPlugin); /// Returns all Dart package folders (typically, plugin + example) of the /// plugins involved in this command execution. Stream getPackages() async* { await for (final Directory plugin in getPlugins()) { yield plugin; yield* plugin .list(recursive: true, followLinks: false) .where(_isDartPackage) .cast(); } } /// Returns the files contained, recursively, within the plugins /// involved in this command execution. Stream getFiles() { return getPlugins().asyncExpand((Directory folder) => folder .list(recursive: true, followLinks: false) .where((FileSystemEntity entity) => entity is File) .cast()); } /// Returns whether the specified entity is a directory containing a /// `pubspec.yaml` file. bool _isDartPackage(FileSystemEntity entity) { return entity is Directory && fileSystem.file(p.join(entity.path, 'pubspec.yaml')).existsSync(); } /// Returns the example Dart packages contained in the specified plugin, or /// an empty List, if the plugin has no examples. Iterable getExamplesForPlugin(Directory plugin) { final Directory exampleFolder = fileSystem.directory(p.join(plugin.path, 'example')); if (!exampleFolder.existsSync()) { return []; } if (isFlutterPackage(exampleFolder, fileSystem)) { return [exampleFolder]; } // Only look at the subdirectories of the example directory if the example // directory itself is not a Dart package, and only look one level below the // example directory for other dart packages. return exampleFolder .listSync() .where( (FileSystemEntity entity) => isFlutterPackage(entity, fileSystem)) .cast(); } /// Retrieve an instance of [GitVersionFinder] based on `_kBaseSha` and [gitDir]. /// /// Throws tool exit if [gitDir] nor root directory is a git directory. Future retrieveVersionFinder() async { final String rootDir = packagesDir.parent.absolute.path; final String baseSha = getStringArg(_kBaseSha); GitDir? baseGitDir = gitDir; if (baseGitDir == null) { if (!await GitDir.isGitDir(rootDir)) { printErrorAndExit( errorMessage: '$rootDir is not a valid Git repository.', exitCode: 2); } baseGitDir = await GitDir.fromExisting(rootDir); } final GitVersionFinder gitVersionFinder = GitVersionFinder(baseGitDir, baseSha); return gitVersionFinder; } Future> _getChangedPackages() async { final GitVersionFinder gitVersionFinder = await retrieveVersionFinder(); final List allChangedFiles = await gitVersionFinder.getChangedFiles(); final Set packages = {}; for (final String path in allChangedFiles) { final List pathComponents = path.split('/'); final int packagesIndex = pathComponents.indexWhere((String element) => element == 'packages'); if (packagesIndex != -1) { packages.add(pathComponents[packagesIndex + 1]); } } if (packages.isNotEmpty) { final String changedPackages = packages.join(','); print(changedPackages); } else { print('No changed packages.'); } return packages; } } /// A class used to run processes. /// /// We use this instead of directly running the process so it can be overridden /// in tests. class ProcessRunner { /// Creates a new process runner. const ProcessRunner(); /// Run the [executable] with [args] and stream output to stderr and stdout. /// /// The current working directory of [executable] can be overridden by /// passing [workingDir]. /// /// If [exitOnError] is set to `true`, then this will throw an error if /// the [executable] terminates with a non-zero exit code. /// /// Returns the exit code of the [executable]. Future runAndStream( String executable, List args, { Directory? workingDir, bool exitOnError = false, }) async { print( 'Running command: "$executable ${args.join(' ')}" in ${workingDir?.path ?? io.Directory.current.path}'); final io.Process process = await io.Process.start(executable, args, workingDirectory: workingDir?.path); await io.stdout.addStream(process.stdout); await io.stderr.addStream(process.stderr); if (exitOnError && await process.exitCode != 0) { final String error = _getErrorString(executable, args, workingDir: workingDir); print('$error See above for details.'); throw ToolExit(await process.exitCode); } return process.exitCode; } /// Run the [executable] with [args]. /// /// The current working directory of [executable] can be overridden by /// passing [workingDir]. /// /// If [exitOnError] is set to `true`, then this will throw an error if /// the [executable] terminates with a non-zero exit code. /// Defaults to `false`. /// /// If [logOnError] is set to `true`, it will print a formatted message about the error. /// Defaults to `false` /// /// Returns the [io.ProcessResult] of the [executable]. Future run(String executable, List args, {Directory? workingDir, bool exitOnError = false, bool logOnError = false, Encoding stdoutEncoding = io.systemEncoding, Encoding stderrEncoding = io.systemEncoding}) async { final io.ProcessResult result = await io.Process.run(executable, args, workingDirectory: workingDir?.path, stdoutEncoding: stdoutEncoding, stderrEncoding: stderrEncoding); if (result.exitCode != 0) { if (logOnError) { final String error = _getErrorString(executable, args, workingDir: workingDir); print('$error Stderr:\n${result.stdout}'); } if (exitOnError) { throw ToolExit(result.exitCode); } } return result; } /// Starts the [executable] with [args]. /// /// The current working directory of [executable] can be overridden by /// passing [workingDir]. /// /// Returns the started [io.Process]. Future start(String executable, List args, {Directory? workingDirectory}) async { final io.Process process = await io.Process.start(executable, args, workingDirectory: workingDirectory?.path); return process; } String _getErrorString(String executable, List args, {Directory? workingDir}) { final String workdir = workingDir == null ? '' : ' in ${workingDir.path}'; return 'ERROR: Unable to execute "$executable ${args.join(' ')}"$workdir.'; } } /// Finding version of [package] that is published on pub. class PubVersionFinder { /// Constructor. /// /// Note: you should manually close the [httpClient] when done using the finder. PubVersionFinder({this.pubHost = defaultPubHost, required this.httpClient}); /// The default pub host to use. static const String defaultPubHost = 'https://pub.dev'; /// The pub host url, defaults to `https://pub.dev`. final String pubHost; /// The http client. /// /// You should manually close this client when done using this finder. final http.Client httpClient; /// Get the package version on pub. Future getPackageVersion( {required String package}) async { assert(package.isNotEmpty); final Uri pubHostUri = Uri.parse(pubHost); final Uri url = pubHostUri.replace(path: '/packages/$package.json'); final http.Response response = await httpClient.get(url); if (response.statusCode == 404) { return PubVersionFinderResponse( versions: null, result: PubVersionFinderResult.noPackageFound, httpResponse: response); } else if (response.statusCode != 200) { return PubVersionFinderResponse( versions: null, result: PubVersionFinderResult.fail, httpResponse: response); } final List versions = (json.decode(response.body)['versions'] as List) .map((final dynamic versionString) => Version.parse(versionString as String)) .toList(); return PubVersionFinderResponse( versions: versions, result: PubVersionFinderResult.success, httpResponse: response); } } /// Represents a response for [PubVersionFinder]. class PubVersionFinderResponse { /// Constructor. PubVersionFinderResponse({this.versions, this.result, this.httpResponse}) { if (versions != null && versions!.isNotEmpty) { versions!.sort((Version a, Version b) { // TODO(cyanglaz): Think about how to handle pre-release version with [Version.prioritize]. // https://github.com/flutter/flutter/issues/82222 return b.compareTo(a); }); } } /// The versions found in [PubVersionFinder]. /// /// This is sorted by largest to smallest, so the first element in the list is the largest version. /// Might be `null` if the [result] is not [PubVersionFinderResult.success]. final List? versions; /// The result of the version finder. final PubVersionFinderResult? result; /// The response object of the http request. final http.Response? httpResponse; } /// An enum representing the result of [PubVersionFinder]. enum PubVersionFinderResult { /// The version finder successfully found a version. success, /// The version finder failed to find a valid version. /// /// This might due to http connection errors or user errors. fail, /// The version finder failed to locate the package. /// /// This indicates the package is new. noPackageFound, } /// Finding diffs based on `baseGitDir` and `baseSha`. class GitVersionFinder { /// Constructor GitVersionFinder(this.baseGitDir, this.baseSha); /// The top level directory of the git repo. /// /// That is where the .git/ folder exists. final GitDir baseGitDir; /// The base sha used to get diff. final String? baseSha; static bool _isPubspec(String file) { return file.trim().endsWith('pubspec.yaml'); } /// Get a list of all the pubspec.yaml file that is changed. Future> getChangedPubSpecs() async { return (await getChangedFiles()).where(_isPubspec).toList(); } /// Get a list of all the changed files. Future> getChangedFiles() async { final String baseSha = await _getBaseSha(); final io.ProcessResult changedFilesCommand = await baseGitDir .runCommand(['diff', '--name-only', baseSha, 'HEAD']); print('Determine diff with base sha: $baseSha'); final String changedFilesStdout = changedFilesCommand.stdout.toString(); if (changedFilesStdout.isEmpty) { return []; } final List changedFiles = changedFilesStdout.split('\n') ..removeWhere((String element) => element.isEmpty); return changedFiles.toList(); } /// Get the package version specified in the pubspec file in `pubspecPath` and /// at the revision of `gitRef` (defaulting to the base if not provided). Future getPackageVersion(String pubspecPath, {String? gitRef}) async { final String ref = gitRef ?? (await _getBaseSha()); io.ProcessResult gitShow; try { gitShow = await baseGitDir.runCommand(['show', '$ref:$pubspecPath']); } on io.ProcessException { return null; } final String fileContent = gitShow.stdout as String; final String? versionString = loadYaml(fileContent)['version'] as String?; return versionString == null ? null : Version.parse(versionString); } Future _getBaseSha() async { if (baseSha != null && baseSha!.isNotEmpty) { return baseSha!; } io.ProcessResult baseShaFromMergeBase = await baseGitDir.runCommand( ['merge-base', '--fork-point', 'FETCH_HEAD', 'HEAD'], throwOnError: false); if (baseShaFromMergeBase.stderr != null || baseShaFromMergeBase.stdout == null) { baseShaFromMergeBase = await baseGitDir .runCommand(['merge-base', 'FETCH_HEAD', 'HEAD']); } return (baseShaFromMergeBase.stdout as String).trim(); } }