packages/script/tool/lib/src/common.dart

// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io' as io;
import 'dart:math';

import 'package:args/command_runner.dart';
import 'package:colorize/colorize.dart';
import 'package:file/file.dart';
import 'package:git/git.dart';
import 'package:http/http.dart' as http;
import 'package:path/path.dart' as p;
import 'package:pub_semver/pub_semver.dart';
import 'package:yaml/yaml.dart';

/// The signature for a print handler for commands that allow overriding the
/// print destination.
typedef Print = void Function(Object? object);

/// Key for windows platform.
const String kWindows = 'windows';

/// Key for macos platform.
const String kMacos = 'macos';

/// Key for linux platform.
const String kLinux = 'linux';

/// Key for IPA (iOS) platform.
const String kIos = 'ios';

/// Key for APK (Android) platform.
const String kAndroid = 'android';

/// Key for Web platform.
const String kWeb = 'web';

/// Key for IPA.
const String kIpa = 'ipa';

/// Key for APK.
const String kApk = 'apk';

/// Key for enable experiment.
const String kEnableExperiment = 'enable-experiment';

/// Returns whether the given directory contains a Flutter package.
bool isFlutterPackage(FileSystemEntity entity, FileSystem fileSystem) {
  if (entity is! Directory) {
    return false;
  }

  try {
    final File pubspecFile =
        fileSystem.file(p.join(entity.path, 'pubspec.yaml'));
    final YamlMap pubspecYaml =
        loadYaml(pubspecFile.readAsStringSync()) as YamlMap;
    final YamlMap? dependencies = pubspecYaml['dependencies'] as YamlMap?;
    if (dependencies == null) {
      return false;
    }
    return dependencies.containsKey('flutter');
  } on FileSystemException {
    return false;
  } on YamlException {
    return false;
  }
}

/// Returns whether the given directory contains a Flutter [platform] plugin.
///
/// It checks this by looking for the following pattern in the pubspec:
///
///     flutter:
///       plugin:
///         platforms:
///           [platform]:
bool pluginSupportsPlatform(
    String platform, FileSystemEntity entity, FileSystem fileSystem) {
  assert(platform == kIos ||
      platform == kAndroid ||
      platform == kWeb ||
      platform == kMacos ||
      platform == kWindows ||
      platform == kLinux);
  if (entity is! Directory) {
    return false;
  }

  try {
    final File pubspecFile =
        fileSystem.file(p.join(entity.path, 'pubspec.yaml'));
    final YamlMap pubspecYaml =
        loadYaml(pubspecFile.readAsStringSync()) as YamlMap;
    final YamlMap? flutterSection = pubspecYaml['flutter'] as YamlMap?;
    if (flutterSection == null) {
      return false;
    }
    final YamlMap? pluginSection = flutterSection['plugin'] as YamlMap?;
    if (pluginSection == null) {
      return false;
    }
    final YamlMap? platforms = pluginSection['platforms'] as YamlMap?;
    if (platforms == null) {
      // Legacy plugin specs are assumed to support iOS and Android.
      if (!pluginSection.containsKey('platforms')) {
        return platform == kIos || platform == kAndroid;
      }
      return false;
    }
    return platforms.containsKey(platform);
  } on FileSystemException {
    return false;
  } on YamlException {
    return false;
  }
}

/// Returns whether the given directory contains a Flutter Android plugin.
bool isAndroidPlugin(FileSystemEntity entity, FileSystem fileSystem) {
  return pluginSupportsPlatform(kAndroid, entity, fileSystem);
}

/// Returns whether the given directory contains a Flutter iOS plugin.
bool isIosPlugin(FileSystemEntity entity, FileSystem fileSystem) {
  return pluginSupportsPlatform(kIos, entity, fileSystem);
}

/// Returns whether the given directory contains a Flutter web plugin.
bool isWebPlugin(FileSystemEntity entity, FileSystem fileSystem) {
  return pluginSupportsPlatform(kWeb, entity, fileSystem);
}

/// Returns whether the given directory contains a Flutter Windows plugin.
bool isWindowsPlugin(FileSystemEntity entity, FileSystem fileSystem) {
  return pluginSupportsPlatform(kWindows, entity, fileSystem);
}

/// Returns whether the given directory contains a Flutter macOS plugin.
bool isMacOsPlugin(FileSystemEntity entity, FileSystem fileSystem) {
  return pluginSupportsPlatform(kMacos, entity, fileSystem);
}

/// Returns whether the given directory contains a Flutter linux plugin.
bool isLinuxPlugin(FileSystemEntity entity, FileSystem fileSystem) {
  return pluginSupportsPlatform(kLinux, entity, fileSystem);
}

/// Throws a [ToolExit] with `exitCode` and log the `errorMessage` in red.
void printErrorAndExit({required String errorMessage, int exitCode = 1}) {
  final Colorize redError = Colorize(errorMessage)..red();
  print(redError);
  throw ToolExit(exitCode);
}

/// Error thrown when a command needs to exit with a non-zero exit code.
class ToolExit extends Error {
  /// Creates a tool exit with the given [exitCode].
  ToolExit(this.exitCode);

  /// The code that the process should exit with.
  final int exitCode;
}

/// Interface definition for all commands in this tool.
abstract class PluginCommand extends Command<void> {
  /// Creates a command to operate on [packagesDir] with the given environment.
  PluginCommand(
    this.packagesDir,
    this.fileSystem, {
    this.processRunner = const ProcessRunner(),
    this.gitDir,
  }) {
    argParser.addMultiOption(
      _pluginsArg,
      splitCommas: true,
      help:
          'Specifies which plugins the command should run on (before sharding).',
      valueHelp: 'plugin1,plugin2,...',
    );
    argParser.addOption(
      _shardIndexArg,
      help: 'Specifies the zero-based index of the shard to '
          'which the command applies.',
      valueHelp: 'i',
      defaultsTo: '0',
    );
    argParser.addOption(
      _shardCountArg,
      help: 'Specifies the number of shards into which plugins are divided.',
      valueHelp: 'n',
      defaultsTo: '1',
    );
    argParser.addMultiOption(
      _excludeArg,
      abbr: 'e',
      help: 'Exclude packages from this command.',
      defaultsTo: <String>[],
    );
    argParser.addFlag(_runOnChangedPackagesArg,
        help: 'Run the command on changed packages/plugins.\n'
            'If the $_pluginsArg is specified, this flag is ignored.\n'
            'If no plugins have changed, the command runs on all plugins.\n'
            'The packages excluded with $_excludeArg is also excluded even if changed.\n'
            'See $_kBaseSha if a custom base is needed to determine the diff.');
    argParser.addOption(_kBaseSha,
        help: 'The base sha used to determine git diff. \n'
            'This is useful when $_runOnChangedPackagesArg is specified.\n'
            'If not specified, merge-base is used as base sha.');
  }

  static const String _pluginsArg = 'plugins';
  static const String _shardIndexArg = 'shardIndex';
  static const String _shardCountArg = 'shardCount';
  static const String _excludeArg = 'exclude';
  static const String _runOnChangedPackagesArg = 'run-on-changed-packages';
  static const String _kBaseSha = 'base-sha';

  /// The directory containing the plugin packages.
  final Directory packagesDir;

  /// The file system.
  ///
  /// This can be overridden for testing.
  final FileSystem fileSystem;

  /// The process runner.
  ///
  /// This can be overridden for testing.
  final ProcessRunner processRunner;

  /// The git directory to use. By default it uses the parent directory.
  ///
  /// This can be mocked for testing.
  final GitDir? gitDir;

  int? _shardIndex;
  int? _shardCount;

  /// The shard of the overall command execution that this instance should run.
  int get shardIndex {
    if (_shardIndex == null) {
      _checkSharding();
    }
    return _shardIndex!;
  }

  /// The number of shards this command is divided into.
  int get shardCount {
    if (_shardCount == null) {
      _checkSharding();
    }
    return _shardCount!;
  }

  /// Convenience accessor for boolean arguments.
  bool getBoolArg(String key) {
    return (argResults![key] as bool?) ?? false;
  }

  /// Convenience accessor for String arguments.
  String getStringArg(String key) {
    return (argResults![key] as String?) ?? '';
  }

  /// Convenience accessor for List<String> arguments.
  List<String> getStringListArg(String key) {
    return (argResults![key] as List<String>?) ?? <String>[];
  }

  void _checkSharding() {
    final int? shardIndex = int.tryParse(getStringArg(_shardIndexArg));
    final int? shardCount = int.tryParse(getStringArg(_shardCountArg));
    if (shardIndex == null) {
      usageException('$_shardIndexArg must be an integer');
    }
    if (shardCount == null) {
      usageException('$_shardCountArg must be an integer');
    }
    if (shardCount < 1) {
      usageException('$_shardCountArg must be positive');
    }
    if (shardIndex < 0 || shardCount <= shardIndex) {
      usageException(
          '$_shardIndexArg must be in the half-open range [0..$shardCount[');
    }
    _shardIndex = shardIndex;
    _shardCount = shardCount;
  }

  /// Returns the root Dart package folders of the plugins involved in this
  /// command execution.
  Stream<Directory> getPlugins() async* {
    // To avoid assuming consistency of `Directory.list` across command
    // invocations, we collect and sort the plugin folders before sharding.
    // This is considered an implementation detail which is why the API still
    // uses streams.
    final List<Directory> allPlugins = await _getAllPlugins().toList();
    allPlugins.sort((Directory d1, Directory d2) => d1.path.compareTo(d2.path));
    // Sharding 10 elements into 3 shards should yield shard sizes 4, 4, 2.
    // Sharding  9 elements into 3 shards should yield shard sizes 3, 3, 3.
    // Sharding  2 elements into 3 shards should yield shard sizes 1, 1, 0.
    final int shardSize = allPlugins.length ~/ shardCount +
        (allPlugins.length % shardCount == 0 ? 0 : 1);
    final int start = min(shardIndex * shardSize, allPlugins.length);
    final int end = min(start + shardSize, allPlugins.length);

    for (final Directory plugin in allPlugins.sublist(start, end)) {
      yield plugin;
    }
  }

  /// Returns the root Dart package folders of the plugins involved in this
  /// command execution, assuming there is only one shard.
  ///
  /// Plugin packages can exist in the following places relative to the packages
  /// directory:
  ///
  /// 1. As a Dart package in a directory which is a direct child of the
  ///    packages directory. This is a plugin where all of the implementations
  ///    exist in a single Dart package.
  /// 2. Several plugin packages may live in a directory which is a direct
  ///    child of the packages directory. This directory groups several Dart
  ///    packages which implement a single plugin. This directory contains a
  ///    "client library" package, which declares the API for the plugin, as
  ///    well as one or more platform-specific implementations.
  /// 3./4. Either of the above, but in a third_party/packages/ directory that
  ///    is a sibling of the packages directory. This is used for a small number
  ///    of packages in the flutter/packages repository.
  Stream<Directory> _getAllPlugins() async* {
    Set<String> plugins = Set<String>.from(getStringListArg(_pluginsArg));
    final Set<String> excludedPlugins =
        Set<String>.from(getStringListArg(_excludeArg));
    final bool runOnChangedPackages = getBoolArg(_runOnChangedPackagesArg);
    if (plugins.isEmpty && runOnChangedPackages) {
      plugins = await _getChangedPackages();
    }

    final Directory thirdPartyPackagesDirectory = packagesDir.parent
        .childDirectory('third_party')
        .childDirectory('packages');

    for (final Directory dir in <Directory>[
      packagesDir,
      if (thirdPartyPackagesDirectory.existsSync()) thirdPartyPackagesDirectory,
    ]) {
      await for (final FileSystemEntity entity
          in dir.list(followLinks: false)) {
        // A top-level Dart package is a plugin package.
        if (_isDartPackage(entity)) {
          if (!excludedPlugins.contains(entity.basename) &&
              (plugins.isEmpty || plugins.contains(p.basename(entity.path)))) {
            yield entity as Directory;
          }
        } else if (entity is Directory) {
          // Look for Dart packages under this top-level directory.
          await for (final FileSystemEntity subdir
              in entity.list(followLinks: false)) {
            if (_isDartPackage(subdir)) {
              // If --plugin=my_plugin is passed, then match all federated
              // plugins under 'my_plugin'. Also match if the exact plugin is
              // passed.
              final String relativePath =
                  p.relative(subdir.path, from: dir.path);
              final String packageName = p.basename(subdir.path);
              final String basenamePath = p.basename(entity.path);
              if (!excludedPlugins.contains(basenamePath) &&
                  !excludedPlugins.contains(packageName) &&
                  !excludedPlugins.contains(relativePath) &&
                  (plugins.isEmpty ||
                      plugins.contains(relativePath) ||
                      plugins.contains(basenamePath))) {
                yield subdir as Directory;
              }
            }
          }
        }
      }
    }
  }

  /// Returns the example Dart package folders of the plugins involved in this
  /// command execution.
  Stream<Directory> getExamples() =>
      getPlugins().expand<Directory>(getExamplesForPlugin);

  /// Returns all Dart package folders (typically, plugin + example) of the
  /// plugins involved in this command execution.
  Stream<Directory> getPackages() async* {
    await for (final Directory plugin in getPlugins()) {
      yield plugin;
      yield* plugin
          .list(recursive: true, followLinks: false)
          .where(_isDartPackage)
          .cast<Directory>();
    }
  }

  /// Returns the files contained, recursively, within the plugins
  /// involved in this command execution.
  Stream<File> getFiles() {
    return getPlugins().asyncExpand<File>((Directory folder) => folder
        .list(recursive: true, followLinks: false)
        .where((FileSystemEntity entity) => entity is File)
        .cast<File>());
  }

  /// Returns whether the specified entity is a directory containing a
  /// `pubspec.yaml` file.
  bool _isDartPackage(FileSystemEntity entity) {
    return entity is Directory &&
        fileSystem.file(p.join(entity.path, 'pubspec.yaml')).existsSync();
  }

  /// Returns the example Dart packages contained in the specified plugin, or
  /// an empty List, if the plugin has no examples.
  Iterable<Directory> getExamplesForPlugin(Directory plugin) {
    final Directory exampleFolder =
        fileSystem.directory(p.join(plugin.path, 'example'));
    if (!exampleFolder.existsSync()) {
      return <Directory>[];
    }
    if (isFlutterPackage(exampleFolder, fileSystem)) {
      return <Directory>[exampleFolder];
    }
    // Only look at the subdirectories of the example directory if the example
    // directory itself is not a Dart package, and only look one level below the
    // example directory for other dart packages.
    return exampleFolder
        .listSync()
        .where(
            (FileSystemEntity entity) => isFlutterPackage(entity, fileSystem))
        .cast<Directory>();
  }

  /// Retrieve an instance of [GitVersionFinder] based on `_kBaseSha` and [gitDir].
  ///
  /// Throws tool exit if [gitDir] nor root directory is a git directory.
  Future<GitVersionFinder> retrieveVersionFinder() async {
    final String rootDir = packagesDir.parent.absolute.path;
    final String baseSha = getStringArg(_kBaseSha);

    GitDir? baseGitDir = gitDir;
    if (baseGitDir == null) {
      if (!await GitDir.isGitDir(rootDir)) {
        printErrorAndExit(
            errorMessage: '$rootDir is not a valid Git repository.',
            exitCode: 2);
      }
      baseGitDir = await GitDir.fromExisting(rootDir);
    }

    final GitVersionFinder gitVersionFinder =
        GitVersionFinder(baseGitDir, baseSha);
    return gitVersionFinder;
  }

  Future<Set<String>> _getChangedPackages() async {
    final GitVersionFinder gitVersionFinder = await retrieveVersionFinder();

    final List<String> allChangedFiles =
        await gitVersionFinder.getChangedFiles();
    final Set<String> packages = <String>{};
    for (final String path in allChangedFiles) {
      final List<String> pathComponents = path.split('/');
      final int packagesIndex =
          pathComponents.indexWhere((String element) => element == 'packages');
      if (packagesIndex != -1) {
        packages.add(pathComponents[packagesIndex + 1]);
      }
    }
    if (packages.isNotEmpty) {
      final String changedPackages = packages.join(',');
      print(changedPackages);
    } else {
      print('No changed packages.');
    }
    return packages;
  }
}

/// A class used to run processes.
///
/// We use this instead of directly running the process so it can be overridden
/// in tests.
class ProcessRunner {
  /// Creates a new process runner.
  const ProcessRunner();

  /// Run the [executable] with [args] and stream output to stderr and stdout.
  ///
  /// The current working directory of [executable] can be overridden by
  /// passing [workingDir].
  ///
  /// If [exitOnError] is set to `true`, then this will throw an error if
  /// the [executable] terminates with a non-zero exit code.
  ///
  /// Returns the exit code of the [executable].
  Future<int> runAndStream(
    String executable,
    List<String> args, {
    Directory? workingDir,
    bool exitOnError = false,
  }) async {
    print(
        'Running command: "$executable ${args.join(' ')}" in ${workingDir?.path ?? io.Directory.current.path}');
    final io.Process process = await io.Process.start(executable, args,
        workingDirectory: workingDir?.path);
    await io.stdout.addStream(process.stdout);
    await io.stderr.addStream(process.stderr);
    if (exitOnError && await process.exitCode != 0) {
      final String error =
          _getErrorString(executable, args, workingDir: workingDir);
      print('$error See above for details.');
      throw ToolExit(await process.exitCode);
    }
    return process.exitCode;
  }

  /// Run the [executable] with [args].
  ///
  /// The current working directory of [executable] can be overridden by
  /// passing [workingDir].
  ///
  /// If [exitOnError] is set to `true`, then this will throw an error if
  /// the [executable] terminates with a non-zero exit code.
  /// Defaults to `false`.
  ///
  /// If [logOnError] is set to `true`, it will print a formatted message about the error.
  /// Defaults to `false`
  ///
  /// Returns the [io.ProcessResult] of the [executable].
  Future<io.ProcessResult> run(String executable, List<String> args,
      {Directory? workingDir,
      bool exitOnError = false,
      bool logOnError = false,
      Encoding stdoutEncoding = io.systemEncoding,
      Encoding stderrEncoding = io.systemEncoding}) async {
    final io.ProcessResult result = await io.Process.run(executable, args,
        workingDirectory: workingDir?.path,
        stdoutEncoding: stdoutEncoding,
        stderrEncoding: stderrEncoding);
    if (result.exitCode != 0) {
      if (logOnError) {
        final String error =
            _getErrorString(executable, args, workingDir: workingDir);
        print('$error Stderr:\n${result.stdout}');
      }
      if (exitOnError) {
        throw ToolExit(result.exitCode);
      }
    }
    return result;
  }

  /// Starts the [executable] with [args].
  ///
  /// The current working directory of [executable] can be overridden by
  /// passing [workingDir].
  ///
  /// Returns the started [io.Process].
  Future<io.Process?> start(String executable, List<String> args,
      {Directory? workingDirectory}) async {
    final io.Process process = await io.Process.start(executable, args,
        workingDirectory: workingDirectory?.path);
    return process;
  }

  String _getErrorString(String executable, List<String> args,
      {Directory? workingDir}) {
    final String workdir = workingDir == null ? '' : ' in ${workingDir.path}';
    return 'ERROR: Unable to execute "$executable ${args.join(' ')}"$workdir.';
  }
}

/// Finding version of [package] that is published on pub.
class PubVersionFinder {
  /// Constructor.
  ///
  /// Note: you should manually close the [httpClient] when done using the finder.
  PubVersionFinder({this.pubHost = defaultPubHost, required this.httpClient});

  /// The default pub host to use.
  static const String defaultPubHost = 'https://pub.dev';

  /// The pub host url, defaults to `https://pub.dev`.
  final String pubHost;

  /// The http client.
  ///
  /// You should manually close this client when done using this finder.
  final http.Client httpClient;

  /// Get the package version on pub.
  Future<PubVersionFinderResponse> getPackageVersion(
      {required String package}) async {
    assert(package.isNotEmpty);
    final Uri pubHostUri = Uri.parse(pubHost);
    final Uri url = pubHostUri.replace(path: '/packages/$package.json');
    final http.Response response = await httpClient.get(url);

    if (response.statusCode == 404) {
      return PubVersionFinderResponse(
          versions: null,
          result: PubVersionFinderResult.noPackageFound,
          httpResponse: response);
    } else if (response.statusCode != 200) {
      return PubVersionFinderResponse(
          versions: null,
          result: PubVersionFinderResult.fail,
          httpResponse: response);
    }
    final List<Version> versions =
        (json.decode(response.body)['versions'] as List<dynamic>)
            .map<Version>((final dynamic versionString) =>
                Version.parse(versionString as String))
            .toList();

    return PubVersionFinderResponse(
        versions: versions,
        result: PubVersionFinderResult.success,
        httpResponse: response);
  }
}

/// Represents a response for [PubVersionFinder].
class PubVersionFinderResponse {
  /// Constructor.
  PubVersionFinderResponse({this.versions, this.result, this.httpResponse}) {
    if (versions != null && versions!.isNotEmpty) {
      versions!.sort((Version a, Version b) {
        // TODO(cyanglaz): Think about how to handle pre-release version with [Version.prioritize].
        // https://github.com/flutter/flutter/issues/82222
        return b.compareTo(a);
      });
    }
  }

  /// The versions found in [PubVersionFinder].
  ///
  /// This is sorted by largest to smallest, so the first element in the list is the largest version.
  /// Might be `null` if the [result] is not [PubVersionFinderResult.success].
  final List<Version>? versions;

  /// The result of the version finder.
  final PubVersionFinderResult? result;

  /// The response object of the http request.
  final http.Response? httpResponse;
}

/// An enum representing the result of [PubVersionFinder].
enum PubVersionFinderResult {
  /// The version finder successfully found a version.
  success,

  /// The version finder failed to find a valid version.
  ///
  /// This might due to http connection errors or user errors.
  fail,

  /// The version finder failed to locate the package.
  ///
  /// This indicates the package is new.
  noPackageFound,
}

/// Finding diffs based on `baseGitDir` and `baseSha`.
class GitVersionFinder {
  /// Constructor
  GitVersionFinder(this.baseGitDir, this.baseSha);

  /// The top level directory of the git repo.
  ///
  /// That is where the .git/ folder exists.
  final GitDir baseGitDir;

  /// The base sha used to get diff.
  final String? baseSha;

  static bool _isPubspec(String file) {
    return file.trim().endsWith('pubspec.yaml');
  }

  /// Get a list of all the pubspec.yaml file that is changed.
  Future<List<String>> getChangedPubSpecs() async {
    return (await getChangedFiles()).where(_isPubspec).toList();
  }

  /// Get a list of all the changed files.
  Future<List<String>> getChangedFiles() async {
    final String baseSha = await _getBaseSha();
    final io.ProcessResult changedFilesCommand = await baseGitDir
        .runCommand(<String>['diff', '--name-only', baseSha, 'HEAD']);
    print('Determine diff with base sha: $baseSha');
    final String changedFilesStdout = changedFilesCommand.stdout.toString();
    if (changedFilesStdout.isEmpty) {
      return <String>[];
    }
    final List<String> changedFiles = changedFilesStdout.split('\n')
      ..removeWhere((String element) => element.isEmpty);
    return changedFiles.toList();
  }

  /// Get the package version specified in the pubspec file in `pubspecPath` and
  /// at the revision of `gitRef` (defaulting to the base if not provided).
  Future<Version?> getPackageVersion(String pubspecPath,
      {String? gitRef}) async {
    final String ref = gitRef ?? (await _getBaseSha());

    io.ProcessResult gitShow;
    try {
      gitShow =
          await baseGitDir.runCommand(<String>['show', '$ref:$pubspecPath']);
    } on io.ProcessException {
      return null;
    }
    final String fileContent = gitShow.stdout as String;
    final String? versionString = loadYaml(fileContent)['version'] as String?;
    return versionString == null ? null : Version.parse(versionString);
  }

  Future<String> _getBaseSha() async {
    if (baseSha != null && baseSha!.isNotEmpty) {
      return baseSha!;
    }

    io.ProcessResult baseShaFromMergeBase = await baseGitDir.runCommand(
        <String>['merge-base', '--fork-point', 'FETCH_HEAD', 'HEAD'],
        throwOnError: false);
    if (baseShaFromMergeBase.stderr != null ||
        baseShaFromMergeBase.stdout == null) {
      baseShaFromMergeBase = await baseGitDir
          .runCommand(<String>['merge-base', 'FETCH_HEAD', 'HEAD']);
    }
    return (baseShaFromMergeBase.stdout as String).trim();
  }
}