mirror of
https://github.com/flutter/packages.git
synced 2025-08-15 02:48:37 +08:00
[flutter_plugin_tools] Reimplements the excerpt system inline in the tool, rather than relying on a separate package. (#4417)
* Allows excerpts to come from any package, not just examples. * Fixes a bug in the excerpting logic that was causing a stray `}` to appear in one example. * Removes the need for `build.excerpt.yaml` files. * Remove the dependency on build_runner for excerpts. * Reduces the time to generate the excerpts from about 10 minutes to about 5 seconds. * Almost certainly fixes https://github.com/flutter/flutter/issues/107180 (untested). The new logic is not quite backwards compatible; the `path-base` feature now specifies a real path to the actual source directories, rather than a path into the invisible generated `excerpts/` directory with its special structure. Also, a number of features from the previous package that were not actually used in this repository are no longer supported (such as having multiple section names per `#docregion` pragma).
This commit is contained in:
Ian Hickson
@ -2,18 +2,20 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:io' as io;
|
||||
|
||||
import 'package:file/file.dart';
|
||||
import 'package:meta/meta.dart';
|
||||
import 'package:yaml/yaml.dart';
|
||||
import 'package:yaml_edit/yaml_edit.dart';
|
||||
|
||||
import 'common/output_utils.dart';
|
||||
import 'common/package_looping_command.dart';
|
||||
import 'common/pub_utils.dart';
|
||||
import 'common/repository_package.dart';
|
||||
|
||||
class _UpdateResult {
|
||||
const _UpdateResult(this.changed, this.errors);
|
||||
final bool changed;
|
||||
final List<String> errors;
|
||||
}
|
||||
|
||||
enum _ExcerptParseMode { normal, pragma, injecting }
|
||||
|
||||
/// A command to update .md code excerpts from code files.
|
||||
class UpdateExcerptsCommand extends PackageLoopingCommand {
|
||||
/// Creates a excerpt updater command instance.
|
||||
@ -23,232 +25,295 @@ class UpdateExcerptsCommand extends PackageLoopingCommand {
|
||||
super.platform,
|
||||
super.gitDir,
|
||||
}) {
|
||||
argParser.addFlag(_failOnChangeFlag, hide: true);
|
||||
argParser.addFlag(_noCleanupFlag,
|
||||
help: 'Skips the step of cleaning up the excerpt extraction output. '
|
||||
'This can be useful when debugging extraction or checking paths to '
|
||||
'reference in snippets.');
|
||||
argParser.addFlag(
|
||||
_failOnChangeFlag,
|
||||
help: 'Fail if the command does anything. '
|
||||
'(Used in CI to ensure excerpts are up to date.)',
|
||||
);
|
||||
}
|
||||
|
||||
static const String _failOnChangeFlag = 'fail-on-change';
|
||||
static const String _noCleanupFlag = 'no-cleanup';
|
||||
|
||||
static const String _buildRunnerConfigName = 'excerpt';
|
||||
// The name of the build_runner configuration file that will be in an example
|
||||
// directory if the package is set up to use `code-excerpt`.
|
||||
static const String _buildRunnerConfigFile =
|
||||
'build.$_buildRunnerConfigName.yaml';
|
||||
|
||||
/// The relative directory path to put the extracted excerpt yaml files.
|
||||
@visibleForTesting
|
||||
static const String excerptOutputDir = 'excerpts';
|
||||
|
||||
// The filename to store the pre-modification copy of the pubspec.
|
||||
static const String _originalPubspecFilename =
|
||||
'pubspec.plugin_tools_original.yaml';
|
||||
|
||||
@override
|
||||
final String name = 'update-excerpts';
|
||||
|
||||
@override
|
||||
final String description = 'Updates code excerpts in .md files, based '
|
||||
'on code from code files, via code-excerpt';
|
||||
'on code from code files, via <?code-excerpt?> pragmas.';
|
||||
|
||||
@override
|
||||
Future<PackageResult> runForPackage(RepositoryPackage package) async {
|
||||
final Iterable<RepositoryPackage> configuredExamples = package
|
||||
.getExamples()
|
||||
.where((RepositoryPackage example) =>
|
||||
example.directory.childFile(_buildRunnerConfigFile).existsSync());
|
||||
|
||||
if (configuredExamples.isEmpty) {
|
||||
return PackageResult.skip(
|
||||
'No $_buildRunnerConfigFile found in example(s).');
|
||||
}
|
||||
|
||||
final Directory repoRoot =
|
||||
packagesDir.fileSystem.directory((await gitDir).path);
|
||||
|
||||
for (final RepositoryPackage example in configuredExamples) {
|
||||
_addSubmoduleDependencies(example, repoRoot: repoRoot);
|
||||
|
||||
try {
|
||||
// Ensure that dependencies are available.
|
||||
if (!await runPubGet(example, processRunner, platform)) {
|
||||
return PackageResult.fail(
|
||||
<String>['Unable to get script dependencies']);
|
||||
}
|
||||
|
||||
// Update the excerpts.
|
||||
if (!await _extractSnippets(example)) {
|
||||
return PackageResult.fail(<String>['Unable to extract excerpts']);
|
||||
}
|
||||
if (!await _injectSnippets(example, targetPackage: package)) {
|
||||
return PackageResult.fail(<String>['Unable to inject excerpts']);
|
||||
}
|
||||
if (!await _injectSnippets(example, targetPackage: example)) {
|
||||
return PackageResult.fail(
|
||||
<String>['Unable to inject example excerpts']);
|
||||
}
|
||||
} finally {
|
||||
// Clean up the pubspec changes and extracted excerpts directory.
|
||||
_undoPubspecChanges(example);
|
||||
final Directory excerptDirectory =
|
||||
example.directory.childDirectory(excerptOutputDir);
|
||||
if (excerptDirectory.existsSync()) {
|
||||
if (getBoolArg(_noCleanupFlag)) {
|
||||
final String relativeDir =
|
||||
getRelativePosixPath(excerptDirectory, from: package.directory);
|
||||
print(
|
||||
'\n\nSKIPPING CLEANUP: Extraction output is in $relativeDir/');
|
||||
} else {
|
||||
excerptDirectory.deleteSync(recursive: true);
|
||||
}
|
||||
}
|
||||
final List<File> changedFiles = <File>[];
|
||||
final List<String> errors = <String>[];
|
||||
final List<File> markdownFiles = package.directory
|
||||
.listSync(recursive: true)
|
||||
.where((FileSystemEntity entity) {
|
||||
return entity is File &&
|
||||
entity.basename != 'CHANGELOG.md' &&
|
||||
entity.basename.toLowerCase().endsWith('.md');
|
||||
})
|
||||
.cast<File>()
|
||||
.toList();
|
||||
for (final File file in markdownFiles) {
|
||||
final _UpdateResult result = _updateExcerptsIn(file);
|
||||
if (result.changed) {
|
||||
changedFiles.add(file);
|
||||
}
|
||||
if (result.errors.isNotEmpty) {
|
||||
errors.addAll(result.errors);
|
||||
}
|
||||
}
|
||||
|
||||
if (getBoolArg(_failOnChangeFlag)) {
|
||||
final String? stateError = await _validateRepositoryState(package);
|
||||
if (stateError != null) {
|
||||
printError('One or more .md files are out of sync with their source '
|
||||
'excerpts.\n\n'
|
||||
'If you edited code in a .md file directly, you should instead '
|
||||
'edit the example source files. If you edited source files, run '
|
||||
'the repository tooling\'s "$name" command on this package, and '
|
||||
'update your PR with the resulting changes.\n\n'
|
||||
'For more information, see '
|
||||
'https://github.com/flutter/flutter/wiki/Contributing-to-Plugins-and-Packages#readme-code');
|
||||
return PackageResult.fail(<String>[stateError]);
|
||||
}
|
||||
if (errors.isNotEmpty) {
|
||||
printError('Injecting excerpts failed:');
|
||||
printError(errors.join('\n'));
|
||||
return PackageResult.fail();
|
||||
}
|
||||
|
||||
if (getBoolArg(_failOnChangeFlag) && changedFiles.isNotEmpty) {
|
||||
printError(
|
||||
'The following files have out of date excerpts:\n'
|
||||
' ${changedFiles.map((File file) => file.path).join("\n ")}\n'
|
||||
'\n'
|
||||
'If you edited code in a .md file directly, you should instead edit the '
|
||||
'files that contain the sources of the excerpts.\n'
|
||||
'If you did edit those source files, run the repository tooling\'s "$name" '
|
||||
'command on this package, and update your PR with the resulting changes.\n'
|
||||
'\n'
|
||||
'For more information, see '
|
||||
'https://github.com/flutter/flutter/wiki/Contributing-to-Plugins-and-Packages#readme-code',
|
||||
);
|
||||
return PackageResult.fail();
|
||||
}
|
||||
|
||||
return PackageResult.success();
|
||||
}
|
||||
|
||||
/// Runs the extraction step to create the excerpt files for the given
|
||||
/// example, returning true on success.
|
||||
Future<bool> _extractSnippets(RepositoryPackage example) async {
|
||||
final int exitCode = await processRunner.runAndStream(
|
||||
'dart',
|
||||
<String>[
|
||||
'run',
|
||||
'build_runner',
|
||||
'build',
|
||||
'--config',
|
||||
_buildRunnerConfigName,
|
||||
'--output',
|
||||
excerptOutputDir,
|
||||
'--delete-conflicting-outputs',
|
||||
],
|
||||
workingDir: example.directory);
|
||||
return exitCode == 0;
|
||||
static const String _pragma = '<?code-excerpt';
|
||||
static final RegExp _basePattern =
|
||||
RegExp(r'^ *<\?code-excerpt path-base="([^"]+)"\?>$');
|
||||
static final RegExp _injectPattern = RegExp(
|
||||
r'^ *<\?code-excerpt "(?<path>[^ ]+) \((?<section>[^)]+)\)"(?: plaster="(?<plaster>[^"]*)")?\?>$',
|
||||
);
|
||||
|
||||
_UpdateResult _updateExcerptsIn(File file) {
|
||||
bool detectedChange = false;
|
||||
final List<String> errors = <String>[];
|
||||
Directory pathBase = file.parent;
|
||||
final StringBuffer output = StringBuffer();
|
||||
final StringBuffer existingBlock = StringBuffer();
|
||||
String? language;
|
||||
String? excerpt;
|
||||
_ExcerptParseMode mode = _ExcerptParseMode.normal;
|
||||
int lineNumber = 0;
|
||||
for (final String line in file.readAsLinesSync()) {
|
||||
lineNumber += 1;
|
||||
switch (mode) {
|
||||
case _ExcerptParseMode.normal:
|
||||
if (line.contains(_pragma)) {
|
||||
RegExpMatch? match = _basePattern.firstMatch(line);
|
||||
if (match != null) {
|
||||
pathBase =
|
||||
file.parent.childDirectory(path.normalize(match.group(1)!));
|
||||
} else {
|
||||
match = _injectPattern.firstMatch(line);
|
||||
if (match != null) {
|
||||
final String excerptPath =
|
||||
path.normalize(match.namedGroup('path')!);
|
||||
final File excerptSourceFile = pathBase.childFile(excerptPath);
|
||||
final String extension = path.extension(excerptSourceFile.path);
|
||||
switch (extension) {
|
||||
case '':
|
||||
language = 'txt';
|
||||
break;
|
||||
case '.kt':
|
||||
language = 'kotlin';
|
||||
break;
|
||||
case '.cc':
|
||||
case '.cpp':
|
||||
language = 'c++';
|
||||
break;
|
||||
default:
|
||||
language = extension.substring(1);
|
||||
break;
|
||||
}
|
||||
final String section = match.namedGroup('section')!;
|
||||
final String plaster = match.namedGroup('plaster') ?? '···';
|
||||
if (!excerptSourceFile.existsSync()) {
|
||||
errors.add(
|
||||
'${file.path}:$lineNumber: specified file "$excerptPath" (resolved to "${excerptSourceFile.path}") does not exist');
|
||||
} else {
|
||||
excerpt = _extractExcerpt(
|
||||
excerptSourceFile, section, plaster, language, errors);
|
||||
}
|
||||
mode = _ExcerptParseMode.pragma;
|
||||
} else {
|
||||
errors.add(
|
||||
'${file.path}:$lineNumber: $_pragma?> pragma does not match expected syntax or is not alone on the line');
|
||||
}
|
||||
}
|
||||
}
|
||||
output.writeln(line);
|
||||
break;
|
||||
case _ExcerptParseMode.pragma:
|
||||
if (!line.startsWith('```')) {
|
||||
errors.add(
|
||||
'${file.path}:$lineNumber: expected code block but did not find one');
|
||||
mode = _ExcerptParseMode.normal;
|
||||
} else {
|
||||
if (line.startsWith('``` ')) {
|
||||
errors.add(
|
||||
'${file.path}:$lineNumber: code block was followed by a space character instead of the language (expected "$language")');
|
||||
mode = _ExcerptParseMode.injecting;
|
||||
} else if (line != '```$language' && line != '```rfwtxt' && line != '```json') {
|
||||
// We special-case rfwtxt and json because the rfw package extracts such sections from Dart files.
|
||||
// If we get more special cases we should think about a more general solution.
|
||||
errors.add(
|
||||
'${file.path}:$lineNumber: code block has wrong language');
|
||||
}
|
||||
mode = _ExcerptParseMode.injecting;
|
||||
}
|
||||
output.writeln(line);
|
||||
break;
|
||||
case _ExcerptParseMode.injecting:
|
||||
if (line == '```') {
|
||||
if (existingBlock.toString() != excerpt) {
|
||||
detectedChange = true;
|
||||
}
|
||||
output.write(excerpt);
|
||||
output.writeln(line);
|
||||
mode = _ExcerptParseMode.normal;
|
||||
language = null;
|
||||
excerpt = null;
|
||||
existingBlock.clear();
|
||||
} else {
|
||||
existingBlock.writeln(line);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (detectedChange) {
|
||||
if (errors.isNotEmpty) {
|
||||
errors.add('${file.path}: skipped updating file due to errors');
|
||||
} else {
|
||||
try {
|
||||
file.writeAsStringSync(output.toString());
|
||||
} catch (e) {
|
||||
errors.add(
|
||||
'${file.path}: failed to update file (${e.runtimeType}: $e)');
|
||||
}
|
||||
}
|
||||
}
|
||||
return _UpdateResult(detectedChange, errors);
|
||||
}
|
||||
|
||||
/// Runs the injection step to update [targetPackage]'s top-level .md files
|
||||
/// with the latest excerpts from [example], returning true on success.
|
||||
Future<bool> _injectSnippets(
|
||||
RepositoryPackage example, {
|
||||
required RepositoryPackage targetPackage,
|
||||
}) async {
|
||||
final List<String> relativeMdPaths = targetPackage.directory
|
||||
.listSync()
|
||||
.whereType<File>()
|
||||
.where((File f) =>
|
||||
f.basename.toLowerCase().endsWith('.md') &&
|
||||
// Exclude CHANGELOG since it should never have excerpts.
|
||||
f.basename != 'CHANGELOG.md')
|
||||
.map((File f) => getRelativePosixPath(f, from: example.directory))
|
||||
.toList();
|
||||
if (relativeMdPaths.isEmpty) {
|
||||
return true;
|
||||
String _extractExcerpt(File excerptSourceFile, String section,
|
||||
String plasterInside, String language, List<String> errors) {
|
||||
final List<String> buffer = <String>[];
|
||||
bool extracting = false;
|
||||
int lineNumber = 0;
|
||||
int maxLength = 0;
|
||||
bool found = false;
|
||||
String prefix = '';
|
||||
String suffix = '';
|
||||
String padding = '';
|
||||
switch (language) {
|
||||
case 'cc':
|
||||
case 'c++':
|
||||
case 'dart':
|
||||
case 'js':
|
||||
case 'kotlin':
|
||||
case 'rfwtxt':
|
||||
case 'swift':
|
||||
prefix = '// ';
|
||||
break;
|
||||
case 'css':
|
||||
prefix = '/* ';
|
||||
suffix = ' */';
|
||||
break;
|
||||
case 'html':
|
||||
case 'xml':
|
||||
prefix = '<!--';
|
||||
suffix = '-->';
|
||||
padding = ' ';
|
||||
break;
|
||||
case 'yaml':
|
||||
prefix = '# ';
|
||||
break;
|
||||
}
|
||||
final int exitCode = await processRunner.runAndStream(
|
||||
'dart',
|
||||
<String>[
|
||||
'run',
|
||||
'code_excerpt_updater',
|
||||
'--write-in-place',
|
||||
'--yaml',
|
||||
'--no-escape-ng-interpolation',
|
||||
...relativeMdPaths,
|
||||
],
|
||||
workingDir: example.directory);
|
||||
return exitCode == 0;
|
||||
}
|
||||
|
||||
/// Adds `code_excerpter` and `code_excerpt_updater` to [package]'s
|
||||
/// `dev_dependencies` using path-based references to the submodule copies.
|
||||
///
|
||||
/// This is done on the fly rather than being checked in so that:
|
||||
/// - Just building examples don't require everyone to check out submodules.
|
||||
/// - Examples can be analyzed/built even on versions of Flutter that these
|
||||
/// submodules do not support.
|
||||
void _addSubmoduleDependencies(RepositoryPackage package,
|
||||
{required Directory repoRoot}) {
|
||||
final String pubspecContents = package.pubspecFile.readAsStringSync();
|
||||
// Save aside a copy of the current pubspec state. This allows restoration
|
||||
// to the previous state regardless of its git status at the time the script
|
||||
// ran.
|
||||
package.directory
|
||||
.childFile(_originalPubspecFilename)
|
||||
.writeAsStringSync(pubspecContents);
|
||||
|
||||
// Update the actual pubspec.
|
||||
final YamlEditor editablePubspec = YamlEditor(pubspecContents);
|
||||
const String devDependenciesKey = 'dev_dependencies';
|
||||
final YamlNode root = editablePubspec.parseAt(<String>[]);
|
||||
// Ensure that there's a `dev_dependencies` entry to update.
|
||||
if ((root as YamlMap)[devDependenciesKey] == null) {
|
||||
editablePubspec.update(<String>['dev_dependencies'], YamlMap());
|
||||
final String startRegionMarker = '$prefix#docregion $section$suffix';
|
||||
final String endRegionMarker = '$prefix#enddocregion $section$suffix';
|
||||
final String plaster = '$prefix$padding$plasterInside$padding$suffix';
|
||||
int? indentation;
|
||||
for (final String excerptLine in excerptSourceFile.readAsLinesSync()) {
|
||||
final String trimmedLine = excerptLine.trimLeft();
|
||||
lineNumber += 1;
|
||||
if (extracting) {
|
||||
if (trimmedLine == endRegionMarker) {
|
||||
extracting = false;
|
||||
indentation = excerptLine.length - trimmedLine.length;
|
||||
} else {
|
||||
if (trimmedLine == startRegionMarker) {
|
||||
errors.add(
|
||||
'${excerptSourceFile.path}:$lineNumber: saw "$startRegionMarker" pragma while already in a "$section" doc region');
|
||||
}
|
||||
if (excerptLine.length > maxLength) {
|
||||
maxLength = excerptLine.length;
|
||||
}
|
||||
if (!excerptLine.contains('$prefix#docregion ') &&
|
||||
!excerptLine.contains('$prefix#enddocregion ')) {
|
||||
buffer.add(excerptLine);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (trimmedLine == startRegionMarker) {
|
||||
found = true;
|
||||
extracting = true;
|
||||
if (buffer.isNotEmpty && plasterInside != 'none') {
|
||||
assert(indentation != null);
|
||||
buffer.add('${" " * indentation!}$plaster');
|
||||
indentation = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
final Set<String> submoduleDependencies = <String>{
|
||||
'code_excerpter',
|
||||
'code_excerpt_updater',
|
||||
};
|
||||
final String relativeRootPath =
|
||||
getRelativePosixPath(repoRoot, from: package.directory);
|
||||
for (final String dependency in submoduleDependencies) {
|
||||
editablePubspec.update(<String>[
|
||||
devDependenciesKey,
|
||||
dependency
|
||||
], <String, String>{
|
||||
'path': '$relativeRootPath/site-shared/packages/$dependency'
|
||||
});
|
||||
if (extracting) {
|
||||
errors
|
||||
.add('${excerptSourceFile.path}: missing "$endRegionMarker" pragma');
|
||||
}
|
||||
package.pubspecFile.writeAsStringSync(editablePubspec.toString());
|
||||
}
|
||||
|
||||
/// Restores the version of the pubspec that was present before running
|
||||
/// [_addSubmoduleDependencies].
|
||||
void _undoPubspecChanges(RepositoryPackage package) {
|
||||
package.directory
|
||||
.childFile(_originalPubspecFilename)
|
||||
.renameSync(package.pubspecFile.path);
|
||||
}
|
||||
|
||||
/// Checks the git state, returning an error string if any .md files have
|
||||
/// changed.
|
||||
Future<String?> _validateRepositoryState(RepositoryPackage package) async {
|
||||
final io.ProcessResult checkFiles = await processRunner.run(
|
||||
'git',
|
||||
<String>['ls-files', '--modified'],
|
||||
workingDir: package.directory,
|
||||
logOnError: true,
|
||||
);
|
||||
if (checkFiles.exitCode != 0) {
|
||||
return 'Unable to determine local file state';
|
||||
if (!found) {
|
||||
errors.add(
|
||||
'${excerptSourceFile.path}: did not find a "$startRegionMarker" pragma');
|
||||
return '';
|
||||
}
|
||||
|
||||
final String stdout = checkFiles.stdout as String;
|
||||
final List<String> changedFiles = stdout.trim().split('\n');
|
||||
final Iterable<String> changedMDFiles =
|
||||
changedFiles.where((String filePath) => filePath.endsWith('.md'));
|
||||
if (changedMDFiles.isNotEmpty) {
|
||||
return 'Snippets are out of sync in the following files: '
|
||||
'${changedMDFiles.join(', ')}';
|
||||
if (buffer.isEmpty) {
|
||||
errors.add('${excerptSourceFile.path}: region "$section" is empty');
|
||||
return '';
|
||||
}
|
||||
|
||||
return null;
|
||||
int indent = maxLength;
|
||||
for (final String line in buffer) {
|
||||
if (indent == 0) {
|
||||
break;
|
||||
}
|
||||
if (line.isEmpty) {
|
||||
continue;
|
||||
}
|
||||
for (int index = 0; index < line.length; index += 1) {
|
||||
if (line[index] != ' ') {
|
||||
if (index < indent) {
|
||||
indent = index;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
final StringBuffer excerpt = StringBuffer();
|
||||
for (final String line in buffer) {
|
||||
if (line.isEmpty) {
|
||||
excerpt.writeln();
|
||||
} else {
|
||||
excerpt.writeln(line.substring(indent));
|
||||
}
|
||||
}
|
||||
return excerpt.toString();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user