diff --git a/.gitignore b/.gitignore index 851f68f5e2..7d3cdbcbd9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,8 @@ GeneratedPluginRegistrant.m GeneratedPluginRegistrant.java +packages/measure/result.json +packages/measure/resources +*instrumentscli*.trace +*.cipd + diff --git a/packages/measure/CHANGELOG.md b/packages/measure/CHANGELOG.md new file mode 100644 index 0000000000..802b87e002 --- /dev/null +++ b/packages/measure/CHANGELOG.md @@ -0,0 +1,4 @@ +## 0.1.0 + +* Initial release. + diff --git a/packages/measure/LICENSE b/packages/measure/LICENSE new file mode 100644 index 0000000000..4611350993 --- /dev/null +++ b/packages/measure/LICENSE @@ -0,0 +1,27 @@ +Copyright 2019 The Chromium Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/measure/README.md b/packages/measure/README.md new file mode 100644 index 0000000000..8033f42a00 --- /dev/null +++ b/packages/measure/README.md @@ -0,0 +1,37 @@ +Tools for measuring some performance metrics. + +Currently there's only one tool to measure iOS CPU/GPU usages for Flutter's CI +tests. + +# Install + +First install [depot_tools][1] (we used its `cipd`). + +Then install [dart](https://dart.dev/get-dart) and make sure that `pub` is on +your path. + +Finally run: +```shell +pub global activate measure +``` + +# Run +Connect an iPhone, run a Flutter app on it, and +```shell +measure ioscpugpu new +``` + +Sample output: +``` +gpu: 12.4%, cpu: 22.525% +``` + +For more information, try +```shell +measure help +measure help ioscpugpu +measure help ioscpugpu new +measure help ioscpugpu parse +``` + +[1]: https://commondatastorage.googleapis.com/chrome-infra-docs/flat/depot_tools/docs/html/depot_tools_tutorial.html#_setting_up diff --git a/packages/measure/bin/measure.dart b/packages/measure/bin/measure.dart new file mode 100644 index 0000000000..b00eae74cf --- /dev/null +++ b/packages/measure/bin/measure.dart @@ -0,0 +1,16 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/command_runner.dart'; + +import 'package:measure/commands/ioscpugpu.dart'; + +void main(List<String> args) { + final CommandRunner<void> runner = CommandRunner<void>( + 'measure', + 'Tools for measuring some performance metrics.', + ); + runner.addCommand(IosCpuGpu()); + runner.run(args); +} diff --git a/packages/measure/cipd.yaml b/packages/measure/cipd.yaml new file mode 100644 index 0000000000..31ef025eb3 --- /dev/null +++ b/packages/measure/cipd.yaml @@ -0,0 +1,5 @@ +package: flutter/packages/measure/resources +description: Binaries and other resources for measuring performance metrics. +install_mode: copy +data: + - dir: resources diff --git a/packages/measure/lib/commands/base.dart b/packages/measure/lib/commands/base.dart new file mode 100644 index 0000000000..504739659b --- /dev/null +++ b/packages/measure/lib/commands/base.dart @@ -0,0 +1,105 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:meta/meta.dart'; + +const String kOptionResourcesRoot = 'resources-root'; +const String kOptionTimeLimitMs = 'time-limit-ms'; +const String kOptionDevice = 'device'; +const String kOptionProessName = 'process-name'; +const String kOptionOutJson = 'out-json'; +const String kFlagVerbose = 'verbose'; + +const String kDefaultProccessName = 'Runner'; // Flutter app's default process + +abstract class BaseCommand extends Command<void> { + BaseCommand() { + argParser.addFlag(kFlagVerbose); + argParser.addOption( + kOptionOutJson, + abbr: 'o', + help: 'Specifies the json file for result output.', + defaultsTo: 'result.json', + ); + argParser.addOption( + kOptionResourcesRoot, + abbr: 'r', + help: 'Specifies the path to download resources', + defaultsTo: defaultResourcesRoot, + ); + } + + static String get defaultResourcesRoot => + '${Platform.environment['HOME']}/.measure'; + + static Future<void> doEnsureResources(String rootPath, + {bool isVerbose}) async { + final Directory root = await Directory(rootPath).create(recursive: true); + final Directory previous = Directory.current; + Directory.current = root; + final File ensureFile = File('ensure_file.txt'); + ensureFile.writeAsStringSync('flutter/packages/measure/resources latest'); + if (isVerbose) { + print('Downloading resources from CIPD...'); + } + final ProcessResult result = Process.runSync( + 'cipd', + <String>[ + 'ensure', + '-ensure-file', + 'ensure_file.txt', + '-root', + '.', + ], + ); + if (result.exitCode != 0) { + print('cipd ensure stdout:\n${result.stdout}\n'); + print('cipd ensure stderr:\n${result.stderr}\n'); + throw Exception('Failed to download the CIPD package.'); + } + if (isVerbose) { + print('Download completes.'); + } + Directory.current = previous; + } + + @protected + Future<void> ensureResources() async { + doEnsureResources(resourcesRoot, isVerbose: isVerbose); + } + + @protected + void checkRequiredOption(String option) { + if (argResults[option] == null) { + throw Exception('Option $option is required.'); + } + } + + @protected + bool get isVerbose => argResults[kFlagVerbose]; + @protected + String get outJson => argResults[kOptionOutJson]; + @protected + String get resourcesRoot => argResults[kOptionResourcesRoot]; +} + +abstract class IosCpuGpuSubcommand extends BaseCommand { + IosCpuGpuSubcommand() { + argParser.addOption( + kOptionProessName, + abbr: 'p', + help: 'Specifies the process name used for filtering the instruments CPU ' + 'measurements.', + defaultsTo: kDefaultProccessName, + ); + } + + @protected + String get processName => argResults[kOptionProessName]; + @protected + String get traceUtilityPath => '$resourcesRoot/resources/TraceUtility'; +} diff --git a/packages/measure/lib/commands/ioscpugpu.dart b/packages/measure/lib/commands/ioscpugpu.dart new file mode 100644 index 0000000000..5e1c7ae079 --- /dev/null +++ b/packages/measure/lib/commands/ioscpugpu.dart @@ -0,0 +1,19 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/command_runner.dart'; +import 'package:measure/commands/ioscpugpu/new.dart'; +import 'package:measure/commands/ioscpugpu/parse.dart'; + +class IosCpuGpu extends Command<void> { + IosCpuGpu() { + addSubcommand(IosCpuGpuNew()); + addSubcommand(IosCpuGpuParse()); + } + + @override + String get name => 'ioscpugpu'; + @override + String get description => 'Measure the iOS CPU/GPU percentage.'; +} diff --git a/packages/measure/lib/commands/ioscpugpu/new.dart b/packages/measure/lib/commands/ioscpugpu/new.dart new file mode 100644 index 0000000000..73139f9cbd --- /dev/null +++ b/packages/measure/lib/commands/ioscpugpu/new.dart @@ -0,0 +1,111 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:measure/commands/base.dart'; +import 'package:measure/parser.dart'; + +class IosCpuGpuNew extends IosCpuGpuSubcommand { + IosCpuGpuNew() { + argParser.addOption( + kOptionTimeLimitMs, + abbr: 'l', + defaultsTo: '5000', + help: 'time limit (in ms) to run instruments for measuring', + ); + argParser.addOption( + kOptionDevice, + abbr: 'w', + help: 'device identifier recognizable by instruments ' + '(e.g., 00008020-000364CE0AF8003A)', + ); + } + + @override + String get name => 'new'; + @override + String get description => 'Take a new measurement on the iOS CPU/GPU ' + 'percentage (of Flutter Runner).'; + + String get _timeLimit => argResults[kOptionTimeLimitMs]; + + String get _templatePath => + '$resourcesRoot/resources/CpuGpuTemplate.tracetemplate'; + + @override + Future<void> run() async { + _checkDevice(); + await ensureResources(); + + print('Running instruments on iOS device $_device for ${_timeLimit}ms'); + + final List<String> args = <String>[ + '-l', + _timeLimit, + '-t', + _templatePath, + '-w', + _device, + ]; + if (isVerbose) { + print('instruments args: $args'); + } + final ProcessResult processResult = Process.runSync('instruments', args); + _parseTraceFilename(processResult.stdout.toString()); + + print('Parsing $_traceFilename'); + + final IosTraceParser parser = IosTraceParser(isVerbose, traceUtilityPath); + final CpuGpuResult result = parser.parseCpuGpu(_traceFilename, processName); + result.writeToJsonFile(outJson); + print('$result\nThe result has been written into $outJson'); + } + + String _traceFilename; + Future<void> _parseTraceFilename(String out) async { + const String kPrefix = 'Instruments Trace Complete: '; + final int prefixIndex = out.indexOf(kPrefix); + if (prefixIndex == -1) { + throw Exception('Failed to parse instruments output:\n$out'); + } + _traceFilename = out.substring(prefixIndex + kPrefix.length).trim(); + } + + String _device; + void _checkDevice() { + _device = argResults[kOptionDevice]; + if (_device != null) { + return; + } + final ProcessResult result = Process.runSync( + 'instruments', + <String>['-s', 'devices'], + ); + for (String line in result.stdout.toString().split('\n')) { + if (line.contains('iPhone') && !line.contains('Simulator')) { + _device = RegExp(r'\[(.+)\]').firstMatch(line).group(1); + break; + } + } + if (_device == null) { + print(''' + Option device (-w) is not provided, and failed to find an iPhone(not a + simulator) from `instruments -s devices`. + + stdout of `instruments -s device`: + =========================== + ${result.stdout} + =========================== + + stderr of `instruments -s device`: + =========================== + ${result.stderr} + =========================== + '''); + throw Exception('Failed to determine the device.'); + } + } +} diff --git a/packages/measure/lib/commands/ioscpugpu/parse.dart b/packages/measure/lib/commands/ioscpugpu/parse.dart new file mode 100644 index 0000000000..ed74a4e842 --- /dev/null +++ b/packages/measure/lib/commands/ioscpugpu/parse.dart @@ -0,0 +1,40 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:measure/commands/base.dart'; +import 'package:measure/parser.dart'; + +class IosCpuGpuParse extends IosCpuGpuSubcommand { + @override + String get name => 'parse'; + @override + String get description => + 'parse an existing instruments trace with CPU/GPU measurements.'; + + @override + String get usage { + final List<String> lines = super.usage.split('\n'); + lines[0] = 'Usage: measure ioscpugpu -u <trace-utility-path> ' + 'parse <trace-file-path>'; + return lines.join('\n'); + } + + @override + Future<void> run() async { + if (argResults.rest.length != 1) { + print(usage); + throw Exception('exactly one argument <trace-file-path> expected'); + } + final String path = argResults.rest[0]; + + await ensureResources(); + + final CpuGpuResult result = IosTraceParser(isVerbose, traceUtilityPath) + .parseCpuGpu(path, processName); + result.writeToJsonFile(outJson); + print('$result\nThe result has been written into $outJson'); + } +} diff --git a/packages/measure/lib/parser.dart b/packages/measure/lib/parser.dart new file mode 100644 index 0000000000..a13e7a74c9 --- /dev/null +++ b/packages/measure/lib/parser.dart @@ -0,0 +1,116 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +class CpuGpuResult { + CpuGpuResult(this.gpuPercentage, this.cpuPercentage); + + final double gpuPercentage; + final double cpuPercentage; + + @override + String toString() { + return 'gpu: $gpuPercentage%, cpu: $cpuPercentage%'; + } + + void writeToJsonFile(String filename) { + final String output = json.encode(<String, double>{ + 'gpu_percentage': gpuPercentage, + 'cpu_percentage': cpuPercentage + }); + File(filename).writeAsStringSync(output); + } +} + +class IosTraceParser { + IosTraceParser(this.isVerbose, this.traceUtilityPath); + + final bool isVerbose; + final String traceUtilityPath; + + List<String> _gpuMeasurements; + List<String> _cpuMeasurements; + + CpuGpuResult parseCpuGpu(String filename, String processName) { + final ProcessResult result = Process.runSync( + traceUtilityPath, + <String>[filename], + ); + if (result.exitCode != 0) { + print('TraceUtility stdout:\n${result.stdout.toString}\n\n'); + print('TraceUtility stderr:\n${result.stderr.toString}\n\n'); + throw Exception('TraceUtility failed with exit code ${result.exitCode}'); + } + final List<String> lines = result.stderr.toString().split('\n'); + + // toSet to remove duplicates + _gpuMeasurements = + lines.where((String s) => s.contains('GPU')).toSet().toList(); + _cpuMeasurements = + lines.where((String s) => s.contains(processName)).toSet().toList(); + _gpuMeasurements.sort(); + _cpuMeasurements.sort(); + + if (isVerbose) { + _gpuMeasurements.forEach(print); + _cpuMeasurements.forEach(print); + } + + return CpuGpuResult(_computeGpuPercent(), _computeCpuPercent()); + } + + static final RegExp _percentagePattern = RegExp(r'(\d+(\.\d*)?)%'); + double _parseSingleGpuMeasurement(String line) { + return double.parse(_percentagePattern.firstMatch(line).group(1)); + } + + double _computeGpuPercent() { + return _average(_gpuMeasurements.map(_parseSingleGpuMeasurement)); + } + + // The return is a list of 2: the 1st is the time key string, the 2nd is the + // double percentage + List<dynamic> _parseSingleCpuMeasurement(String line) { + final String timeKey = line.substring(0, line.indexOf(',')); + final RegExpMatch match = _percentagePattern.firstMatch(line); + return <dynamic>[ + timeKey, + match == null + ? 0 + : double.parse(_percentagePattern.firstMatch(line).group(1)) + ]; + } + + double _computeCpuPercent() { + final Iterable<List<dynamic>> results = + _cpuMeasurements.map(_parseSingleCpuMeasurement); + final Map<String, double> sums = <String, double>{}; + for (List<dynamic> pair in results) { + sums[pair[0]] = 0; + } + for (List<dynamic> pair in results) { + sums[pair[0]] += pair[1]; + } + + // This key always has 0% usage. Remove it. + assert(sums['00:00.000.000'] == 0); + sums.remove('00:00.000.000'); + + if (isVerbose) { + print('CPU maps: $sums'); + } + return _average(sums.values); + } + + double _average(Iterable<double> values) { + if (values == null || values.isEmpty) { + _gpuMeasurements.forEach(print); + _cpuMeasurements.forEach(print); + throw Exception('No valid measurements found.'); + } + return values.reduce((double a, double b) => a + b) / values.length; + } +} diff --git a/packages/measure/pubspec.yaml b/packages/measure/pubspec.yaml new file mode 100644 index 0000000000..e3d748b3cd --- /dev/null +++ b/packages/measure/pubspec.yaml @@ -0,0 +1,18 @@ +name: measure +description: Tools for measuring some performance metrics. +author: Flutter Team <flutter-dev@googlegroups.com> +homepage: https://github.com/flutter/packages/tree/master/packages/measure +version: 0.1.0 + +executables: + measure: measure + +dependencies: + args: ^1.5.2 + meta: ^1.1.7 + +dev_dependencies: + test: ^1.6.8 + +environment: + sdk: ">=2.2.2 <3.0.0" diff --git a/packages/measure/test/measure_test.dart b/packages/measure/test/measure_test.dart new file mode 100644 index 0000000000..0ff12fe86d --- /dev/null +++ b/packages/measure/test/measure_test.dart @@ -0,0 +1,88 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@TestOn('mac-os') + +import 'dart:io'; + +import 'package:test/test.dart'; + +import 'package:measure/commands/base.dart'; + +void main() { + const String measureRootPath = '.'; + final String resourcesRootPath = BaseCommand.defaultResourcesRoot; + BaseCommand.doEnsureResources(resourcesRootPath, isVerbose: true); + + test('help works', () { + final ProcessResult result = Process.runSync( + 'dart', + <String>['$measureRootPath/bin/measure.dart', 'help'], + ); + expect( + result.stdout.toString(), + contains( + 'Tools for measuring some performance metrics.', + )); + }); + + ProcessResult _testIosCpuGpu(List<String> extraArgs) { + return Process.runSync( + 'dart', + <String>[ + '$measureRootPath/bin/measure.dart', + 'ioscpugpu', + ...extraArgs, + '-r', + resourcesRootPath, + ], + ); + } + + ProcessResult _testParse(List<String> extraArgs) { + return _testIosCpuGpu(<String>[ + 'parse', + '$resourcesRootPath/resources/example_instrumentscli.trace/', + ...extraArgs, + ]); + } + + test('ioscpugpu parse works', () { + final ProcessResult result = _testParse(<String>[]); + expect( + result.stdout.toString(), + contains( + 'gpu: 12.6%, cpu: 18.15%', + )); + expect( + File('result.json').readAsStringSync(), + contains( + '{"gpu_percentage":12.6,"cpu_percentage":18.15}', + )); + }); + + test('ioscpugpu parse works with verbose', () { + final ProcessResult result = _testParse(<String>['--verbose']); + expect( + result.stdout.toString(), + contains( + '00:00.000.000 0 FPS 13.0% GPU', + )); + expect( + result.stdout.toString(), + contains( + '00:00.477.632, 1.55 s, Runner (2209), n/a, 2209, mobile, 23.7%', + )); + }); + + test('ioscpugpu new works', () { + final ProcessResult result = _testIosCpuGpu(<String>['new']); + expect( + result.stdout.toString(), + contains('The result has been written into result.json'), + reason: '\n\nioscpugpu new failed. Do you have a single connected iPhone ' + 'that has a Flutter app running?', + ); + }); +}