Add a new package: measure (#31)

For https://github.com/flutter/flutter/pull/39439 and https://github.com/flutter/flutter/issues/33899

.gitignore

@@ -14,3 +14,8 @@ GeneratedPluginRegistrant.m
 
 GeneratedPluginRegistrant.java
 
+packages/measure/result.json
+packages/measure/resources
+*instrumentscli*.trace
+*.cipd
+

packages/measure/CHANGELOG.md

@@ -0,0 +1,4 @@
+## 0.1.0
+
+* Initial release.
+

packages/measure/LICENSE

@@ -0,0 +1,27 @@
+Copyright 2019 The Chromium Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/measure/README.md

@@ -0,0 +1,37 @@
+Tools for measuring some performance metrics.
+
+Currently there's only one tool to measure iOS CPU/GPU usages for Flutter's CI
+tests.
+
+# Install
+
+First install [depot_tools][1] (we used its `cipd`).
+
+Then install [dart](https://dart.dev/get-dart) and make sure that `pub` is on
+your path.
+
+Finally run:
+```shell
+pub global activate measure
+```
+
+# Run
+Connect an iPhone, run a Flutter app on it, and
+```shell
+measure ioscpugpu new
+```
+
+Sample output:
+```
+gpu: 12.4%, cpu: 22.525%
+```
+
+For more information, try
+```shell
+measure help
+measure help ioscpugpu
+measure help ioscpugpu new
+measure help ioscpugpu parse
+```
+
+[1]: https://commondatastorage.googleapis.com/chrome-infra-docs/flat/depot_tools/docs/html/depot_tools_tutorial.html#_setting_up

packages/measure/bin/measure.dart

@@ -0,0 +1,16 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:args/command_runner.dart';
+
+import 'package:measure/commands/ioscpugpu.dart';
+
+void main(List<String> args) {
+  final CommandRunner<void> runner = CommandRunner<void>(
+    'measure',
+    'Tools for measuring some performance metrics.',
+  );
+  runner.addCommand(IosCpuGpu());
+  runner.run(args);
+}

packages/measure/cipd.yaml

@@ -0,0 +1,5 @@
+package: flutter/packages/measure/resources
+description: Binaries and other resources for measuring performance metrics.
+install_mode: copy
+data:
+  - dir: resources

packages/measure/lib/commands/base.dart

@@ -0,0 +1,105 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/command_runner.dart';
+import 'package:meta/meta.dart';
+
+const String kOptionResourcesRoot = 'resources-root';
+const String kOptionTimeLimitMs = 'time-limit-ms';
+const String kOptionDevice = 'device';
+const String kOptionProessName = 'process-name';
+const String kOptionOutJson = 'out-json';
+const String kFlagVerbose = 'verbose';
+
+const String kDefaultProccessName = 'Runner'; // Flutter app's default process
+
+abstract class BaseCommand extends Command<void> {
+  BaseCommand() {
+    argParser.addFlag(kFlagVerbose);
+    argParser.addOption(
+      kOptionOutJson,
+      abbr: 'o',
+      help: 'Specifies the json file for result output.',
+      defaultsTo: 'result.json',
+    );
+    argParser.addOption(
+      kOptionResourcesRoot,
+      abbr: 'r',
+      help: 'Specifies the path to download resources',
+      defaultsTo: defaultResourcesRoot,
+    );
+  }
+
+  static String get defaultResourcesRoot =>
+      '${Platform.environment['HOME']}/.measure';
+
+  static Future<void> doEnsureResources(String rootPath,
+      {bool isVerbose}) async {
+    final Directory root = await Directory(rootPath).create(recursive: true);
+    final Directory previous = Directory.current;
+    Directory.current = root;
+    final File ensureFile = File('ensure_file.txt');
+    ensureFile.writeAsStringSync('flutter/packages/measure/resources latest');
+    if (isVerbose) {
+      print('Downloading resources from CIPD...');
+    }
+    final ProcessResult result = Process.runSync(
+      'cipd',
+      <String>[
+        'ensure',
+        '-ensure-file',
+        'ensure_file.txt',
+        '-root',
+        '.',
+      ],
+    );
+    if (result.exitCode != 0) {
+      print('cipd ensure stdout:\n${result.stdout}\n');
+      print('cipd ensure stderr:\n${result.stderr}\n');
+      throw Exception('Failed to download the CIPD package.');
+    }
+    if (isVerbose) {
+      print('Download completes.');
+    }
+    Directory.current = previous;
+  }
+
+  @protected
+  Future<void> ensureResources() async {
+    doEnsureResources(resourcesRoot, isVerbose: isVerbose);
+  }
+
+  @protected
+  void checkRequiredOption(String option) {
+    if (argResults[option] == null) {
+      throw Exception('Option $option is required.');
+    }
+  }
+
+  @protected
+  bool get isVerbose => argResults[kFlagVerbose];
+  @protected
+  String get outJson => argResults[kOptionOutJson];
+  @protected
+  String get resourcesRoot => argResults[kOptionResourcesRoot];
+}
+
+abstract class IosCpuGpuSubcommand extends BaseCommand {
+  IosCpuGpuSubcommand() {
+    argParser.addOption(
+      kOptionProessName,
+      abbr: 'p',
+      help: 'Specifies the process name used for filtering the instruments CPU '
+          'measurements.',
+      defaultsTo: kDefaultProccessName,
+    );
+  }
+
+  @protected
+  String get processName => argResults[kOptionProessName];
+  @protected
+  String get traceUtilityPath => '$resourcesRoot/resources/TraceUtility';
+}

packages/measure/lib/commands/ioscpugpu.dart

@@ -0,0 +1,19 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:args/command_runner.dart';
+import 'package:measure/commands/ioscpugpu/new.dart';
+import 'package:measure/commands/ioscpugpu/parse.dart';
+
+class IosCpuGpu extends Command<void> {
+  IosCpuGpu() {
+    addSubcommand(IosCpuGpuNew());
+    addSubcommand(IosCpuGpuParse());
+  }
+
+  @override
+  String get name => 'ioscpugpu';
+  @override
+  String get description => 'Measure the iOS CPU/GPU percentage.';
+}

packages/measure/lib/commands/ioscpugpu/new.dart

@@ -0,0 +1,111 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:measure/commands/base.dart';
+import 'package:measure/parser.dart';
+
+class IosCpuGpuNew extends IosCpuGpuSubcommand {
+  IosCpuGpuNew() {
+    argParser.addOption(
+      kOptionTimeLimitMs,
+      abbr: 'l',
+      defaultsTo: '5000',
+      help: 'time limit (in ms) to run instruments for measuring',
+    );
+    argParser.addOption(
+      kOptionDevice,
+      abbr: 'w',
+      help: 'device identifier recognizable by instruments '
+          '(e.g., 00008020-000364CE0AF8003A)',
+    );
+  }
+
+  @override
+  String get name => 'new';
+  @override
+  String get description => 'Take a new measurement on the iOS CPU/GPU '
+      'percentage (of Flutter Runner).';
+
+  String get _timeLimit => argResults[kOptionTimeLimitMs];
+
+  String get _templatePath =>
+      '$resourcesRoot/resources/CpuGpuTemplate.tracetemplate';
+
+  @override
+  Future<void> run() async {
+    _checkDevice();
+    await ensureResources();
+
+    print('Running instruments on iOS device $_device for ${_timeLimit}ms');
+
+    final List<String> args = <String>[
+      '-l',
+      _timeLimit,
+      '-t',
+      _templatePath,
+      '-w',
+      _device,
+    ];
+    if (isVerbose) {
+      print('instruments args: $args');
+    }
+    final ProcessResult processResult = Process.runSync('instruments', args);
+    _parseTraceFilename(processResult.stdout.toString());
+
+    print('Parsing $_traceFilename');
+
+    final IosTraceParser parser = IosTraceParser(isVerbose, traceUtilityPath);
+    final CpuGpuResult result = parser.parseCpuGpu(_traceFilename, processName);
+    result.writeToJsonFile(outJson);
+    print('$result\nThe result has been written into $outJson');
+  }
+
+  String _traceFilename;
+  Future<void> _parseTraceFilename(String out) async {
+    const String kPrefix = 'Instruments Trace Complete: ';
+    final int prefixIndex = out.indexOf(kPrefix);
+    if (prefixIndex == -1) {
+      throw Exception('Failed to parse instruments output:\n$out');
+    }
+    _traceFilename = out.substring(prefixIndex + kPrefix.length).trim();
+  }
+
+  String _device;
+  void _checkDevice() {
+    _device = argResults[kOptionDevice];
+    if (_device != null) {
+      return;
+    }
+    final ProcessResult result = Process.runSync(
+      'instruments',
+      <String>['-s', 'devices'],
+    );
+    for (String line in result.stdout.toString().split('\n')) {
+      if (line.contains('iPhone') && !line.contains('Simulator')) {
+        _device = RegExp(r'\[(.+)\]').firstMatch(line).group(1);
+        break;
+      }
+    }
+    if (_device == null) {
+      print('''
+  Option device (-w) is not provided, and failed to find an iPhone(not a
+  simulator) from `instruments -s devices`.
+
+  stdout of `instruments -s device`:
+  ===========================
+  ${result.stdout}
+  ===========================
+
+  stderr of `instruments -s device`:
+  ===========================
+  ${result.stderr}
+  ===========================
+  ''');
+      throw Exception('Failed to determine the device.');
+    }
+  }
+}

packages/measure/lib/commands/ioscpugpu/parse.dart

@@ -0,0 +1,40 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:measure/commands/base.dart';
+import 'package:measure/parser.dart';
+
+class IosCpuGpuParse extends IosCpuGpuSubcommand {
+  @override
+  String get name => 'parse';
+  @override
+  String get description =>
+      'parse an existing instruments trace with CPU/GPU measurements.';
+
+  @override
+  String get usage {
+    final List<String> lines = super.usage.split('\n');
+    lines[0] = 'Usage: measure ioscpugpu -u <trace-utility-path> '
+        'parse <trace-file-path>';
+    return lines.join('\n');
+  }
+
+  @override
+  Future<void> run() async {
+    if (argResults.rest.length != 1) {
+      print(usage);
+      throw Exception('exactly one argument <trace-file-path> expected');
+    }
+    final String path = argResults.rest[0];
+
+    await ensureResources();
+
+    final CpuGpuResult result = IosTraceParser(isVerbose, traceUtilityPath)
+        .parseCpuGpu(path, processName);
+    result.writeToJsonFile(outJson);
+    print('$result\nThe result has been written into $outJson');
+  }
+}

packages/measure/lib/parser.dart

@@ -0,0 +1,116 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+class CpuGpuResult {
+  CpuGpuResult(this.gpuPercentage, this.cpuPercentage);
+
+  final double gpuPercentage;
+  final double cpuPercentage;
+
+  @override
+  String toString() {
+    return 'gpu: $gpuPercentage%, cpu: $cpuPercentage%';
+  }
+
+  void writeToJsonFile(String filename) {
+    final String output = json.encode(<String, double>{
+      'gpu_percentage': gpuPercentage,
+      'cpu_percentage': cpuPercentage
+    });
+    File(filename).writeAsStringSync(output);
+  }
+}
+
+class IosTraceParser {
+  IosTraceParser(this.isVerbose, this.traceUtilityPath);
+
+  final bool isVerbose;
+  final String traceUtilityPath;
+
+  List<String> _gpuMeasurements;
+  List<String> _cpuMeasurements;
+
+  CpuGpuResult parseCpuGpu(String filename, String processName) {
+    final ProcessResult result = Process.runSync(
+      traceUtilityPath,
+      <String>[filename],
+    );
+    if (result.exitCode != 0) {
+      print('TraceUtility stdout:\n${result.stdout.toString}\n\n');
+      print('TraceUtility stderr:\n${result.stderr.toString}\n\n');
+      throw Exception('TraceUtility failed with exit code ${result.exitCode}');
+    }
+    final List<String> lines = result.stderr.toString().split('\n');
+
+    // toSet to remove duplicates
+    _gpuMeasurements =
+        lines.where((String s) => s.contains('GPU')).toSet().toList();
+    _cpuMeasurements =
+        lines.where((String s) => s.contains(processName)).toSet().toList();
+    _gpuMeasurements.sort();
+    _cpuMeasurements.sort();
+
+    if (isVerbose) {
+      _gpuMeasurements.forEach(print);
+      _cpuMeasurements.forEach(print);
+    }
+
+    return CpuGpuResult(_computeGpuPercent(), _computeCpuPercent());
+  }
+
+  static final RegExp _percentagePattern = RegExp(r'(\d+(\.\d*)?)%');
+  double _parseSingleGpuMeasurement(String line) {
+    return double.parse(_percentagePattern.firstMatch(line).group(1));
+  }
+
+  double _computeGpuPercent() {
+    return _average(_gpuMeasurements.map(_parseSingleGpuMeasurement));
+  }
+
+  // The return is a list of 2: the 1st is the time key string, the 2nd is the
+  // double percentage
+  List<dynamic> _parseSingleCpuMeasurement(String line) {
+    final String timeKey = line.substring(0, line.indexOf(','));
+    final RegExpMatch match = _percentagePattern.firstMatch(line);
+    return <dynamic>[
+      timeKey,
+      match == null
+          ? 0
+          : double.parse(_percentagePattern.firstMatch(line).group(1))
+    ];
+  }
+
+  double _computeCpuPercent() {
+    final Iterable<List<dynamic>> results =
+        _cpuMeasurements.map(_parseSingleCpuMeasurement);
+    final Map<String, double> sums = <String, double>{};
+    for (List<dynamic> pair in results) {
+      sums[pair[0]] = 0;
+    }
+    for (List<dynamic> pair in results) {
+      sums[pair[0]] += pair[1];
+    }
+
+    // This key always has 0% usage. Remove it.
+    assert(sums['00:00.000.000'] == 0);
+    sums.remove('00:00.000.000');
+
+    if (isVerbose) {
+      print('CPU maps: $sums');
+    }
+    return _average(sums.values);
+  }
+
+  double _average(Iterable<double> values) {
+    if (values == null || values.isEmpty) {
+      _gpuMeasurements.forEach(print);
+      _cpuMeasurements.forEach(print);
+      throw Exception('No valid measurements found.');
+    }
+    return values.reduce((double a, double b) => a + b) / values.length;
+  }
+}

packages/measure/pubspec.yaml

@@ -0,0 +1,18 @@
+name: measure
+description: Tools for measuring some performance metrics.
+author: Flutter Team <flutter-dev@googlegroups.com>
+homepage: https://github.com/flutter/packages/tree/master/packages/measure
+version: 0.1.0
+
+executables:
+  measure: measure
+
+dependencies:
+  args: ^1.5.2
+  meta: ^1.1.7
+
+dev_dependencies:
+  test: ^1.6.8
+
+environment:
+  sdk: ">=2.2.2 <3.0.0"

packages/measure/test/measure_test.dart

@@ -0,0 +1,88 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@TestOn('mac-os')
+
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+import 'package:measure/commands/base.dart';
+
+void main() {
+  const String measureRootPath = '.';
+  final String resourcesRootPath = BaseCommand.defaultResourcesRoot;
+  BaseCommand.doEnsureResources(resourcesRootPath, isVerbose: true);
+
+  test('help works', () {
+    final ProcessResult result = Process.runSync(
+      'dart',
+      <String>['$measureRootPath/bin/measure.dart', 'help'],
+    );
+    expect(
+        result.stdout.toString(),
+        contains(
+          'Tools for measuring some performance metrics.',
+        ));
+  });
+
+  ProcessResult _testIosCpuGpu(List<String> extraArgs) {
+    return Process.runSync(
+      'dart',
+      <String>[
+        '$measureRootPath/bin/measure.dart',
+        'ioscpugpu',
+        ...extraArgs,
+        '-r',
+        resourcesRootPath,
+      ],
+    );
+  }
+
+  ProcessResult _testParse(List<String> extraArgs) {
+    return _testIosCpuGpu(<String>[
+      'parse',
+      '$resourcesRootPath/resources/example_instrumentscli.trace/',
+      ...extraArgs,
+    ]);
+  }
+
+  test('ioscpugpu parse works', () {
+    final ProcessResult result = _testParse(<String>[]);
+    expect(
+        result.stdout.toString(),
+        contains(
+          'gpu: 12.6%, cpu: 18.15%',
+        ));
+    expect(
+        File('result.json').readAsStringSync(),
+        contains(
+          '{"gpu_percentage":12.6,"cpu_percentage":18.15}',
+        ));
+  });
+
+  test('ioscpugpu parse works with verbose', () {
+    final ProcessResult result = _testParse(<String>['--verbose']);
+    expect(
+        result.stdout.toString(),
+        contains(
+          '00:00.000.000  0 FPS 13.0% GPU',
+        ));
+    expect(
+        result.stdout.toString(),
+        contains(
+          '00:00.477.632, 1.55 s, Runner (2209), n/a, 2209, mobile, 23.7%',
+        ));
+  });
+
+  test('ioscpugpu new works', () {
+    final ProcessResult result = _testIosCpuGpu(<String>['new']);
+    expect(
+      result.stdout.toString(),
+      contains('The result has been written into result.json'),
+      reason: '\n\nioscpugpu new failed. Do you have a single connected iPhone '
+          'that has a Flutter app running?',
+    );
+  });
+}