From ca4992ecd016ea06159f8ebf273378cf7dd5d0e1 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Fri, 22 Jan 2021 16:19:34 -0800 Subject: [PATCH] Import metrics_center (#268) --- analysis_options.yaml | 2 +- packages/metrics_center/.gitignore | 1 + packages/metrics_center/CHANGELOG.md | 3 + packages/metrics_center/LICENSE | 25 + packages/metrics_center/README.md | 9 + .../metrics_center/lib/metrics_center.dart | 8 + packages/metrics_center/lib/src/common.dart | 65 ++ .../metrics_center/lib/src/constants.dart | 55 ++ packages/metrics_center/lib/src/flutter.dart | 60 ++ packages/metrics_center/lib/src/gcs_lock.dart | 90 +++ .../metrics_center/lib/src/github_helper.dart | 35 ++ .../lib/src/google_benchmark.dart | 74 +++ .../lib/src/legacy_datastore.dart | 33 + .../lib/src/legacy_flutter.dart | 91 +++ packages/metrics_center/lib/src/skiaperf.dart | 438 +++++++++++++ packages/metrics_center/pubspec.yaml | 25 + packages/metrics_center/test/common.dart | 27 + .../test/example_google_benchmark.json | 32 + .../metrics_center/test/flutter_test.dart | 38 ++ .../metrics_center/test/gcs_lock_test.dart | 105 ++++ .../test/github_helper_test.dart | 41 ++ .../test/google_benchmark_test.dart | 39 ++ .../test/legacy_flutter_test.dart | 41 ++ .../metrics_center/test/skiaperf_test.dart | 573 ++++++++++++++++++ packages/metrics_center/test/utility.dart | 23 + 25 files changed, 1932 insertions(+), 1 deletion(-) create mode 100644 packages/metrics_center/.gitignore create mode 100644 packages/metrics_center/CHANGELOG.md create mode 100644 packages/metrics_center/LICENSE create mode 100644 packages/metrics_center/README.md create mode 100644 packages/metrics_center/lib/metrics_center.dart create mode 100644 packages/metrics_center/lib/src/common.dart create mode 100644 packages/metrics_center/lib/src/constants.dart create mode 100644 packages/metrics_center/lib/src/flutter.dart create mode 100644 packages/metrics_center/lib/src/gcs_lock.dart create mode 100644 packages/metrics_center/lib/src/github_helper.dart create mode 100644 packages/metrics_center/lib/src/google_benchmark.dart create mode 100644 packages/metrics_center/lib/src/legacy_datastore.dart create mode 100644 packages/metrics_center/lib/src/legacy_flutter.dart create mode 100644 packages/metrics_center/lib/src/skiaperf.dart create mode 100644 packages/metrics_center/pubspec.yaml create mode 100644 packages/metrics_center/test/common.dart create mode 100644 packages/metrics_center/test/example_google_benchmark.json create mode 100644 packages/metrics_center/test/flutter_test.dart create mode 100644 packages/metrics_center/test/gcs_lock_test.dart create mode 100644 packages/metrics_center/test/github_helper_test.dart create mode 100644 packages/metrics_center/test/google_benchmark_test.dart create mode 100644 packages/metrics_center/test/legacy_flutter_test.dart create mode 100644 packages/metrics_center/test/skiaperf_test.dart create mode 100644 packages/metrics_center/test/utility.dart diff --git a/analysis_options.yaml b/analysis_options.yaml index d7ec8e54ad..02f94ca6d7 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -38,7 +38,7 @@ linter: - always_specify_types - annotate_overrides # - avoid_annotating_with_dynamic # conflicts with always_specify_types - - avoid_as + # - avoid_as # conflicts with NNBD - avoid_bool_literals_in_conditional_expressions # - avoid_catches_without_on_clauses # we do this commonly # - avoid_catching_errors # we do this commonly diff --git a/packages/metrics_center/.gitignore b/packages/metrics_center/.gitignore new file mode 100644 index 0000000000..d97c5eada5 --- /dev/null +++ b/packages/metrics_center/.gitignore @@ -0,0 +1 @@ +secret diff --git a/packages/metrics_center/CHANGELOG.md b/packages/metrics_center/CHANGELOG.md new file mode 100644 index 0000000000..beea2ff637 --- /dev/null +++ b/packages/metrics_center/CHANGELOG.md @@ -0,0 +1,3 @@ +# 0.0.4+1 + +- Moved to the `flutter/packages` repository diff --git a/packages/metrics_center/LICENSE b/packages/metrics_center/LICENSE new file mode 100644 index 0000000000..922fc0c19f --- /dev/null +++ b/packages/metrics_center/LICENSE @@ -0,0 +1,25 @@ +Copyright 2014 The Flutter Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/packages/metrics_center/README.md b/packages/metrics_center/README.md new file mode 100644 index 0000000000..8f1418d4a8 --- /dev/null +++ b/packages/metrics_center/README.md @@ -0,0 +1,9 @@ +# Metrics Center + +Metrics center is a minimal set of code and services to support multiple perf +metrics generators (e.g., Cocoon device lab, Cirrus bots, LUCI bots, Firebase +Test Lab) and destinations (e.g., old Cocoon perf dashboard, Skia perf +dashboard). The work and maintenance it requires is very close to that of just +supporting a single generator and destination (e.g., engine bots to Skia perf), +and the small amount of extra work is designed to make it easy to support more +generators and destinations in the future. diff --git a/packages/metrics_center/lib/metrics_center.dart b/packages/metrics_center/lib/metrics_center.dart new file mode 100644 index 0000000000..beebe8ecb9 --- /dev/null +++ b/packages/metrics_center/lib/metrics_center.dart @@ -0,0 +1,8 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/common.dart'; +export 'src/flutter.dart'; +export 'src/google_benchmark.dart'; +export 'src/skiaperf.dart'; diff --git a/packages/metrics_center/lib/src/common.dart b/packages/metrics_center/lib/src/common.dart new file mode 100644 index 0000000000..e4864e2d64 --- /dev/null +++ b/packages/metrics_center/lib/src/common.dart @@ -0,0 +1,65 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:collection'; +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; +import 'package:equatable/equatable.dart'; + +import 'package:googleapis_auth/auth.dart'; +import 'package:googleapis_auth/auth_io.dart'; +import 'package:http/http.dart'; + +/// Common format of a metric data point. +class MetricPoint extends Equatable { + /// Creates a new data point. + MetricPoint( + this.value, + Map tags, + ) : _tags = SplayTreeMap.from(tags); + + /// Can store integer values. + final double value; + + /// Test name, unit, timestamp, configs, git revision, ..., in sorted order. + UnmodifiableMapView get tags => + UnmodifiableMapView(_tags); + + /// Unique identifier for updating existing data point. + /// + /// We shouldn't have to worry about hash collisions until we have about + /// 2^128 points. + /// + /// This id should stay constant even if the [tags.keys] are reordered. + /// (Because we are using an ordered SplayTreeMap to generate the id.) + String get id => sha256.convert(utf8.encode('$_tags')).toString(); + + @override + String toString() { + return 'MetricPoint(value=$value, tags=$_tags)'; + } + + final SplayTreeMap _tags; + + @override + List get props => [value, tags]; +} + +/// Interface to write [MetricPoint]. +abstract class MetricDestination { + /// Insert new data points or modify old ones with matching id. + Future update(List points); +} + +/// Create `AuthClient` in case we only have an access token without the full +/// credentials json. It's currently the case for Chrmoium LUCI bots. +AuthClient authClientFromAccessToken(String token, List scopes) { + final DateTime anHourLater = DateTime.now().add(const Duration(hours: 1)); + final AccessToken accessToken = + AccessToken('Bearer', token, anHourLater.toUtc()); + final AccessCredentials accessCredentials = + AccessCredentials(accessToken, null, scopes); + return authenticatedClient(Client(), accessCredentials); +} diff --git a/packages/metrics_center/lib/src/constants.dart b/packages/metrics_center/lib/src/constants.dart new file mode 100644 index 0000000000..ba32171da6 --- /dev/null +++ b/packages/metrics_center/lib/src/constants.dart @@ -0,0 +1,55 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Strings used for MetricPoint tag keys +const String kGithubRepoKey = 'gitRepo'; + +/// Strings used for MetricPoint tag keys +const String kGitRevisionKey = 'gitRevision'; + +/// Strings used for MetricPoint tag keys +const String kUnitKey = 'unit'; + +/// Strings used for MetricPoint tag keys +const String kNameKey = 'name'; + +/// Strings used for MetricPoint tag keys +const String kSubResultKey = 'subResult'; + +/// Flutter repo name. +const String kFlutterFrameworkRepo = 'flutter/flutter'; + +/// Engine repo name. +const String kFlutterEngineRepo = 'flutter/engine'; + +/// The key for the GCP project id in the credentials json. +const String kProjectId = 'project_id'; + +/// Timeline key in JSON. +const String kSourceTimeMicrosName = 'sourceTimeMicros'; + +/// The size of 500 is currently limited by Google datastore. It cannot write +/// more than 500 entities in a single call. +const int kMaxBatchSize = 500; + +/// The prod bucket name for Flutter's instance of Skia Perf. +const String kBucketName = 'flutter-skia-perf-prod'; + +/// The test bucket name for Flutter's instance of Skia Perf. +const String kTestBucketName = 'flutter-skia-perf-test'; + +/// JSON key for Skia Perf's git hash entry. +const String kSkiaPerfGitHashKey = 'gitHash'; + +/// JSON key for Skia Perf's results entry. +const String kSkiaPerfResultsKey = 'results'; + +/// JSON key for Skia Perf's value entry. +const String kSkiaPerfValueKey = 'value'; + +/// JSON key for Skia Perf's options entry. +const String kSkiaPerfOptionsKey = 'options'; + +/// JSON key for Skia Perf's default config entry. +const String kSkiaPerfDefaultConfig = 'default'; diff --git a/packages/metrics_center/lib/src/flutter.dart b/packages/metrics_center/lib/src/flutter.dart new file mode 100644 index 0000000000..3641135d28 --- /dev/null +++ b/packages/metrics_center/lib/src/flutter.dart @@ -0,0 +1,60 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'common.dart'; +import 'constants.dart'; +import 'legacy_datastore.dart'; +import 'legacy_flutter.dart'; + +/// Convenient class to capture the benchmarks in the Flutter engine repo. +class FlutterEngineMetricPoint extends MetricPoint { + /// Creates a metric point for the Flutter engine repository. + /// + /// The `name`, `value`, and `gitRevision` parameters must not be null. + FlutterEngineMetricPoint( + String name, + double value, + String gitRevision, { + Map moreTags = const {}, + }) : super( + value, + { + kNameKey: name, + kGithubRepoKey: kFlutterEngineRepo, + kGitRevisionKey: gitRevision, + }..addAll(moreTags), + ); +} + +/// All Flutter performance metrics (framework, engine, ...) should be written +/// to this destination. +class FlutterDestination extends MetricDestination { + // TODO(liyuqian): change the implementation of this class (without changing + // its public APIs) to remove `LegacyFlutterDestination` and directly use + // `SkiaPerfDestination` once the migration is fully done. + FlutterDestination._(this._legacyDestination); + + /// Creates a [FlutterDestination] from service account JSON. + static Future makeFromCredentialsJson( + Map json) async { + final LegacyFlutterDestination legacyDestination = + LegacyFlutterDestination(await datastoreFromCredentialsJson(json)); + return FlutterDestination._(legacyDestination); + } + + /// Creates a [FlutterDestination] from an OAuth access token. + static FlutterDestination makeFromAccessToken( + String accessToken, String projectId) { + final LegacyFlutterDestination legacyDestination = LegacyFlutterDestination( + datastoreFromAccessToken(accessToken, projectId)); + return FlutterDestination._(legacyDestination); + } + + @override + Future update(List points) async { + await _legacyDestination.update(points); + } + + final LegacyFlutterDestination _legacyDestination; +} diff --git a/packages/metrics_center/lib/src/gcs_lock.dart b/packages/metrics_center/lib/src/gcs_lock.dart new file mode 100644 index 0000000000..1150e22d84 --- /dev/null +++ b/packages/metrics_center/lib/src/gcs_lock.dart @@ -0,0 +1,90 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:googleapis/storage/v1.dart'; +import 'package:googleapis_auth/auth.dart'; + +/// Global (in terms of earth) mutex using Google Cloud Storage. +class GcsLock { + /// Create a lock with an authenticated client and a GCS bucket name. + /// + /// The client is used to communicate with Google Cloud Storage APIs. + GcsLock(this._client, this._bucketName) + : assert(_client != null), + assert(_bucketName != null) { + _api = StorageApi(_client); + } + + /// Create a temporary lock file in GCS, and use it as a mutex mechanism to + /// run a piece of code exclusively. + /// + /// There must be no existing lock file with the same name in order to + /// proceed. If multiple [GcsLock]s with the same `bucketName` and + /// `lockFileName` try [protectedRun] simultaneously, only one will proceed + /// and create the lock file. All others will be blocked. + /// + /// When [protectedRun] finishes, the lock file is deleted, and other blocked + /// [protectedRun] may proceed. + /// + /// If the lock file is stuck (e.g., `_unlock` is interrupted unexpectedly), + /// one may need to manually delete the lock file from GCS to unblock any + /// [protectedRun] that may depend on it. + Future protectedRun(String lockFileName, Future f()) async { + await _lock(lockFileName); + try { + await f(); + } catch (e, stacktrace) { + print(stacktrace); + rethrow; + } finally { + await _unlock(lockFileName); + } + } + + Future _lock(String lockFileName) async { + final Object object = Object(); + object.bucket = _bucketName; + object.name = lockFileName; + final Media content = Media(const Stream>.empty(), 0); + + Duration waitPeriod = const Duration(milliseconds: 10); + bool locked = false; + while (!locked) { + try { + await _api.objects.insert(object, _bucketName, + ifGenerationMatch: '0', uploadMedia: content); + locked = true; + } on DetailedApiRequestError catch (e) { + if (e.status == 412) { + // Status 412 means that the lock file already exists. Wait until + // that lock file is deleted. + await Future.delayed(waitPeriod); + waitPeriod *= 2; + if (waitPeriod >= _kWarningThreshold) { + print( + 'The lock is waiting for a long time: $waitPeriod. ' + 'If the lock file $lockFileName in bucket $_bucketName ' + 'seems to be stuck (i.e., it was created a long time ago and ' + 'no one seems to be owning it currently), delete it manually ' + 'to unblock this.', + ); + } + } else { + rethrow; + } + } + } + } + + Future _unlock(String lockFileName) async { + await _api.objects.delete(_bucketName, lockFileName); + } + + StorageApi _api; + + final String _bucketName; + final AuthClient _client; + + static const Duration _kWarningThreshold = Duration(seconds: 10); +} diff --git a/packages/metrics_center/lib/src/github_helper.dart b/packages/metrics_center/lib/src/github_helper.dart new file mode 100644 index 0000000000..9e33b487f3 --- /dev/null +++ b/packages/metrics_center/lib/src/github_helper.dart @@ -0,0 +1,35 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:github/github.dart'; + +/// Singleton class to query some Github info with an in-memory cache. +class GithubHelper { + /// Return the singleton helper. + factory GithubHelper() { + return _singleton; + } + + GithubHelper._internal(); + + /// The result is cached in memory so querying the same thing again in the + /// same process is fast. + /// + /// Our unit test requires that calling this method 1000 times for the same + /// `githubRepo` and `sha` should be done in 1 second. + Future getCommitDateTime(String githubRepo, String sha) async { + final String key = '$githubRepo/commit/$sha'; + if (_commitDateTimeCache[key] == null) { + final RepositoryCommit commit = await _github.repositories + .getCommit(RepositorySlug.full(githubRepo), sha); + _commitDateTimeCache[key] = commit.commit.committer.date; + } + return _commitDateTimeCache[key]; + } + + static final GithubHelper _singleton = GithubHelper._internal(); + + final GitHub _github = GitHub(auth: findAuthenticationFromEnvironment()); + final Map _commitDateTimeCache = {}; +} diff --git a/packages/metrics_center/lib/src/google_benchmark.dart b/packages/metrics_center/lib/src/google_benchmark.dart new file mode 100644 index 0000000000..ab4ae221dc --- /dev/null +++ b/packages/metrics_center/lib/src/google_benchmark.dart @@ -0,0 +1,74 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'common.dart'; +import 'constants.dart'; + +const String _kTimeUnitKey = 'time_unit'; + +const List _kNonNumericalValueSubResults = [ + kNameKey, + _kTimeUnitKey, + 'iterations', + 'big_o', +]; + +/// Parse the json result of https://github.com/google/benchmark. +class GoogleBenchmarkParser { + /// Given a Google benchmark json output, parse its content into a list of [MetricPoint]. + static Future> parse(String jsonFileName) async { + final Map jsonResult = + jsonDecode(File(jsonFileName).readAsStringSync()) + as Map; + + final Map rawContext = + jsonResult['context'] as Map; + final Map context = rawContext.map( + (String k, dynamic v) => MapEntry(k, v.toString()), + ); + final List points = []; + for (final dynamic item in jsonResult['benchmarks']) { + _parseAnItem(item as Map, points, context); + } + return points; + } +} + +void _parseAnItem( + Map item, + List points, + Map context, +) { + final String name = item[kNameKey] as String; + final Map timeUnitMap = { + kUnitKey: item[_kTimeUnitKey] as String + }; + for (final String subResult in item.keys) { + if (!_kNonNumericalValueSubResults.contains(subResult)) { + num rawValue; + try { + rawValue = item[subResult] as num; + } catch (e) { + print( + '$subResult: ${item[subResult]} (${item[subResult].runtimeType}) is not a number'); + rethrow; + } + + final double value = + rawValue is int ? rawValue.toDouble() : rawValue as double; + points.add( + MetricPoint( + value, + {kNameKey: name, kSubResultKey: subResult} + ..addAll(context) + ..addAll( + subResult.endsWith('time') ? timeUnitMap : {}), + ), + ); + } + } +} diff --git a/packages/metrics_center/lib/src/legacy_datastore.dart b/packages/metrics_center/lib/src/legacy_datastore.dart new file mode 100644 index 0000000000..1d12d7ab8b --- /dev/null +++ b/packages/metrics_center/lib/src/legacy_datastore.dart @@ -0,0 +1,33 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO(liyuqian): Remove this file once the migration is fully done and we no +// longer need to fall back to the datastore. + +import 'package:gcloud/db.dart'; +import 'package:googleapis_auth/auth.dart'; +import 'package:googleapis_auth/auth_io.dart'; + +// The official pub.dev/packages/gcloud documentation uses datastore_impl +// so we have to ignore implementation_imports here. +// ignore: implementation_imports +import 'package:gcloud/src/datastore_impl.dart'; + +import 'common.dart'; +import 'constants.dart'; + +/// Creates a [DatastoreDB] connection from JSON service account credentials. +Future datastoreFromCredentialsJson( + Map json) async { + final AutoRefreshingAuthClient client = await clientViaServiceAccount( + ServiceAccountCredentials.fromJson(json), DatastoreImpl.SCOPES); + return DatastoreDB(DatastoreImpl(client, json[kProjectId] as String)); +} + +/// Creates a [DatastoreDB] from an auth token. +DatastoreDB datastoreFromAccessToken(String token, String projectId) { + final AuthClient client = + authClientFromAccessToken(token, DatastoreImpl.SCOPES); + return DatastoreDB(DatastoreImpl(client, projectId)); +} diff --git a/packages/metrics_center/lib/src/legacy_flutter.dart b/packages/metrics_center/lib/src/legacy_flutter.dart new file mode 100644 index 0000000000..8bc7f1fc9c --- /dev/null +++ b/packages/metrics_center/lib/src/legacy_flutter.dart @@ -0,0 +1,91 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO(liyuqian): Remove this legacy file once the migration is fully done. +// See go/flutter-metrics-center-migration for detailed plans. +import 'dart:convert'; +import 'dart:math'; + +import 'package:gcloud/db.dart'; + +import 'common.dart'; +import 'constants.dart'; +import 'legacy_datastore.dart'; + +/// This model corresponds to the existing data model 'MetricPoint' used in the +/// flutter-cirrus GCP project. +/// +/// The originId and sourceTimeMicros fields are no longer used but we are still +/// providing valid values to them so it's compatible with old code and services +/// during the migration. +@Kind(name: 'MetricPoint', idType: IdType.String) +class LegacyMetricPointModel extends Model { + /// Initializes a metrics point data model for the flutter-cirrus GCP project. + LegacyMetricPointModel({MetricPoint fromMetricPoint}) { + if (fromMetricPoint != null) { + id = fromMetricPoint.id; + value = fromMetricPoint.value; + originId = 'legacy-flutter'; + sourceTimeMicros = null; + tags = fromMetricPoint.tags.keys + .map((String key) => + jsonEncode({key: fromMetricPoint.tags[key]})) + .toList(); + } + } + + /// The value of this metric. + @DoubleProperty(required: true, indexed: false) + double value; + + /// Any tags associated with this metric. + @StringListProperty() + List tags; + + /// The origin of this metric, which is no longer used. + @StringProperty(required: true) + String originId; + + /// The sourceTimeMicros field, which is no longer used. + @IntProperty(propertyName: kSourceTimeMicrosName) + int sourceTimeMicros; +} + +/// A [FlutterDestination] that is backwards compatible with the flutter-cirrus +/// GCP project. +class LegacyFlutterDestination extends MetricDestination { + /// Creates a legacy destination compatible with the flutter-cirrus GCP + /// project. + LegacyFlutterDestination(this._db); + + /// Creates this destination from a service account credentials JSON file. + static Future makeFromCredentialsJson( + Map json) async { + return LegacyFlutterDestination(await datastoreFromCredentialsJson(json)); + } + + /// Creates this destination to authorize with an OAuth access token. + static LegacyFlutterDestination makeFromAccessToken( + String accessToken, String projectId) { + return LegacyFlutterDestination( + datastoreFromAccessToken(accessToken, projectId)); + } + + @override + Future update(List points) async { + final List flutterCenterPoints = points + .map((MetricPoint p) => LegacyMetricPointModel(fromMetricPoint: p)) + .toList(); + + for (int start = 0; start < points.length; start += kMaxBatchSize) { + final int end = min(start + kMaxBatchSize, points.length); + await _db.withTransaction((Transaction tx) async { + tx.queueMutations(inserts: flutterCenterPoints.sublist(start, end)); + await tx.commit(); + }); + } + } + + final DatastoreDB _db; +} diff --git a/packages/metrics_center/lib/src/skiaperf.dart b/packages/metrics_center/lib/src/skiaperf.dart new file mode 100644 index 0000000000..b663de1d05 --- /dev/null +++ b/packages/metrics_center/lib/src/skiaperf.dart @@ -0,0 +1,438 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:gcloud/storage.dart'; +import 'package:googleapis/storage/v1.dart' show DetailedApiRequestError; +import 'package:googleapis_auth/auth.dart'; +import 'package:googleapis_auth/auth_io.dart'; + +import 'common.dart'; +import 'constants.dart'; +import 'gcs_lock.dart'; +import 'github_helper.dart'; + +/// A [MetricPoint] modeled after the format that Skia Perf expects. +/// +/// Skia Perf Format is a JSON file that looks like: +/// ```json +/// { +/// "gitHash": "fe4a4029a080bc955e9588d05a6cd9eb490845d4", +/// "key": { +/// "arch": "x86", +/// "gpu": "GTX660", +/// "model": "ShuttleA", +/// "os": "Ubuntu12" +/// }, +/// "results": { +/// "ChunkAlloc_PushPop_640_480": { +/// "nonrendering": { +/// "min_ms": 0.01485466666666667, +/// "options": { +/// "source_type": "bench" +/// } +/// } +/// }, +/// "DeferredSurfaceCopy_discardable_640_480": { +/// "565": { +/// "min_ms": 2.215988, +/// "options": { +/// "source_type": "bench" +/// } +/// } +/// } +/// } +/// } +/// } +/// ``` +class SkiaPerfPoint extends MetricPoint { + SkiaPerfPoint._(this.githubRepo, this.gitHash, this.testName, this.subResult, + double value, this._options, this.jsonUrl) + : assert(_options[kGithubRepoKey] == null), + assert(_options[kGitRevisionKey] == null), + assert(_options[kNameKey] == null), + super( + value, + {} + ..addAll(_options) + ..addAll({ + kGithubRepoKey: githubRepo, + kGitRevisionKey: gitHash, + kNameKey: testName, + kSubResultKey: subResult, + }), + ) { + assert(tags[kGithubRepoKey] != null); + assert(tags[kGitRevisionKey] != null); + assert(tags[kNameKey] != null); + } + + /// Construct [SkiaPerfPoint] from a well-formed [MetricPoint]. + /// + /// The [MetricPoint] must have [kGithubRepoKey], [kGitRevisionKey], + /// [kNameKey] in its tags for this to be successful. + /// + /// If the [MetricPoint] has a tag 'date', that tag will be removed so Skia + /// perf can plot multiple metrics with different date as a single trace. + /// Skia perf will use the git revision's date instead of this date tag in + /// the time axis. + factory SkiaPerfPoint.fromPoint(MetricPoint p) { + final String githubRepo = p.tags[kGithubRepoKey]; + final String gitHash = p.tags[kGitRevisionKey]; + final String name = p.tags[kNameKey]; + + if (githubRepo == null || gitHash == null || name == null) { + throw '$kGithubRepoKey, $kGitRevisionKey, $kNameKey must be set in' + ' the tags of $p.'; + } + + final String subResult = p.tags[kSubResultKey] ?? kSkiaPerfValueKey; + + final Map options = {}..addEntries( + p.tags.entries.where( + (MapEntry entry) => + entry.key != kGithubRepoKey && + entry.key != kGitRevisionKey && + entry.key != kNameKey && + entry.key != kSubResultKey && + // https://github.com/google/benchmark automatically generates a + // 'date' field. If it's included in options, the Skia perf won't + // be able to connect different points in a single trace because + // the date is always different. + entry.key != 'date', + ), + ); + + return SkiaPerfPoint._( + githubRepo, gitHash, name, subResult, p.value, options, null); + } + + /// In the format of '/' such as 'flutter/flutter' or + /// 'flutter/engine'. + final String githubRepo; + + /// SHA such as 'ad20d368ffa09559754e4b2b5c12951341ca3b2d' + final String gitHash; + + /// For Flutter devicelab, this is the task name (e.g., + /// 'flutter_gallery__transition_perf'); for Google benchmark, this is the + /// benchmark name (e.g., 'BM_ShellShutdown'). + /// + /// In Skia perf web dashboard, this value can be queried and filtered by + /// "test". + final String testName; + + /// The name of "subResult" comes from the special treatment of "sub_result" + /// in SkiaPerf. If not provided, its value will be set to kSkiaPerfValueKey. + /// + /// When Google benchmarks are converted to SkiaPerfPoint, this subResult + /// could be "cpu_time" or "real_time". + /// + /// When devicelab benchmarks are converted to SkiaPerfPoint, this subResult + /// is often the metric name such as "average_frame_build_time_millis" whereas + /// the [testName] is the benchmark or task name such as + /// "flutter_gallery__transition_perf". + final String subResult; + + /// The url to the Skia perf json file in the Google Cloud Storage bucket. + /// + /// This can be null if the point has been stored in the bucket yet. + final String jsonUrl; + + Map _toSubResultJson() { + return { + subResult: value, + kSkiaPerfOptionsKey: _options, + }; + } + + /// Convert a list of SkiaPoints with the same git repo and git revision into + /// a single json file in the Skia perf format. + /// + /// The list must be non-empty. + static Map toSkiaPerfJson(List points) { + assert(points.isNotEmpty); + assert(() { + for (final SkiaPerfPoint p in points) { + if (p.githubRepo != points[0].githubRepo || + p.gitHash != points[0].gitHash) { + return false; + } + } + return true; + }(), 'All points must have same githubRepo and gitHash'); + + final Map results = {}; + for (final SkiaPerfPoint p in points) { + final Map subResultJson = p._toSubResultJson(); + if (results[p.testName] == null) { + results[p.testName] = { + kSkiaPerfDefaultConfig: subResultJson, + }; + } else { + // Flutter currently doesn't support having the same name but different + // options/configurations. If this actually happens in the future, we + // probably can use different values of config (currently there's only + // one kSkiaPerfDefaultConfig) to resolve the conflict. + assert(results[p.testName][kSkiaPerfDefaultConfig][kSkiaPerfOptionsKey] + .toString() == + subResultJson[kSkiaPerfOptionsKey].toString()); + assert( + results[p.testName][kSkiaPerfDefaultConfig][p.subResult] == null); + results[p.testName][kSkiaPerfDefaultConfig][p.subResult] = p.value; + } + } + + return { + kSkiaPerfGitHashKey: points[0].gitHash, + kSkiaPerfResultsKey: results, + }; + } + + // Equivalent to tags without git repo, git hash, and name because those two + // are already stored somewhere else. + final Map _options; +} + +/// Handle writing and updates of Skia perf GCS buckets. +class SkiaPerfGcsAdaptor { + /// Construct the adaptor given the associated GCS bucket where the data is + /// read from and written to. + SkiaPerfGcsAdaptor(this._gcsBucket) : assert(_gcsBucket != null); + + /// Used by Skia to differentiate json file format versions. + static const int version = 1; + + /// Write a list of SkiaPerfPoint into a GCS file with name `objectName` in + /// the proper json format that's understandable by Skia perf services. + /// + /// The `objectName` must be a properly formatted string returned by + /// [computeObjectName]. + /// + /// The read may retry multiple times if transient network errors with code + /// 504 happens. + Future writePoints( + String objectName, List points) async { + final String jsonString = jsonEncode(SkiaPerfPoint.toSkiaPerfJson(points)); + final List content = utf8.encode(jsonString); + + // Retry multiple times as GCS may return 504 timeout. + for (int retry = 0; retry < 5; retry += 1) { + try { + await _gcsBucket.writeBytes(objectName, content); + return; + } catch (e) { + if (e is DetailedApiRequestError && e.status == 504) { + continue; + } + rethrow; + } + } + // Retry one last time and let the exception go through. + await _gcsBucket.writeBytes(objectName, content); + } + + /// Read a list of `SkiaPerfPoint` that have been previously written to the + /// GCS file with name `objectName`. + /// + /// The Github repo and revision of those points will be inferred from the + /// `objectName`. + /// + /// Return an empty list if the object does not exist in the GCS bucket. + /// + /// The read may retry multiple times if transient network errors with code + /// 504 happens. + Future> readPoints(String objectName) async { + // Retry multiple times as GCS may return 504 timeout. + for (int retry = 0; retry < 5; retry += 1) { + try { + return await _readPointsWithoutRetry(objectName); + } catch (e) { + if (e is DetailedApiRequestError && e.status == 504) { + continue; + } + rethrow; + } + } + // Retry one last time and let the exception go through. + return await _readPointsWithoutRetry(objectName); + } + + Future> _readPointsWithoutRetry(String objectName) async { + ObjectInfo info; + + try { + info = await _gcsBucket.info(objectName); + } catch (e) { + if (e.toString().contains('No such object')) { + return []; + } else { + rethrow; + } + } + + final Stream> stream = _gcsBucket.read(objectName); + final Stream byteStream = stream.expand((List x) => x); + final Map decodedJson = + jsonDecode(utf8.decode(await byteStream.toList())) + as Map; + + final List points = []; + + final String firstGcsNameComponent = objectName.split('/')[0]; + _populateGcsNameToGithubRepoMapIfNeeded(); + final String githubRepo = _gcsNameToGithubRepo[firstGcsNameComponent]; + assert(githubRepo != null); + + final String gitHash = decodedJson[kSkiaPerfGitHashKey] as String; + final Map results = + decodedJson[kSkiaPerfResultsKey] as Map; + for (final String name in results.keys) { + final Map subResultMap = + results[name][kSkiaPerfDefaultConfig] as Map; + for (final String subResult + in subResultMap.keys.where((String s) => s != kSkiaPerfOptionsKey)) { + points.add(SkiaPerfPoint._( + githubRepo, + gitHash, + name, + subResult, + subResultMap[subResult] as double, + (subResultMap[kSkiaPerfOptionsKey] as Map) + .cast(), + info.downloadLink.toString(), + )); + } + } + return points; + } + + /// Compute the GCS file name that's used to store metrics for a given commit + /// (git revision). + /// + /// Skia perf needs all directory names to be well formatted. The final name + /// of the json file (currently `values.json`) can be arbitrary, and multiple + /// json files can be put in that leaf directory. We intend to use multiple + /// json files in the future to scale up the system if too many writes are + /// competing for the same json file. + static Future computeObjectName(String githubRepo, String revision, + {GithubHelper githubHelper}) async { + assert(_githubRepoToGcsName[githubRepo] != null); + final String topComponent = _githubRepoToGcsName[githubRepo]; + final DateTime t = await (githubHelper ?? GithubHelper()) + .getCommitDateTime(githubRepo, revision); + final String month = t.month.toString().padLeft(2, '0'); + final String day = t.day.toString().padLeft(2, '0'); + final String hour = t.hour.toString().padLeft(2, '0'); + final String dateComponents = '${t.year}/$month/$day/$hour'; + return '$topComponent/$dateComponents/$revision/values.json'; + } + + static final Map _githubRepoToGcsName = { + kFlutterFrameworkRepo: 'flutter-flutter', + kFlutterEngineRepo: 'flutter-engine', + }; + static final Map _gcsNameToGithubRepo = {}; + + static void _populateGcsNameToGithubRepoMapIfNeeded() { + if (_gcsNameToGithubRepo.isEmpty) { + for (final String repo in _githubRepoToGcsName.keys) { + final String gcsName = _githubRepoToGcsName[repo]; + assert(_gcsNameToGithubRepo[gcsName] == null); + _gcsNameToGithubRepo[gcsName] = repo; + } + } + } + + final Bucket _gcsBucket; +} + +/// A [MetricDestination] that conforms to Skia Perf's protocols. +class SkiaPerfDestination extends MetricDestination { + /// Creates a new [SkiaPerfDestination]. + SkiaPerfDestination(this._gcs, this._lock); + + /// Create from a full credentials json (of a service account). + static Future makeFromGcpCredentials( + Map credentialsJson, + {bool isTesting = false}) async { + final AutoRefreshingAuthClient client = await clientViaServiceAccount( + ServiceAccountCredentials.fromJson(credentialsJson), Storage.SCOPES); + return make( + client, + credentialsJson[kProjectId] as String, + isTesting: isTesting, + ); + } + + /// Create from an access token and its project id. + static Future makeFromAccessToken( + String token, String projectId, + {bool isTesting = false}) async { + final AuthClient client = authClientFromAccessToken(token, Storage.SCOPES); + return make(client, projectId, isTesting: isTesting); + } + + /// Create from an [AuthClient] and a GCP project id. + /// + /// [AuthClient] can be obtained from functions like `clientViaUserConsent`. + static Future make(AuthClient client, String projectId, + {bool isTesting = false}) async { + final Storage storage = Storage(client, projectId); + final String bucketName = isTesting ? kTestBucketName : kBucketName; + if (!await storage.bucketExists(bucketName)) { + throw 'Bucket $bucketName does not exist.'; + } + final SkiaPerfGcsAdaptor adaptor = + SkiaPerfGcsAdaptor(storage.bucket(bucketName)); + final GcsLock lock = GcsLock(client, bucketName); + return SkiaPerfDestination(adaptor, lock); + } + + @override + Future update(List points) async { + // 1st, create a map based on git repo, git revision, and point id. Git repo + // and git revision are the top level components of the Skia perf GCS object + // name. + final Map>> pointMap = + >>{}; + for (final SkiaPerfPoint p + in points.map((MetricPoint x) => SkiaPerfPoint.fromPoint(x))) { + if (p != null) { + pointMap[p.githubRepo] ??= >{}; + pointMap[p.githubRepo][p.gitHash] ??= {}; + pointMap[p.githubRepo][p.gitHash][p.id] = p; + } + } + + // 2nd, read existing points from the gcs object and update with new ones. + for (final String repo in pointMap.keys) { + for (final String revision in pointMap[repo].keys) { + final String objectName = + await SkiaPerfGcsAdaptor.computeObjectName(repo, revision); + final Map newPoints = pointMap[repo][revision]; + // If too many bots are writing the metrics of a git revision into this + // single json file (with name `objectName`), the contention on the lock + // might be too high. In that case, break the json file into multiple + // json files according to bot names or task names. Skia perf read all + // json files in the directory so one can use arbitrary names for those + // sharded json file names. + _lock.protectedRun('$objectName.lock', () async { + final List oldPoints = + await _gcs.readPoints(objectName); + for (final SkiaPerfPoint p in oldPoints) { + if (newPoints[p.id] == null) { + newPoints[p.id] = p; + } + } + await _gcs.writePoints(objectName, newPoints.values.toList()); + }); + } + } + } + + final SkiaPerfGcsAdaptor _gcs; + final GcsLock _lock; +} diff --git a/packages/metrics_center/pubspec.yaml b/packages/metrics_center/pubspec.yaml new file mode 100644 index 0000000000..6fe34daefb --- /dev/null +++ b/packages/metrics_center/pubspec.yaml @@ -0,0 +1,25 @@ +name: metrics_center +version: 0.0.4+1 +description: + Support multiple performance metrics sources/formats and destinations. +homepage: + https://github.com/flutter/packages/tree/master/packages/metrics_center + +environment: + sdk: '>=2.10.0 <3.0.0' + +dependencies: + args: ^1.6.0 + crypto: ^2.1.5 + equatable: ^1.2.5 + gcloud: ^0.7.3 + github: ^7.0.4 + googleapis: ^0.56.1 + googleapis_auth: ^0.2.12 + http: ^0.12.2 + +dev_dependencies: + fake_async: ^1.2.0-nullsafety.3 + mockito: ^4.1.1 + pedantic: ^1.10.0-nullsafety.3 + test: ^1.16.0-nullsafety.9 diff --git a/packages/metrics_center/test/common.dart b/packages/metrics_center/test/common.dart new file mode 100644 index 0000000000..6ca543fdc0 --- /dev/null +++ b/packages/metrics_center/test/common.dart @@ -0,0 +1,27 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:test/test.dart' hide TypeMatcher, isInstanceOf; +import 'package:test/test.dart' as test_package show TypeMatcher; + +export 'package:test/test.dart' hide TypeMatcher, isInstanceOf; + +// Defines a 'package:test' shim. +// TODO(ianh): Remove this file once https://github.com/dart-lang/matcher/issues/98 is fixed + +/// A matcher that compares the type of the actual value to the type argument T. +test_package.TypeMatcher isInstanceOf() => isA(); + +void tryToDelete(Directory directory) { + // This should not be necessary, but it turns out that + // on Windows it's common for deletions to fail due to + // bogus (we think) "access denied" errors. + try { + directory.deleteSync(recursive: true); + } on FileSystemException catch (error) { + print('Failed to delete ${directory.path}: $error'); + } +} diff --git a/packages/metrics_center/test/example_google_benchmark.json b/packages/metrics_center/test/example_google_benchmark.json new file mode 100644 index 0000000000..212ce9cf37 --- /dev/null +++ b/packages/metrics_center/test/example_google_benchmark.json @@ -0,0 +1,32 @@ +{ + "context": { + "date": "2019-12-17 15:14:14", + "num_cpus": 56, + "mhz_per_cpu": 2594, + "cpu_scaling_enabled": true, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_PaintRecordInit", + "iterations": 6749079, + "real_time": 101, + "cpu_time": 101, + "time_unit": "ns" + }, + { + "name": "BM_ParagraphShortLayout", + "iterations": 151761, + "real_time": 4460, + "cpu_time": 4460, + "time_unit": "ns" + }, + { + "name": "BM_ParagraphStylesBigO_BigO", + "cpu_coefficient": 6548, + "real_coefficient": 6548, + "big_o": "N", + "time_unit": "ns" + } + ] +} diff --git a/packages/metrics_center/test/flutter_test.dart b/packages/metrics_center/test/flutter_test.dart new file mode 100644 index 0000000000..341b0ff6da --- /dev/null +++ b/packages/metrics_center/test/flutter_test.dart @@ -0,0 +1,38 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:metrics_center/src/constants.dart'; +import 'package:metrics_center/src/flutter.dart'; + +import 'common.dart'; + +void main() { + test('FlutterEngineMetricPoint works.', () { + const String gitRevision = 'ca799fa8b2254d09664b78ee80c43b434788d112'; + final FlutterEngineMetricPoint simplePoint = FlutterEngineMetricPoint( + 'BM_ParagraphLongLayout', + 287235, + gitRevision, + ); + expect(simplePoint.value, equals(287235)); + expect(simplePoint.tags[kGithubRepoKey], kFlutterEngineRepo); + expect(simplePoint.tags[kGitRevisionKey], gitRevision); + expect(simplePoint.tags[kNameKey], 'BM_ParagraphLongLayout'); + + final FlutterEngineMetricPoint detailedPoint = FlutterEngineMetricPoint( + 'BM_ParagraphLongLayout', + 287224, + 'ca799fa8b2254d09664b78ee80c43b434788d112', + moreTags: const { + 'executable': 'txt_benchmarks', + 'sub_result': 'CPU', + kUnitKey: 'ns', + }, + ); + expect(detailedPoint.value, equals(287224)); + expect(detailedPoint.tags['executable'], equals('txt_benchmarks')); + expect(detailedPoint.tags['sub_result'], equals('CPU')); + expect(detailedPoint.tags[kUnitKey], equals('ns')); + }); +} diff --git a/packages/metrics_center/test/gcs_lock_test.dart b/packages/metrics_center/test/gcs_lock_test.dart new file mode 100644 index 0000000000..d682d950c2 --- /dev/null +++ b/packages/metrics_center/test/gcs_lock_test.dart @@ -0,0 +1,105 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:googleapis/storage/v1.dart'; +import 'package:fake_async/fake_async.dart'; +import 'package:gcloud/storage.dart'; +import 'package:googleapis_auth/auth_io.dart'; +import 'package:metrics_center/src/constants.dart'; +import 'package:metrics_center/src/gcs_lock.dart'; +import 'package:mockito/mockito.dart'; + +import 'common.dart'; +import 'utility.dart'; + +enum TestPhase { + run1, + run2, +} + +class MockClient extends Mock implements AuthClient {} + +void main() { + const Duration kDelayStep = Duration(milliseconds: 10); + final Map credentialsJson = getTestGcpCredentialsJson(); + + test('GcsLock prints warnings for long waits', () { + // Capture print to verify error messages. + final List prints = []; + final ZoneSpecification spec = + ZoneSpecification(print: (_, __, ___, String msg) => prints.add(msg)); + + Zone.current.fork(specification: spec).run(() { + fakeAsync((FakeAsync fakeAsync) { + final MockClient mockClient = MockClient(); + final GcsLock lock = GcsLock(mockClient, 'mockBucket'); + when(mockClient.send(any)).thenThrow(DetailedApiRequestError(412, '')); + final Future runFinished = + lock.protectedRun('mock.lock', () async {}); + fakeAsync.elapse(const Duration(seconds: 10)); + when(mockClient.send(any)).thenThrow(AssertionError('Stop!')); + runFinished.catchError((dynamic e) { + final AssertionError error = e as AssertionError; + expect(error.message, 'Stop!'); + print('${error.message}'); + }); + fakeAsync.elapse(const Duration(seconds: 20)); + }); + }); + + const String kExpectedErrorMessage = 'The lock is waiting for a long time: ' + '0:00:10.240000. If the lock file mock.lock in bucket mockBucket ' + 'seems to be stuck (i.e., it was created a long time ago and no one ' + 'seems to be owning it currently), delete it manually to unblock this.'; + expect(prints, equals([kExpectedErrorMessage, 'Stop!'])); + }); + + test('GcsLock integration test: single protectedRun is successful', () async { + final AutoRefreshingAuthClient client = await clientViaServiceAccount( + ServiceAccountCredentials.fromJson(credentialsJson), Storage.SCOPES); + final GcsLock lock = GcsLock(client, kTestBucketName); + int testValue = 0; + await lock.protectedRun('test.lock', () async { + testValue = 1; + }); + expect(testValue, 1); + }, skip: credentialsJson == null); + + test('GcsLock integration test: protectedRun is exclusive', () async { + final AutoRefreshingAuthClient client = await clientViaServiceAccount( + ServiceAccountCredentials.fromJson(credentialsJson), Storage.SCOPES); + final GcsLock lock1 = GcsLock(client, kTestBucketName); + final GcsLock lock2 = GcsLock(client, kTestBucketName); + + TestPhase phase = TestPhase.run1; + final Completer started1 = Completer(); + final Future finished1 = lock1.protectedRun('test.lock', () async { + started1.complete(); + while (phase == TestPhase.run1) { + await Future.delayed(kDelayStep); + } + }); + + await started1.future; + + final Completer started2 = Completer(); + final Future finished2 = lock2.protectedRun('test.lock', () async { + started2.complete(); + }); + + // started2 should not be set even after a long wait because lock1 is + // holding the GCS lock file. + await Future.delayed(kDelayStep * 10); + expect(started2.isCompleted, false); + + // When phase is switched to run2, lock1 should be released soon and + // lock2 should soon be able to proceed its protectedRun. + phase = TestPhase.run2; + await started2.future; + await finished1; + await finished2; + }, skip: credentialsJson == null); +} diff --git a/packages/metrics_center/test/github_helper_test.dart b/packages/metrics_center/test/github_helper_test.dart new file mode 100644 index 0000000000..3d53248b7d --- /dev/null +++ b/packages/metrics_center/test/github_helper_test.dart @@ -0,0 +1,41 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@Timeout(Duration(seconds: 3600)) + +import 'package:metrics_center/src/github_helper.dart'; + +import 'common.dart'; + +void main() { + test('GithubHelper gets correct commit date time', () async { + final GithubHelper helper = GithubHelper(); + expect( + await helper.getCommitDateTime( + 'flutter/flutter', + 'ad20d368ffa09559754e4b2b5c12951341ca3b2d', + ), + equals(DateTime.parse('2019-12-06 03:33:01.000Z')), + ); + }); + + test('GithubHelper is a singleton', () { + final GithubHelper helper1 = GithubHelper(); + final GithubHelper helper2 = GithubHelper(); + expect(helper1, equals(helper2)); + }); + + test('GithubHelper can query the same commit 1000 times within 1 second', + () async { + final DateTime start = DateTime.now(); + for (int i = 0; i < 1000; i += 1) { + await GithubHelper().getCommitDateTime( + 'flutter/flutter', + 'ad20d368ffa09559754e4b2b5c12951341ca3b2d', + ); + } + final Duration duration = DateTime.now().difference(start); + expect(duration, lessThan(const Duration(seconds: 1))); + }); +} diff --git a/packages/metrics_center/test/google_benchmark_test.dart b/packages/metrics_center/test/google_benchmark_test.dart new file mode 100644 index 0000000000..11883ae331 --- /dev/null +++ b/packages/metrics_center/test/google_benchmark_test.dart @@ -0,0 +1,39 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:metrics_center/src/constants.dart'; +import 'package:metrics_center/src/common.dart'; +import 'package:metrics_center/src/google_benchmark.dart'; + +import 'common.dart'; +import 'utility.dart'; + +void main() { + test('GoogleBenchmarkParser parses example json.', () async { + final List points = + await GoogleBenchmarkParser.parse('test/example_google_benchmark.json'); + expect(points.length, 6); + expectSetMatch( + points.map((MetricPoint p) => p.value), + [101, 101, 4460, 4460, 6548, 6548], + ); + expectSetMatch( + points.map((MetricPoint p) => p.tags[kSubResultKey]), + [ + 'cpu_time', + 'real_time', + 'cpu_coefficient', + 'real_coefficient', + ], + ); + expectSetMatch( + points.map((MetricPoint p) => p.tags[kNameKey]), + [ + 'BM_PaintRecordInit', + 'BM_ParagraphShortLayout', + 'BM_ParagraphStylesBigO_BigO', + ], + ); + }); +} diff --git a/packages/metrics_center/test/legacy_flutter_test.dart b/packages/metrics_center/test/legacy_flutter_test.dart new file mode 100644 index 0000000000..a67220a5dd --- /dev/null +++ b/packages/metrics_center/test/legacy_flutter_test.dart @@ -0,0 +1,41 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:gcloud/src/datastore_impl.dart'; +import 'package:googleapis_auth/auth_io.dart'; +import 'package:metrics_center/src/constants.dart'; +import 'package:metrics_center/src/common.dart'; +import 'package:metrics_center/src/legacy_flutter.dart'; + +import 'common.dart'; +import 'utility.dart'; + +const String kTestSourceId = 'test'; + +void main() { + final Map credentialsJson = getTestGcpCredentialsJson(); + test( + 'LegacyFlutterDestination integration test: ' + 'update does not crash.', () async { + final LegacyFlutterDestination dst = + await LegacyFlutterDestination.makeFromCredentialsJson(credentialsJson); + await dst.update([MetricPoint(1.0, const {})]); + }, skip: credentialsJson == null); + + test( + 'LegacyFlutterDestination integration test: ' + 'can update with an access token.', () async { + final AutoRefreshingAuthClient client = await clientViaServiceAccount( + ServiceAccountCredentials.fromJson(credentialsJson), + DatastoreImpl.SCOPES, + ); + final String token = client.credentials.accessToken.data; + final LegacyFlutterDestination dst = + LegacyFlutterDestination.makeFromAccessToken( + token, + credentialsJson[kProjectId] as String, + ); + await dst.update([MetricPoint(1.0, const {})]); + }, skip: credentialsJson == null); +} diff --git a/packages/metrics_center/test/skiaperf_test.dart b/packages/metrics_center/test/skiaperf_test.dart new file mode 100644 index 0000000000..41d873709e --- /dev/null +++ b/packages/metrics_center/test/skiaperf_test.dart @@ -0,0 +1,573 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@Timeout(Duration(seconds: 3600)) + +import 'dart:convert'; + +import 'package:gcloud/storage.dart'; +import 'package:googleapis/storage/v1.dart' show DetailedApiRequestError; +import 'package:googleapis_auth/auth_io.dart'; +import 'package:metrics_center/metrics_center.dart'; +import 'package:mockito/mockito.dart'; + +import 'package:metrics_center/src/constants.dart'; +import 'package:metrics_center/src/github_helper.dart'; +import 'package:metrics_center/src/gcs_lock.dart'; + +import 'common.dart'; +import 'utility.dart'; + +class MockBucket extends Mock implements Bucket {} + +class MockObjectInfo extends Mock implements ObjectInfo {} + +class MockGithubHelper extends Mock implements GithubHelper {} + +class MockGcsLock implements GcsLock { + @override + Future protectedRun( + String exclusiveObjectName, Future Function() f) async { + await f(); + } +} + +class MockSkiaPerfGcsAdaptor implements SkiaPerfGcsAdaptor { + @override + Future> readPoints(String objectName) async { + return _storage[objectName] ?? []; + } + + @override + Future writePoints( + String objectName, List points) async { + _storage[objectName] = points.toList(); + } + + // Map from the object name to the list of SkiaPoint that mocks the GCS. + final Map> _storage = + >{}; +} + +Future main() async { + const double kValue1 = 1.0; + const double kValue2 = 2.0; + const double kValue3 = 3.0; + + const String kFrameworkRevision1 = '9011cece2595447eea5dd91adaa241c1c9ef9a33'; + const String kFrameworkRevision2 = '372fe290e4d4f3f97cbf02a57d235771a9412f10'; + const String kEngineRevision1 = '617938024315e205f26ed72ff0f0647775fa6a71'; + const String kEngineRevision2 = '5858519139c22484aaff1cf5b26bdf7951259344'; + const String kTaskName = 'analyzer_benchmark'; + const String kMetric1 = 'flutter_repo_batch_maximum'; + const String kMetric2 = 'flutter_repo_watch_maximum'; + + final MetricPoint cocoonPointRev1Metric1 = MetricPoint( + kValue1, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: kFrameworkRevision1, + kNameKey: kTaskName, + kSubResultKey: kMetric1, + kUnitKey: 's', + }, + ); + + final MetricPoint cocoonPointRev1Metric2 = MetricPoint( + kValue2, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: kFrameworkRevision1, + kNameKey: kTaskName, + kSubResultKey: kMetric2, + kUnitKey: 's', + }, + ); + + final MetricPoint cocoonPointRev2Metric1 = MetricPoint( + kValue3, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: kFrameworkRevision2, + kNameKey: kTaskName, + kSubResultKey: kMetric1, + kUnitKey: 's', + }, + ); + + final MetricPoint cocoonPointBetaRev1Metric1 = MetricPoint( + kValue1, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: kFrameworkRevision1, + kNameKey: 'beta/$kTaskName', + kSubResultKey: kMetric1, + kUnitKey: 's', + 'branch': 'beta', + }, + ); + + final MetricPoint cocoonPointBetaRev1Metric1BadBranch = MetricPoint( + kValue1, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: kFrameworkRevision1, + kNameKey: kTaskName, + kSubResultKey: kMetric1, + kUnitKey: 's', + + // If we only add this 'branch' tag without changing the test or sub-result name, an exception + // would be thrown as Skia Perf currently only supports the same set of tags for a pair of + // kNameKey and kSubResultKey values. So to support branches, one also has to add the branch + // name to the test name. + 'branch': 'beta', + }, + ); + + const String engineMetricName = 'BM_PaintRecordInit'; + const String engineRevision = 'ca799fa8b2254d09664b78ee80c43b434788d112'; + const double engineValue1 = 101; + const double engineValue2 = 102; + + final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint( + engineMetricName, + engineValue1, + engineRevision, + moreTags: const { + kSubResultKey: 'cpu_time', + kUnitKey: 'ns', + 'date': '2019-12-17 15:14:14', + 'num_cpus': '56', + 'mhz_per_cpu': '2594', + 'cpu_scaling_enabled': 'true', + 'library_build_type': 'release', + }, + ); + + final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint( + engineMetricName, + engineValue2, + engineRevision, + moreTags: const { + kSubResultKey: 'real_time', + kUnitKey: 'ns', + 'date': '2019-12-17 15:14:14', + 'num_cpus': '56', + 'mhz_per_cpu': '2594', + 'cpu_scaling_enabled': 'true', + 'library_build_type': 'release', + }, + ); + + test('Throw if invalid points are converted to SkiaPoint', () { + final MetricPoint noGithubRepoPoint = MetricPoint( + kValue1, + const { + kGitRevisionKey: kFrameworkRevision1, + kNameKey: kTaskName, + }, + ); + + final MetricPoint noGitRevisionPoint = MetricPoint( + kValue1, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kNameKey: kTaskName, + }, + ); + + final MetricPoint noTestNamePoint = MetricPoint( + kValue1, + const { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: kFrameworkRevision1, + }, + ); + + expect(() => SkiaPerfPoint.fromPoint(noGithubRepoPoint), throwsA(anything)); + expect( + () => SkiaPerfPoint.fromPoint(noGitRevisionPoint), throwsA(anything)); + expect(() => SkiaPerfPoint.fromPoint(noTestNamePoint), throwsA(anything)); + }); + + test('Correctly convert a metric point from cocoon to SkiaPoint', () { + final SkiaPerfPoint skiaPoint1 = + SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1); + expect(skiaPoint1, isNotNull); + expect(skiaPoint1.testName, equals(kTaskName)); + expect(skiaPoint1.subResult, equals(kMetric1)); + expect(skiaPoint1.value, equals(cocoonPointRev1Metric1.value)); + expect(skiaPoint1.jsonUrl, isNull); // Not inserted yet + }); + + test('Cocoon points correctly encode into Skia perf json format', () { + final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1); + final SkiaPerfPoint p2 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric2); + final SkiaPerfPoint p3 = + SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1); + + const JsonEncoder encoder = JsonEncoder.withIndent(' '); + + expect( + encoder + .convert(SkiaPerfPoint.toSkiaPerfJson([p1, p2, p3])), + equals(''' +{ + "gitHash": "9011cece2595447eea5dd91adaa241c1c9ef9a33", + "results": { + "analyzer_benchmark": { + "default": { + "flutter_repo_batch_maximum": 1.0, + "options": { + "unit": "s" + }, + "flutter_repo_watch_maximum": 2.0 + } + }, + "beta/analyzer_benchmark": { + "default": { + "flutter_repo_batch_maximum": 1.0, + "options": { + "branch": "beta", + "unit": "s" + } + } + } + } +}''')); + }); + + test('Engine metric points correctly encode into Skia perf json format', () { + const JsonEncoder encoder = JsonEncoder.withIndent(' '); + expect( + encoder.convert(SkiaPerfPoint.toSkiaPerfJson([ + SkiaPerfPoint.fromPoint(enginePoint1), + SkiaPerfPoint.fromPoint(enginePoint2), + ])), + equals( + ''' +{ + "gitHash": "ca799fa8b2254d09664b78ee80c43b434788d112", + "results": { + "BM_PaintRecordInit": { + "default": { + "cpu_time": 101.0, + "options": { + "cpu_scaling_enabled": "true", + "library_build_type": "release", + "mhz_per_cpu": "2594", + "num_cpus": "56", + "unit": "ns" + }, + "real_time": 102.0 + } + } + } +}''', + ), + ); + }); + + test( + 'Throw if engine points with the same test name but different options are converted to ' + 'Skia perf points', () { + final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint( + 'BM_PaintRecordInit', + 101, + 'ca799fa8b2254d09664b78ee80c43b434788d112', + moreTags: const { + kSubResultKey: 'cpu_time', + kUnitKey: 'ns', + 'cpu_scaling_enabled': 'true', + }, + ); + final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint( + 'BM_PaintRecordInit', + 102, + 'ca799fa8b2254d09664b78ee80c43b434788d112', + moreTags: const { + kSubResultKey: 'real_time', + kUnitKey: 'ns', + 'cpu_scaling_enabled': 'false', + }, + ); + + const JsonEncoder encoder = JsonEncoder.withIndent(' '); + expect( + () => encoder.convert(SkiaPerfPoint.toSkiaPerfJson([ + SkiaPerfPoint.fromPoint(enginePoint1), + SkiaPerfPoint.fromPoint(enginePoint2), + ])), + throwsA(anything), + ); + }); + + test( + 'Throw if two Cocoon metric points with the same name and subResult keys ' + 'but different options are converted to Skia perf points', () { + final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1); + final SkiaPerfPoint p2 = + SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1BadBranch); + + expect( + () => SkiaPerfPoint.toSkiaPerfJson([p1, p2]), + throwsA(anything), + ); + }); + + test('SkiaPerfGcsAdaptor computes name correctly', () async { + final MockGithubHelper mockHelper = MockGithubHelper(); + when(mockHelper.getCommitDateTime( + kFlutterFrameworkRepo, kFrameworkRevision1)) + .thenAnswer((_) => Future.value(DateTime(2019, 12, 4, 23))); + expect( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, + kFrameworkRevision1, + githubHelper: mockHelper, + ), + equals('flutter-flutter/2019/12/04/23/$kFrameworkRevision1/values.json'), + ); + when(mockHelper.getCommitDateTime(kFlutterEngineRepo, kEngineRevision1)) + .thenAnswer((_) => Future.value(DateTime(2019, 12, 3, 20))); + expect( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterEngineRepo, + kEngineRevision1, + githubHelper: mockHelper, + ), + equals('flutter-engine/2019/12/03/20/$kEngineRevision1/values.json'), + ); + when(mockHelper.getCommitDateTime(kFlutterEngineRepo, kEngineRevision2)) + .thenAnswer((_) => Future.value(DateTime(2020, 1, 3, 15))); + expect( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterEngineRepo, + kEngineRevision2, + githubHelper: mockHelper, + ), + equals('flutter-engine/2020/01/03/15/$kEngineRevision2/values.json'), + ); + }); + + test('Successfully read mock GCS that fails 1st time with 504', () async { + final MockBucket testBucket = MockBucket(); + final SkiaPerfGcsAdaptor skiaPerfGcs = SkiaPerfGcsAdaptor(testBucket); + + final String testObjectName = await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision1); + + final List writePoints = [ + SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1), + ]; + final String skiaPerfJson = + jsonEncode(SkiaPerfPoint.toSkiaPerfJson(writePoints)); + await skiaPerfGcs.writePoints(testObjectName, writePoints); + verify(testBucket.writeBytes(testObjectName, utf8.encode(skiaPerfJson))); + + // Emulate the first network request to fail with 504. + when(testBucket.info(testObjectName)) + .thenThrow(DetailedApiRequestError(504, 'Test Failure')); + + final MockObjectInfo mockObjectInfo = MockObjectInfo(); + when(mockObjectInfo.downloadLink) + .thenReturn(Uri.https('test.com', 'mock.json')); + when(testBucket.info(testObjectName)) + .thenAnswer((_) => Future.value(mockObjectInfo)); + when(testBucket.read(testObjectName)) + .thenAnswer((_) => Stream>.value(utf8.encode(skiaPerfJson))); + + final List readPoints = + await skiaPerfGcs.readPoints(testObjectName); + expect(readPoints.length, equals(1)); + expect(readPoints[0].testName, kTaskName); + expect(readPoints[0].subResult, kMetric1); + expect(readPoints[0].value, kValue1); + expect(readPoints[0].githubRepo, kFlutterFrameworkRepo); + expect(readPoints[0].gitHash, kFrameworkRevision1); + expect(readPoints[0].jsonUrl, 'https://test.com/mock.json'); + }); + + test('Return empty list if the GCS file does not exist', () async { + final MockBucket testBucket = MockBucket(); + final SkiaPerfGcsAdaptor skiaPerfGcs = SkiaPerfGcsAdaptor(testBucket); + final String testObjectName = await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision1); + when(testBucket.info(testObjectName)) + .thenThrow(Exception('No such object')); + expect((await skiaPerfGcs.readPoints(testObjectName)).length, 0); + }); + + // The following is for integration tests. + Bucket testBucket; + GcsLock testLock; + final Map credentialsJson = getTestGcpCredentialsJson(); + if (credentialsJson != null) { + final ServiceAccountCredentials credentials = + ServiceAccountCredentials.fromJson(credentialsJson); + + final AutoRefreshingAuthClient client = + await clientViaServiceAccount(credentials, Storage.SCOPES); + final Storage storage = + Storage(client, credentialsJson['project_id'] as String); + + const String kTestBucketName = 'flutter-skia-perf-test'; + + assert(await storage.bucketExists(kTestBucketName)); + testBucket = storage.bucket(kTestBucketName); + testLock = GcsLock(client, kTestBucketName); + } + + Future skiaPerfGcsAdapterIntegrationTest() async { + final SkiaPerfGcsAdaptor skiaPerfGcs = SkiaPerfGcsAdaptor(testBucket); + + final String testObjectName = await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision1); + + await skiaPerfGcs.writePoints(testObjectName, [ + SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1), + SkiaPerfPoint.fromPoint(cocoonPointRev1Metric2), + ]); + + final List points = + await skiaPerfGcs.readPoints(testObjectName); + expect(points.length, equals(2)); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.testName), [kTaskName]); + expectSetMatch(points.map((SkiaPerfPoint p) => p.subResult), + [kMetric1, kMetric2]); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.value), [kValue1, kValue2]); + expectSetMatch(points.map((SkiaPerfPoint p) => p.githubRepo), + [kFlutterFrameworkRepo]); + expectSetMatch(points.map((SkiaPerfPoint p) => p.gitHash), + [kFrameworkRevision1]); + for (int i = 0; i < 2; i += 1) { + expect(points[0].jsonUrl, startsWith('https://')); + } + } + + Future skiaPerfGcsIntegrationTestWithEnginePoints() async { + final SkiaPerfGcsAdaptor skiaPerfGcs = SkiaPerfGcsAdaptor(testBucket); + + final String testObjectName = await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterEngineRepo, engineRevision); + + await skiaPerfGcs.writePoints(testObjectName, [ + SkiaPerfPoint.fromPoint(enginePoint1), + SkiaPerfPoint.fromPoint(enginePoint2), + ]); + + final List points = + await skiaPerfGcs.readPoints(testObjectName); + expect(points.length, equals(2)); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.testName), + [engineMetricName, engineMetricName], + ); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.value), + [engineValue1, engineValue2], + ); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.githubRepo), + [kFlutterEngineRepo], + ); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.gitHash), [engineRevision]); + for (int i = 0; i < 2; i += 1) { + expect(points[0].jsonUrl, startsWith('https://')); + } + } + + // To run the following integration tests, there must be a valid Google Cloud + // Project service account credentials in secret/test_gcp_credentials.json so + // `testBucket` won't be null. Currently, these integration tests are skipped + // in the CI, and only verified locally. + test( + 'SkiaPerfGcsAdaptor passes integration test with Google Cloud Storage', + skiaPerfGcsAdapterIntegrationTest, + skip: testBucket == null, + ); + + test( + 'SkiaPerfGcsAdaptor integration test with engine points', + skiaPerfGcsIntegrationTestWithEnginePoints, + skip: testBucket == null, + ); + + // `SkiaPerfGcsAdaptor.computeObjectName` uses `GithubHelper` which requires + // network connections. Hence we put them as integration tests instead of unit + // tests. + test( + 'SkiaPerfGcsAdaptor integration test for name computations', + () async { + expect( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision1), + equals( + 'flutter-flutter/2019/12/04/23/$kFrameworkRevision1/values.json'), + ); + expect( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterEngineRepo, kEngineRevision1), + equals('flutter-engine/2019/12/03/20/$kEngineRevision1/values.json'), + ); + expect( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterEngineRepo, kEngineRevision2), + equals('flutter-engine/2020/01/03/15/$kEngineRevision2/values.json'), + ); + }, + skip: testBucket == null, + ); + + test('SkiaPerfDestination correctly updates points', () async { + final SkiaPerfGcsAdaptor mockGcs = MockSkiaPerfGcsAdaptor(); + final GcsLock mockLock = MockGcsLock(); + final SkiaPerfDestination dst = SkiaPerfDestination(mockGcs, mockLock); + await dst.update([cocoonPointRev1Metric1]); + await dst.update([cocoonPointRev1Metric2]); + List points = await mockGcs.readPoints( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision1)); + expect(points.length, equals(2)); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.testName), [kTaskName]); + expectSetMatch(points.map((SkiaPerfPoint p) => p.subResult), + [kMetric1, kMetric2]); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.value), [kValue1, kValue2]); + + final MetricPoint updated = + MetricPoint(kValue3, cocoonPointRev1Metric1.tags); + + await dst.update([updated, cocoonPointRev2Metric1]); + + points = await mockGcs.readPoints( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision2)); + expect(points.length, equals(1)); + expect(points[0].gitHash, equals(kFrameworkRevision2)); + expect(points[0].value, equals(kValue3)); + + points = await mockGcs.readPoints( + await SkiaPerfGcsAdaptor.computeObjectName( + kFlutterFrameworkRepo, kFrameworkRevision1)); + expectSetMatch( + points.map((SkiaPerfPoint p) => p.value), [kValue2, kValue3]); + }); + + Future skiaPerfDestinationIntegrationTest() async { + final SkiaPerfDestination destination = + SkiaPerfDestination(SkiaPerfGcsAdaptor(testBucket), testLock); + await destination.update([cocoonPointRev1Metric1]); + } + + test( + 'SkiaPerfDestination integration test', + skiaPerfDestinationIntegrationTest, + skip: testBucket == null, + ); +} diff --git a/packages/metrics_center/test/utility.dart b/packages/metrics_center/test/utility.dart new file mode 100644 index 0000000000..6dacd48b0a --- /dev/null +++ b/packages/metrics_center/test/utility.dart @@ -0,0 +1,23 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'common.dart'; + +// This will be used in many of our unit tests. +void expectSetMatch(Iterable actual, Iterable expected) { + expect(Set.from(actual), equals(Set.from(expected))); +} + +// May return null if the credentials file doesn't exist. +Map getTestGcpCredentialsJson() { + final File f = File('secret/test_gcp_credentials.json'); + if (!f.existsSync()) { + return null; + } + return jsonDecode(File('secret/test_gcp_credentials.json').readAsStringSync()) + as Map; +}