[imitatioin_game] Made the imitation_game start reporting its results in the README.md. (#193)

packages/imitation_game/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+* Initial version (TBD).

packages/imitation_game/LICENSE

@@ -0,0 +1,27 @@
+Copyright 2019 The Chromium Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/imitation_game/README.md

@@ -34,7 +34,7 @@ tested.  As new tests are added please add to this list:
 ```text
 ./
 ├─ imitation_game.dart
-└─ tests/
+└─ imitation_tests/
    ├─ smiley/
    │  ├─ README.md
    │  ├─ flutter/
@@ -91,3 +91,13 @@ An implementation has to follow these rules:
 ```
 
 A single test run can report multiple numbers.
+
+## Results
+Date created: 2020-08-17 23:57:16.702500Z
+
+- smiley
+    - flutter
+      - startupTime: 0.561475s
+    - uikit
+      - startupTime: 0.373102068901062s
+

packages/imitation_game/bin/imitation_game.dart

@@ -1,6 +1,8 @@
 import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
+import 'package:mustache/mustache.dart';
+import 'package:imitation_game/README_template.dart';
 
 const int _port = 4040;
 
@@ -33,6 +35,14 @@ Future<List<FileSystemEntity>> findFiles(Directory dir, {FileFilter where}) {
   return completer.future;
 }
 
+String _makeMarkdownOutput(Map<String, dynamic> results) {
+  final Template template = Template(readmeTemplate, name: 'README.md');
+  final Map<String, dynamic> values = Map<String, dynamic>.from(results);
+  values['date'] = DateTime.now().toUtc();
+  final String output = template.renderString(values);
+  return output;
+}
+
 class _Script {
   _Script({this.path});
   String path;
@@ -75,6 +85,33 @@ class _ScriptRunner {
   }
 }
 
+/// Recursively converts a map of maps to a map of lists of maps.
+///
+/// For example:
+/// _map2List({'a': {'b': 123}}, ['foo', 'bar']) ->
+/// {
+///   'foo':[
+///     {
+///       'name': 'a',
+///       'bar': [{'name': 'b', 'value': 123}]
+///     }
+///   ]
+/// }
+Map<String, dynamic> _map2List(Map<String, dynamic> map, List<String> names) {
+  final List<Map<String, dynamic>> returnList = <Map<String, dynamic>>[];
+  final List<String> tail = names.sublist(1);
+  map.forEach((String key, dynamic value) {
+    final Map<String, dynamic> testResult = <String, dynamic>{'name': key};
+    if (tail.isEmpty) {
+      testResult['value'] = value;
+    } else {
+      testResult[tail.first] = _map2List(value, tail)[tail.first];
+    }
+    returnList.add(testResult);
+  });
+  return <String, dynamic>{names.first: returnList};
+}
+
 class _ImitationGame {
   final Map<String, dynamic> results = <String, dynamic>{};
   _ScriptRunner _scriptRunner;
@@ -95,6 +132,7 @@ class _ImitationGame {
       results[test][platform] = <String, dynamic>{};
     }
     data['results'].forEach((String k, dynamic v) {
+      // ignore: avoid_as
       results[test][platform][k] = v as double;
     });
     return _runNext();
@@ -148,6 +186,7 @@ Future<void> main() async {
         if (request.method == 'POST') {
           final String content = await utf8.decoder.bind(request).join();
           final Map<String, dynamic> data =
+              // ignore: avoid_as
               jsonDecode(content) as Map<String, dynamic>;
           print('$data');
           keepRunning = await game.handleResult(data);
@@ -163,8 +202,9 @@ Future<void> main() async {
       keepRunning = await game.handleTimeout();
     }
   }
-  const JsonEncoder encoder = JsonEncoder.withIndent('  ');
-  final String jsonResults = encoder.convert(game.results);
-  print('$jsonResults');
+
+  final Map<String, dynamic> markdownValues =
+      _map2List(game.results, <String>['tests', 'platforms', 'measurements']);
+  File('README.md').writeAsStringSync(_makeMarkdownOutput(markdownValues));
   await server.close(force: true);
 }

packages/imitation_game/imitation_tests/smiley/flutter/smiley/lib/main.dart

@@ -26,9 +26,7 @@ Future<void> _sendResult(double result) async {
     body: jsonEncode(<String, dynamic>{
       'test': 'smiley',
       'platform': 'flutter',
-      'results': <String, double>{
-        'startupTime': result
-      },
+      'results': <String, double>{'startupTime': result},
     }),
   );
   if (response.statusCode != 200) {

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/AppDelegate.h

@@ -4,8 +4,6 @@
 
 #import <UIKit/UIKit.h>
 
-@interface AppDelegate : UIResponder <UIApplicationDelegate>
-
+@interface AppDelegate : UIResponder<UIApplicationDelegate>
 
 @end
-

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/AppDelegate.m

@@ -10,28 +10,29 @@
 
 @implementation AppDelegate
 
-
-- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
   // Override point for customization after application launch.
   return YES;
 }
 
-
 #pragma mark - UISceneSession lifecycle
 
-
-- (UISceneConfiguration *)application:(UIApplication *)application configurationForConnectingSceneSession:(UISceneSession *)connectingSceneSession options:(UISceneConnectionOptions *)options {
+- (UISceneConfiguration *)application:(UIApplication *)application
+    configurationForConnectingSceneSession:(UISceneSession *)connectingSceneSession
+                                   options:(UISceneConnectionOptions *)options {
   // Called when a new scene session is being created.
   // Use this method to select a configuration to create the new scene with.
-  return [[UISceneConfiguration alloc] initWithName:@"Default Configuration" sessionRole:connectingSceneSession.role];
+  return [[UISceneConfiguration alloc] initWithName:@"Default Configuration"
+                                        sessionRole:connectingSceneSession.role];
 }
 
-
-- (void)application:(UIApplication *)application didDiscardSceneSessions:(NSSet<UISceneSession *> *)sceneSessions {
+- (void)application:(UIApplication *)application
+    didDiscardSceneSessions:(NSSet<UISceneSession *> *)sceneSessions {
   // Called when the user discards a scene session.
-  // If any sessions were discarded while the application was not running, this will be called shortly after application:didFinishLaunchingWithOptions.
-  // Use this method to release any resources that were specific to the discarded scenes, as they will not return.
+  // If any sessions were discarded while the application was not running, this will be called
+  // shortly after application:didFinishLaunchingWithOptions. Use this method to release any
+  // resources that were specific to the discarded scenes, as they will not return.
 }
 
-
 @end

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/SceneDelegate.h

@@ -4,9 +4,8 @@
 
 #import <UIKit/UIKit.h>
 
-@interface SceneDelegate : UIResponder <UIWindowSceneDelegate>
+@interface SceneDelegate : UIResponder<UIWindowSceneDelegate>
 
-@property (strong, nonatomic) UIWindow * window;
+@property(strong, nonatomic) UIWindow* window;
 
 @end
-

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/SceneDelegate.m

@@ -10,45 +10,43 @@
 
 @implementation SceneDelegate
 
-
-- (void)scene:(UIScene *)scene willConnectToSession:(UISceneSession *)session options:(UISceneConnectionOptions *)connectionOptions {
-  // Use this method to optionally configure and attach the UIWindow `window` to the provided UIWindowScene `scene`.
-  // If using a storyboard, the `window` property will automatically be initialized and attached to the scene.
-  // This delegate does not imply the connecting scene or session are new (see `application:configurationForConnectingSceneSession` instead).
+- (void)scene:(UIScene *)scene
+    willConnectToSession:(UISceneSession *)session
+                 options:(UISceneConnectionOptions *)connectionOptions {
+  // Use this method to optionally configure and attach the UIWindow `window` to the provided
+  // UIWindowScene `scene`. If using a storyboard, the `window` property will automatically be
+  // initialized and attached to the scene. This delegate does not imply the connecting scene or
+  // session are new (see `application:configurationForConnectingSceneSession` instead).
 }
 
-
 - (void)sceneDidDisconnect:(UIScene *)scene {
   // Called as the scene is being released by the system.
   // This occurs shortly after the scene enters the background, or when its session is discarded.
-  // Release any resources associated with this scene that can be re-created the next time the scene connects.
-  // The scene may re-connect later, as its session was not neccessarily discarded (see `application:didDiscardSceneSessions` instead).
+  // Release any resources associated with this scene that can be re-created the next time the scene
+  // connects. The scene may re-connect later, as its session was not neccessarily discarded (see
+  // `application:didDiscardSceneSessions` instead).
 }
 
-
 - (void)sceneDidBecomeActive:(UIScene *)scene {
   // Called when the scene has moved from an inactive state to an active state.
-  // Use this method to restart any tasks that were paused (or not yet started) when the scene was inactive.
+  // Use this method to restart any tasks that were paused (or not yet started) when the scene was
+  // inactive.
 }
 
-
 - (void)sceneWillResignActive:(UIScene *)scene {
   // Called when the scene will move from an active state to an inactive state.
   // This may occur due to temporary interruptions (ex. an incoming phone call).
 }
 
-
 - (void)sceneWillEnterForeground:(UIScene *)scene {
   // Called as the scene transitions from the background to the foreground.
   // Use this method to undo the changes made on entering the background.
 }
 
-
 - (void)sceneDidEnterBackground:(UIScene *)scene {
   // Called as the scene transitions from the foreground to the background.
-  // Use this method to save data, release shared resources, and store enough scene-specific state information
-  // to restore the scene back to its current state.
+  // Use this method to save data, release shared resources, and store enough scene-specific state
+  // information to restore the scene back to its current state.
 }
 
-
 @end

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/ViewController.h

@@ -8,4 +8,3 @@
 @property(nonatomic, strong) IBOutlet UIImageView* imageView;
 
 @end
-

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/ViewController.m

@@ -24,8 +24,7 @@ static int64_t loadStartupTime(NSError **error) {
   struct timeval startTime = proc.kp_proc.p_starttime;
   int64_t microsecondsInSecond = 1000000LL;
   int64_t microsecondsSinceEpoch =
-      (int64_t)(startTime.tv_sec * microsecondsInSecond) +
-      (int64_t)startTime.tv_usec;
+      (int64_t)(startTime.tv_sec * microsecondsInSecond) + (int64_t)startTime.tv_usec;
 
   return microsecondsSinceEpoch;
 }
@@ -34,16 +33,13 @@ static NSString *loadIpAddress() {
 #if TARGET_IPHONE_SIMULATOR
   return @"127.0.0.1:4040";
 #else
-  NSString *ipPath = [[NSBundle mainBundle] pathForResource:@"ip"
-                                                     ofType:@"txt"];
+  NSString *ipPath = [[NSBundle mainBundle] pathForResource:@"ip" ofType:@"txt"];
   NSError *err;
-  NSString *ipAddress = [NSString stringWithContentsOfFile:ipPath
-                                                  encoding:NSUTF8StringEncoding
-                                                     error:&err];
+  NSString *ipAddress =
+      [NSString stringWithContentsOfFile:ipPath encoding:NSUTF8StringEncoding error:&err];
   assert(err == nil);
-  return [ipAddress
-      stringByTrimmingCharactersInSet:[NSCharacterSet
-                                          whitespaceAndNewlineCharacterSet]];
+  return
+      [ipAddress stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]];
 #endif
 }
 
@@ -54,30 +50,27 @@ static void sendResults(NSTimeInterval result) {
   NSMutableURLRequest *urlRequest =
       [[NSMutableURLRequest alloc] initWithURL:[NSURL URLWithString:url]];
   NSDictionary *payload = @{
-    @"test": @"smiley",
-    @"platform": @"uikit",
+    @"test" : @"smiley",
+    @"platform" : @"uikit",
     @"results" : @{
       @"startupTime" : @(result),
     }
   };
   NSError *error;
-  NSData *jsonData = [NSJSONSerialization dataWithJSONObject:payload
-                                                     options:0
-                                                       error:&error];
+  NSData *jsonData = [NSJSONSerialization dataWithJSONObject:payload options:0 error:&error];
   assert(error == nil && jsonData);
   [urlRequest setHTTPMethod:@"POST"];
   [urlRequest setHTTPBody:jsonData];
 
   NSURLSession *session = [NSURLSession sharedSession];
-  NSURLSessionDataTask *dataTask = [session
-      dataTaskWithRequest:urlRequest
-        completionHandler:^(NSData *data, NSURLResponse *response,
-                            NSError *error) {
-          NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
-          if (httpResponse.statusCode != 200) {
-            NSLog(@"Error %@ '%@'", error, url);
-          }
-        }];
+  NSURLSessionDataTask *dataTask =
+      [session dataTaskWithRequest:urlRequest
+                 completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
+                   NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
+                   if (httpResponse.statusCode != 200) {
+                     NSLog(@"Error %@ '%@'", error, url);
+                   }
+                 }];
   [dataTask resume];
 }
 
@@ -95,10 +88,8 @@ static void sendResults(NSTimeInterval result) {
 
 - (void)loadView {
   [super loadView];
-  self.displayLink = [CADisplayLink displayLinkWithTarget:self
-                                                 selector:@selector(onTick:)];
-  [self.displayLink addToRunLoop:[NSRunLoop mainRunLoop]
-                         forMode:NSRunLoopCommonModes];
+  self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(onTick:)];
+  [self.displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes];
   self.imageView.image = [UIImage imageNamed:@"smiley"];
   _waitingForDraw = YES;
 }

packages/imitation_game/imitation_tests/smiley/uikit/smiley/smiley/main.m

@@ -5,11 +5,11 @@
 #import <UIKit/UIKit.h>
 #import "AppDelegate.h"
 
-int main(int argc, char * argv[]) {
-  NSString * appDelegateClassName;
+int main(int argc, char* argv[]) {
+  NSString* appDelegateClassName;
   @autoreleasepool {
-      // Setup code that might create autoreleased objects goes here.
-      appDelegateClassName = NSStringFromClass([AppDelegate class]);
+    // Setup code that might create autoreleased objects goes here.
+    appDelegateClassName = NSStringFromClass([AppDelegate class]);
   }
   return UIApplicationMain(argc, argv, nil, appDelegateClassName);
 }

packages/imitation_game/lib/README_template.dart

@@ -0,0 +1,109 @@
+/// Mustache template used for generating README.md.
+String readmeTemplate = """# Imitation Game
+
+## Description
+
+`imitation_game` is a platform for performing automated tests to compare the
+performance of different UI frameworks.  For example, how much memory does the
+same app written in Flutter and UIKit take?
+
+## Running all the tests
+
+You need a mobile device plugged into your computer and setup for development.
+The mobile device and the computer need to be on the same network, one that
+allows communication between computers since that's how the mobile phone will
+report its results to the computer.
+
+```sh
+dart imitation_game.dart
+```
+
+## Dependencies
+
+In order to run the tests you will need the union of all the platforms being
+tested.  As new tests are added please add to this list:
+
+### iOS
+
+- Flutter
+- Xcode
+- [ios_deploy](https://github.com/ios-control/ios-deploy) - used to launch apps
+  on the attached iOS device.
+
+## Example File Layout
+
+```text
+./
+├─ imitation_game.dart
+└─ tests/
+   ├─ smiley/
+   │  ├─ README.md
+   │  ├─ flutter/
+   │  │  ├─ run_ios.sh
+   │  │  └─ <flutter project files>
+   │  └─ uikit/
+   │     ├─ run_ios.sh
+   │     └─ <uikit project files>
+   └─ memory/
+      ├─ README.md
+      ├─ flutter/
+      │  ├─ run_ios.sh
+      │  └─ <flutter project files>
+      └─ uikit/
+         ├─ run_ios.sh
+         └─ <uikit project files>
+```
+
+Here there are 2 different tests with 2 different platform implementations.  The
+tests are named `smiley` and `memory`, they are both implemented on the
+platforms `flutter` and `uikit`.
+
+### Adding a test
+
+Tests should comprise of implementations on one or more platform.  The directory
+for the test should be added to `./tests`.  Inside that directory there should
+be a directory of implementations and a `README.md` file that explains the test.
+
+### Adding an implementation to a test
+
+An implementation has to follow these rules:
+
+- It needs to perform the same operations as the other implementations and
+  follow the description in the test's `README.md`.
+- It needs to contain a `run_ios.sh` script that will build and launch the test
+  on the connected device.
+- It should contain a file named `ip.txt` which will be overwritten by
+  `imitation_game.dart` with the ip address and port that should be used to
+  report results to.
+- It needs to report its results to the ip and port in the `ip.txt` via an HTTP
+  POST of JSON data.
+
+## Data format for results
+
+```json
+{
+  "test": "name_of_test",
+  "platform": "name_of_platform",
+  "results": {
+    "some_result_name": 1.23,
+    "some_result_name2": 4.56,
+  }
+}
+```
+
+A single test run can report multiple numbers.
+
+## Results
+Date created: {{date}}
+
+{{#tests}}
+- {{name}}
+  {{#platforms}}
+    - {{name}}
+    {{#measurements}}
+      - {{name}}: {{value}}s
+    {{/measurements}}
+  {{/platforms}}
+{{/tests}}
+
+""";

packages/imitation_game/lib/imitation_game.dart

@@ -0,0 +1 @@
+

packages/imitation_game/pubspec.yaml

@@ -0,0 +1,8 @@
+name: imitation_game
+version: 0.0.1
+description: Testing framework for comparing multiple frameworks' performance.
+homepage: https://github.com/flutter/packages/tree/master/packages/imitation_game
+dependencies:
+  mustache: ^1.1.1
+environment:
+  sdk: ">=1.8.0 <3.0.0"

packages/imitation_game/run.sh

@@ -0,0 +1 @@
+pub run imitation_game