chore(screenshot): local & CI integration (#15825)

core/package.json

@@ -39,6 +39,7 @@
     "@types/swiper": "4.2.1",
     "agadoo": "^1.0.0",
     "autoprefixer": "^9.0.2",
+    "aws-sdk": "^2.320.0",
     "chromedriver": "^2.38.3",
     "clean-css-cli": "^4.1.11",
     "fs-extra": "^7.0.0",
@@ -77,7 +78,7 @@
     "test": "stencil test --spec --e2e",
     "test.spec": "stencil test --spec",
     "test.e2e": "stencil test --e2e",
-    "test.screenshot": "stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/local.js",
+    "test.screenshot": "stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/dev.js",
     "test.screenshot.ci": "stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/ci.js --ci",
     "test.watch": "jest --watch --no-cache",
     "theme-app-build": "stencil build --dev --config scripts/theme-builder/stencil.config.js",

core/screenshot/.gitignore

@@ -1,7 +1,3 @@
-# only master screenshot data should be committed
 images
-local
-compare.html
-
-# temporarily ignoring master
-master
+builds
+compare.html

core/scripts/screenshot/ci.js

@@ -1,12 +1,70 @@
-const LocalScreenshotConnector = require('./local');
+const IonicConnector = require('./ionic');
 const fs = require('fs');
 const path = require('path');
+const S3 = require('aws-sdk/clients/s3');
+const execa = require('execa');
+const stream = require('stream');
+
+const S3_BUCKET = 'screenshot.ionicframework.com';
+const s3 = new S3({ apiVersion: '2006-03-01' });
 
 
-class CIScreenshotConnector extends LocalScreenshotConnector {
+class CIScreenshotConnector extends IonicConnector {
 
-  async publishBuild() {
+  async initBuild(opts) {
+    const result = await execa.stdout('git', ['log', '-1', '--format=%h%n%an <%ae>%n%ct%n%s']);
+    const [ sha1short, author, timestamp, msg ] = result.split('\n');
+
+    opts.buildId = sha1short;
+    opts.buildMessage = msg;
+    opts.buildAuthor = author;
+    opts.buildTimestamp = (timestamp * 1000);
+
+    await super.initBuild(opts);
+  }
+
+  async uploadImage(image) {
+    const file = path.join(this.imagesDir, image);
+    const stream = fs.createReadStream(file);
+    const key = `data/images/${image}`;
+
+    await this.uploadStream(stream, key);
+  }
+
+  async uploadStream(stream, key, extra = {}) {
+    try {
+      await s3.headObject({ Bucket: S3_BUCKET, Key: key }).promise();
+    } catch (e) {
+      if (e.statusCode !== 404) {
+        throw e;
+      }
+
+      this.logger.debug(`uploading: ${key}`);
+      await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: stream, ...extra }).promise();
+    }
+  }
+
+  async pullMasterBuild() {
+    await super.pullIonicMasterBuild();
+  }
+
+  async publishBuild(build) {
     const timespan = this.logger.createTimeSpan(`publishing build started`);
+    const images = build.screenshots.map(screenshot => screenshot.image);
+    const buildBuffer = Buffer.from(JSON.stringify(build, undefined, 2));
+    const buildStream = new stream.PassThrough();
+    buildStream.end(buildBuffer);
+
+    await Promise.all(images.map(async image => this.uploadImage(image)));
+    await this.uploadStream(buildStream, `data/builds/${build.id}.json`, { ContentType: 'application/json' });
+
+    if (this.updateMaster) {
+      const buildStream = new stream.PassThrough();
+      buildStream.end(buildBuffer);
+      const key = `data/builds/master.json`;
+      this.logger.debug(`uploading: ${key}`);
+      await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: buildStream, ContentType: 'application/json' }).promise();
+    }
 
     timespan.finish(`publishing build finished`);
   }

core/scripts/screenshot/dev.js

@@ -0,0 +1,14 @@
+const IonicConnector = require('./ionic');
+
+class DevConnector extends IonicConnector {
+
+  async pullMasterBuild() {
+    await super.pullIonicMasterBuild();
+
+    const masterBuild = await super.getMasterBuild();
+    await this.generateJsonpDataUris(masterBuild);
+  }
+
+}
+
+module.exports = DevConnector;

core/scripts/screenshot/ionic.js

@@ -0,0 +1,76 @@
+const ScreenshotConnector = require('@stencil/core/screenshot/local-connector');
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+
+class IonicConnector extends ScreenshotConnector {
+
+  async pullIonicMasterBuild() {
+    const timespan = this.logger.createTimeSpan(`pull master screenshot images started`);
+
+    const ws = fs.createWriteStream(this.masterBuildFilePath);
+    const p = `/data/builds/master.json`;
+    await this.downloadToStream(ws, p);
+    const masterBuild = await this.getMasterBuild();
+
+    const masterImageNames = masterBuild.screenshots.map(s => s.image);
+    const missingImages = masterImageNames.filter(masterImageName => {
+      try {
+        const masterImagePath = path.join(this.imagesDir, masterImageName);
+        fs.accessSync(masterImagePath);
+        return false;
+      } catch (e) {}
+      return true
+    });
+
+    if (missingImages.length > 0) {
+      await Promise.all(missingImages.map(async image => {
+        this.logger.debug(`downloading: ${image}`);
+
+        try {
+          await this.downloadImage(image);
+        } catch (e) {
+          this.logger.error(`Error with ${image}: ${e}`);
+          throw e;
+        }
+      }));
+    }
+
+    timespan.finish(`pull master screenshot images finished`);
+  }
+
+  async downloadToStream(stream, p) {
+    return new Promise((resolve, reject) => {
+      const req = https.request({
+        method: 'GET',
+        hostname: 'screenshot.ionicframework.com',
+        path: p,
+      });
+
+      req.on('response', res => {
+        if (res.statusCode !== 200) {
+          return reject(new Error(`Bad Status Code: ${res.statusCode}`));
+        }
+
+        stream.on('error', reject);
+        stream.on('close', resolve);
+
+        res.on('error', reject);
+        res.pipe(stream);
+      });
+
+      req.on('error', reject);
+      req.end();
+    });
+  }
+
+  async downloadImage(image) {
+    const stream = fs.createWriteStream(path.join(this.imagesDir, image));
+    const p = `/data/images/${image}`;
+
+    await this.downloadToStream(stream, p);
+  }
+
+}
+
+module.exports = IonicConnector;

core/scripts/screenshot/local.js

@@ -1,33 +0,0 @@
-const ScreenshotConnector = require('@stencil/core/screenshot/screenshot-connector');
-const fs = require('fs');
-const path = require('path');
-
-
-class LocalScreenshotConnector extends ScreenshotConnector {
-
-  async pullMasterImages() {
-    const timespan = this.logger.createTimeSpan(`pull master screenshot images started`);
-
-    const masterFilePaths = (fs.readdirSync(this.masterDir)).map(f => path.join(this.masterDir, f)).filter(f => f.endsWith('.json'));
-    const masterScreenshots = masterFilePaths.map(f => JSON.parse(fs.readFileSync(f, 'utf-8')));
-    const masterImageNames = masterScreenshots.map(s => s.image);
-    const missingImages = masterImageNames.filter(masterImageName => {
-      try {
-        const masterImagePath = path.join(this.imagesDir, masterImageName);
-        fs.accessSync(masterImagePath);
-        return false;
-      } catch (e) {}
-      return true
-    });
-
-    missingImages.forEach(missingImage => {
-      const url = missingImage;
-      this.logger.info(`downloading: ${url}`);
-    });
-
-    timespan.finish(`pull master screenshot images finished`);
-  }
-
-}
-
-module.exports = LocalScreenshotConnector;

core/stencil.config.ts

@@ -108,21 +108,25 @@ export const config: Config = {
     emulate: [
       {
         userAgent: 'iPhone',
-        width: 400,
-        height: 800,
-        deviceScaleFactor: 2,
-        isMobile: true,
-        hasTouch: true,
-        isLandscape: false
+        viewport: {
+          width: 400,
+          height: 800,
+          deviceScaleFactor: 2,
+          isMobile: true,
+          hasTouch: true,
+          isLandscape: false
+        }
       },
       {
         userAgent: 'Android',
-        width: 400,
-        height: 800,
-        deviceScaleFactor: 2,
-        isMobile: true,
-        hasTouch: true,
-        isLandscape: false
+        viewport: {
+          width: 400,
+          height: 800,
+          deviceScaleFactor: 2,
+          isMobile: true,
+          hasTouch: true,
+          isLandscape: false
+        }
       }
     ]
   },