From a87a8738ed1d76c8f2bbe03b0a2069d9edcc6993 Mon Sep 17 00:00:00 2001 From: dwieeb Date: Mon, 8 Oct 2018 10:28:39 -0500 Subject: [PATCH] chore(screenshot): local & CI integration (#15825) --- core/package.json | 3 +- core/screenshot/.gitignore | 8 +--- core/scripts/screenshot/ci.js | 64 +++++++++++++++++++++++++-- core/scripts/screenshot/dev.js | 14 ++++++ core/scripts/screenshot/ionic.js | 76 ++++++++++++++++++++++++++++++++ core/scripts/screenshot/local.js | 33 -------------- core/stencil.config.ts | 28 +++++++----- 7 files changed, 171 insertions(+), 55 deletions(-) create mode 100644 core/scripts/screenshot/dev.js create mode 100644 core/scripts/screenshot/ionic.js delete mode 100644 core/scripts/screenshot/local.js diff --git a/core/package.json b/core/package.json index 415e52c036..d86423e26d 100644 --- a/core/package.json +++ b/core/package.json @@ -39,6 +39,7 @@ "@types/swiper": "4.2.1", "agadoo": "^1.0.0", "autoprefixer": "^9.0.2", + "aws-sdk": "^2.320.0", "chromedriver": "^2.38.3", "clean-css-cli": "^4.1.11", "fs-extra": "^7.0.0", @@ -77,7 +78,7 @@ "test": "stencil test --spec --e2e", "test.spec": "stencil test --spec", "test.e2e": "stencil test --e2e", - "test.screenshot": "stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/local.js", + "test.screenshot": "stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/dev.js", "test.screenshot.ci": "stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/ci.js --ci", "test.watch": "jest --watch --no-cache", "theme-app-build": "stencil build --dev --config scripts/theme-builder/stencil.config.js", diff --git a/core/screenshot/.gitignore b/core/screenshot/.gitignore index c28b6b84c4..563eaa94e4 100644 --- a/core/screenshot/.gitignore +++ b/core/screenshot/.gitignore @@ -1,7 +1,3 @@ -# only master screenshot data should be committed images -local -compare.html - -# temporarily ignoring master -master \ No newline at end of file +builds +compare.html \ No newline at end of file diff --git a/core/scripts/screenshot/ci.js b/core/scripts/screenshot/ci.js index 1c9b7b49f1..20104047d9 100644 --- a/core/scripts/screenshot/ci.js +++ b/core/scripts/screenshot/ci.js @@ -1,12 +1,70 @@ -const LocalScreenshotConnector = require('./local'); +const IonicConnector = require('./ionic'); const fs = require('fs'); const path = require('path'); +const S3 = require('aws-sdk/clients/s3'); +const execa = require('execa'); +const stream = require('stream'); + +const S3_BUCKET = 'screenshot.ionicframework.com'; +const s3 = new S3({ apiVersion: '2006-03-01' }); -class CIScreenshotConnector extends LocalScreenshotConnector { +class CIScreenshotConnector extends IonicConnector { - async publishBuild() { + async initBuild(opts) { + const result = await execa.stdout('git', ['log', '-1', '--format=%h%n%an <%ae>%n%ct%n%s']); + const [ sha1short, author, timestamp, msg ] = result.split('\n'); + + opts.buildId = sha1short; + opts.buildMessage = msg; + opts.buildAuthor = author; + opts.buildTimestamp = (timestamp * 1000); + + await super.initBuild(opts); + } + + async uploadImage(image) { + const file = path.join(this.imagesDir, image); + const stream = fs.createReadStream(file); + const key = `data/images/${image}`; + + await this.uploadStream(stream, key); + } + + async uploadStream(stream, key, extra = {}) { + try { + await s3.headObject({ Bucket: S3_BUCKET, Key: key }).promise(); + } catch (e) { + if (e.statusCode !== 404) { + throw e; + } + + this.logger.debug(`uploading: ${key}`); + await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: stream, ...extra }).promise(); + } + } + + async pullMasterBuild() { + await super.pullIonicMasterBuild(); + } + + async publishBuild(build) { const timespan = this.logger.createTimeSpan(`publishing build started`); + const images = build.screenshots.map(screenshot => screenshot.image); + const buildBuffer = Buffer.from(JSON.stringify(build, undefined, 2)); + const buildStream = new stream.PassThrough(); + buildStream.end(buildBuffer); + + await Promise.all(images.map(async image => this.uploadImage(image))); + await this.uploadStream(buildStream, `data/builds/${build.id}.json`, { ContentType: 'application/json' }); + + if (this.updateMaster) { + const buildStream = new stream.PassThrough(); + buildStream.end(buildBuffer); + const key = `data/builds/master.json`; + this.logger.debug(`uploading: ${key}`); + await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: buildStream, ContentType: 'application/json' }).promise(); + } timespan.finish(`publishing build finished`); } diff --git a/core/scripts/screenshot/dev.js b/core/scripts/screenshot/dev.js new file mode 100644 index 0000000000..89627eebf9 --- /dev/null +++ b/core/scripts/screenshot/dev.js @@ -0,0 +1,14 @@ +const IonicConnector = require('./ionic'); + +class DevConnector extends IonicConnector { + + async pullMasterBuild() { + await super.pullIonicMasterBuild(); + + const masterBuild = await super.getMasterBuild(); + await this.generateJsonpDataUris(masterBuild); + } + +} + +module.exports = DevConnector; diff --git a/core/scripts/screenshot/ionic.js b/core/scripts/screenshot/ionic.js new file mode 100644 index 0000000000..0a59a720c2 --- /dev/null +++ b/core/scripts/screenshot/ionic.js @@ -0,0 +1,76 @@ +const ScreenshotConnector = require('@stencil/core/screenshot/local-connector'); +const fs = require('fs'); +const path = require('path'); +const https = require('https'); + +class IonicConnector extends ScreenshotConnector { + + async pullIonicMasterBuild() { + const timespan = this.logger.createTimeSpan(`pull master screenshot images started`); + + const ws = fs.createWriteStream(this.masterBuildFilePath); + const p = `/data/builds/master.json`; + await this.downloadToStream(ws, p); + const masterBuild = await this.getMasterBuild(); + + const masterImageNames = masterBuild.screenshots.map(s => s.image); + const missingImages = masterImageNames.filter(masterImageName => { + try { + const masterImagePath = path.join(this.imagesDir, masterImageName); + fs.accessSync(masterImagePath); + return false; + } catch (e) {} + return true + }); + + if (missingImages.length > 0) { + await Promise.all(missingImages.map(async image => { + this.logger.debug(`downloading: ${image}`); + + try { + await this.downloadImage(image); + } catch (e) { + this.logger.error(`Error with ${image}: ${e}`); + throw e; + } + })); + } + + timespan.finish(`pull master screenshot images finished`); + } + + async downloadToStream(stream, p) { + return new Promise((resolve, reject) => { + const req = https.request({ + method: 'GET', + hostname: 'screenshot.ionicframework.com', + path: p, + }); + + req.on('response', res => { + if (res.statusCode !== 200) { + return reject(new Error(`Bad Status Code: ${res.statusCode}`)); + } + + stream.on('error', reject); + stream.on('close', resolve); + + res.on('error', reject); + res.pipe(stream); + }); + + req.on('error', reject); + req.end(); + }); + } + + async downloadImage(image) { + const stream = fs.createWriteStream(path.join(this.imagesDir, image)); + const p = `/data/images/${image}`; + + await this.downloadToStream(stream, p); + } + +} + +module.exports = IonicConnector; diff --git a/core/scripts/screenshot/local.js b/core/scripts/screenshot/local.js deleted file mode 100644 index c8249ee950..0000000000 --- a/core/scripts/screenshot/local.js +++ /dev/null @@ -1,33 +0,0 @@ -const ScreenshotConnector = require('@stencil/core/screenshot/screenshot-connector'); -const fs = require('fs'); -const path = require('path'); - - -class LocalScreenshotConnector extends ScreenshotConnector { - - async pullMasterImages() { - const timespan = this.logger.createTimeSpan(`pull master screenshot images started`); - - const masterFilePaths = (fs.readdirSync(this.masterDir)).map(f => path.join(this.masterDir, f)).filter(f => f.endsWith('.json')); - const masterScreenshots = masterFilePaths.map(f => JSON.parse(fs.readFileSync(f, 'utf-8'))); - const masterImageNames = masterScreenshots.map(s => s.image); - const missingImages = masterImageNames.filter(masterImageName => { - try { - const masterImagePath = path.join(this.imagesDir, masterImageName); - fs.accessSync(masterImagePath); - return false; - } catch (e) {} - return true - }); - - missingImages.forEach(missingImage => { - const url = missingImage; - this.logger.info(`downloading: ${url}`); - }); - - timespan.finish(`pull master screenshot images finished`); - } - -} - -module.exports = LocalScreenshotConnector; diff --git a/core/stencil.config.ts b/core/stencil.config.ts index ab1be31093..2d253c31bc 100644 --- a/core/stencil.config.ts +++ b/core/stencil.config.ts @@ -108,21 +108,25 @@ export const config: Config = { emulate: [ { userAgent: 'iPhone', - width: 400, - height: 800, - deviceScaleFactor: 2, - isMobile: true, - hasTouch: true, - isLandscape: false + viewport: { + width: 400, + height: 800, + deviceScaleFactor: 2, + isMobile: true, + hasTouch: true, + isLandscape: false + } }, { userAgent: 'Android', - width: 400, - height: 800, - deviceScaleFactor: 2, - isMobile: true, - hasTouch: true, - isLandscape: false + viewport: { + width: 400, + height: 800, + deviceScaleFactor: 2, + isMobile: true, + hasTouch: true, + isLandscape: false + } } ] },