ionic-framework/core/scripts/screenshot/ci.js

const IonicConnector = require('./ionic');
const fs = require('fs');
const path = require('path');
const S3 = require('aws-sdk/clients/s3');
const execa = require('execa');
const stream = require('stream');

const BUILD_URL = 'https://github.com/ionic-team/ionic/commit/';
const S3_BUCKET = 'screenshot.ionicframework.com';
const s3 = new S3({ apiVersion: '2006-03-01' });


class CIScreenshotConnector extends IonicConnector {

  async initBuild(opts) {
    const result = await execa.stdout('git', ['log', '-1', '--format=%H%n%an <%ae>%n%ct%n%s']);
    const [ sha1, author, timestamp, msg ] = result.split('\n');
    const sha1short = sha1.slice(0, 7);

    opts.buildId = sha1short;
    opts.buildMessage = msg;
    opts.buildAuthor = author;
    opts.buildUrl = BUILD_URL + sha1short;
    opts.previewUrl = `https://${S3_BUCKET}/${sha1short}`;
    opts.buildTimestamp = (timestamp * 1000);

    await super.initBuild(opts);
  }

  async uploadImage(image) {
    const file = path.join(this.imagesDir, image);
    const stream = fs.createReadStream(file);
    const key = `data/images/${image}`;

    await this.uploadStream(stream, key, { ContentType: 'image/png' });
  }

  async uploadStream(stream, key, extra = {}) {
    try {
      await s3.headObject({ Bucket: S3_BUCKET, Key: key }).promise();
    } catch (e) {
      if (e.statusCode !== 404) {
        throw e;
      }

      this.logger.debug(`uploading: ${key}`);
      await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: stream, ...extra }).promise();
    }
  }

  async pullMasterBuild() {
    await super.pullIonicMasterBuild();
  }

  async publishBuild(results) {
    const currentBuild = results.currentBuild;

    const timespan = this.logger.createTimeSpan(`publishing build started`);
    const images = currentBuild.screenshots.map(screenshot => screenshot.image);
    const imageBatches = [];

    while (images.length > 0) {
      imageBatches.push(images.splice(0, 10));
    }

    for (const batch of imageBatches) {
      await Promise.all(batch.map(async image => this.uploadImage(image)));
    }

    const buildBuffer = Buffer.from(JSON.stringify(currentBuild, undefined, 2));
    const buildStream = new stream.PassThrough();
    buildStream.end(buildBuffer);

    const uploads = [
      this.uploadStream(buildStream, `data/builds/${currentBuild.id}.json`, { ContentType: 'application/json' }),
    ];

    if (this.updateMaster) {
      // master build
      // update the master data with this current build
      // no need to upload a compare data
      const buildStream = new stream.PassThrough();
      buildStream.end(buildBuffer);
      const key = `data/builds/master.json`;
      this.logger.debug(`uploading: ${key}`);
      uploads.push(
        s3.upload({ Bucket: S3_BUCKET, Key: key, Body: buildStream, ContentType: 'application/json' }).promise()
      );

    } else {
      // PR build
      // not updating master
      // upload compare data of the PR against the master data
      const compare = results.compare;
      compare.url = `https://${S3_BUCKET}/${compare.a.id}/${compare.b.id}`;

      const compareBuffer = Buffer.from(JSON.stringify(compare, undefined, 2));
      const compareStream = new stream.PassThrough();
      compareStream.end(compareBuffer);
      uploads.push(
        this.uploadStream(compareStream, `data/compares/${compare.id}.json`, { ContentType: 'application/json' })
      );
    }

    await Promise.all(uploads);

    timespan.finish(`publishing build finished`);

    await this.uploadTests(results);

    return results;
  }

  async uploadTests(results) {
    const timespan = this.logger.createTimeSpan(`uploading tests started`);

    const appRoot = path.join(__dirname, '..', '..');
    let uploadPaths = [];

    const cssDir = path.join(appRoot, 'css');
    fs.readdirSync(cssDir).forEach(cssFile => {
      uploadPaths.push(path.join(cssDir, cssFile));
    });

    uploadPaths.push(path.join(appRoot, 'scripts', 'testing', 'styles.css'));

    const distDir = path.join(appRoot, 'dist');
    uploadPaths.push(path.join(distDir, 'ionic.js'));

    const distIonicDir = path.join(distDir, 'ionic');
    fs.readdirSync(distIonicDir).forEach(distIonicFile => {
      uploadPaths.push(path.join(distIonicDir, distIonicFile));
    });

    const distIonicSvgDir = path.join(distIonicDir, 'svg');
    fs.readdirSync(distIonicSvgDir).forEach(distIonicSvgFile => {
      uploadPaths.push(path.join(distIonicSvgDir, distIonicSvgFile));
    });

    results.currentBuild.screenshots.forEach(screenshot => {
      const testDir = path.dirname(screenshot.testPath);
      const testIndexHtml = path.join(appRoot, testDir, 'index.html');
      if (!uploadPaths.includes(testIndexHtml)) {
        uploadPaths.push(testIndexHtml);
      }
    });

    uploadPaths = uploadPaths.filter(p => p.endsWith('.js') || p.endsWith('.css') || p.endsWith('.html') || p.endsWith('.svg'));

    const fileCount = uploadPaths.length;

    const uploadBatches = [];
    while (uploadPaths.length > 0) {
      uploadBatches.push(uploadPaths.splice(0, 20));
    }

    for (const batch of uploadBatches) {
      await Promise.all(batch.map(async uploadPath => {
        const stream = fs.createReadStream(uploadPath);
        const relPath = path.relative(appRoot, uploadPath);
        const key = `data/tests/${results.currentBuild.id}/${relPath}`;

        let contentType = 'text/plain';
        if (uploadPath.endsWith('.js')) {
          contentType = 'application/javascript'
        } else if (uploadPath.endsWith('.css')) {
          contentType = 'text/css'
        } else if (uploadPath.endsWith('.html')) {
          contentType = 'text/html'
        } else if (uploadPath.endsWith('.svg')) {
          contentType = 'image/svg+xml'
        }

        this.logger.debug(`uploading: ${key} ${contentType}`);
        await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: stream, ContentType: contentType }).promise();
      }));
    }

    timespan.finish(`uploading tests finished: ${fileCount} files`);
  }

  async getScreenshotCache() {
    const timespan = this.logger.createTimeSpan(`get screenshot cache started`, true);

    try {
      const ws = fs.createWriteStream(this.screenshotCacheFilePath);
      const p = `/data/compares/screenshot-cache.json?ts=${Date.now()}`;
      await this.downloadToStream(ws, p);

    } catch (e) {
      this.logger.debug(e);
    }

    timespan.finish(`get screenshot cache finished`);

    return super.getScreenshotCache();
  }

  async updateScreenshotCache(cache, buildResults) {
    const timespan = this.logger.createTimeSpan(`update screenshot cache started`, true);

    cache = await super.updateScreenshotCache(cache, buildResults);

    const cacheBuffer = Buffer.from(JSON.stringify(cache, undefined, 2));
    const cacheStream = new stream.PassThrough();
    cacheStream.end(cacheBuffer);

    const key = `data/compares/screenshot-cache.json`;
    this.logger.debug(`uploading: ${key}`);

    await s3.upload({ Bucket: S3_BUCKET, Key: key, Body: cacheStream, ContentType: 'application/json' }).promise();

    timespan.finish(`update screenshot cache finished`);

    return cache;
  }

}

module.exports = CIScreenshotConnector;