From 2c8809d3cf8de83ab018fb0fa86de45a7084428d Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Thu, 18 Jul 2019 13:48:35 +0200 Subject: [PATCH] @grafana/toolkit: integrate latest improvements (#18168) * @grafana/toolkit: make ts-loader ignore files that are not bundled * @grafana/toolkit: improve the circleci task (#18133) This PR makes some minor improvements to the circle task Adds build info to plugin.json adds dependencies to deployed artifacts Makes sure prettier has content before writing (avoid writing empty files) renames 'bundle' to 'package' and saves stuff in a 'packages' file * @grafana/toolkit: enable plugin themes to work with common stylesheet (#18160) This makes it possible to use themes styleshheet files and stylesheet imports at the same time. The problem occurred when trying to migrate polystat panel to toolkit: grafana/grafana-polystat-panel#62 --- lerna.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-runtime/package.json | 2 +- packages/grafana-toolkit/README.md | 3 + packages/grafana-toolkit/package.json | 2 +- packages/grafana-toolkit/src/cli/index.ts | 16 +- .../src/cli/tasks/package.build.ts | 17 ++ .../src/cli/tasks/plugin.build.ts | 19 +- .../src/cli/tasks/plugin.ci.ts | 245 ++++++------------ .../src/cli/tasks/plugin/ci.ts | 214 +++++++++++++++ .../src/cli/utils/fileHelper.ts | 72 +++++ .../src/config/utils/pluginValidation.ts | 14 +- .../src/config/webpack.plugin.config.ts | 6 +- .../src/config/webpack/loaders.ts | 35 ++- packages/grafana-toolkit/tsconfig.json | 4 +- packages/grafana-ui/package.json | 4 +- 16 files changed, 437 insertions(+), 220 deletions(-) create mode 100644 packages/grafana-toolkit/src/cli/tasks/plugin/ci.ts create mode 100644 packages/grafana-toolkit/src/cli/utils/fileHelper.ts diff --git a/lerna.json b/lerna.json index f72b0b06cd2..87f521493db 100644 --- a/lerna.json +++ b/lerna.json @@ -2,5 +2,5 @@ "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], - "version": "6.4.0-alpha.12" + "version": "6.4.0-alpha.22" } diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index c42aeb8f25f..0981296b990 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/data", - "version": "6.4.0-alpha.12", + "version": "6.4.0-alpha.22", "description": "Grafana Data Library", "keywords": [ "typescript" diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index a73858a9b09..919e8766c44 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/runtime", - "version": "6.4.0-alpha.12", + "version": "6.4.0-alpha.22", "description": "Grafana Runtime Library", "keywords": [ "grafana" diff --git a/packages/grafana-toolkit/README.md b/packages/grafana-toolkit/README.md index ad38b44caea..db4e548b58a 100644 --- a/packages/grafana-toolkit/README.md +++ b/packages/grafana-toolkit/README.md @@ -90,6 +90,7 @@ Adidtionaly, you can also provide additional Jest config via package.json file. We support pure css, SASS and CSS in JS approach (via Emotion). 1. Single css/sass file + Create your css/sass file and import it in your plugin entry point (typically module.ts): ```ts @@ -100,6 +101,7 @@ The styles will be injected via `style` tag during runtime. Note, that imported static assets will be inlined as base64 URIs. *This can be a subject of change in the future!* 2. Theme specific css/sass files + If you want to provide different stylesheets for dark/light theme, create `dark.[css|scss]` and `light.[css|scss]` files in `src/styles` directory of your plugin. Based on that we will generate stylesheets that will end up in `dist/styles` directory. TODO: add note about loadPluginCss @@ -107,6 +109,7 @@ TODO: add note about loadPluginCss Note that static files (png, svg, json, html) are all copied to dist directory when the plugin is bundled. Relative paths to those files does not change. 3. Emotion + Starting from Grafana 6.2 our suggested way of styling plugins is by using [Emotion](https://emotion.sh). It's a css-in-js library that we use internaly at Grafana. The biggest advantage of using Emotion is that you will get access to Grafana Theme variables. To use start using Emotion you first need to add it to your plugin dependencies: diff --git a/packages/grafana-toolkit/package.json b/packages/grafana-toolkit/package.json index bdc90c75e85..ab2ba692124 100644 --- a/packages/grafana-toolkit/package.json +++ b/packages/grafana-toolkit/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/toolkit", - "version": "6.4.0-alpha.12", + "version": "6.4.0-alpha.22", "description": "Grafana Toolkit", "keywords": [ "grafana", diff --git a/packages/grafana-toolkit/src/cli/index.ts b/packages/grafana-toolkit/src/cli/index.ts index 24c34a725b4..3d8d3ec58ac 100644 --- a/packages/grafana-toolkit/src/cli/index.ts +++ b/packages/grafana-toolkit/src/cli/index.ts @@ -16,10 +16,9 @@ import { pluginDevTask } from './tasks/plugin.dev'; import { ciBuildPluginTask, ciBuildPluginDocsTask, - ciBundlePluginTask, + ciPackagePluginTask, ciTestPluginTask, ciPluginReportTask, - ciDeployPluginTask, } from './tasks/plugin.ci'; import { buildPackageTask } from './tasks/package.build'; @@ -171,10 +170,10 @@ export const run = (includeInternalScripts = false) => { }); program - .command('plugin:ci-bundle') - .description('Create a zip artifact for the plugin') + .command('plugin:ci-package') + .description('Create a zip packages for the plugin') .action(async cmd => { - await execTask(ciBundlePluginTask)({}); + await execTask(ciPackagePluginTask)({}); }); program @@ -194,13 +193,6 @@ export const run = (includeInternalScripts = false) => { await execTask(ciPluginReportTask)({}); }); - program - .command('plugin:ci-deploy') - .description('Publish plugin CI results') - .action(async cmd => { - await execTask(ciDeployPluginTask)({}); - }); - program.on('command:*', () => { console.error('Invalid command: %s\nSee --help for a list of available commands.', program.args.join(' ')); process.exit(1); diff --git a/packages/grafana-toolkit/src/cli/tasks/package.build.ts b/packages/grafana-toolkit/src/cli/tasks/package.build.ts index c0d65f9c39e..8f210713786 100644 --- a/packages/grafana-toolkit/src/cli/tasks/package.build.ts +++ b/packages/grafana-toolkit/src/cli/tasks/package.build.ts @@ -43,6 +43,23 @@ export const savePackage = useSpinner( const preparePackage = async (pkg: any) => { pkg.main = 'index.js'; pkg.types = 'index.d.ts'; + + const version: string = pkg.version; + const name: string = pkg.name; + const deps: any = pkg.dependencies; + + // Below we are adding cross-dependencies to Grafana's packages + // with the version being published + if (name.endsWith('/ui')) { + deps['@grafana/data'] = version; + } else if (name.endsWith('/runtime')) { + deps['@grafana/data'] = version; + deps['@grafana/ui'] = version; + } else if (name.endsWith('/toolkit')) { + deps['@grafana/data'] = version; + deps['@grafana/ui'] = version; + } + await savePackage({ path: `${cwd}/dist/package.json`, pkg, diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.build.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.build.ts index b87ad56e537..5b92a6f54d6 100644 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.build.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.build.ts @@ -82,12 +82,19 @@ export const prettierCheckPlugin = useSpinner('Prettier check', async ( if (!prettier.check(data.toString(), opts)) { if (fix) { const fixed = prettier.format(data.toString(), opts); - fs.writeFile(s, fixed, err => { - if (err) { - console.log('Error fixing ' + s, err); - failed = true; - } - }); + if (fixed && fixed.length > 10) { + fs.writeFile(s, fixed, err => { + if (err) { + console.log('Error fixing ' + s, err); + failed = true; + } else { + console.log('Fixed: ' + s); + } + }); + } else { + console.log('No automatic fix for: ' + s); + failed = true; + } } else { failed = true; } diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts index 0f8ab0bf58d..e3d92fb6689 100644 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts @@ -7,89 +7,34 @@ import { getPluginJson } from '../../config/utils/pluginValidation'; import execa = require('execa'); import path = require('path'); import fs = require('fs'); +import { getPackageDetails } from '../utils/fileHelper'; +import { + job, + getJobFolder, + writeJobStats, + getCiFolder, + agregateWorkflowInfo, + agregateCoverageInfo, + getPluginSourceInfo, + TestResultInfo, + agregateTestInfo, +} from './plugin/ci'; export interface PluginCIOptions { backend?: string; full?: boolean; } -const calcJavascriptSize = (base: string, files?: string[]): number => { - files = files || fs.readdirSync(base); - let size = 0; - - if (files) { - files.forEach(file => { - const newbase = path.join(base, file); - const stat = fs.statSync(newbase); - if (stat.isDirectory()) { - size += calcJavascriptSize(newbase, fs.readdirSync(newbase)); - } else { - if (file.endsWith('.js')) { - size += stat.size; - } - } - }); - } - return size; -}; - -const getJobFromProcessArgv = () => { - const arg = process.argv[2]; - if (arg && arg.startsWith('plugin:ci-')) { - const task = arg.substring('plugin:ci-'.length); - if ('build' === task) { - if ('--platform' === process.argv[3] && process.argv[4]) { - return task + '_' + process.argv[4]; - } - return 'build_nodejs'; - } - return task; - } - return 'unknown_job'; -}; - -const job = process.env.CIRCLE_JOB || getJobFromProcessArgv(); - -const getJobFolder = () => { - const dir = path.resolve(process.cwd(), 'ci', 'jobs', job); - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - return dir; -}; - -const getCiFolder = () => { - const dir = path.resolve(process.cwd(), 'ci'); - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - return dir; -}; - -const writeJobStats = (startTime: number, workDir: string) => { - const stats = { - job, - startTime, - endTime: Date.now(), - }; - const f = path.resolve(workDir, 'stats.json'); - fs.writeFile(f, JSON.stringify(stats, null, 2), err => { - if (err) { - throw new Error('Unable to stats: ' + f); - } - }); -}; - /** * 1. BUILD * * when platform exists it is building backend, otherwise frontend * * Each build writes data: - * ~/work/build_xxx/ + * ~/ci/jobs/build_xxx/ * * Anything that should be put into the final zip file should be put in: - * ~/work/build_xxx/dist + * ~/ci/jobs/build_xxx/dist */ const buildPluginRunner: TaskRunner = async ({ backend }) => { const start = Date.now(); @@ -158,23 +103,22 @@ const buildPluginDocsRunner: TaskRunner = async () => { export const ciBuildPluginDocsTask = new Task('Build Plugin Docs', buildPluginDocsRunner); /** - * 2. BUNDLE + * 2. Package * * Take everything from `~/ci/job/{any}/dist` and * 1. merge it into: `~/ci/dist` - * 2. zip it into artifacts in `~/ci/artifacts` + * 2. zip it into packages in `~/ci/packages` * 3. prepare grafana environment in: `~/ci/grafana-test-env` - * */ -const bundlePluginRunner: TaskRunner = async () => { +const packagePluginRunner: TaskRunner = async () => { const start = Date.now(); const ciDir = getCiFolder(); - const artifactsDir = path.resolve(ciDir, 'artifacts'); + const packagesDir = path.resolve(ciDir, 'packages'); const distDir = path.resolve(ciDir, 'dist'); const docsDir = path.resolve(ciDir, 'docs'); const grafanaEnvDir = path.resolve(ciDir, 'grafana-test-env'); - await execa('rimraf', [artifactsDir, distDir, grafanaEnvDir]); - fs.mkdirSync(artifactsDir); + await execa('rimraf', [packagesDir, distDir, grafanaEnvDir]); + fs.mkdirSync(packagesDir); fs.mkdirSync(distDir); fs.mkdirSync(grafanaEnvDir); @@ -199,10 +143,19 @@ const bundlePluginRunner: TaskRunner = async () => { } } + console.log('Save the source info in plugin.json'); + const pluginJsonFile = path.resolve(distDir, 'plugin.json'); + const pluginInfo = getPluginJson(pluginJsonFile); + (pluginInfo.info as any).source = await getPluginSourceInfo(); + fs.writeFile(pluginJsonFile, JSON.stringify(pluginInfo, null, 2), err => { + if (err) { + throw new Error('Error writing: ' + pluginJsonFile); + } + }); + console.log('Building ZIP'); - const pluginInfo = getPluginJson(`${distDir}/plugin.json`); let zipName = pluginInfo.id + '-' + pluginInfo.info.version + '.zip'; - let zipFile = path.resolve(artifactsDir, zipName); + let zipFile = path.resolve(packagesDir, zipName); process.chdir(distDir); await execa('zip', ['-r', zipFile, '.']); restoreCwd(); @@ -212,58 +165,31 @@ const bundlePluginRunner: TaskRunner = async () => { throw new Error('Invalid zip file: ' + zipFile); } - const zipInfo: any = { - name: zipName, - size: zipStats.size, - }; const info: any = { - plugin: zipInfo, + plugin: await getPackageDetails(zipFile, distDir), }; - try { - const exe = await execa('shasum', [zipFile]); - const idx = exe.stdout.indexOf(' '); - const sha1 = exe.stdout.substring(0, idx); - fs.writeFile(zipFile + '.sha1', sha1, err => {}); - zipInfo.sha1 = sha1; - } catch { - console.warn('Unable to read SHA1 Checksum'); - } console.log('Setup Grafan Environment'); let p = path.resolve(grafanaEnvDir, 'plugins', pluginInfo.id); fs.mkdirSync(p, { recursive: true }); await execa('unzip', [zipFile, '-d', p]); - // If docs exist, zip them into artifacts + // If docs exist, zip them into packages if (fs.existsSync(docsDir)) { console.log('Creating documentation zip'); zipName = pluginInfo.id + '-' + pluginInfo.info.version + '-docs.zip'; - zipFile = path.resolve(artifactsDir, zipName); + zipFile = path.resolve(packagesDir, zipName); process.chdir(docsDir); await execa('zip', ['-r', zipFile, '.']); restoreCwd(); - const zipStats = fs.statSync(zipFile); - const zipInfo: any = { - name: zipName, - size: zipStats.size, - }; - try { - const exe = await execa('shasum', [zipFile]); - const idx = exe.stdout.indexOf(' '); - const sha1 = exe.stdout.substring(0, idx); - fs.writeFile(zipFile + '.sha1', sha1, err => {}); - zipInfo.sha1 = sha1; - } catch { - console.warn('Unable to read SHA1 Checksum'); - } - info.docs = zipInfo; + info.docs = await getPackageDetails(zipFile, docsDir); } - p = path.resolve(artifactsDir, 'info.json'); + p = path.resolve(packagesDir, 'info.json'); fs.writeFile(p, JSON.stringify(info, null, 2), err => { if (err) { - throw new Error('Error writing artifact info: ' + p); + throw new Error('Error writing package info: ' + p); } }); @@ -283,7 +209,7 @@ const bundlePluginRunner: TaskRunner = async () => { writeJobStats(start, getJobFolder()); }; -export const ciBundlePluginTask = new Task('Bundle Plugin', bundlePluginRunner); +export const ciPackagePluginTask = new Task('Bundle Plugin', packagePluginRunner); /** * 3. Test (end-to-end) @@ -294,11 +220,11 @@ export const ciBundlePluginTask = new Task('Bundle Plugin', bun const testPluginRunner: TaskRunner = async ({ full }) => { const start = Date.now(); const workDir = getJobFolder(); - const pluginInfo = getPluginJson(`${process.cwd()}/src/plugin.json`); - + const pluginInfo = getPluginJson(`${process.cwd()}/ci/dist/plugin.json`); + const results: TestResultInfo = { job }; const args = { withCredentials: true, - baseURL: process.env.GRAFANA_URL || 'http://localhost:3000/', + baseURL: process.env.BASE_URL || 'http://localhost:3000/', responseType: 'json', auth: { username: 'admin', @@ -306,40 +232,32 @@ const testPluginRunner: TaskRunner = async ({ full }) => { }, }; - const axios = require('axios'); - const frontendSettings = await axios.get('api/frontend/settings', args); + try { + const axios = require('axios'); + const frontendSettings = await axios.get('api/frontend/settings', args); + results.grafana = frontendSettings.data.buildInfo; - console.log('Grafana Version: ' + JSON.stringify(frontendSettings.data.buildInfo, null, 2)); + console.log('Grafana: ' + JSON.stringify(results.grafana, null, 2)); - const allPlugins: any[] = await axios.get('api/plugins', args).data; - // for (const plugin of allPlugins) { - // if (plugin.id === pluginInfo.id) { - // console.log('------------'); - // console.log(plugin); - // console.log('------------'); - // } else { - // console.log('Plugin:', plugin.id, plugin.latestVersion); - // } - // } - console.log('PLUGINS:', allPlugins); - - if (full) { const pluginSettings = await axios.get(`api/plugins/${pluginInfo.id}/settings`, args); console.log('Plugin Info: ' + JSON.stringify(pluginSettings.data, null, 2)); + + console.log('TODO Puppeteer Tests', workDir); + + results.status = 'TODO... puppeteer'; + } catch (err) { + results.error = err; + results.status = 'EXCEPTION Thrown'; + console.log('Test Error', err); } - console.log('TODO puppeteer'); + const f = path.resolve(workDir, 'results.json'); + fs.writeFile(f, JSON.stringify(results, null, 2), err => { + if (err) { + throw new Error('Error saving: ' + f); + } + }); - const elapsed = Date.now() - start; - const stats = { - job, - sha1: `${process.env.CIRCLE_SHA1}`, - startTime: start, - buildTime: elapsed, - endTime: Date.now(), - }; - - console.log('TODO Puppeteer Tests', stats); writeJobStats(start, workDir); }; @@ -352,35 +270,28 @@ export const ciTestPluginTask = new Task('Test Plugin (e2e)', t * */ const pluginReportRunner: TaskRunner = async () => { - const start = Date.now(); - const workDir = getJobFolder(); - const reportDir = path.resolve(process.cwd(), 'ci', 'report'); - await execa('rimraf', [reportDir]); - fs.mkdirSync(reportDir); + const ciDir = path.resolve(process.cwd(), 'ci'); + const packageInfo = require(path.resolve(ciDir, 'packages', 'info.json')); - const file = path.resolve(reportDir, `report.txt`); - fs.writeFile(file, `TODO... actually make a report (csv etc)`, err => { + console.log('Save the source info in plugin.json'); + const pluginJsonFile = path.resolve(ciDir, 'dist', 'plugin.json'); + const report = { + plugin: getPluginJson(pluginJsonFile), + packages: packageInfo, + workflow: agregateWorkflowInfo(), + coverage: agregateCoverageInfo(), + tests: agregateTestInfo(), + }; + + console.log('REPORT', report); + + const file = path.resolve(ciDir, 'report.json'); + fs.writeFile(file, JSON.stringify(report, null, 2), err => { if (err) { throw new Error('Unable to write: ' + file); } }); - - console.log('TODO... real report'); - writeJobStats(start, workDir); + console.log('TODO... notify some service'); }; -export const ciPluginReportTask = new Task('Deploy plugin', pluginReportRunner); - -/** - * 5. Deploy - * - * deploy the zip to a running grafana instance - * - */ -const deployPluginRunner: TaskRunner = async () => { - console.log('TODO DEPLOY??'); - console.log(' if PR => write a comment to github with difference '); - console.log(' if master | vXYZ ==> upload artifacts to some repo '); -}; - -export const ciDeployPluginTask = new Task('Deploy plugin', deployPluginRunner); +export const ciPluginReportTask = new Task('Generate Plugin Report', pluginReportRunner); diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin/ci.ts b/packages/grafana-toolkit/src/cli/tasks/plugin/ci.ts new file mode 100644 index 00000000000..32450daa760 --- /dev/null +++ b/packages/grafana-toolkit/src/cli/tasks/plugin/ci.ts @@ -0,0 +1,214 @@ +import execa = require('execa'); +import path = require('path'); +import fs = require('fs'); + +export interface PluginSourceInfo { + time?: number; + repo?: string; + branch?: string; + hash?: string; +} + +export interface JobInfo { + job?: string; + startTime: number; + endTime: number; + elapsed: number; + status?: string; + buildNumber?: number; +} + +export interface WorkflowInfo extends JobInfo { + workflowId?: string; + jobs: JobInfo[]; + user?: string; + repo?: string; +} + +const getJobFromProcessArgv = () => { + const arg = process.argv[2]; + if (arg && arg.startsWith('plugin:ci-')) { + const task = arg.substring('plugin:ci-'.length); + if ('build' === task) { + if ('--backend' === process.argv[3] && process.argv[4]) { + return task + '_' + process.argv[4]; + } + return 'build_plugin'; + } + return task; + } + return 'unknown_job'; +}; + +export const job = process.env.CIRCLE_JOB || getJobFromProcessArgv(); + +export const getPluginSourceInfo = async (): Promise => { + if (process.env.CIRCLE_SHA1) { + return Promise.resolve({ + time: Date.now(), + repo: process.env.CIRCLE_REPOSITORY_URL, + branch: process.env.CIRCLE_BRANCH, + hash: process.env.CIRCLE_SHA1, + }); + } + const exe = await execa('git', ['rev-parse', 'HEAD']); + return { + time: Date.now(), + hash: exe.stdout, + }; +}; + +const getBuildNumber = (): number | undefined => { + if (process.env.CIRCLE_BUILD_NUM) { + return parseInt(process.env.CIRCLE_BUILD_NUM, 10); + } + return undefined; +}; + +export const getJobFolder = () => { + const dir = path.resolve(process.cwd(), 'ci', 'jobs', job); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + return dir; +}; + +export const getCiFolder = () => { + const dir = path.resolve(process.cwd(), 'ci'); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + return dir; +}; + +export const writeJobStats = (startTime: number, workDir: string) => { + const endTime = Date.now(); + const stats: JobInfo = { + job, + startTime, + endTime, + elapsed: endTime - startTime, + buildNumber: getBuildNumber(), + }; + const f = path.resolve(workDir, 'job.json'); + fs.writeFile(f, JSON.stringify(stats, null, 2), err => { + if (err) { + throw new Error('Unable to stats: ' + f); + } + }); +}; + +export const agregateWorkflowInfo = (): WorkflowInfo => { + const now = Date.now(); + const workflow: WorkflowInfo = { + jobs: [], + startTime: now, + endTime: now, + workflowId: process.env.CIRCLE_WORKFLOW_ID, + repo: process.env.CIRCLE_PROJECT_REPONAME, + user: process.env.CIRCLE_PROJECT_USERNAME, + buildNumber: getBuildNumber(), + elapsed: 0, + }; + + const jobsFolder = path.resolve(getCiFolder(), 'jobs'); + if (fs.existsSync(jobsFolder)) { + const files = fs.readdirSync(jobsFolder); + if (files && files.length) { + files.forEach(file => { + const p = path.resolve(jobsFolder, file, 'job.json'); + if (fs.existsSync(p)) { + const job = require(p) as JobInfo; + workflow.jobs.push(job); + if (job.startTime < workflow.startTime) { + workflow.startTime = job.startTime; + } + if (job.endTime > workflow.endTime) { + workflow.endTime = job.endTime; + } + } else { + console.log('Missing Job info: ', p); + } + }); + } else { + console.log('NO JOBS IN: ', jobsFolder); + } + } + + workflow.elapsed = workflow.endTime - workflow.startTime; + return workflow; +}; + +export interface CoverageDetails { + total: number; + covered: number; + skipped: number; + pct: number; +} + +export interface CoverageInfo { + job: string; + summary: { [key: string]: CoverageDetails }; + report?: string; // path to report +} + +export const agregateCoverageInfo = (): CoverageInfo[] => { + const coverage: CoverageInfo[] = []; + const ciDir = getCiFolder(); + const jobsFolder = path.resolve(ciDir, 'jobs'); + if (fs.existsSync(jobsFolder)) { + const files = fs.readdirSync(jobsFolder); + if (files && files.length) { + files.forEach(file => { + const dir = path.resolve(jobsFolder, file, 'coverage'); + if (fs.existsSync(dir)) { + const s = path.resolve(dir, 'coverage-summary.json'); + const r = path.resolve(dir, 'lcov-report', 'index.html'); + if (fs.existsSync(s)) { + const raw = require(s); + const info: CoverageInfo = { + job: file, + summary: raw.total, + }; + if (fs.existsSync(r)) { + info.report = r.substring(ciDir.length); + } + coverage.push(info); + } + } + }); + } else { + console.log('NO JOBS IN: ', jobsFolder); + } + } + return coverage; +}; + +export interface TestResultInfo { + job: string; + grafana?: any; + status?: string; + error?: string; +} + +export const agregateTestInfo = (): TestResultInfo[] => { + const tests: TestResultInfo[] = []; + const ciDir = getCiFolder(); + const jobsFolder = path.resolve(ciDir, 'jobs'); + if (fs.existsSync(jobsFolder)) { + const files = fs.readdirSync(jobsFolder); + if (files && files.length) { + files.forEach(file => { + if (file.startsWith('test')) { + const summary = path.resolve(jobsFolder, file, 'results.json'); + if (fs.existsSync(summary)) { + tests.push(require(summary) as TestResultInfo); + } + } + }); + } else { + console.log('NO Jobs IN: ', jobsFolder); + } + } + return tests; +}; diff --git a/packages/grafana-toolkit/src/cli/utils/fileHelper.ts b/packages/grafana-toolkit/src/cli/utils/fileHelper.ts new file mode 100644 index 00000000000..39d8a72cc10 --- /dev/null +++ b/packages/grafana-toolkit/src/cli/utils/fileHelper.ts @@ -0,0 +1,72 @@ +import execa = require('execa'); +import path = require('path'); +import fs = require('fs'); + +interface ExtensionBytes { + [key: string]: number; +} + +export function getFileSizeReportInFolder(dir: string, info?: ExtensionBytes): ExtensionBytes { + if (!info) { + info = {}; + } + + const files = fs.readdirSync(dir); + if (files) { + files.forEach(file => { + const newbase = path.join(dir, file); + const stat = fs.statSync(newbase); + if (stat.isDirectory()) { + getFileSizeReportInFolder(newbase, info); + } else { + let ext = ''; + const idx = file.lastIndexOf('.'); + if (idx > 0) { + ext = file.substring(idx + 1).toLowerCase(); + } + const current = info![ext] || 0; + info![ext] = current + stat.size; + } + }); + } + return info; +} + +interface ZipFileInfo { + name: string; + size: number; + contents: ExtensionBytes; + sha1?: string; + md5?: string; +} + +export async function getPackageDetails(zipFile: string, zipSrc: string, writeChecksum = true): Promise { + const zipStats = fs.statSync(zipFile); + if (zipStats.size < 100) { + throw new Error('Invalid zip file: ' + zipFile); + } + const info: ZipFileInfo = { + name: path.basename(zipFile), + size: zipStats.size, + contents: getFileSizeReportInFolder(zipSrc), + }; + try { + const exe = await execa('shasum', [zipFile]); + const idx = exe.stdout.indexOf(' '); + const sha1 = exe.stdout.substring(0, idx); + if (writeChecksum) { + fs.writeFile(zipFile + '.sha1', sha1, err => {}); + } + info.sha1 = sha1; + } catch { + console.warn('Unable to read SHA1 Checksum'); + } + try { + const exe = await execa('md5sum', [zipFile]); + const idx = exe.stdout.indexOf(' '); + info.md5 = exe.stdout.substring(0, idx); + } catch { + console.warn('Unable to read MD5 Checksum'); + } + return info; +} diff --git a/packages/grafana-toolkit/src/config/utils/pluginValidation.ts b/packages/grafana-toolkit/src/config/utils/pluginValidation.ts index 1947d74a190..ccc3db2990c 100644 --- a/packages/grafana-toolkit/src/config/utils/pluginValidation.ts +++ b/packages/grafana-toolkit/src/config/utils/pluginValidation.ts @@ -1,12 +1,4 @@ -// See: packages/grafana-ui/src/types/plugin.ts -interface PluginJSONSchema { - id: string; - info: PluginMetaInfo; -} - -interface PluginMetaInfo { - version: string; -} +import { PluginMeta } from '@grafana/ui'; export const validatePluginJson = (pluginJson: any) => { if (!pluginJson.id) { @@ -32,7 +24,7 @@ export const validatePluginJson = (pluginJson: any) => { } }; -export const getPluginJson = (path: string): PluginJSONSchema => { +export const getPluginJson = (path: string): PluginMeta => { let pluginJson; try { pluginJson = require(path); @@ -42,5 +34,5 @@ export const getPluginJson = (path: string): PluginJSONSchema => { validatePluginJson(pluginJson); - return pluginJson as PluginJSONSchema; + return pluginJson as PluginMeta; }; diff --git a/packages/grafana-toolkit/src/config/webpack.plugin.config.ts b/packages/grafana-toolkit/src/config/webpack.plugin.config.ts index c654d6cb067..e3d6f95ae84 100644 --- a/packages/grafana-toolkit/src/config/webpack.plugin.config.ts +++ b/packages/grafana-toolkit/src/config/webpack.plugin.config.ts @@ -185,8 +185,10 @@ export const getWebpackConfig: WebpackConfigurationGetter = options => { plugins: ['angularjs-annotate'], }, }, - - 'ts-loader', + { + loader: 'ts-loader', + options: { onlyCompileBundledFiles: true }, + }, ], exclude: /(node_modules)/, }, diff --git a/packages/grafana-toolkit/src/config/webpack/loaders.ts b/packages/grafana-toolkit/src/config/webpack/loaders.ts index aab0846b45d..950c7dfd5ac 100644 --- a/packages/grafana-toolkit/src/config/webpack/loaders.ts +++ b/packages/grafana-toolkit/src/config/webpack/loaders.ts @@ -63,16 +63,12 @@ export const hasThemeStylesheets = (root: string = process.cwd()) => { }; export const getStyleLoaders = () => { - const shouldExtractCss = hasThemeStylesheets(); - - const executiveLoader = shouldExtractCss - ? { - loader: MiniCssExtractPlugin.loader, - options: { - publicPath: '../', - }, - } - : 'style-loader'; + const extractionLoader = { + loader: MiniCssExtractPlugin.loader, + options: { + publicPath: '../', + }, + }; const cssLoaders = [ { @@ -95,21 +91,32 @@ export const getStyleLoaders = () => { }, ]; - return [ + const rules = [ + { + test: /(dark|light)\.css$/, + use: [extractionLoader, ...cssLoaders], + }, + { + test: /(dark|light)\.scss$/, + use: [extractionLoader, ...cssLoaders, 'sass-loader'], + }, { test: /\.css$/, - use: [executiveLoader, ...cssLoaders], + use: ['style-loader', ...cssLoaders, 'sass-loader'], + exclude: [`${process.cwd()}/src/styles/light.css`, `${process.cwd()}/src/styles/dark.css`], }, { test: /\.scss$/, - use: [executiveLoader, ...cssLoaders, 'sass-loader'], + use: ['style-loader', ...cssLoaders, 'sass-loader'], + exclude: [`${process.cwd()}/src/styles/light.scss`, `${process.cwd()}/src/styles/dark.scss`], }, ]; + + return rules; }; export const getFileLoaders = () => { const shouldExtractCss = hasThemeStylesheets(); - // const pluginJson = getPluginJson(); return [ { diff --git a/packages/grafana-toolkit/tsconfig.json b/packages/grafana-toolkit/tsconfig.json index d7c48a886b9..8e0aa7ae2b3 100644 --- a/packages/grafana-toolkit/tsconfig.json +++ b/packages/grafana-toolkit/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "../tsconfig.json", - "include": ["src/**/*.ts"], + "include": ["src/**/*.ts", "../../public/app/types/jquery/*.ts"], "exclude": ["dist", "node_modules"], "compilerOptions": { "module": "commonjs", @@ -9,6 +9,6 @@ "declaration": false, "typeRoots": ["./node_modules/@types"], "esModuleInterop": true, - "lib": ["es2015", "es2017.string"] + "lib": ["es2015", "es2017.string", "dom"] } } diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 6b8879a6dcd..f848de262a7 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -1,6 +1,6 @@ { "name": "@grafana/ui", - "version": "6.4.0-alpha.12", + "version": "6.4.0-alpha.22", "description": "Grafana Components Library", "keywords": [ "grafana", @@ -20,7 +20,7 @@ "author": "Grafana Labs", "license": "Apache-2.0", "dependencies": { - "@grafana/data": "^6.4.0-alpha.8", + "@grafana/data": "^6.4.0-alpha.22", "@torkelo/react-select": "2.1.1", "@types/react-color": "2.17.0", "classnames": "2.2.6",