diff --git a/bin/sitespeed.js b/bin/sitespeed.js index 7f07d0403..94ae48376 100755 --- a/bin/sitespeed.js +++ b/bin/sitespeed.js @@ -4,34 +4,17 @@ 'use strict'; -const cli = require('../lib/cli/cli'), - sitespeed = require('../lib/sitespeed'), - Promise = require('bluebird'); +const cli = require('../lib/cli/cli'); +const sitespeed = require('../lib/sitespeed'); -if (process.env.NODE_ENV !== 'production') { - require('longjohn'); -} - -Promise.config({ - warnings: true, - longStackTraces: true -}); - -process.exitCode = 1; - -let parsed = cli.parseCommandLine(); -let budgetFailing = false; -// hack for getting in the unchanged cli options -parsed.options.explicitOptions = parsed.explicitOptions; -parsed.options.urls = parsed.urls; -parsed.options.urlsMetaData = parsed.urlsMetaData; - -sitespeed - .run(parsed.options) - .then(result => { +async function run(options) { + process.exitCode = 1; + try { + const result = await sitespeed.run(options); if (result.errors.length > 0) { throw new Error('Errors while running:\n' + result.errors.join('\n')); } + if ( parsed.options.budget && Object.keys(result.budgetResult.failing).length > 0 @@ -39,16 +22,24 @@ sitespeed process.exitCode = 1; budgetFailing = true; } - }) - .then(() => { + if ( !budgetFailing || (parsed.options.budget && parsed.options.budget.suppressExitCode) ) { process.exitCode = 0; } - }) - .catch(() => { + } catch (e) { process.exitCode = 1; - }) - .finally(() => process.exit()); + } finally { + process.exit(); + } +} +let parsed = cli.parseCommandLine(); +let budgetFailing = false; +// hack for getting in the unchanged cli options +parsed.options.explicitOptions = parsed.explicitOptions; +parsed.options.urls = parsed.urls; +parsed.options.urlsMetaData = parsed.urlsMetaData; + +run(parsed.options); diff --git a/lib/core/pluginLoader.js b/lib/core/pluginLoader.js index 7e2508df6..7ea500070 100644 --- a/lib/core/pluginLoader.js +++ b/lib/core/pluginLoader.js @@ -1,10 +1,9 @@ 'use strict'; -const Promise = require('bluebird'), - path = require('path'), - fs = require('fs'); - -Promise.promisifyAll(fs); +const path = require('path'); +const fs = require('fs'); +const { promisify } = require('util'); +const readdir = promisify(fs.readdir); const defaultPlugins = new Set([ 'browsertime', @@ -23,7 +22,7 @@ const defaultPlugins = new Set([ const pluginsDir = path.join(__dirname, '..', 'plugins'); module.exports = { - parsePluginNames(options) { + async parsePluginNames(options) { // if we don't use the cli, this will work out fine as long // we configure only what we need const possibleConfiguredPlugins = options.explicitOptions || options; @@ -39,31 +38,30 @@ module.exports = { return pluginNames; }; - return fs - .readdirAsync(pluginsDir) - .map(name => path.basename(name, '.js')) - .then(builtins => { - let plugins = builtins.filter(isDefaultOrConfigured); - return addMessageLoggerIfDebug(plugins); - }); + const files = await readdir(pluginsDir); + const builtins = files.map(name => path.basename(name, '.js')); + const plugins = builtins.filter(isDefaultOrConfigured); + return addMessageLoggerIfDebug(plugins); }, - loadPlugins(pluginNames) { - return Promise.resolve(pluginNames).map(name => { + async loadPlugins(pluginNames) { + const plugins = []; + for (let name of pluginNames) { try { const plugin = require(path.join(pluginsDir, name)); if (!plugin.name) { plugin.name = () => name; } - return plugin; + plugins.push(plugin); } catch (err) { try { - return require(path.resolve(process.cwd(), name)); + plugins.push(require(path.resolve(process.cwd(), name))); } catch (error) { console.error("Couldn't load plugin %s: %s", name, err); // eslint-disable-line no-console // if it fails here, let it fail hard throw error; } } - }); + } + return plugins; } }; diff --git a/lib/core/queueHandler.js b/lib/core/queueHandler.js index dcee3847e..52c1c83be 100644 --- a/lib/core/queueHandler.js +++ b/lib/core/queueHandler.js @@ -2,11 +2,10 @@ /* eslint no-console:0 */ -const cq = require('concurrent-queue'), - Promise = require('bluebird'), - log = require('intel').getLogger('sitespeedio.queuehandler'), - messageMaker = require('../support/messageMaker'), - queueStats = require('./queueStatistics'); +const cq = require('concurrent-queue'); +const log = require('intel').getLogger('sitespeedio.queuehandler'); +const messageMaker = require('../support/messageMaker'); +const queueStats = require('./queueStatistics'); const make = messageMaker('queueHandler').make; @@ -165,7 +164,7 @@ class QueueHandler { return { plugin, queue }; }); } - run(sources) { + async run(sources) { /* setup - plugins chance to talk to each other or setup what they need. url - urls passed around to analyze @@ -176,7 +175,11 @@ class QueueHandler { return this.startProcessingQueues() .then(() => this.postMessage(make('sitespeedio.setup'))) .then(() => this.drainAllQueues()) - .then(() => Promise.map(sources, source => source.findUrls(this))) + .then(async () => { + for (let source of sources) { + await source.findUrls(this); + } + }) .then(() => this.drainAllQueues()) .then(() => this.postMessage(make('sitespeedio.summarize'))) .then(() => this.drainAllQueues()) @@ -186,8 +189,8 @@ class QueueHandler { if (this.options.queueStats) { log.info(JSON.stringify(queueStats.generateStatistics(), null, 2)); } - }) - .return(this.errors); + return this.errors; + }); } postMessage(message) { @@ -200,17 +203,17 @@ class QueueHandler { } } - startProcessingQueues() { - return Promise.each(this.queues, item => { + async startProcessingQueues() { + for (let item of this.queues) { const queue = item.queue, plugin = item.plugin; queue.process(message => Promise.resolve(plugin.processMessage(message, this)) ); - }); + } } - drainAllQueues() { + async drainAllQueues() { const queues = this.queues; return new Promise(resolve => { queues.forEach(item => diff --git a/lib/core/resultsStorage/storageManager.js b/lib/core/resultsStorage/storageManager.js index 0e5f7a29c..7308af65f 100644 --- a/lib/core/resultsStorage/storageManager.js +++ b/lib/core/resultsStorage/storageManager.js @@ -1,12 +1,10 @@ 'use strict'; const fs = require('fs-extra'); -const Promise = require('bluebird'); const path = require('path'); - const pathToFolder = require('./pathToFolder'); - -const mkdirp = Promise.promisify(require('mkdirp')); +const { promisify } = require('util'); +const mkdirp = promisify(require('mkdirp')); function write(dirPath, filename, data) { return fs.writeFile(path.join(dirPath, filename), data); diff --git a/lib/plugins/browsertime/analyzer.js b/lib/plugins/browsertime/analyzer.js index 78e9ceab8..8151358bd 100644 --- a/lib/plugins/browsertime/analyzer.js +++ b/lib/plugins/browsertime/analyzer.js @@ -1,14 +1,13 @@ 'use strict'; -const merge = require('lodash.merge'), - forEach = require('lodash.foreach'), - path = require('path'), - Promise = require('bluebird'), - browsertime = require('browsertime'), - log = require('intel').getLogger('sitespeedio.plugin.browsertime'), - set = require('lodash.set'), - get = require('lodash.get'), - coach = require('webcoach'); +const merge = require('lodash.merge'); +const forEach = require('lodash.foreach'); +const path = require('path'); +const browsertime = require('browsertime'); +const log = require('intel').getLogger('sitespeedio.plugin.browsertime'); +const set = require('lodash.set'); +const get = require('lodash.get'); +const coach = require('webcoach'); const browserScripts = browsertime.browserScripts; @@ -20,44 +19,36 @@ const iphone6UserAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 ' + '(KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25'; -function parseUserScripts(scripts) { +async function parseUserScripts(scripts) { if (!Array.isArray(scripts)) scripts = [scripts]; - - return Promise.reduce( - scripts, - (results, script) => - browserScripts - .findAndParseScripts(path.resolve(script), 'custom') - .then(scripts => merge(results, scripts)), - {} - ); + const allUserScripts = {}; + for (let script of scripts) { + const myScript = await browserScripts.findAndParseScripts( + path.resolve(script), + 'custom' + ); + merge(allUserScripts, myScript); + } + return allUserScripts; } -function addCoachScripts(scripts) { - const coachAdvice = coach.getDomAdvice(); - return Promise.join(scripts, coachAdvice, (scripts, advice) => { - scripts.coach = { - coachAdvice: advice - }; - return scripts; - }); +async function addCoachScripts(scripts) { + const coachAdvice = await coach.getDomAdvice(); + scripts.coach = { + coachAdvice: coachAdvice + }; + return scripts; } function addExtraScripts(scriptsByCategory, pluginScripts) { - return Promise.join( - scriptsByCategory, - pluginScripts, - (scriptsByCategory, pluginScripts) => { - // For all different script in the array - for (var scripts of pluginScripts) { - // and then for all scripts in that category - forEach(scripts.scripts, function(script, name) { - set(scriptsByCategory, scripts.category + '.' + name, script); - }); - } - return scriptsByCategory; - } - ); + // For all different script in the array + for (let scripts of pluginScripts) { + // and then for all scripts in that category + forEach(scripts.scripts, function(script, name) { + set(scriptsByCategory, scripts.category + '.' + name, script); + }); + } + return scriptsByCategory; } function setupAsynScripts(asyncScripts) { @@ -93,18 +84,13 @@ module.exports = { } } const scriptCategories = await browserScripts.allScriptCategories; - let scriptsByCategory = browserScripts.getScriptsForCategories( + let scriptsByCategory = await browserScripts.getScriptsForCategories( scriptCategories ); if (btOptions.script) { - const userScripts = parseUserScripts(btOptions.script); - scriptsByCategory = await Promise.join( - scriptsByCategory, - userScripts, - (scriptsByCategory, userScripts) => - merge(scriptsByCategory, userScripts) - ); + const userScripts = await parseUserScripts(btOptions.script); + scriptsByCategory = merge(scriptsByCategory, userScripts); } if (btOptions.coach) { diff --git a/lib/plugins/crawler/index.js b/lib/plugins/crawler/index.js index fe6dda9dc..bbfa859f1 100644 --- a/lib/plugins/crawler/index.js +++ b/lib/plugins/crawler/index.js @@ -1,6 +1,5 @@ 'use strict'; -const Promise = require('bluebird'); const path = require('path'); const merge = require('lodash.merge'); const log = require('intel').getLogger('sitespeedio.plugin.crawler'); diff --git a/lib/plugins/grafana/send-annotation.js b/lib/plugins/grafana/send-annotation.js index cbfd8f43a..4d8b503dc 100644 --- a/lib/plugins/grafana/send-annotation.js +++ b/lib/plugins/grafana/send-annotation.js @@ -2,7 +2,6 @@ const http = require('http'); const https = require('https'); const log = require('intel').getLogger('sitespeedio.plugin.grafana'); -const Promise = require('bluebird'); const tsdbUtil = require('../../support/tsdbUtil'); const annotationsHelper = require('../../support/annotationsHelper'); const util = require('../../support/util'); diff --git a/lib/plugins/graphite/graphite-sender.js b/lib/plugins/graphite/graphite-sender.js index c979a93cb..fae525ae1 100644 --- a/lib/plugins/graphite/graphite-sender.js +++ b/lib/plugins/graphite/graphite-sender.js @@ -1,8 +1,7 @@ 'use strict'; -const net = require('net'), - Promise = require('bluebird'), - Sender = require('./sender'); +const net = require('net'); +const Sender = require('./sender'); class GraphiteSender extends Sender { get facility() { diff --git a/lib/plugins/graphite/send-annotation.js b/lib/plugins/graphite/send-annotation.js index af1b99faf..40e33925b 100644 --- a/lib/plugins/graphite/send-annotation.js +++ b/lib/plugins/graphite/send-annotation.js @@ -2,7 +2,6 @@ const http = require('http'); const https = require('https'); const log = require('intel').getLogger('sitespeedio.plugin.graphite'); -const Promise = require('bluebird'); const graphiteUtil = require('../../support/tsdbUtil'); const annotationsHelper = require('../../support/annotationsHelper'); const util = require('../../support/util'); diff --git a/lib/plugins/graphite/sender.js b/lib/plugins/graphite/sender.js index 5252628c5..0c3738d23 100644 --- a/lib/plugins/graphite/sender.js +++ b/lib/plugins/graphite/sender.js @@ -1,7 +1,6 @@ 'use strict'; -const Promise = require('bluebird'), - log = require('intel').getLogger('sitespeedio.plugin.graphite'); +const log = require('intel').getLogger('sitespeedio.plugin.graphite'); class Sender { constructor(host, port, bulkSize) { diff --git a/lib/plugins/graphite/statsd-sender.js b/lib/plugins/graphite/statsd-sender.js index aa5745f08..ac4bf0e14 100644 --- a/lib/plugins/graphite/statsd-sender.js +++ b/lib/plugins/graphite/statsd-sender.js @@ -1,8 +1,7 @@ 'use strict'; -const dgram = require('dgram'), - Promise = require('bluebird'), - Sender = require('./sender'); +const dgram = require('dgram'); +const Sender = require('./sender'); class StatsDSender extends Sender { get facility() { diff --git a/lib/plugins/harstorer/index.js b/lib/plugins/harstorer/index.js index 1c97f00df..d323cf336 100644 --- a/lib/plugins/harstorer/index.js +++ b/lib/plugins/harstorer/index.js @@ -1,9 +1,8 @@ 'use strict'; -const Promise = require('bluebird'); const zlib = require('zlib'); - -Promise.promisifyAll(zlib); +const { promisify } = require('util'); +const gzip = promisify(zlib.gzip); module.exports = { open(context, options) { @@ -17,15 +16,15 @@ module.exports = { const json = JSON.stringify(message.data); if (this.gzipHAR) { - return zlib - .gzipAsync(Buffer.from(json), { level: 1 }) - .then(gziped => - this.storageManager.writeDataForUrl( - gziped, - `${message.type}.gz`, - message.url - ) - ); + return gzip(Buffer.from(json), { + level: 1 + }).then(gziped => + this.storageManager.writeDataForUrl( + gziped, + `${message.type}.gz`, + message.url + ) + ); } else { return this.storageManager.writeDataForUrl( json, diff --git a/lib/plugins/html/htmlBuilder.js b/lib/plugins/html/htmlBuilder.js index 58fe6caaa..7b94104fb 100644 --- a/lib/plugins/html/htmlBuilder.js +++ b/lib/plugins/html/htmlBuilder.js @@ -1,17 +1,16 @@ 'use strict'; -const helpers = require('../../support/helpers'), - Promise = require('bluebird'), - path = require('path'), - merge = require('lodash.merge'), - get = require('lodash.get'), - log = require('intel').getLogger('sitespeedio.plugin.html'), - chunk = require('lodash.chunk'), - packageInfo = require('../../../package'), - renderer = require('./renderer'), - metricHelper = require('./metricHelper'), - markdown = require('markdown').markdown, - isEmpty = require('lodash.isempty'); +const helpers = require('../../support/helpers'); +const path = require('path'); +const merge = require('lodash.merge'); +const get = require('lodash.get'); +const log = require('intel').getLogger('sitespeedio.plugin.html'); +const chunk = require('lodash.chunk'); +const packageInfo = require('../../../package'); +const renderer = require('./renderer'); +const metricHelper = require('./metricHelper'); +const markdown = require('markdown').markdown; +const isEmpty = require('lodash.isempty'); const summaryBoxesSetup = require('./setup/summaryBoxes'), detailedSetup = require('./setup/detailed'); @@ -55,7 +54,7 @@ class HTMLBuilder { this.inlineCss.push(css); } - render(dataCollector) { + async render(dataCollector) { const options = this.options; const name = this.context.name; const timestamp = this.timestamp; @@ -170,7 +169,8 @@ class HTMLBuilder { ) ); - const urlPageRenders = Promise.resolve(Object.keys(validPages)).map(url => { + const urlPageRenders = []; + for (let url of Object.keys(validPages)) { const pageInfo = validPages[url]; const runPages = dataCollector.getURLRuns(url); const medianRun = metricHelper.pickMedianRun(runPages, pageInfo); @@ -181,7 +181,10 @@ class HTMLBuilder { name: summaryPageHAR.log.browser.name, version: summaryPageHAR.log.browser.version } - : { name: '', version: '' }; + : { + name: '', + version: '' + }; // if we are on the summary page we inline the HAR and then make sure // we only pick one HAR run (medianRun). But you can also choose to // fetch the HAR in the HTML, then it isn't included. @@ -249,59 +252,60 @@ class HTMLBuilder { } data.pugs = pugs; - return this._renderUrlPage(url, 'index', data).tap(() => - Promise.resolve(Object.keys(runPages)).map(runIndex => { - const iteration = Number(runIndex) + 1; - const pugs = {}; - const pageInfo = runPages[runIndex]; - const runTimestamp = get( - pageInfo, - 'data.browsertime.run.timestamp', - this.timestamp - ); + await this._renderUrlPage(url, 'index', data); + for (let runIndex of Object.keys(runPages)) { + const iteration = Number(runIndex) + 1; + const pugs = {}; + const pageInfo = runPages[runIndex]; + const runTimestamp = get( + pageInfo, + 'data.browsertime.run.timestamp', + this.timestamp + ); - const pageRuns = this.pageRuns.filter( - run => !!get(pageInfo.data, [run.id, 'run']) - ); + const pageRuns = this.pageRuns.filter( + run => !!get(pageInfo.data, [run.id, 'run']) + ); - let data = { - daurl: url, - daurlAlias, - iteration, - runIndex, - pageInfo, - options, - runPages, - browser, - hasScreenShots: dataCollector.browsertimeScreenshots, - screenShotType: dataCollector.browsertimeScreenshotsType, - css, - h: helpers, - urlLink: './index.html', - JSON: JSON, - markdown: markdown, - rootPath: this.storageManager.rootPathFromUrl(url), - menu: 'pages', - pageTitle: `Run ${parseInt(runIndex) + - 1} for ${url} at ${runTimestamp}`, - pageDescription: `${metricHelper.getMetricsFromRun( - pageInfo - )} collected by sitespeed.io ${packageInfo.version}`, - headers: this.summary, - version: packageInfo.version, - timestamp: runTimestamp, - context: this.context, - pageRuns - }; - // Add pugs for extra plugins - for (const run of pageRuns) { - pugs[run.id] = renderer.renderTemplate(run.id, data); - } - data.pugs = pugs; - return this._renderUrlRunPage(url, parseInt(runIndex) + 1, data); - }) - ); - }); + let data = { + daurl: url, + daurlAlias, + iteration, + runIndex, + pageInfo, + options, + runPages, + browser, + hasScreenShots: dataCollector.browsertimeScreenshots, + screenShotType: dataCollector.browsertimeScreenshotsType, + css, + h: helpers, + urlLink: './index.html', + JSON: JSON, + markdown: markdown, + rootPath: this.storageManager.rootPathFromUrl(url), + menu: 'pages', + pageTitle: `Run ${parseInt(runIndex) + + 1} for ${url} at ${runTimestamp}`, + pageDescription: `${metricHelper.getMetricsFromRun( + pageInfo + )} collected by sitespeed.io ${packageInfo.version}`, + headers: this.summary, + version: packageInfo.version, + timestamp: runTimestamp, + context: this.context, + pageRuns + }; + // Add pugs for extra plugins + for (const run of pageRuns) { + pugs[run.id] = renderer.renderTemplate(run.id, data); + } + data.pugs = pugs; + urlPageRenders.push( + this._renderUrlRunPage(url, parseInt(runIndex) + 1, data) + ); + } + } // Aggregate/summarize data and write additional files return this.storageManager .copyToResultDir(path.join(__dirname, 'assets')) @@ -314,7 +318,7 @@ class HTMLBuilder { ); } - _renderUrlPage(url, name, locals) { + async _renderUrlPage(url, name, locals) { log.debug('Render URL page %s', name); return this.storageManager.writeHtmlForUrl( @@ -324,7 +328,7 @@ class HTMLBuilder { ); } - _renderUrlRunPage(url, name, locals) { + async _renderUrlRunPage(url, name, locals) { log.debug('Render URL run page %s', name); return this.storageManager.writeHtmlForUrl( renderer.renderTemplate('url/iteration/index', locals), @@ -333,7 +337,7 @@ class HTMLBuilder { ); } - _renderSummaryPage(name, locals) { + async _renderSummaryPage(name, locals) { log.debug('Render summary page %s', name); return this.storageManager.writeHtml( diff --git a/lib/plugins/influxdb/send-annotation.js b/lib/plugins/influxdb/send-annotation.js index 586f0ef29..84ebce9a4 100644 --- a/lib/plugins/influxdb/send-annotation.js +++ b/lib/plugins/influxdb/send-annotation.js @@ -2,7 +2,6 @@ const http = require('http'); const https = require('https'); const log = require('intel').getLogger('sitespeedio.plugin.influxdb'); -const Promise = require('bluebird'); const querystring = require('querystring'); const tsdbUtil = require('../../support/tsdbUtil'); const annotationsHelper = require('../../support/annotationsHelper'); diff --git a/lib/plugins/influxdb/sender.js b/lib/plugins/influxdb/sender.js index d12f76be2..f724cd322 100644 --- a/lib/plugins/influxdb/sender.js +++ b/lib/plugins/influxdb/sender.js @@ -1,7 +1,6 @@ 'use strict'; -const Influx = require('influx'), - Promise = require('bluebird'); +const Influx = require('influx'); class InfluxDBSender { constructor({ protocol, host, port, database, username, password }) { diff --git a/lib/plugins/slack/index.js b/lib/plugins/slack/index.js index a9a738882..d073cb395 100644 --- a/lib/plugins/slack/index.js +++ b/lib/plugins/slack/index.js @@ -1,7 +1,6 @@ 'use strict'; const throwIfMissing = require('../../support/util').throwIfMissing; -const Promise = require('bluebird'); const log = require('intel').getLogger('sitespeedio.plugin.slack'); const Slack = require('node-slack'); const merge = require('lodash.merge'); @@ -9,8 +8,7 @@ const set = require('lodash.set'); const DataCollector = require('./dataCollector'); const getAttachments = require('./attachements'); const getSummary = require('./summary'); - -Promise.promisifyAll(Slack.prototype); +const { promisify } = require('util'); const defaultConfig = { userName: 'Sitespeed.io', @@ -23,6 +21,7 @@ const defaultConfig = { function send(options, dataCollector, context, screenshotType) { const slackOptions = merge({}, defaultConfig, options.slack); const slack = new Slack(slackOptions.hookUrl); + const send = promisify(slack.send.bind(slack)); const type = slackOptions.type; const pageErrors = []; let logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack.png'; @@ -69,22 +68,20 @@ function send(options, dataCollector, context, screenshotType) { slackOptions.channel, slackOptions.userName ); - return slack - .sendAsync({ - text, - icon_url: logo, - channel, - mrkdwn: true, - username: slackOptions.userName, - attachments - }) - .catch(e => { - if (e.errno === 'ETIMEDOUT') { - log.warn('Timeout sending Slack message.'); - } else { - throw e; - } - }); + return send({ + text, + icon_url: logo, + channel, + mrkdwn: true, + username: slackOptions.userName, + attachments + }).catch(e => { + if (e.errno === 'ETIMEDOUT') { + log.warn('Timeout sending Slack message.'); + } else { + throw e; + } + }); } } diff --git a/lib/plugins/tracestorer/index.js b/lib/plugins/tracestorer/index.js index 05883f89f..eb17c398e 100644 --- a/lib/plugins/tracestorer/index.js +++ b/lib/plugins/tracestorer/index.js @@ -1,9 +1,8 @@ 'use strict'; const zlib = require('zlib'); -const Promise = require('bluebird'); - -Promise.promisifyAll(zlib); +const { promisify } = require('util'); +const gzip = promisify(zlib.gzip); module.exports = { open(context) { @@ -14,15 +13,13 @@ module.exports = { case 'webpagetest.chrometrace': { const json = JSON.stringify(message.data); - return zlib - .gzipAsync(Buffer.from(json), { level: 1 }) - .then(gziped => - this.storageManager.writeDataForUrl( - gziped, - `${message.name}.gz`, - message.url - ) - ); + return gzip(Buffer.from(json), { level: 1 }).then(gziped => + this.storageManager.writeDataForUrl( + gziped, + `${message.name}.gz`, + message.url + ) + ); } } } diff --git a/lib/plugins/webpagetest/analyzer.js b/lib/plugins/webpagetest/analyzer.js index 2ac3c125b..5f12f2539 100644 --- a/lib/plugins/webpagetest/analyzer.js +++ b/lib/plugins/webpagetest/analyzer.js @@ -1,15 +1,13 @@ 'use strict'; -const Promise = require('bluebird'); const clone = require('lodash.clonedeep'); const get = require('lodash.get'); const WebPageTest = require('webpagetest'); const WPTAPIError = require('webpagetest/lib/helper').WPTAPIError; - -Promise.promisifyAll(WebPageTest.prototype); +const { promisify } = require('util'); module.exports = { - analyzeUrl(url, storageManager, log, wptOptions) { + async analyzeUrl(url, storageManager, log, wptOptions) { const wptClient = new WebPageTest(wptOptions.host, wptOptions.key); wptOptions.firstViewOnly = !wptOptions.includeRepeatView; let urlOrScript = url; @@ -19,9 +17,24 @@ module.exports = { urlOrScript = wptOptions.script.split('{{{URL}}}').join(url); } + // Setup WebPageTest methods + const runTest = promisify(wptClient.runTest.bind(wptClient)); + const getHARData = promisify(wptClient.getHARData.bind(wptClient)); + const getScreenshotImage = promisify( + wptClient.getScreenshotImage.bind(wptClient) + ); + const getWaterfallImage = promisify( + wptClient.getWaterfallImage.bind(wptClient) + ); + const getChromeTraceData = promisify( + wptClient.getChromeTraceData.bind(wptClient) + ); + // See https://github.com/sitespeedio/sitespeed.io/issues/1367 const options = clone(wptOptions); - return wptClient.runTestAsync(urlOrScript, options).then(function(data) { + + try { + const data = await runTest(urlOrScript, options); const id = data.data.id; log.info('Got %s analysed with id %s from %s', url, id, options.host); log.verbose('Got JSON from WebPageTest :%:2j', data); @@ -60,8 +73,7 @@ module.exports = { const promises = []; let har; promises.push( - wptClient - .getHARDataAsync(id, {}) + getHARData(id, {}) .then(theHar => (har = theHar)) .catch(WPTAPIError, error => log.warn( @@ -82,8 +94,10 @@ module.exports = { const repeatView = view === 'repeatView'; promises.push( - wptClient - .getScreenshotImageAsync(id, { run, repeatView }) + getScreenshotImage(id, { + run, + repeatView + }) .then(img => storageManager.writeDataForUrl( img, @@ -102,8 +116,10 @@ module.exports = { ); promises.push( - wptClient - .getWaterfallImageAsync(id, { run, repeatView }) + getWaterfallImage(id, { + run, + repeatView + }) .then(img => storageManager.writeDataForUrl( img, @@ -122,12 +138,11 @@ module.exports = { ); promises.push( - wptClient - .getWaterfallImageAsync(id, { - run, - chartType: 'connection', - repeatView - }) + getWaterfallImage(id, { + run, + chartType: 'connection', + repeatView + }) .then(img => storageManager.writeDataForUrl( img, @@ -147,8 +162,10 @@ module.exports = { if (wptOptions.timeline) { promises.push( - wptClient - .getChromeTraceDataAsync(id, { run, repeatView }) + getChromeTraceData(id, { + run, + repeatView + }) .then( trace => (traces['trace-' + run + '-wpt-' + view] = trace) ) @@ -171,6 +188,8 @@ module.exports = { myResult.trace = traces; return myResult; }); - }); + } catch (e) { + log.error('Could not run test for WebPageTest', e); + } } }; diff --git a/lib/plugins/webpagetest/index.js b/lib/plugins/webpagetest/index.js index 9e2ab579d..7ddead9e2 100644 --- a/lib/plugins/webpagetest/index.js +++ b/lib/plugins/webpagetest/index.js @@ -173,7 +173,7 @@ module.exports = { const group = message.group; return analyzer .analyzeUrl(url, this.storageManager, this.log, wptOptions) - .tap(result => { + .then(result => { addCustomMetric(result, filterRegistry); if (result.trace) { forEach(result.trace, (value, key) => { diff --git a/lib/sitespeed.js b/lib/sitespeed.js index 5686937a1..faf8bbebb 100644 --- a/lib/sitespeed.js +++ b/lib/sitespeed.js @@ -1,19 +1,18 @@ 'use strict'; -const Promise = require('bluebird'), - dayjs = require('dayjs'), - log = require('intel').getLogger('sitespeedio'), - intel = require('intel'), - os = require('os'), - process = require('process'), - logging = require('./core/logging'), - toArray = require('./support/util').toArray, - pullAll = require('lodash.pullall'), - union = require('lodash.union'), - messageMaker = require('./support/messageMaker'), - filterRegistry = require('./support/filterRegistry'), - statsHelpers = require('./support/statsHelpers'), - packageInfo = require('../package'); +const dayjs = require('dayjs'); +const log = require('intel').getLogger('sitespeedio'); +const intel = require('intel'); +const os = require('os'); +const process = require('process'); +const logging = require('./core/logging'); +const toArray = require('./support/util').toArray; +const pullAll = require('lodash.pullall'); +const union = require('lodash.union'); +const messageMaker = require('./support/messageMaker'); +const filterRegistry = require('./support/filterRegistry'); +const statsHelpers = require('./support/statsHelpers'); +const packageInfo = require('../package'); const QueueHandler = require('./core/queueHandler'), resultsStorage = require('./core/resultsStorage'), @@ -35,13 +34,13 @@ function runOptionalFunction(objects, fN) { for (let i = 2; i < arguments.length; i++) { args[i - 2] = arguments[i]; } - return Promise.resolve(objects) + return objects .filter(hasFunctionFilter(fN)) .map(plugin => Promise.resolve(plugin[fN].apply(plugin, args))); } module.exports = { - run(options) { + async run(options) { const url = options.urls[0]; const timestamp = dayjs(); @@ -57,96 +56,93 @@ module.exports = { options.useHash ); - return storageManager - .createDirectory('logs') - .then(logDir => { - logging.configure(options, logDir); - }) - .then(() => { - if (log.isEnabledFor(log.VERBOSE)) { - Promise.longStackTraces(); - } - log.info( - 'Versions OS: %s nodejs: %s sitespeed.io: %s browsertime: %s coach: %s', - os.platform() + ' ' + os.release(), - process.version, - packageInfo.version, - packageInfo.dependencies.browsertime, - packageInfo.dependencies.webcoach + // Setup logging + const logDir = await storageManager.createDirectory('logs'); + logging.configure(options, logDir); + + // Tell the world what we are using + log.info( + 'Versions OS: %s nodejs: %s sitespeed.io: %s browsertime: %s coach: %s', + os.platform() + ' ' + os.release(), + process.version, + packageInfo.version, + packageInfo.dependencies.browsertime, + packageInfo.dependencies.webcoach + ); + log.verbose('Config options: %:2j', options); + + let pluginNames = await loader.parsePluginNames(options); + + const plugins = options.plugins; + + // Deprecated setup + if (plugins) { + if (plugins.disable) { + log.warn( + '--plugins.disable is deprecated, use plugins.remove instead.' ); - log.verbose('Config options: %:2j', options); - }) - .then(() => { - return loader.parsePluginNames(options); - }) - .then(pluginNames => { - const plugins = options.plugins; - if (plugins) { - if (plugins.disable) { - log.warn( - '--plugins.disable is deprecated, use plugins.remove instead.' - ); - plugins.remove = plugins.disable; - } - if (plugins.load) { - log.warn('--plugins.load is deprecated, use plugins.add instead.'); - plugins.add = plugins.load; - } + plugins.remove = plugins.disable; + } + if (plugins.load) { + log.warn('--plugins.load is deprecated, use plugins.add instead.'); + plugins.add = plugins.load; + } - pullAll(pluginNames, toArray(plugins.remove)); - pluginNames = union(pluginNames, toArray(plugins.add)); + // Finalize the plugins that we wanna run + pullAll(pluginNames, toArray(plugins.remove)); + pluginNames = union(pluginNames, toArray(plugins.add)); - if (plugins.list) { - log.info( - 'The following plugins are enabled: %s', - pluginNames.join(', ') - ); - } - } - return pluginNames; - }) - .then(pluginNames => { - return loader.loadPlugins(pluginNames).then(plugins => { - let urlSources = [urlSource]; - const allPlugins = urlSources.concat(plugins), - queueHandler = new QueueHandler(plugins, options); + if (plugins.list) { + log.info( + 'The following plugins are enabled: %s', + pluginNames.join(', ') + ); + } + } - const context = { - storageManager, - resultUrls, - timestamp, - budget: budgetResult, - name: url, - log, - intel, - messageMaker, - statsHelpers, - filterRegistry - }; - return runOptionalFunction(allPlugins, 'open', context, options) - .then(() => queueHandler.run(urlSources)) - .tap(errors => - runOptionalFunction(allPlugins, 'close', options, errors) - ); - }); - }) - .then(errors => { - log.info('Finished analysing %s', url); - if (resultUrls.hasBaseUrl()) { - log.info('Find the result at %s', resultUrls.reportSummaryUrl()); - } + const runningPlugins = await loader.loadPlugins(pluginNames); + let urlSources = [urlSource]; + const allPlugins = urlSources.concat(runningPlugins); + const queueHandler = new QueueHandler(runningPlugins, options); - if (options.summary && options.summary.out) { - console.log(options.summary.out); // eslint-disable-line no-console - } - return { - errors, - budgetResult - }; - }) - .catch(err => { - log.error(err); - throw err; - }); + // This is the contect where we wanna run our tests + const context = { + storageManager, + resultUrls, + timestamp, + budget: budgetResult, + name: url, + log, + intel, + messageMaker, + statsHelpers, + filterRegistry + }; + + // Open/start each and every plugin + try { + await runOptionalFunction(allPlugins, 'open', context, options); + + // Pass the URLs + const errors = await queueHandler.run(urlSources); + + // Close the plugins + await runOptionalFunction(allPlugins, 'close', options, errors); + + if (resultUrls.hasBaseUrl()) { + log.info('Find the result at %s', resultUrls.reportSummaryUrl()); + } + + if (options.summary && options.summary.out) { + console.log(options.summary.out); // eslint-disable-line no-console + } + return { + errors, + budgetResult + }; + } catch (err) { + log.error(err); + throw err; + } } }; diff --git a/package-lock.json b/package-lock.json index 79aaee4fb..4436dcee3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -511,7 +511,8 @@ "bluebird": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", - "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==" + "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==", + "dev": true }, "brace-expansion": { "version": "1.1.11", @@ -4884,14 +4885,6 @@ "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=" }, - "longjohn": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/longjohn/-/longjohn-0.2.12.tgz", - "integrity": "sha1-fKdEawg2VcN351EiE9x1TVKmSn4=", - "requires": { - "source-map-support": "0.3.2 - 1.0.0" - } - }, "loud-rejection": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", @@ -7004,20 +6997,6 @@ "is-plain-obj": "^1.0.0" } }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" - }, - "source-map-support": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.6.tgz", - "integrity": "sha512-N4KXEz7jcKqPf2b2vZF11lQIz9W5ZMuUcIOGj243lduidkf2fjkVKJS9vNxVWn3u/uxX38AcE8U9nnH9FPcq+g==", - "requires": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "sparkles": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/sparkles/-/sparkles-1.0.1.tgz", diff --git a/package.json b/package.json index 18d5ab3a1..098bfd832 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "node": ">=8.9.0" }, "devDependencies": { + "bluebird": "3.5.2", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "clean-css-cli": "^4.1.11", @@ -68,7 +69,6 @@ "main": "./lib/sitespeed.js", "dependencies": { "aws-sdk": "2.327.0", - "bluebird": "3.5.2", "browsertime": "3.12.0", "cli-color": "1.3.0", "concurrent-queue": "7.0.2", @@ -91,7 +91,6 @@ "lodash.reduce": "4.6.0", "lodash.set": "4.3.2", "lodash.union": "4.6.0", - "longjohn": "0.2.12", "markdown": "0.5.0", "mkdirp": "0.5.1", "node-slack": "0.0.7",