mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Format code using the Prettier formatter. (#1677)

This commit is contained in:
Tobias Lidskog 2017-07-20 21:24:12 +02:00 committed by Peter Hedenskog
parent b7e8a709a5
commit 3debfec0b4
94 changed files with 9507 additions and 1802 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.eslintrc
View File

@ -1,11 +0,0 @@
root: true
env:
node: true
es6: true
extends:
"eslint:recommended"
rules:
"no-bitwise": error

21
.eslintrc.json Normal file
View File

@ -0,0 +1,21 @@
{
"root": true,
"env": {
"node": true,
"es6": true
},
"plugins": [
"prettier"
],
"extends": "eslint:recommended",
"rules": {
"prettier/prettier": [
"error",
{
"singleQuote": true
}
],
"no-extra-semi": 0,
"no-mixed-spaces-and-tabs": 0
}
}

17
lib/plugins/analysisstorer/index.js
View File

@ -6,7 +6,16 @@ const fs = require('fs');
Promise.promisifyAll(fs);
function shouldIgnoreMessage(message) {
return ['url', 'error', 'summarize','browsertime.screenshot', 'browsertime.har', 'webpagetest.har'].indexOf(message.type) >= 0;
return (
[
'url',
'error',
'summarize',
'browsertime.screenshot',
'browsertime.har',
'webpagetest.har'
].indexOf(message.type) >= 0
);
}
module.exports = {
@ -26,7 +35,11 @@ module.exports = {
}
if (message.url) {
return this.storageManager.writeDataForUrl(jsonData, fileName, message.url);
return this.storageManager.writeDataForUrl(
jsonData,
fileName,
message.url
);
} else {
return this.storageManager.writeData(fileName, jsonData);
}

18
lib/plugins/assets/aggregator.js
View File

@ -13,7 +13,6 @@ module.exports = {
largestThirdPartyAssetsByGroup: {},
slowestThirdPartyAssetsByGroup: {},
addToAggregate(data, group, url, resultUrls, runIndex, options) {
const maxSize = get(options, 'html.topListSize', 10);
let page = resultUrls.relativeSummaryPageUrl(url);
let runPage = page + runIndex;
@ -36,13 +35,17 @@ module.exports = {
if (this.largestAssets[asset.type]) {
this.largestAssets[asset.type].add(asset, page, runPage);
} else {
this.largestAssets[asset.type] = new AssetsBySize(maxSize , asset, page);
this.largestAssets[asset.type] = new AssetsBySize(maxSize, asset, page);
}
if (this.largestAssetsByGroup[group][asset.type]) {
this.largestAssetsByGroup[group][asset.type].add(asset, page, runPage);
} else {
this.largestAssetsByGroup[group][asset.type] = new AssetsBySize(maxSize, asset, page);
this.largestAssetsByGroup[group][asset.type] = new AssetsBySize(
maxSize,
asset,
page
);
}
this.slowestAssets.add(asset, page, runPage);
@ -88,7 +91,6 @@ module.exports = {
urlInfoGroup.requestCount++;
this.groups[group][url] = urlInfoGroup;
}
},
summarize() {
@ -111,13 +113,17 @@ module.exports = {
summary.size[group] = {};
summary.timing[group] = {};
for (let assetTypes of Object.keys(this.largestAssetsByGroup[group])) {
summary.size[group][assetTypes] = this.largestAssetsByGroup[group][assetTypes].getItems();
summary.size[group][assetTypes] = this.largestAssetsByGroup[group][
assetTypes
].getItems();
}
summary.timing[group] = this.slowestAssetsByGroup[group].getItems();
}
for (let assetTypes of Object.keys(this.largestAssets)) {
summary.size.total[assetTypes] = this.largestAssets[assetTypes].getItems();
summary.size.total[assetTypes] = this.largestAssets[
assetTypes
].getItems();
}
summary.timing.total = this.slowestAssets.getItems();

2
lib/plugins/assets/assetsBySpeed.js
View File

@ -1,6 +1,6 @@
'use strict';
class AssetsBySpeed{
class AssetsBySpeed {
constructor(maxSize) {
this.maxSize = maxSize;
this.items = [];

60
lib/plugins/assets/index.js
View File

@ -7,15 +7,16 @@ const aggregator = require('./aggregator');
const make = messageMaker('assets').make;
const DEFAULT_METRICS_LARGEST_ASSETS = [
'image.0.transferSize'
];
const DEFAULT_METRICS_LARGEST_ASSETS = ['image.0.transferSize'];
module.exports = {
open(context, options) {
this.options = options;
this.resultUrls = context.resultUrls;
filterRegistry.registerFilterForType(DEFAULT_METRICS_LARGEST_ASSETS, 'largestassets.summary');
filterRegistry.registerFilterForType(
DEFAULT_METRICS_LARGEST_ASSETS,
'largestassets.summary'
);
filterRegistry.registerFilterForType([], 'slowestassets.summary');
filterRegistry.registerFilterForType([], 'aggregateassets.summary');
filterRegistry.registerFilterForType([], 'slowestthirdpartyassets.summary');
@ -23,37 +24,64 @@ module.exports = {
},
processMessage(message, queue) {
switch (message.type) {
case 'pagexray.run':
{
aggregator.addToAggregate(message.data, message.group, message.url, this.resultUrls, message.runIndex, this.options);
case 'pagexray.run': {
aggregator.addToAggregate(
message.data,
message.group,
message.url,
this.resultUrls,
message.runIndex,
this.options
);
break;
}
case 'summarize':
{
case 'summarize': {
const summary = aggregator.summarize();
if (!isEmpty(summary)) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(make('aggregateassets.summary', summary.groups[group], {group}));
queue.postMessage(
make('aggregateassets.summary', summary.groups[group], { group })
);
for (let contentType of Object.keys(summary.size[group])) {
const d = {};
d[contentType] = summary.size[group][contentType];
queue.postMessage(make('largestassets.summary', d, {group}));
queue.postMessage(make('largestassets.summary', d, { group }));
}
queue.postMessage(make('slowestassets.summary', summary.timing[group], {group}));
queue.postMessage(
make('slowestassets.summary', summary.timing[group], { group })
);
}
for (let contentType of Object.keys(summary.size.total)) {
queue.postMessage(make('largestassets.summary', summary.size.total[contentType], {group: 'total'}));
queue.postMessage(
make('largestassets.summary', summary.size.total[contentType], {
group: 'total'
})
);
}
queue.postMessage(make('slowestassets.summary', summary.timing.total, {group: 'total'}));
queue.postMessage(
make('slowestassets.summary', summary.timing.total, {
group: 'total'
})
);
if (this.options.firstParty) {
queue.postMessage(make('slowestthirdpartyassets.summary', summary.timing.thirdParty, {group: 'total'}));
queue.postMessage(make('largestthirdpartyassets.summary', summary.size.thirdParty, {group: 'total'}));
queue.postMessage(
make(
'slowestthirdpartyassets.summary',
summary.timing.thirdParty,
{ group: 'total' }
)
);
queue.postMessage(
make('largestthirdpartyassets.summary', summary.size.thirdParty, {
group: 'total'
})
);
}
}
break;

65
lib/plugins/browsertime/aggregator.js
View File

@ -10,44 +10,77 @@ module.exports = {
groups: {},
addToAggregate(browsertimeRunData, group) {
if (this.groups[group] === undefined) {
this.groups[group] = {};
}
forEach(timings, (timing) => {
forEach(timings, timing => {
if (browsertimeRunData.timings[timing]) {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], timing, browsertimeRunData.timings[timing]);
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
timing,
browsertimeRunData.timings[timing]
);
}
});
forEach(browsertimeRunData.timings.navigationTiming, (value, name) => {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], ['navigationTiming', name], value);
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
['navigationTiming', name],
value
);
});
// pick up one level of custom metrics
forEach(browsertimeRunData.custom, (value, name) => {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], ['custom', name], value);
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
['custom', name],
value
);
});
forEach(browsertimeRunData.timings.pageTimings, (value, name) => {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], ['pageTimings', name], value);
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
['pageTimings', name],
value
);
});
forEach(browsertimeRunData.timings.userTimings.marks, (timing) => {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], ['userTimings', 'marks', timing.name], timing.startTime);
forEach(browsertimeRunData.timings.userTimings.marks, timing => {
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
['userTimings', 'marks', timing.name],
timing.startTime
);
});
forEach(browsertimeRunData.timings.userTimings.measures, (timing) => {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], ['userTimings', 'measures', timing.name], timing.duration);
forEach(browsertimeRunData.timings.userTimings.measures, timing => {
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
['userTimings', 'measures', timing.name],
timing.duration
);
});
forEach(browsertimeRunData.visualMetrics, (value, name) => {
if (name !== 'VisualProgress') {
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], ['visualMetrics', name], value);
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
['visualMetrics', name],
value
);
}
});
},
summarize() {
if (Object.keys(this.statsPerType).length === 0) {
@ -69,23 +102,23 @@ module.exports = {
summarizePerObject(obj) {
return Object.keys(obj).reduce((summary, name) => {
if (timings.indexOf(name) > -1) {
statsHelpers.setStatsSummary(summary, name, obj[name])
statsHelpers.setStatsSummary(summary, name, obj[name]);
} else if ('userTimings'.indexOf(name) > -1) {
summary.userTimings = {};
const marksData = {},
measuresData = {};
forEach(obj.userTimings.marks, (stats, timingName) => {
statsHelpers.setStatsSummary(marksData, timingName, stats)
statsHelpers.setStatsSummary(marksData, timingName, stats);
});
forEach(obj.userTimings.measures, (stats, timingName) => {
statsHelpers.setStatsSummary(measuresData, timingName, stats)
statsHelpers.setStatsSummary(measuresData, timingName, stats);
});
summary.userTimings.marks = marksData;
summary.userTimings.measures = measuresData;
} else {
const categoryData = {};
forEach(obj[name], (stats, timingName) => {
statsHelpers.setStatsSummary(categoryData, timingName, stats)
statsHelpers.setStatsSummary(categoryData, timingName, stats);
});
summary[name] = categoryData;
}

53
lib/plugins/browsertime/analyzer.js
View File

@ -19,22 +19,25 @@ const chromeIphoneEmulationOptions = {
}
};
const iphone6UserAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 ' +
const iphone6UserAgent =
'Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 ' +
'(KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25';
function parseUserScripts(scripts) {
if (!Array.isArray(scripts))
scripts = [scripts];
if (!Array.isArray(scripts)) scripts = [scripts];
return Promise.reduce(scripts, (results, script) =>
browserScripts.findAndParseScripts(path.resolve(script), 'custom')
.then((scripts) => merge(results, scripts)),
{});
return Promise.reduce(
scripts,
(results, script) =>
browserScripts
.findAndParseScripts(path.resolve(script), 'custom')
.then(scripts => merge(results, scripts)),
{}
);
}
function addCoachScripts(scripts) {
return Promise.join(scripts, coach.getDomAdvice(),
(scripts, advice) => {
return Promise.join(scripts, coach.getDomAdvice(), (scripts, advice) => {
scripts.coach = {
coachAdvice: advice
};
@ -45,25 +48,35 @@ function addCoachScripts(scripts) {
module.exports = {
analyzeUrl(url, options) {
const btOptions = merge({}, defaultBrowsertimeOptions, options);
merge(btOptions, {verbose: options.verbose});
merge(btOptions, { verbose: options.verbose });
// set mobile options
if (options.mobile) {
btOptions.viewPort = '360x640';
if (btOptions.browser === 'chrome') {
btOptions.chrome = merge({}, btOptions.chrome, chromeIphoneEmulationOptions);
btOptions.chrome = merge(
{},
btOptions.chrome,
chromeIphoneEmulationOptions
);
} else {
btOptions.userAgent = iphone6UserAgent;
}
}
const scriptCategories = browserScripts.allScriptCategories;
let scriptsByCategory = browserScripts.getScriptsForCategories(scriptCategories);
let scriptsByCategory = browserScripts.getScriptsForCategories(
scriptCategories
);
if (btOptions.script) {
const userScripts = parseUserScripts(btOptions.script);
scriptsByCategory = Promise.join(scriptsByCategory, userScripts,
(scriptsByCategory, userScripts) => merge(scriptsByCategory, userScripts));
scriptsByCategory = Promise.join(
scriptsByCategory,
userScripts,
(scriptsByCategory, userScripts) =>
merge(scriptsByCategory, userScripts)
);
}
if (btOptions.coach) {
@ -71,9 +84,15 @@ module.exports = {
}
let engine = new browsertime.Engine(btOptions);
log.info('Starting %s for analysing %s %s time(s)', btOptions.browser, url, btOptions.iterations);
return engine.start()
log.info(
'Starting %s for analysing %s %s time(s)',
btOptions.browser,
url,
btOptions.iterations
);
return engine
.start()
.then(() => engine.run(url, scriptsByCategory))
.finally(() => engine.stop())
.finally(() => engine.stop());
}
};

194
lib/plugins/browsertime/index.js
View File

@ -35,7 +35,7 @@ const defaultConfig = {
},
viewPort: '1366x708',
delay: 0
}
};
const DEFAULT_METRICS_PAGE_SUMMARY = [
'statistics.timings.pageTimings',
@ -66,15 +66,16 @@ const DEFAULT_METRICS_SUMMARY = [
'custom.*'
];
function jsonifyVisualProgress(visualProgress) {
// Original data looks like
// "0=0%, 1500=81%, 1516=81%, 1533=84%, 1550=84%, 1566=84%, 1600=95%, 1683=95%, 1833=100%"
const progress = visualProgress.split(',');
const visualProgressJSON = {};
forEach(progress, (value) => {
forEach(progress, value => {
const eachMetric = value.split('=');
visualProgressJSON[eachMetric[0].replace(' ','')] = Number(eachMetric[1].replace('%',''));
visualProgressJSON[eachMetric[0].replace(' ', '')] = Number(
eachMetric[1].replace('%', '')
);
});
return visualProgressJSON;
}
@ -90,26 +91,46 @@ module.exports = {
browsertime.logging.configure(options);
// hack for disabling viewport on Android that's not supported
if (this.options.chrome && this.options.chrome.android && this.options.chrome.android.package) {
if (
this.options.chrome &&
this.options.chrome.android &&
this.options.chrome.android.package
) {
this.options.viewPort = undefined;
}
filterRegistry.registerFilterForType(DEFAULT_METRICS_PAGE_SUMMARY, 'browsertime.pageSummary');
filterRegistry.registerFilterForType(DEFAULT_METRICS_SUMMARY, 'browsertime.summary');
filterRegistry.registerFilterForType(
DEFAULT_METRICS_PAGE_SUMMARY,
'browsertime.pageSummary'
);
filterRegistry.registerFilterForType(
DEFAULT_METRICS_SUMMARY,
'browsertime.summary'
);
},
processMessage(message, queue) {
function processCoachOutput(url, group, results) {
return Promise.resolve(results.browserScripts)
.each((run, runIndex) => {
return Promise.resolve(results.browserScripts).each((run, runIndex) => {
const coachAdvice = run.coach.coachAdvice;
// check if the coach has error(s)
if (!isEmpty(coachAdvice.errors)) {
log.error('%s generated the following errors in the coach %:2j', url, coachAdvice.errors);
queue.postMessage(make('error', 'The coach got the following errors: ' + JSON.stringify(coachAdvice.errors), {
log.error(
'%s generated the following errors in the coach %:2j',
url,
coachAdvice.errors
);
queue.postMessage(
make(
'error',
'The coach got the following errors: ' +
JSON.stringify(coachAdvice.errors),
{
url,
runIndex
}));
}
)
);
}
// if we miss the HAR from Firefox
@ -117,47 +138,58 @@ module.exports = {
// make sure to get the right run in the HAR
const myHar = api.pickAPage(results.har, runIndex);
return api.runHarAdvice(myHar)
.then((harResult) => api.merge(coachAdvice, harResult))
.then((total) => queue.postMessage(make('coach.run', total, {
return api
.runHarAdvice(myHar)
.then(harResult => api.merge(coachAdvice, harResult))
.then(total =>
queue.postMessage(
make('coach.run', total, {
url,
group,
runIndex
})));
})
)
);
} else {
return queue.postMessage(make('coach.run', coachAdvice, {
return queue.postMessage(
make('coach.run', coachAdvice, {
url,
group,
runIndex
}));
})
);
}
});
}
switch (message.type) {
case 'url':
{
case 'url': {
const url = message.url;
const group = message.group;
visitedUrls.add(url);
// manually set the resultBaseDir
// it's used in BT when we record a video
return this.storageManager.createDirForUrl(message.url, 'data').then((dir) => {
this.options.resultDir = dir
}).then(() => analyzer.analyzeUrl(url, this.options))
.tap((results) => {
log.verbose('Result from Browsertime for %s with %:2j', url, results);
return this.storageManager
.createDirForUrl(message.url, 'data')
.then(dir => {
this.options.resultDir = dir;
})
.tap((results) => {
.then(() => analyzer.analyzeUrl(url, this.options))
.tap(results => {
log.verbose(
'Result from Browsertime for %s with %:2j',
url,
results
);
})
.tap(results => {
results.browserScripts.forEach((run, runIndex) => {
// take the HAR from this run and add it to the
// run data
// sometimes Firefox can't create the HAR + in the future
// we may wanna use Safari (without HAR)
if (results.har) {
// Add meta data to be used when we compare multiple HARs
results.har.log.pages[runIndex]._meta = {};
const _meta = results.har.log.pages[runIndex]._meta;
@ -174,29 +206,40 @@ module.exports = {
// Setup visual metrics to use when we compare hars
results.har.log.pages[runIndex]._visualMetrics = {};
const _visualMetrics = results.har.log.pages[runIndex]._visualMetrics;
const _visualMetrics =
results.har.log.pages[runIndex]._visualMetrics;
const pageTimings = results.har.log.pages[runIndex].pageTimings;
// if we have first, complete85 and last visual change add it to the HAR file
// so we can see it in the waterfall graph (PerfCascade automatically picks it up)
if (results.visualMetrics && results.visualMetrics[runIndex]) {
pageTimings._firstVisualChange = results.visualMetrics[runIndex].FirstVisualChange;
pageTimings._lastVisualChange = results.visualMetrics[runIndex].LastVisualChange;
pageTimings._visualComplete85 = results.visualMetrics[runIndex].VisualComplete85;
_visualMetrics._FirstVisualChange = results.visualMetrics[runIndex].FirstVisualChange;
_visualMetrics._SpeedIndex = results.visualMetrics[runIndex].SpeedIndex;
_visualMetrics._VisualComplete85 = results.visualMetrics[runIndex].VisualComplete85;
_visualMetrics._LastVisualChange = results.visualMetrics[runIndex].LastVisualChange;
_visualMetrics._VisualProgress = jsonifyVisualProgress(results.visualMetrics[runIndex].VisualProgress);
}
pageTimings._firstVisualChange =
results.visualMetrics[runIndex].FirstVisualChange;
pageTimings._lastVisualChange =
results.visualMetrics[runIndex].LastVisualChange;
pageTimings._visualComplete85 =
results.visualMetrics[runIndex].VisualComplete85;
_visualMetrics._FirstVisualChange =
results.visualMetrics[runIndex].FirstVisualChange;
_visualMetrics._SpeedIndex =
results.visualMetrics[runIndex].SpeedIndex;
_visualMetrics._VisualComplete85 =
results.visualMetrics[runIndex].VisualComplete85;
_visualMetrics._LastVisualChange =
results.visualMetrics[runIndex].LastVisualChange;
_visualMetrics._VisualProgress = jsonifyVisualProgress(
results.visualMetrics[runIndex].VisualProgress
);
} else if (run.timings.firstPaint) {
// only add first paint if we don't have visual metrics
else if (run.timings.firstPaint) {
pageTimings._firstPaint = run.timings.firstPaint;
}
if (run.timings.pageTimings) {
pageTimings._domInteractiveTime = run.timings.pageTimings.domInteractiveTime;
pageTimings._domContentLoadedTime = run.timings.pageTimings.domContentLoadedTime;
pageTimings._domInteractiveTime =
run.timings.pageTimings.domInteractiveTime;
pageTimings._domContentLoadedTime =
run.timings.pageTimings.domContentLoadedTime;
}
run.har = api.pickAPage(results.har, runIndex);
@ -209,69 +252,90 @@ module.exports = {
run.visualMetrics = results.visualMetrics[runIndex];
}
run.timestamp = moment(results.timestamps[runIndex]).format(TIME_FORMAT);
run.timestamp = moment(results.timestamps[runIndex]).format(
TIME_FORMAT
);
queue.postMessage(make('browsertime.run', run, {
queue.postMessage(
make('browsertime.run', run, {
url,
group,
runIndex
}));
})
);
aggregator.addToAggregate(run, group);
});
// Let take the first runs timestamp and use that as the summary timestamp
results.timestamp = moment(results.timestamps[0]).format(TIME_FORMAT);
queue.postMessage(make('browsertime.pageSummary', results, {
results.timestamp = moment(results.timestamps[0]).format(
TIME_FORMAT
);
queue.postMessage(
make('browsertime.pageSummary', results, {
url,
group
}));
})
.tap((results) => {
);
})
.tap(results => {
if (results.har) {
queue.postMessage(make('browsertime.har', results.har, {
queue.postMessage(
make('browsertime.har', results.har, {
url,
group
}));
})
);
}
})
.tap((results) => {
.tap(results => {
if (results.extraJson) {
forEach(results.extraJson, (value, key) => {
if (key.indexOf('trace' > -1)) {
queue.postMessage(make('browsertime.chrometrace', value, {
queue.postMessage(
make('browsertime.chrometrace', value, {
url,
group,
name: key
}));
})
);
}
});
}
})
}
})
.tap((results) => {
.tap(results => {
if (results.screenshots) {
queue.postMessage(make('browsertime.screenshot', results.screenshots, {
queue.postMessage(
make('browsertime.screenshot', results.screenshots, {
url,
group
}));
})
);
}
})
.tap((results) => {
.tap(results => {
if (this.options.coach) {
return processCoachOutput(url, group, results);
}
})
.catch(BrowsertimeError, (e) => {
log.error('%s generated the following error in Browsertime %s', url, e);
queue.postMessage(make('error', e.message, merge({url}, e.extra)));
.catch(BrowsertimeError, e => {
log.error(
'%s generated the following error in Browsertime %s',
url,
e
);
queue.postMessage(
make('error', e.message, merge({ url }, e.extra))
);
});
}
case 'summarize':
{
case 'summarize': {
const summary = aggregator.summarize();
if (summary) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(make('browsertime.summary', summary.groups[group], {group}));
queue.postMessage(
make('browsertime.summary', summary.groups[group], { group })
);
}
}

16
lib/plugins/budget/index.js
View File

@ -19,8 +19,7 @@ module.exports = {
case 'browsertime.pageSummary':
case 'webpagetest.pageSummary':
case 'pagexray.pageSummary':
case 'coach.pageSummary':
{
case 'coach.pageSummary': {
verify(message, this.result, this.options.budget.config);
return;
}
@ -32,13 +31,20 @@ module.exports = {
tap.writeTap(this.result, this.storageManager.getBaseDir());
} else if (this.options.budget.output === 'junit') {
junit.writeJunit(this.result, this.storageManager.getBaseDir());
}
else {
} else {
let failing = 0;
let working = 0;
for (const url of Object.keys(this.result.failing)) {
for (const result of this.result.failing[url]) {
log.error('Failing budget %s.%s for %s with value %s %s limit %s', result.type, result.metric, url, result.value, result.limitType , result.limit);
log.error(
'Failing budget %s.%s for %s with value %s %s limit %s',
result.type,
result.metric,
url,
result.value,
result.limitType,
result.limit
);
failing = failing + 1;
}
}

38
lib/plugins/budget/junit.js
View File

@ -12,29 +12,53 @@ exports.writeJunit = function(results, dir) {
for (const url of urls) {
const parsedUrl = urlParser.parse(url);
const fineUrl = parsedUrl.hostname.replace(/\./g, '_') + '.' + parsedUrl.path.replace(/\./g, '_').replace(/\//g, '_');
const fineUrl =
parsedUrl.hostname.replace(/\./g, '_') +
'.' +
parsedUrl.path.replace(/\./g, '_').replace(/\//g, '_');
const suite = builder.testSuite().name('sitespeed.io' + '.' + fineUrl);
if (results.failing[url]) {
for (const result of results.failing[url]) {
suite.testCase()
suite
.testCase()
.className(fineUrl)
.name(result.type + '.' + result.metric)
.failure(result.metric + ' is ' + result.value + ' and limit ' + result.limitType + ' ' + result.limit + ' ' + url);
.failure(
result.metric +
' is ' +
result.value +
' and limit ' +
result.limitType +
' ' +
result.limit +
' ' +
url
);
}
}
if (results.working[url]) {
for (const result of results.working[url]) {
suite.testCase()
suite
.testCase()
.className(fineUrl)
.name(result.type + '.' + result.metric)
.standardOutput(result.metric + ' is ' + result.value + ' and limit ' + result.limitType + ' ' + result.limit + ' ' + url);
.standardOutput(
result.metric +
' is ' +
result.value +
' and limit ' +
result.limitType +
' ' +
result.limit +
' ' +
url
);
}
}
}
const file = path.join(dir, 'junit.xml');
log.info('Write junit budget to %s', path.resolve(file));
builder.writeTo(file);
}
};

20
lib/plugins/budget/tap.js
View File

@ -7,9 +7,8 @@ const tap = require('tape'),
EOL = require('os').EOL;
exports.writeTap = function(results, dir) {
const file = path.join(dir, 'budget.tap');
log.info('Write budget to %s', path.resolve(file) );
log.info('Write budget to %s', path.resolve(file));
const tapOutput = fs.createWriteStream(file);
tap.createStream().pipe(tapOutput);
@ -23,10 +22,21 @@ exports.writeTap = function(results, dir) {
if (resultType === 'failing') {
extra = ' limit ' + result.limitType + ' ' + result.limit + EOL;
}
t.ok(resultType === 'failing' ? false : true, result.type + '.' + result.metric + ' ' + result.value + ' ' + extra + ' ' + url);
t.ok(
resultType === 'failing' ? false : true,
result.type +
'.' +
result.metric +
' ' +
result.value +
' ' +
extra +
' ' +
url
);
t.end();
})
});
}
}
}
}
};

27
lib/plugins/budget/verify.js
View File

@ -17,12 +17,15 @@ function getItem(url, type, metric, value, limit, limitType) {
}
function getHelperFunction(metric) {
if (metric.indexOf('transferSize') > -1 || metric.indexOf('contentSize') > -1) {
if (
metric.indexOf('transferSize') > -1 ||
metric.indexOf('contentSize') > -1
) {
return size;
} else if (metric.indexOf('timings') > -1) {
return function(time) {
return time + ' ms';
}
};
} else return noop;
}
@ -33,13 +36,19 @@ module.exports = {
// do we have an entry in the budget for this kind of message?
if (budgets[message.type]) {
for (var budget of budgets[message.type]) {
let value = get(message.data, budget.metric);
if (value !== undefined) {
const format = getHelperFunction(budget.metric);
const item = getItem(message.url, message.type, budget.metric, format(value), budget.max !== undefined ? format(budget.max) : format(budget.min), budget.max !== undefined ? 'max': 'min');
const item = getItem(
message.url,
message.type,
budget.metric,
format(value),
budget.max !== undefined ? format(budget.max) : format(budget.min),
budget.max !== undefined ? 'max' : 'min'
);
if (budget.max !== undefined) {
if (value <= budget.max) {
@ -64,11 +73,15 @@ module.exports = {
// group working/failing per URL
if (failing.length > 0) {
result.failing[message.url] = result.failing[message.url] ? result.failing[message.url].concat(failing) : failing;
result.failing[message.url] = result.failing[message.url]
? result.failing[message.url].concat(failing)
: failing;
}
if (working.length > 0) {
result.working[message.url] = result.working[message.url] ? result.working[message.url].concat(working) : working;
result.working[message.url] = result.working[message.url]
? result.working[message.url].concat(working)
: working;
}
}
}
};

26
lib/plugins/coach/aggregator.js
View File

@ -8,23 +8,37 @@ module.exports = {
groups: {},
addToAggregate(coachData, group) {
if (this.groups[group] === undefined) {
this.groups[group] = {};
}
// push the total score
statsHelpers.pushGroupStats(this.statsPerCategory, this.groups[group], 'score', coachData.advice.score);
statsHelpers.pushGroupStats(
this.statsPerCategory,
this.groups[group],
'score',
coachData.advice.score
);
forEach(coachData.advice, (category, categoryName) => {
if (category.score === undefined) {
return;
}
// Push the score per category
statsHelpers.pushGroupStats(this.statsPerCategory, this.groups[group], [categoryName, 'score'], category.score);
statsHelpers.pushGroupStats(
this.statsPerCategory,
this.groups[group],
[categoryName, 'score'],
category.score
);
forEach(category.adviceList, (advice, adviceName) => {
statsHelpers.pushGroupStats(this.statsPerCategory, this.groups[group], [categoryName, adviceName], advice.score);
statsHelpers.pushGroupStats(
this.statsPerCategory,
this.groups[group],
[categoryName, adviceName],
advice.score
);
});
});
},
@ -47,12 +61,12 @@ module.exports = {
summarizePerObject(type) {
return Object.keys(type).reduce((summary, categoryName) => {
if (categoryName === 'score') {
statsHelpers.setStatsSummary(summary, 'score', type[categoryName])
statsHelpers.setStatsSummary(summary, 'score', type[categoryName]);
} else {
const categoryData = {};
forEach(type[categoryName], (stats, name) => {
statsHelpers.setStatsSummary(categoryData, name, stats)
statsHelpers.setStatsSummary(categoryData, name, stats);
});
summary[categoryName] = categoryData;

101
lib/plugins/coach/index.js
View File

@ -8,68 +8,129 @@ const filterRegistry = require('../../support/filterRegistry');
const make = messageMaker('coach').make;
const DEFAULT_METRICS_SUMMARY = ['score.*','performance.score.*', 'bestpractice.score.*', 'accessibility.score.*'];
const DEFAULT_METRICS_PAGESUMMARY = ['advice.score','advice.performance.score','advice.bestpractice.score', 'advice.accessibility.score','advice.info.documentHeight','advice.info.domElements','advice.info.domDepth', 'advice.info.iframes', 'advice.info.scripts','advice.info.localStorageSize'];
const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes','expireStats', 'totalDomains', 'lastModifiedStats', 'cookieStats'];
const DEFAULT_PAGEXRAY_SUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes','expireStats', 'domains', 'lastModifiedStats', 'cookieStats'];
const DEFAULT_METRICS_SUMMARY = [
'score.*',
'performance.score.*',
'bestpractice.score.*',
'accessibility.score.*'
];
const DEFAULT_METRICS_PAGESUMMARY = [
'advice.score',
'advice.performance.score',
'advice.bestpractice.score',
'advice.accessibility.score',
'advice.info.documentHeight',
'advice.info.domElements',
'advice.info.domDepth',
'advice.info.iframes',
'advice.info.scripts',
'advice.info.localStorageSize'
];
const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = [
'contentTypes',
'transferSize',
'contentSize',
'requests',
'firstParty',
'thirdParty',
'responseCodes',
'expireStats',
'totalDomains',
'lastModifiedStats',
'cookieStats'
];
const DEFAULT_PAGEXRAY_SUMMARY_METRICS = [
'contentTypes',
'transferSize',
'contentSize',
'requests',
'firstParty',
'thirdParty',
'responseCodes',
'expireStats',
'domains',
'lastModifiedStats',
'cookieStats'
];
module.exports = {
open(context, options) {
this.options = options;
filterRegistry.registerFilterForType(DEFAULT_METRICS_SUMMARY, 'coach.summary');
filterRegistry.registerFilterForType(DEFAULT_METRICS_PAGESUMMARY, 'coach.pageSummary');
filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS, 'pagexray.pageSummary');
filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_SUMMARY_METRICS, 'pagexray.summary');
filterRegistry.registerFilterForType(
DEFAULT_METRICS_SUMMARY,
'coach.summary'
);
filterRegistry.registerFilterForType(
DEFAULT_METRICS_PAGESUMMARY,
'coach.pageSummary'
);
filterRegistry.registerFilterForType(
DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS,
'pagexray.pageSummary'
);
filterRegistry.registerFilterForType(
DEFAULT_PAGEXRAY_SUMMARY_METRICS,
'pagexray.summary'
);
},
processMessage(message, queue) {
switch (message.type) {
case 'coach.run':
{
case 'coach.run': {
if (message.runIndex === 0) {
// For now, choose the first run to represent the whole page.
// Later we might want to change the median run (based on some metric) similar to the WPT approach.
const url = message.url;
const group = message.group;
queue.postMessage(make('coach.pageSummary', message.data, {url, group}));
queue.postMessage(
make('coach.pageSummary', message.data, { url, group })
);
}
aggregator.addToAggregate(message.data, message.group);
break;
}
case 'browsertime.har':
{
case 'browsertime.har': {
const url = message.url;
const group = message.group;
let config = {
includeAssets: true,
firstParty: this.options.firstParty ? this.options.firstParty : undefined
firstParty: this.options.firstParty
? this.options.firstParty
: undefined
};
const pageSummary = pagexray.convert(message.data, config);
pagexrayAggregator.addToAggregate(pageSummary, group);
queue.postMessage(make('pagexray.pageSummary', pageSummary[0], {url, group}));
queue.postMessage(
make('pagexray.pageSummary', pageSummary[0], { url, group })
);
pageSummary.forEach((run, runIndex) => {
queue.postMessage(make('pagexray.run', run, {url, group, runIndex}));
queue.postMessage(
make('pagexray.run', run, { url, group, runIndex })
);
});
break;
}
case 'summarize':
{
case 'summarize': {
let summary = aggregator.summarize();
if (summary) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(make('coach.summary', summary.groups[group], {group}));
queue.postMessage(
make('coach.summary', summary.groups[group], { group })
);
}
}
let pagexraySummary = pagexrayAggregator.summarize();
if (pagexraySummary) {
for (let group of Object.keys(pagexraySummary.groups)) {
queue.postMessage(make('pagexray.summary', pagexraySummary.groups[group], {group}));
queue.postMessage(
make('pagexray.summary', pagexraySummary.groups[group], { group })
);
}
}
break;

94
lib/plugins/coach/pagexrayAggregator.js
View File

@ -9,7 +9,6 @@ module.exports = {
stats: {},
groups: {},
addToAggregate(pageSummary, group) {
if (this.groups[group] === undefined) {
this.groups[group] = {};
}
@ -20,42 +19,85 @@ module.exports = {
pageSummary.forEach(function(summary) {
// stats for the whole page
METRIC_NAMES.forEach(function(metric) {
statsHelpers.pushGroupStats(stats, groups[group], metric, summary[metric]);
statsHelpers.pushGroupStats(
stats,
groups[group],
metric,
summary[metric]
);
});
Object.keys(summary.contentTypes).forEach(function(contentType) {
METRIC_NAMES.forEach(function(metric) {
statsHelpers.pushGroupStats(stats,groups[group], 'contentTypes.' + contentType + '.' + metric, summary.contentTypes[contentType][metric]);
statsHelpers.pushGroupStats(
stats,
groups[group],
'contentTypes.' + contentType + '.' + metric,
summary.contentTypes[contentType][metric]
);
});
});
Object.keys(summary.responseCodes).forEach(function(responseCode) {
statsHelpers.pushGroupStats(stats, groups[group], 'responseCodes.' + responseCode, summary.responseCodes[responseCode]);
statsHelpers.pushGroupStats(
stats,
groups[group],
'responseCodes.' + responseCode,
summary.responseCodes[responseCode]
);
});
// extras for firstParty vs third
if (summary.firstParty.requests) {
METRIC_NAMES.forEach(function(metric) {
if (summary.firstParty[metric] !== undefined) {
statsHelpers.pushGroupStats(stats, groups[group], 'firstParty' + '.' + metric, summary.firstParty[metric]);
statsHelpers.pushGroupStats(
stats,
groups[group],
'firstParty' + '.' + metric,
summary.firstParty[metric]
);
}
if (summary.thirdParty[metric] !== undefined) {
statsHelpers.pushGroupStats(stats, groups[group], 'thirdParty' + '.' + metric, summary.thirdParty[metric]);
statsHelpers.pushGroupStats(
stats,
groups[group],
'thirdParty' + '.' + metric,
summary.thirdParty[metric]
);
}
})
});
}
// Add the total amount of domains on this page
statsHelpers.pushGroupStats(stats, groups[group], 'domains', (Object.keys(summary.domains)).length);
statsHelpers.pushGroupStats(
stats,
groups[group],
'domains',
Object.keys(summary.domains).length
);
forEach(summary.assets, (asset) => {
statsHelpers.pushGroupStats(stats, groups[group], 'expireStats', asset.expires);
statsHelpers.pushGroupStats(stats, groups[group], 'lastModifiedStats', asset.timeSinceLastModified);
statsHelpers.pushGroupStats(stats, groups[group], 'cookiesStats', asset.cookies);
forEach(summary.assets, asset => {
statsHelpers.pushGroupStats(
stats,
groups[group],
'expireStats',
asset.expires
);
statsHelpers.pushGroupStats(
stats,
groups[group],
'lastModifiedStats',
asset.timeSinceLastModified
);
statsHelpers.pushGroupStats(
stats,
groups[group],
'cookiesStats',
asset.cookies
);
});
});
})
},
summarize() {
if (Object.keys(this.stats).length === 0) {
@ -71,31 +113,37 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]);
}
return summary;
},
summarizePerObject(type) {
return Object.keys(type).reduce((summary, name) => {
if (METRIC_NAMES.indexOf(name) > -1 || name.match(/(^domains|^expireStats|^lastModifiedStats|^cookiesStats)/)) {
statsHelpers.setStatsSummary(summary, name, type[name])
if (
METRIC_NAMES.indexOf(name) > -1 ||
name.match(/(^domains|^expireStats|^lastModifiedStats|^cookiesStats)/)
) {
statsHelpers.setStatsSummary(summary, name, type[name]);
} else {
if (name === 'contentTypes') {
const contentTypeData = {};
forEach(Object.keys(type[name]), (contentType) => {
forEach(Object.keys(type[name]), contentType => {
forEach(type[name][contentType], (stats, metric) => {
statsHelpers.setStatsSummary(contentTypeData, [contentType, metric], stats)
statsHelpers.setStatsSummary(
contentTypeData,
[contentType, metric],
stats
);
});
});
summary[name] = contentTypeData;
} else if(name === 'responseCodes') {
} else if (name === 'responseCodes') {
const responseCodeData = {};
type.responseCodes.forEach((stats, metric) =>{
type.responseCodes.forEach((stats, metric) => {
statsHelpers.setStatsSummary(responseCodeData, metric, stats);
});
summary[name] = responseCodeData;
} else {
const data = {};
forEach(type[name], (stats, metric) => {
statsHelpers.setStatsSummary(data, metric, stats)
statsHelpers.setStatsSummary(data, metric, stats);
});
summary[name] = data;
}

22
lib/plugins/crawler/index.js
View File

@ -9,9 +9,9 @@ const Crawler = require('simplecrawler');
const make = messageMaker('crawler').make;
const defaultOptions = ({
const defaultOptions = {
depth: 3
});
};
module.exports = {
open(context, options) {
@ -25,7 +25,7 @@ module.exports = {
return Promise.resolve();
}
return new Promise((resolve) => {
return new Promise(resolve => {
const redirectedUrls = new Set(),
crawler = new Crawler(message.url);
@ -45,7 +45,7 @@ module.exports = {
redirectedUrls.add(response.headers.location);
});
crawler.on('fetchcomplete', (queueItem) => {
crawler.on('fetchcomplete', queueItem => {
const pageMimeType = /^(text|application)\/x?html/i;
const url = queueItem.url;
@ -55,7 +55,7 @@ module.exports = {
log.verbose('Crawler skipping initial URL %s', url);
} else if (pageMimeType.test(queueItem.stateData.contentType)) {
log.verbose('Crawler found %s URL %s', pageCount, url);
queue.postMessage(make('url', {}, {url, group: message.group}));
queue.postMessage(make('url', {}, { url, group: message.group }));
pageCount++;
if (pageCount >= maxPages) {
@ -70,10 +70,16 @@ module.exports = {
crawler.on('complete', resolve);
log.info('Starting to crawl from ' + message.url + ' with max depth ' + crawler.maxDepth +
' and max count ' + maxPages);
log.info(
'Starting to crawl from ' +
message.url +
' with max depth ' +
crawler.maxDepth +
' and max count ' +
maxPages
);
crawler.start();
})
});
}
}
};

1
lib/plugins/datacollector/dataCollector.js
View File

@ -56,7 +56,6 @@ class DataCollector {
set(this.summaryPages, name, merge({}, data));
}
}
}
module.exports = DataCollector;

58
lib/plugins/datacollector/index.js
View File

@ -15,9 +15,12 @@ module.exports = {
const dataCollector = this.dataCollector;
switch (message.type) {
case 'error':
{
return dataCollector.addErrorForUrl(message.url, message.source, message.data);
case 'error': {
return dataCollector.addErrorForUrl(
message.url,
message.source,
message.data
);
}
case 'browsertime.run':
@ -30,18 +33,25 @@ module.exports = {
case 'pagexray.run':
case 'pagexray.pageSummary':
case 'coach.run':
case 'coach.pageSummary':
{
return dataCollector.addDataForUrl(message.url, message.type, message.data, message.runIndex);
case 'coach.pageSummary': {
return dataCollector.addDataForUrl(
message.url,
message.type,
message.data,
message.runIndex
);
}
case 'aggregateassets.summary':
{
case 'aggregateassets.summary': {
if (message.group === 'total') {
const assetList = reduce(message.data, (assetList, asset) => {
const assetList = reduce(
message.data,
(assetList, asset) => {
assetList.push(asset);
return assetList;
}, []);
},
[]
);
const count = this.maxAssets,
fullCount = Object.keys(assetList).length,
@ -56,8 +66,7 @@ module.exports = {
} else return;
}
case 'largestassets.summary':
{
case 'largestassets.summary': {
if (message.group === 'total') {
const assetsBySize = {};
const contentType = Object.keys(message.data)[0];
@ -67,16 +76,14 @@ module.exports = {
});
} else return;
}
case 'largestthirdpartyassets.summary':
{
case 'largestthirdpartyassets.summary': {
if (message.group === 'total') {
return dataCollector.addDataForSummaryPage('toplist', {
thirdPartyAssetsBySize: message.data
});
} else return;
}
case 'slowestassets.summary':
{
case 'slowestassets.summary': {
if (message.group === 'total') {
const slowestAssets = message.data;
return dataCollector.addDataForSummaryPage('toplist', {
@ -85,8 +92,7 @@ module.exports = {
} else return;
}
case 'slowestthirdpartyassets.summary':
{
case 'slowestthirdpartyassets.summary': {
if (message.group === 'total') {
return dataCollector.addDataForSummaryPage('toplist', {
thirdPartySlowestAssets: message.data
@ -94,13 +100,16 @@ module.exports = {
} else return;
}
case 'domains.summary':
{
case 'domains.summary': {
if (message.group === 'total') {
const domainList = reduce(message.data, (domainList, domainStats) => {
const domainList = reduce(
message.data,
(domainList, domainStats) => {
domainList.push(domainStats);
return domainList;
}, []);
},
[]
);
const count = 200,
fullCount = domainList.length,
@ -113,15 +122,14 @@ module.exports = {
fullCount
});
} else {
return
return;
}
}
case 'webpagetest.summary':
case 'coach.summary':
case 'pagexray.summary':
case 'browsertime.summary':
case 'gpsi.summary':
{
case 'gpsi.summary': {
const data = {};
set(data, message.type, message.data);
dataCollector.addDataForSummaryPage('index', data);

33
lib/plugins/domains/aggregator.js
View File

@ -7,7 +7,15 @@ const Stats = require('fast-stats').Stats,
isEmpty = require('lodash.isempty'),
reduce = require('lodash.reduce');
const timingNames = ['blocked', 'dns', 'connect', 'ssl', 'send', 'wait', 'receive'];
const timingNames = [
'blocked',
'dns',
'connect',
'ssl',
'send',
'wait',
'receive'
];
function parseDomainName(url) {
return urlParser.parse(url).hostname;
@ -26,10 +34,12 @@ function getDomain(domainName) {
wait: new Stats(),
receive: new Stats()
};
}
}
function calc(domains) {
return reduce(domains, (summary, domainStats, domainName) => {
return reduce(
domains,
(summary, domainStats, domainName) => {
const domainSummary = {
requestCount: domainStats.requestCount,
domainName
@ -39,7 +49,7 @@ function calc(domains) {
if (!isEmpty(stats)) {
domainSummary.totalTime = stats;
}
timingNames.forEach((name) => {
timingNames.forEach(name => {
const stats = statsHelpers.summarizeStats(domainStats[name]);
if (!isEmpty(stats)) {
domainSummary[name] = stats;
@ -48,12 +58,14 @@ function calc(domains) {
summary[domainName] = domainSummary;
return summary;
}, {});
},
{}
);
}
function isValidTiming(timing) {
// The HAR format uses -1 to indicate invalid/missing timings
return (typeof timing === 'number' && timing !== -1);
return typeof timing === 'number' && timing !== -1;
}
module.exports = {
@ -62,11 +74,11 @@ module.exports = {
addToAggregate(har, url) {
const mainDomain = parseDomainName(url);
if (this.groups[mainDomain] === undefined) {
this.groups[mainDomain] = {}
this.groups[mainDomain] = {};
}
const firstPageId = har.log.pages[0].id;
har.log.entries.forEach((entry) => {
har.log.entries.forEach(entry => {
if (entry.pageref !== firstPageId) {
// Only pick the first request out of multiple runs.
return;
@ -74,7 +86,8 @@ module.exports = {
const domainName = parseDomainName(entry.request.url),
domain = this.domains[domainName] || getDomain(domainName),
groupDomain = this.groups[mainDomain][domainName] || getDomain(domainName),
groupDomain =
this.groups[mainDomain][domainName] || getDomain(domainName),
totalTime = entry.time;
domain.requestCount++;
@ -87,7 +100,7 @@ module.exports = {
log.debug('Missing time from har entry for url: ' + entry.request.url);
}
timingNames.forEach((name) => {
timingNames.forEach(name => {
const timing = entry.timings[name];
if (isValidTiming(timing)) {

10
lib/plugins/domains/index.js
View File

@ -14,18 +14,18 @@ module.exports = {
},
processMessage(message, queue) {
switch (message.type) {
case 'browsertime.har':
{
case 'browsertime.har': {
aggregator.addToAggregate(message.data, message.url);
break;
}
case 'summarize':
{
case 'summarize': {
const summary = aggregator.summarize();
if (!isEmpty(summary)) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(make('domains.summary', summary.groups[group], {group}));
queue.postMessage(
make('domains.summary', summary.groups[group], { group })
);
}
}
break;

23
lib/plugins/gpsi/aggregator.js
View File

@ -2,7 +2,7 @@
const statsHelpers = require('../../support/statsHelpers');
module.exports = {
module.exports = {
statsPerType: {},
total: {},
groups: {},
@ -10,13 +10,26 @@ const statsHelpers = require('../../support/statsHelpers');
if (this.groups[group] === undefined) {
this.groups[group] = {};
}
statsHelpers.pushGroupStats(this.statsPerType, this.groups[group], 'SPEED.score', gpsiData.ruleGroups['SPEED'].score);
statsHelpers.pushStats(this.total, 'SPEED.score', gpsiData.ruleGroups['SPEED'].score);
statsHelpers.pushGroupStats(
this.statsPerType,
this.groups[group],
'SPEED.score',
gpsiData.ruleGroups['SPEED'].score
);
statsHelpers.pushStats(
this.total,
'SPEED.score',
gpsiData.ruleGroups['SPEED'].score
);
},
summarize() {
const summary = {};
statsHelpers.setStatsSummary(summary, 'groups.total.SPEED.score', this.total.SPEED.score);
statsHelpers.setStatsSummary(
summary,
'groups.total.SPEED.score',
this.total.SPEED.score
);
// TODO add GPSI score per group
return summary;
}
}
};

8
lib/plugins/gpsi/analyzer.js
View File

@ -6,7 +6,7 @@ var log = require('intel').getLogger('sitespeedio.plugin.gpsi'),
module.exports = {
analyzeUrl: function(url, options) {
log.info('Sending url ' + url + ' to test on Page Speed Insights');
const args = {url};
const args = { url };
if (options.gpsi.key) {
args.key = options.gpsi.key;
@ -14,10 +14,10 @@ module.exports = {
args.nokey = true;
}
args.strategy = "desktop";
args.strategy = 'desktop';
if(options.mobile) {
args.strategy = "mobile";
if (options.mobile) {
args.strategy = 'mobile';
}
return gpagespeed(args);

24
lib/plugins/gpsi/index.js
View File

@ -15,30 +15,34 @@ module.exports = {
gpsi: options.gpsi,
mobile: options.mobile
};
filterRegistry.registerFilterForType(DEFAULT_METRICS_PAGESUMMARY, 'gpsi.pageSummary');
filterRegistry.registerFilterForType(
DEFAULT_METRICS_PAGESUMMARY,
'gpsi.pageSummary'
);
},
processMessage(message, queue) {
switch (message.type) {
case 'url':
{
case 'url': {
const url = message.url;
const group = message.group;
return analyzer.analyzeUrl(url, this.options)
.then((result) => {
return analyzer.analyzeUrl(url, this.options).then(result => {
log.info('Got ' + url + ' analysed from Google Page Speed Insights');
log.verbose('Result from Google Page Speed Insights:%:2j', result);
queue.postMessage(make('gpsi.pageSummary', result, {
queue.postMessage(
make('gpsi.pageSummary', result, {
url,
group
}));
})
);
aggregator.addToAggregate(result, group);
});
}
case 'summarize':
{
case 'summarize': {
const summary = aggregator.summarize();
queue.postMessage(make('gpsi.summary', summary.groups.total , {group: 'total'}));
queue.postMessage(
make('gpsi.summary', summary.groups.total, { group: 'total' })
);
}
}
}

32
lib/plugins/graphite/data-generator.js
View File

@ -10,8 +10,11 @@ function keyPathFromMessage(message, options, includeQueryParams) {
typeParts.push(typeParts.shift());
// always have browser and connectivity in Browsertime and related tools
if (message.type.match(/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains)/)) {
if (
message.type.match(
/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains)/
)
) {
// if we have a friendly name for your connectivity, use that!
let connectivity = graphiteUtil.getConnectivity(options);
@ -27,7 +30,16 @@ function keyPathFromMessage(message, options, includeQueryParams) {
}
// if we get a URL type, add the URL
if (message.url) {
typeParts.splice(1, 0, graphiteUtil.getURLAndGroup(options, message.group, message.url, includeQueryParams));
typeParts.splice(
1,
0,
graphiteUtil.getURLAndGroup(
options,
message.group,
message.url,
includeQueryParams
)
);
} else if (message.group) {
// add the group of the summary message
typeParts.splice(1, 0, graphiteUtil.toSafeKey(message.group));
@ -46,13 +58,21 @@ class GraphiteDataGenerator {
dataFromMessage(message, time) {
const timestamp = Math.round(time.valueOf() / 1000);
const keypath = keyPathFromMessage(message, this.options, this.includeQueryParams);
const keypath = keyPathFromMessage(
message,
this.options,
this.includeQueryParams
);
return reduce(flatten.flattenMessageData(message), (entries, value, key) => {
return reduce(
flatten.flattenMessageData(message),
(entries, value, key) => {
const fullKey = util.format('%s.%s.%s', this.namespace, keypath, key);
entries.push(util.format('%s %s %s', fullKey, value, timestamp));
return entries;
}, []);
},
[]
);
}
}

53
lib/plugins/graphite/index.js
View File

@ -21,13 +21,27 @@ module.exports = {
const opts = merge({}, defaultConfig, options.graphite);
this.options = options;
this.sender = new Sender(opts.host, opts.port);
this.dataGenerator = new DataGenerator(opts.namespace, opts.includeQueryParams, options);
log.debug('Setting up Graphite %s:%s for namespace %s', opts.host, opts.port, opts.namespace);
this.dataGenerator = new DataGenerator(
opts.namespace,
opts.includeQueryParams,
options
);
log.debug(
'Setting up Graphite %s:%s for namespace %s',
opts.host,
opts.port,
opts.namespace
);
this.timestamp = context.timestamp;
this.resultUrls = context.resultUrls;
},
processMessage(message) {
if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary')))
if (
!(
message.type.endsWith('.summary') ||
message.type.endsWith('.pageSummary')
)
)
return;
// we only sends individual groups to Graphite, not the
@ -37,27 +51,44 @@ module.exports = {
}
message = filterRegistry.filterMessage(message);
if (isEmpty(message.data))
return;
if (isEmpty(message.data)) return;
// TODO Here we could add logic to either create a new timestamp or
// use the one that we have for that run. Now just use the one for the
// run
const dataPoints = this.dataGenerator.dataFromMessage(message, this.timestamp);
const dataPoints = this.dataGenerator.dataFromMessage(
message,
this.timestamp
);
if (dataPoints.length > 0) {
const data = dataPoints.join('\n') + '\n';
return this.sender.send(data).then(() => {
// make sure we only send once per URL (and browsertime is the most important)
// and you need to configure a base URL where you get the HTML result
if (message.type === 'browsertime.pageSummary' && this.resultUrls.hasBaseUrl()) {
const resultPageUrl = this.resultUrls.absoluteSummaryPageUrl(message.url);
return sendAnnotations.send(this.options, message.group, message.url, resultPageUrl, this.timestamp);
if (
message.type === 'browsertime.pageSummary' &&
this.resultUrls.hasBaseUrl()
) {
const resultPageUrl = this.resultUrls.absoluteSummaryPageUrl(
message.url
);
return sendAnnotations.send(
this.options,
message.group,
message.url,
resultPageUrl,
this.timestamp
);
}
});
} else {
return Promise.reject(new Error('No data to send to graphite for message:\n' +
JSON.stringify(message, null, 2)));
return Promise.reject(
new Error(
'No data to send to graphite for message:\n' +
JSON.stringify(message, null, 2)
)
);
}
},
config: defaultConfig

28
lib/plugins/graphite/send-annotation.js
View File

@ -7,7 +7,6 @@ const graphiteUtil = require('../../support/tsdbUtil');
module.exports = {
send(options, group, url, resultPageUrl, time) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
@ -15,14 +14,18 @@ module.exports = {
const connectivity = graphiteUtil.getConnectivity(options);
const browser = options.browser;
const namespace = options.graphite.namespace.split('.');
const urlAndGroup = graphiteUtil.getURLAndGroup(options, group, url, options.graphite.includeQueryParams).split('.');
const tagsString = `"${connectivity},${browser},${namespace.join(',')},${urlAndGroup.join(',')}"`;
const urlAndGroup = graphiteUtil
.getURLAndGroup(options, group, url, options.graphite.includeQueryParams)
.split('.');
const tagsString = `"${connectivity},${browser},${namespace.join(
','
)},${urlAndGroup.join(',')}"`;
const tagsArray = `["${connectivity}","${browser}","${namespace[0]}","${namespace[1]}","${urlAndGroup[0]}", "${urlAndGroup[1]}"]`;
const message = `<a href='${resultPageUrl}' target='_blank'>Result ${options.browsertime.iterations} run(s)</a>`;
const message = `<a href='${resultPageUrl}' target='_blank'>Result ${options
.browsertime.iterations} run(s)</a>`;
const timestamp = Math.round(time.valueOf() / 1000);
const tags = options.graphite.arrayTags ? tagsArray : tagsString;
const postData =
`{"what": "Sitespeed.io", "tags": ${tags}, "data": "${message}", "when": ${timestamp}}`;
const postData = `{"what": "Sitespeed.io", "tags": ${tags}, "data": "${message}", "when": ${timestamp}}`;
const postOptions = {
hostname: options.graphite.webHost || options.graphite.host,
port: options.graphite.httpPort || 8080,
@ -43,9 +46,12 @@ module.exports = {
log.verbose('Send annotation to Graphite: %j', postData);
// not perfect but maybe work for us
const lib = options.graphite.httpPort === 443 ? https : http;
const req = lib.request(postOptions, (res) => {
const req = lib.request(postOptions, res => {
if (res.statusCode !== 200) {
log.error('Got %s from Graphite when sending annotation', res.statusCode);
log.error(
'Got %s from Graphite when sending annotation',
res.statusCode
);
reject();
} else {
res.setEncoding('utf8');
@ -53,12 +59,12 @@ module.exports = {
resolve();
}
});
req.on('error', (err) => {
req.on('error', err => {
log.error('Got error from Graphite when sending annotation', err);
reject(err)
reject(err);
});
req.write(postData);
req.end();
})
});
}
};

17
lib/plugins/harstorer/index.js
View File

@ -13,13 +13,22 @@ module.exports = {
processMessage(message) {
switch (message.type) {
case 'browsertime.har':
case 'webpagetest.har':
{
case 'webpagetest.har': {
const jsonData = JSON.stringify(message.data);
if (this.gzipHAR) {
return this.storageManager.writeDataForUrl(jsonData, message.type, message.url, '', true);
return this.storageManager.writeDataForUrl(
jsonData,
message.type,
message.url,
'',
true
);
} else {
return this.storageManager.writeDataForUrl(jsonData, message.type, message.url);
return this.storageManager.writeDataForUrl(
jsonData,
message.type,
message.url
);
}
}
}

138
lib/plugins/html/htmlBuilder.js
View File

@ -49,26 +49,38 @@ class HTMLBuilder {
const detailedBoxes = dataCollection.getDetailedBoxes();
this.summary.pages = {
pageTitle: `Overview of ${helpers.plural(nTestedPages,'page')} for ${name} at ${timestamp}`,
pageTitle: `Overview of ${helpers.plural(
nTestedPages,
'page'
)} for ${name} at ${timestamp}`,
pageDescription: 'See all the tested pages on a high level.',
pages: validPages
};
this.summary.index = {
pageTitle: `Executive Summary for ${name} tested ${helpers.plural(nTestedPages,'page')} at ${timestamp}`,
pageDescription: 'Executive summary of the sitespeed.io result. Act on red/yellow/green.',
pageTitle: `Executive Summary for ${name} tested ${helpers.plural(
nTestedPages,
'page'
)} at ${timestamp}`,
pageDescription:
'Executive summary of the sitespeed.io result. Act on red/yellow/green.',
boxes: chunk(summaryBoxes.filter(Boolean), 3)
};
this.summary.detailed = {
pageTitle: `In details summary for ${name} tested ${helpers.plural(nTestedPages,'page')} at ${timestamp}`,
pageDescription: 'Get all the details you need to fast track things you need to change.',
pageTitle: `In details summary for ${name} tested ${helpers.plural(
nTestedPages,
'page'
)} at ${timestamp}`,
pageDescription:
'Get all the details you need to fast track things you need to change.',
metrics: detailedBoxes
};
this.summary.domains = {
pageTitle: `The most used domains for ${name} tested at ${timestamp}`,
pageDescription: 'A list of the most used domains and the respective timings'
pageDescription:
'A list of the most used domains and the respective timings'
};
this.summary.assets = {
@ -84,10 +96,10 @@ class HTMLBuilder {
if (options.budget) {
let totalFailing = 0;
let totalWorking = 0;
for (const url of Object.keys(this.budget.failing )) {
for (const url of Object.keys(this.budget.failing)) {
totalFailing = totalFailing + this.budget.failing[url].length;
}
for (const url of Object.keys(this.budget.working )) {
for (const url of Object.keys(this.budget.working)) {
totalWorking = totalWorking + this.budget.working[url].length;
}
this.summary.budget = {
@ -108,15 +120,21 @@ class HTMLBuilder {
coach: coachData
};
const summaryRenders = Object.keys(this.summary)
.map((name) => this._renderSummaryPage(name, merge({
const summaryRenders = Object.keys(this.summary).map(name =>
this._renderSummaryPage(
name,
merge(
{
options,
noPages: Object.keys(dataCollection.urlPages).length
}, dataCollection.summaryPages[name], this.summary[name])));
},
dataCollection.summaryPages[name],
this.summary[name]
)
)
);
const urlPageRenders = Promise.resolve(Object.keys(validPages))
.map((url) => {
const urlPageRenders = Promise.resolve(Object.keys(validPages)).map(url => {
const pageInfo = validPages[url];
const runPages = dataCollection.urlRunPages[url];
const medianRun = metricHelper.pickMedianRun(runPages, pageInfo);
@ -124,7 +142,7 @@ class HTMLBuilder {
let browser = {
name: summaryPageHAR.log.browser.name,
version: summaryPageHAR.log.browser.version
}
};
// if we are on the summary page we inline the HAR and then make sure
// we only pick one HAR run (medianRun). But you can also choose to
// fetch the HAR in the HTML, then it isn't included.
@ -137,8 +155,12 @@ class HTMLBuilder {
}
let daurlAlias;
if (options.urlsMetaData && options.urlsMetaData[url] && options.urlsMetaData[url].alias) {
daurlAlias = options.urlsMetaData[url].alias
if (
options.urlsMetaData &&
options.urlsMetaData[url] &&
options.urlsMetaData[url].alias
) {
daurlAlias = options.urlsMetaData[url].alias;
}
return this._renderUrlPage(url, 'index', {
@ -150,9 +172,8 @@ class HTMLBuilder {
summaryPageHAR,
medianRun,
browser
})
.tap(() => Promise.resolve(Object.keys(runPages))
.map((runIndex) =>
}).tap(() =>
Promise.resolve(Object.keys(runPages)).map(runIndex =>
this._renderUrlRunPage(url, runIndex, {
daurl: url,
daurlAlias,
@ -161,55 +182,91 @@ class HTMLBuilder {
options,
runPages,
browser
})));
})
)
);
});
// Aggregate/summarize data and write additional files
return Promise.all(summaryRenders)
.then(() => Promise.all(urlPageRenders))
.then(() => this.storageManager.copyToResultDir(path.join(__dirname, 'assets')))
.then(() => log.info('HTML stored in %s', this.storageManager.getBaseDir()));
.then(() =>
this.storageManager.copyToResultDir(path.join(__dirname, 'assets'))
)
.then(() =>
log.info('HTML stored in %s', this.storageManager.getBaseDir())
);
}
_renderUrlPage(url, name, locals) {
const summaryTimestamp = get(locals, 'pageInfo.data.browsertime.pageSummary.timestamp', this.timestamp);
locals = merge({
const summaryTimestamp = get(
locals,
'pageInfo.data.browsertime.pageSummary.timestamp',
this.timestamp
);
locals = merge(
{
JSON: JSON,
rootPath: this.storageManager.rootPathFromUrl(url),
menu: 'pages',
pageTitle: `Summary for ${helpers.plural(this.options.browsertime.iterations,'run')} ${url} at ${summaryTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromPageSummary(locals.pageInfo)} collected by sitespeed.io ${packageInfo.version}`,
pageTitle: `Summary for ${helpers.plural(
this.options.browsertime.iterations,
'run'
)} ${url} at ${summaryTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromPageSummary(
locals.pageInfo
)} collected by sitespeed.io ${packageInfo.version}`,
headers: this.summary,
version: packageInfo.version,
timestamp: summaryTimestamp,
h: helpers,
context: this.context
}, locals);
},
locals
);
return this.storageManager.writeHtmlForUrl(renderer.renderTemplate('url/' + name, locals), name + '.html', url);
return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/' + name, locals),
name + '.html',
url
);
}
_renderUrlRunPage(url, name, locals) {
const runTimestamp = get(locals, 'pageInfo.data.browsertime.run.timestamp', this.timestamp);
locals = merge({
const runTimestamp = get(
locals,
'pageInfo.data.browsertime.run.timestamp',
this.timestamp
);
locals = merge(
{
urlLink: './index.html',
JSON: JSON,
rootPath: this.storageManager.rootPathFromUrl(url),
menu: 'pages',
pageTitle: `Run ${(parseInt(name) + 1)} for ${url} at ${runTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromRun(locals.pageInfo)} collected by sitespeed.io ${packageInfo.version}`,
pageTitle: `Run ${parseInt(name) + 1} for ${url} at ${runTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromRun(
locals.pageInfo
)} collected by sitespeed.io ${packageInfo.version}`,
headers: this.summary,
version: packageInfo.version,
timestamp: runTimestamp,
h: helpers,
context: this.context
}, locals);
},
locals
);
return this.storageManager.writeHtmlForUrl(renderer.renderTemplate('url/run', locals), name + '.html', url);
return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/run', locals),
name + '.html',
url
);
}
_renderSummaryPage(name, locals) {
locals = merge({
locals = merge(
{
rootPath: '',
menu: name,
pageTitle: name,
@ -219,9 +276,14 @@ class HTMLBuilder {
timestamp: this.timestamp,
h: helpers,
context: this.context
}, locals);
},
locals
);
return this.storageManager.writeHtml(name + '.html', renderer.renderTemplate(name, locals));
return this.storageManager.writeHtml(
name + '.html',
renderer.renderTemplate(name, locals)
);
}
}

36
lib/plugins/html/metricHelper.js
View File

@ -6,11 +6,19 @@ module.exports = {
// this configurable through the CLI
// If we have SpeedIndex use that else backup with RUM SpeedIndex
const speedIndexMedian = get(pageInfo, 'data.browsertime.pageSummary.statistics.visualMetrics.SpeedIndex.median');
const rumSpeedIndexMedian = get(pageInfo, 'data.browsertime.pageSummary.statistics.timings.rumSpeedIndex.median');
const speedIndexMedian = get(
pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics.SpeedIndex.median'
);
const rumSpeedIndexMedian = get(
pageInfo,
'data.browsertime.pageSummary.statistics.timings.rumSpeedIndex.median'
);
if (speedIndexMedian) {
for (var run of runs) {
if (speedIndexMedian === run.data.browsertime.run.visualMetrics.SpeedIndex) {
if (
speedIndexMedian === run.data.browsertime.run.visualMetrics.SpeedIndex
) {
return {
name: 'SpeedIndex',
runIndex: run.runIndex
@ -20,7 +28,11 @@ module.exports = {
} else if (rumSpeedIndexMedian) {
for (var rumRuns of runs) {
// make sure we run Browsertime for that run = 3 runs WPT and 2 runs BT
if (rumRuns.data.browsertime && rumSpeedIndexMedian === rumRuns.data.browsertime.run.timings.rumSpeedIndex) {
if (
rumRuns.data.browsertime &&
rumSpeedIndexMedian ===
rumRuns.data.browsertime.run.timings.rumSpeedIndex
) {
return {
name: 'RUMSpeedIndex',
runIndex: rumRuns.runIndex
@ -50,15 +62,23 @@ module.exports = {
}
},
getMetricsFromPageSummary(pageInfo) {
const visualMetrics = get(pageInfo, 'data.browsertime.pageSummary.statistics.visualMetrics');
const timings = get(pageInfo, 'data.browsertime.pageSummary.statistics.timings');
const visualMetrics = get(
pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics'
);
const timings = get(
pageInfo,
'data.browsertime.pageSummary.statistics.timings'
);
if (visualMetrics) {
return `Median First Visual Change: ${visualMetrics.FirstVisualChange.median},
return `Median First Visual Change: ${visualMetrics.FirstVisualChange
.median},
Median Speed Index: ${visualMetrics.SpeedIndex.median},
Median Visual Complete 85%: ${visualMetrics.VisualComplete85.median},
Median Last Visual Change: ${visualMetrics.LastVisualChange.median}`;
} else if (timings) {
return `Median RUMSpeedIndex: ${timings.rumSpeedIndex.median}, Median Fully loaded: ${timings.fullyLoaded.median}`;
return `Median RUMSpeedIndex: ${timings.rumSpeedIndex
.median}, Median Fully loaded: ${timings.fullyLoaded.median}`;
} else {
return '';
}

3
lib/plugins/html/renderer.js
View File

@ -8,8 +8,7 @@ const basePath = path.resolve(__dirname, 'templates');
const templateCache = {};
function getTemplate(templateName) {
if (!templateName.endsWith('.pug'))
templateName = templateName + '.pug';
if (!templateName.endsWith('.pug')) templateName = templateName + '.pug';
const template = templateCache[templateName];
if (template) {

73
lib/plugins/influxdb/data-generator.js
View File

@ -17,7 +17,12 @@ class InfluxDBDataGenerator {
}
dataFromMessage(message, time) {
function getTagsFromMessage(message, includeQueryParams, options, defaultTags) {
function getTagsFromMessage(
message,
includeQueryParams,
options,
defaultTags
) {
const tags = merge({}, defaultTags);
let typeParts = message.type.split('.');
tags.origin = typeParts[0];
@ -41,7 +46,14 @@ class InfluxDBDataGenerator {
// if we get a URL type, add the URL
if (message.url) {
const urlAndGroup = util.getURLAndGroup(options, message.group, message.url, includeQueryParams).split('.');
const urlAndGroup = util
.getURLAndGroup(
options,
message.group,
message.url,
includeQueryParams
)
.split('.');
tags.page = urlAndGroup[1];
tags.group = urlAndGroup[0];
} else if (message.group) {
@ -52,13 +64,23 @@ class InfluxDBDataGenerator {
}
function getFieldAndSeriesName(key) {
const functions = ['min', 'p10', 'median', 'mean', 'avg', 'max', 'p90', 'p99', 'mdev'];
const functions = [
'min',
'p10',
'median',
'mean',
'avg',
'max',
'p90',
'p99',
'mdev'
];
const keyArray = key.split('.');
const end = keyArray.pop();
if (functions.indexOf(end) > -1) {
return {field: end, seriesName: keyArray.pop()}
return { field: end, seriesName: keyArray.pop() };
}
return {field: 'value', seriesName: end};
return { field: 'value', seriesName: end };
}
function getAdditionalTags(key, type) {
@ -73,8 +95,7 @@ class InfluxDBDataGenerator {
// pageTimings.serverResponseTime.max
// visualMetrics.SpeedIndex.median
tags.timings = keyArray[0];
}
else if (type === 'browsertime.pageSummary') {
} else if (type === 'browsertime.pageSummary') {
// statistics.timings.pageTimings.backEndTime.median
// statistics.timings.rumSpeedIndex.median
// statistics.timings.userTimings.marks.logoTime.median
@ -83,15 +104,13 @@ class InfluxDBDataGenerator {
if (keyArray.length >= 5) {
tags[keyArray[2]] = keyArray[3];
}
}
else if (type === 'browsertime.summary') {
} else if (type === 'browsertime.summary') {
// firstPaint.median
// userTimings.marks.logoTime.median
if (key.indexOf('userTimings') > -1) {
tags[keyArray[0]] = keyArray[1];
}
}
else if (type === 'coach.pageSummary') {
} else if (type === 'coach.pageSummary') {
// advice.score
// advice.performance.score
if (keyArray.length > 2) {
@ -123,16 +142,14 @@ class InfluxDBDataGenerator {
// requests.median
// responseCodes.307.max pagexray.summary
// requests.min pagexray.summary
if (key.indexOf('responseCodes') > -1 ) {
if (key.indexOf('responseCodes') > -1) {
tags.responseCodes = 'response';
}
if (key.indexOf('firstParty') > -1 || key.indexOf('thirdParty') > -1) {
tags.party = keyArray[0];
}
}
else if (type === 'pagexray.pageSummary') {
} else if (type === 'pagexray.pageSummary') {
// thirdParty.contentTypes.json.requests pagexray.pageSummary
// thirdParty.requests pagexray.pageSummary
// firstParty.cookieStats.max pagexray.pageSummary
@ -142,26 +159,32 @@ class InfluxDBDataGenerator {
if (key.indexOf('firstParty') > -1 || key.indexOf('thirdParty') > -1) {
tags.party = keyArray[0];
}
if (key.indexOf('responseCodes') > -1 ) {
if (key.indexOf('responseCodes') > -1) {
tags.responseCodes = 'response';
}
if (key.indexOf('contentTypes') > -1 ) {
if (key.indexOf('contentTypes') > -1) {
tags.contentType = keyArray[2];
}
}
else {
} else {
// console.log('Missed added tags to ' + key + ' ' + type);
}
return tags;
}
return reduce(flatten.flattenMessageData(message), (entries, value, key) => {
const fieldAndSeriesName= getFieldAndSeriesName(key);
let tags = getTagsFromMessage(message, this.includeQueryParams, this.options, this.defaultTags);
return reduce(
flatten.flattenMessageData(message),
(entries, value, key) => {
const fieldAndSeriesName = getFieldAndSeriesName(key);
let tags = getTagsFromMessage(
message,
this.includeQueryParams,
this.options,
this.defaultTags
);
tags = merge(getAdditionalTags(key, message.type), tags);
let point = {
time: time.valueOf()
}
};
point[fieldAndSeriesName.field] = value;
entries.push({
tags,
@ -169,7 +192,9 @@ class InfluxDBDataGenerator {
point
});
return entries;
}, []);
},
[]
);
}
}

50
lib/plugins/influxdb/index.js
View File

@ -19,7 +19,11 @@ const defaultConfig = {
module.exports = {
open(context, options) {
throwIfMissing(options.influxdb, ['host', 'database'], 'influxdb');
log.info('Setup InfluxDB host %s and database %s', options.influxdb.host, options.influxdb.database);
log.info(
'Setup InfluxDB host %s and database %s',
options.influxdb.host,
options.influxdb.database
);
const opts = options.influxdb;
this.options = options;
@ -29,11 +33,20 @@ module.exports = {
this.dataGenerator = new DataGenerator(opts.includeQueryParams, options);
},
processMessage(message) {
if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary')))
if (
!(
message.type.endsWith('.summary') ||
message.type.endsWith('.pageSummary')
)
)
return;
// Let us skip this for a while and concentrate on the real deal
if (message.type.match(/(^largestassets|^slowestassets|^aggregateassets|^domains)/))
if (
message.type.match(
/(^largestassets|^slowestassets|^aggregateassets|^domains)/
)
)
return;
// we only sends individual groups to Influx, not the
@ -43,8 +56,7 @@ module.exports = {
}
message = filterRegistry.filterMessage(message);
if (isEmpty(message.data))
return;
if (isEmpty(message.data)) return;
let data = this.dataGenerator.dataFromMessage(message, this.timestamp);
@ -52,13 +64,29 @@ module.exports = {
return this.sender.send(data).then(() => {
// make sure we only send once per URL (and browsertime is the most important)
// and you need to configure a base URL where you get the HTML result
if (message.type === 'browsertime.pageSummary' && this.resultUrls.hasBaseUrl()) {
const resultPageUrl = this.resultUrls.absoluteSummaryPageUrl(message.url);
return sendAnnotations.send(this.options, message.group, message.url, resultPageUrl, this.timestamp);
}})
if (
message.type === 'browsertime.pageSummary' &&
this.resultUrls.hasBaseUrl()
) {
const resultPageUrl = this.resultUrls.absoluteSummaryPageUrl(
message.url
);
return sendAnnotations.send(
this.options,
message.group,
message.url,
resultPageUrl,
this.timestamp
);
}
});
} else {
return Promise.reject(new Error('No data to send to influxdb for message:\n' +
JSON.stringify(message, null, 2)));
return Promise.reject(
new Error(
'No data to send to influxdb for message:\n' +
JSON.stringify(message, null, 2)
)
);
}
},
config: defaultConfig

32
lib/plugins/influxdb/send-annotation.js
View File

@ -14,20 +14,21 @@ module.exports = {
// variables in Grafana.
const connectivity = tsdbUtil.getConnectivity(options);
const browser = options.browser;
const urlAndGroup = tsdbUtil.getURLAndGroup(options, group, url, options.influxdb.includeQueryParams).split('.');
const urlAndGroup = tsdbUtil
.getURLAndGroup(options, group, url, options.influxdb.includeQueryParams)
.split('.');
let tags = `${connectivity},${browser},${urlAndGroup.join(',')}`;
const message = `<a href='${resultPageUrl}' target='_blank'>Result ${options.browsertime.iterations} run(s)</a>`;
const message = `<a href='${resultPageUrl}' target='_blank'>Result ${options
.browsertime.iterations} run(s)</a>`;
const timestamp = Math.round(time.valueOf() / 1000);
// if we have a category, let us send that category too
if (options.influxdb.tags) {
for (var row of options.influxdb.tags.split(',')) {
const keyAndValue = row.split('=');
if (keyAndValue[0] === 'category')
tags += `,${keyAndValue[1]}`;
if (keyAndValue[0] === 'category') tags += `,${keyAndValue[1]}`;
}
}
const postData =
`events title="Sitespeed.io",text="${message}",tags="${tags}" ${timestamp}`;
const postData = `events title="Sitespeed.io",text="${message}",tags="${tags}" ${timestamp}`;
const postOptions = {
hostname: options.influxdb.host,
port: options.influxdb.port,
@ -40,7 +41,10 @@ module.exports = {
};
if (options.influxdb.username) {
postOptions.path = postOptions.path + '&' + querystring.stringify({
postOptions.path =
postOptions.path +
'&' +
querystring.stringify({
u: options.influxdb.username,
p: options.influxdb.password
});
@ -50,9 +54,13 @@ module.exports = {
log.verbose('Send annotation to Influx: %j', postData);
// not perfect but maybe work for us
const lib = options.influxdb.protocol === 'https' ? https : http;
const req = lib.request(postOptions, (res) => {
const req = lib.request(postOptions, res => {
if (res.statusCode !== 204) {
log.error('Got %s from InfluxDB when sending annotation %s', res.statusCode, res.statusMessage);
log.error(
'Got %s from InfluxDB when sending annotation %s',
res.statusCode,
res.statusMessage
);
reject();
} else {
res.setEncoding('utf-8');
@ -60,12 +68,12 @@ module.exports = {
resolve();
}
});
req.on('error', (err) => {
req.on('error', err => {
log.error('Got error from InfluxDB when sending annotation', err);
reject(err)
reject(err);
});
req.write(postData);
req.end();
})
});
}
};

17
lib/plugins/influxdb/sender.js
View File

@ -4,20 +4,27 @@ const Influx = require('influx'),
Promise = require('bluebird');
class InfluxDBSender {
constructor({protocol, host, port, database, username, password}) {
this.client = new Influx.InfluxDB({protocol, host, port, database, username, password});
constructor({ protocol, host, port, database, username, password }) {
this.client = new Influx.InfluxDB({
protocol,
host,
port,
database,
username,
password
});
}
send(data) {
return Promise.resolve(data)
.map((point) => {
.map(point => {
return {
tags: point.tags,
measurement: point.seriesName,
fields: point.point
}
};
})
.then((points) => this.client.writePoints(points));
.then(points => this.client.writePoints(points));
}
}

26
lib/plugins/metrics/index.js
View File

@ -17,8 +17,9 @@ module.exports = {
},
postOpen() {
if (this.options.filter) {
const filters = Array.isArray(this.options.filter) ? this.options.filter : [this.options.filter];
const filters = Array.isArray(this.options.filter)
? this.options.filter
: [this.options.filter];
for (let metric of filters) {
// for all filters
@ -26,18 +27,16 @@ module.exports = {
// metrics are sent
if (metric === '*+') {
filterRegistry.clearAll();
}
} else if (metric === '*-') {
// all registered types will be set as unmatching,
// use it if you want to have a clean filter where
// all types are removed and then you can add your own
else if(metric === '*-') {
let types = filterRegistry.getTypes();
filterRegistry.clearAll();
for (let type of types) {
filterRegistry.registerFilterForType('-', type);
}
}
else {
} else {
let parts = metric.split('.');
// the type is "always" the first two
let type = parts.shift() + '.' + parts.shift();
@ -52,11 +51,15 @@ module.exports = {
}
}
}
},
processMessage(message) {
if (this.options.list) {
if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary')))
if (
!(
message.type.endsWith('.summary') ||
message.type.endsWith('.pageSummary')
)
)
return;
let flattenMess = flatten.flattenMessageData(message);
for (let key of Object.keys(flattenMess)) {
@ -66,7 +69,10 @@ module.exports = {
},
close() {
if (this.options.list) {
this.storageManager.writeData('metrics.txt', Object.keys(this.metrics).join('\n'));
this.storageManager.writeData(
'metrics.txt',
Object.keys(this.metrics).join('\n')
);
}
if (this.options.filterList) {
@ -74,7 +80,7 @@ module.exports = {
let filtersByType = filterRegistry.getFilters();
for (let type of Object.keys(filtersByType)) {
for (let filters of filtersByType[type]) {
output+= type + '.' + filters + '\n';
output += type + '.' + filters + '\n';
}
}
return this.storageManager.writeData('configuredMetrics.txt', output);

18
lib/plugins/s3/index.js
View File

@ -10,8 +10,13 @@ const pick = require('lodash.pick');
Promise.promisifyAll(fs);
function createS3Client(s3Options) {
const clientOptions = pick(s3Options,
['maxAsyncS3', 's3RetryCount', 's3RetryDelay', 'multipartUploadThreshold', 'multipartUploadSize']);
const clientOptions = pick(s3Options, [
'maxAsyncS3',
's3RetryCount',
's3RetryDelay',
'multipartUploadThreshold',
'multipartUploadSize'
]);
clientOptions.s3Options = {
accessKeyId: s3Options.key,
@ -55,7 +60,9 @@ module.exports = {
};
return new Promise((resolve, reject) => {
log.info(`Uploading ${baseDir} to S3 bucket ${s3Options.bucketname}, this can take a while ...`);
log.info(
`Uploading ${baseDir} to S3 bucket ${s3Options.bucketname}, this can take a while ...`
);
const uploader = client.uploadDir(params);
@ -65,12 +72,13 @@ module.exports = {
});
uploader.on('end', () => {
if (s3Options.removeLocalResult) {
fs.removeAsync(baseDir)
fs
.removeAsync(baseDir)
.then(() => {
log.info(`Removed local files and directory ${baseDir}`);
resolve();
})
.catch((e) => reject(e));
.catch(e => reject(e));
} else {
resolve();
}

18
lib/plugins/screenshot/index.js
View File

@ -7,13 +7,19 @@ function getImagesAndName(images) {
return {
data: image,
name: index + '.png'
}
};
});
}
function storeScreenshots(url, imagesAndName, storageManager) {
return Promise.map(imagesAndName, (screenshot) =>
storageManager.writeDataForUrl(screenshot.data, screenshot.name, url, 'screenshots'));
return Promise.map(imagesAndName, screenshot =>
storageManager.writeDataForUrl(
screenshot.data,
screenshot.name,
url,
'screenshots'
)
);
}
module.exports = {
@ -24,7 +30,11 @@ module.exports = {
processMessage(message) {
switch (message.type) {
case 'browsertime.screenshot':
return storeScreenshots(message.url, getImagesAndName(message.data), this.storageManager);
return storeScreenshots(
message.url,
getImagesAndName(message.data),
this.storageManager
);
}
}
};

49
lib/plugins/slack/attachements.js
View File

@ -5,7 +5,7 @@ const h = require('../../support/helpers');
function getMetric(metric) {
if (metric.median) {
return metric.median + ' ms' + ' (' + metric.max + ')'
return metric.median + ' ms' + ' (' + metric.max + ')';
} else {
return metric;
}
@ -22,35 +22,59 @@ module.exports = function(dataCollection, resultUrls, slackOptions) {
const metrics = {
firstPaint: {
name: 'First paint',
metric: get(base.browsertime, 'pageSummary.statistics.timings.firstPaint')
metric: get(
base.browsertime,
'pageSummary.statistics.timings.firstPaint'
)
},
speedIndex: {
name: 'Speed Index',
metric: get(base.browsertime, 'pageSummary.statistics.visualMetrics.SpeedIndex')
metric: get(
base.browsertime,
'pageSummary.statistics.visualMetrics.SpeedIndex'
)
},
firstVisualChange: {
name: 'First Visual Change',
metric: get(base.browsertime, 'pageSummary.statistics.visualMetrics.FirstVisualChange')
metric: get(
base.browsertime,
'pageSummary.statistics.visualMetrics.FirstVisualChange'
)
},
visualComplete85: {
name: 'Visual Complete 85%',
metric: get(base.browsertime, 'pageSummary.statistics.visualMetrics.VisualComplete85')
metric: get(
base.browsertime,
'pageSummary.statistics.visualMetrics.VisualComplete85'
)
},
lastVisualChange: {
name: 'Last Visual Change',
metric: get(base.browsertime, 'pageSummary.statistics.visualMetrics.LastVisualChange')
metric: get(
base.browsertime,
'pageSummary.statistics.visualMetrics.LastVisualChange'
)
},
fullyLoaded: {
name: 'Fully Loaded',
metric: get(base.browsertime, 'pageSummary.statistics.timings.fullyLoaded')
metric: get(
base.browsertime,
'pageSummary.statistics.timings.fullyLoaded'
)
},
domContentLoadedTime: {
name: 'domContentLoadedTime',
metric: get(base.browsertime, 'pageSummary.statistics.timings.pageTimings.domContentLoadedTime')
metric: get(
base.browsertime,
'pageSummary.statistics.timings.pageTimings.domContentLoadedTime'
)
},
rumSpeedIndex: {
name: 'RUM Speed Index',
metric: get(base.browsertime, 'pageSummary.statistics.timings.rumSpeedIndex')
metric: get(
base.browsertime,
'pageSummary.statistics.timings.rumSpeedIndex'
)
},
coachScore: {
name: 'Coach score',
@ -74,7 +98,7 @@ module.exports = function(dataCollection, resultUrls, slackOptions) {
title: metric.name,
value: getMetric(metric.metric),
short: true
})
});
}
}
@ -85,7 +109,7 @@ module.exports = function(dataCollection, resultUrls, slackOptions) {
title: key + ' error',
value: results.errors[key],
short: false
})
});
}
}
@ -102,9 +126,8 @@ module.exports = function(dataCollection, resultUrls, slackOptions) {
} else if (limitMetric.metric < slackOptions.limitWarning) {
color = 'warning';
}
}
} else if (limitMetric.metric > slackOptions.limitError) {
// SpeedIndex/firstVisualChange
else if (limitMetric.metric > slackOptions.limitError) {
color = 'danger';
} else if (limitMetric.metric > slackOptions.limitWarning) {
color = 'warning';

20
lib/plugins/slack/index.js
View File

@ -30,7 +30,7 @@ module.exports = {
const slack = new Slack(slackOptions.hookUrl);
const type = slackOptions.type;
const pageErrors = [];
let logo = "https://www.sitespeed.io/img/slack/sitespeed-logo-slack.png";
let logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack.png';
let channel = slackOptions.channel;
if (channel && !channel.startsWith('#')) {
@ -46,7 +46,13 @@ module.exports = {
let text = '';
if (['summary', 'all', 'error'].includes(type)) {
const sum = getSummary(this.data, pageErrors, this.resultUrls, this.context, options);
const sum = getSummary(
this.data,
pageErrors,
this.resultUrls,
this.context,
options
);
text += sum.summaryText + '\n' + sum.errorText;
logo = sum.logo;
}
@ -56,8 +62,12 @@ module.exports = {
attachments = getAttachments(this.data, this.resultUrls, slackOptions);
}
if (type === 'error' && pageErrors.length > 0 || type !== 'error') {
log.debug('Sending message to Slack channel %s and username %s', slackOptions.channel, slackOptions.userName);
if ((type === 'error' && pageErrors.length > 0) || type !== 'error') {
log.debug(
'Sending message to Slack channel %s and username %s',
slackOptions.channel,
slackOptions.userName
);
return slack.sendAsync({
text,
icon_url: logo,
@ -65,7 +75,7 @@ module.exports = {
mrkdwn: true,
username: slackOptions.userName,
attachments
})
});
}
},
config: defaultConfig

38
lib/plugins/slack/summary.js
View File

@ -3,7 +3,13 @@ const util = require('util');
const get = require('lodash.get');
const h = require('../../support/helpers');
module.exports = function(dataCollection, errors, resultUrls, context, options) {
module.exports = function(
dataCollection,
errors,
resultUrls,
context,
options
) {
const base = dataCollection.summaryPages.index || {};
const metrics = {
firstPaint: {
@ -12,7 +18,10 @@ module.exports = function(dataCollection, errors, resultUrls, context, options)
},
domContentLoadedTime: {
name: 'domContentLoadedTime',
metric: get(base.browsertime, 'summary.pageTimings.domContentLoadedTime.median')
metric: get(
base.browsertime,
'summary.pageTimings.domContentLoadedTime.median'
)
},
speedIndex: {
name: 'Speed Index',
@ -20,15 +29,24 @@ module.exports = function(dataCollection, errors, resultUrls, context, options)
},
firstVisualChange: {
name: 'First Visual Change',
metric: get(base.browsertime, 'summary.visualMetrics.FirstVisualChange.median')
metric: get(
base.browsertime,
'summary.visualMetrics.FirstVisualChange.median'
)
},
visualComplete85: {
name: 'Visual Complete 85%',
metric: get(base.browsertime, 'summary.visualMetrics.VisualComplete85.median')
metric: get(
base.browsertime,
'summary.visualMetrics.VisualComplete85.median'
)
},
lastVisualChange: {
name: 'Last Visual Change',
metric: get(base.browsertime, 'summary.visualMetrics.LastVisualChange.median')
metric: get(
base.browsertime,
'summary.visualMetrics.LastVisualChange.median'
)
},
fullyLoaded: {
name: 'Fully Loaded',
@ -44,9 +62,15 @@ module.exports = function(dataCollection, errors, resultUrls, context, options)
}
};
let summaryText = `${h.plural(Object.keys(dataCollection.urlPages).length, 'page')} analyzed for ${h.short(context.name, 30)} ` +
let summaryText =
`${h.plural(
Object.keys(dataCollection.urlPages).length,
'page'
)} analyzed for ${h.short(context.name, 30)} ` +
`(${h.plural(options.browsertime.iterations, 'run')}, ` +
`${h.cap(options.browsertime.browser)}/${options.mobile ? 'mobile' : 'desktop'}/${options.connectivity})\n`;
`${h.cap(options.browsertime.browser)}/${options.mobile
? 'mobile'
: 'desktop'}/${options.connectivity})\n`;
let message = '';
if (resultUrls.hasBaseUrl()) {

4
lib/plugins/text/index.js
View File

@ -11,7 +11,9 @@ module.exports = {
close(options) {
if (!options.summary) return;
const renderer = this.options.summaryDetail ? textBuilder.renderSummary : textBuilder.renderBriefSummary;
const renderer = this.options.summaryDetail
? textBuilder.renderSummary
: textBuilder.renderBriefSummary;
return renderer(this.dataCollection, this.context, options);
}
};

53
lib/plugins/text/textBuilder.js
View File

@ -6,27 +6,32 @@ const table = require('text-table'),
chunk = require('lodash.chunk'),
h = require('../../support/helpers'),
tableOpts = {
stringLength : color.getStrippedLength
stringLength: color.getStrippedLength
},
drab = color.blackBright;
function getMarker(label) {
return {ok: '√', warning: '!', error: '✗', info: ''}[label] || '';
return { ok: '√', warning: '!', error: '✗', info: '' }[label] || '';
}
function getColor(label) {
// ansi = {ok: 150, warning: 230, error: 217, info: ''}[b.label],
return {ok: 'green', warning: 'yellow', error: 'red', info: 'blackBright'}[label];
return { ok: 'green', warning: 'yellow', error: 'red', info: 'blackBright' }[
label
];
}
function getHeader(dataCollection, context, options) {
const noPages = Object.keys(dataCollection.urlPages).length;
return drab([
`${h.plural(noPages,'page')} analyzed for ${h.short(context.name, 30)} `,
return drab(
[
`${h.plural(noPages, 'page')} analyzed for ${h.short(context.name, 30)} `,
`(${h.plural(options.browsertime.iterations, 'run')}, `,
`${h.cap(options.browsertime.browser)}/${options.mobile ? 'mobile' : 'desktop'}/${options.connectivity})`
].join(''));
`${h.cap(options.browsertime.browser)}/${options.mobile
? 'mobile'
: 'desktop'}/${options.connectivity})`
].join('')
);
}
function getBoxes(dataCollection) {
@ -36,7 +41,7 @@ function getBoxes(dataCollection) {
// foo bar -> fb
function abbr(str) {
if (/total|overall/i.test(str)) return str.trim();
return str.replace(/\w+\s?/g, (a) => a[0]);
return str.replace(/\w+\s?/g, a => a[0]);
}
function noop(a) {
@ -46,38 +51,40 @@ function noop(a) {
module.exports = {
renderSummary(dataCollection, context, options) {
let out = getHeader(dataCollection, context, options);
let rows = getBoxes(dataCollection).map((b) => {
let rows = getBoxes(dataCollection).map(b => {
var marker = getMarker(b.label),
c = getColor(b.label);
// color.xterm(ansi)(label),
return [ color[c](marker), color[c](b.name), color.bold(b.median) ];
return [color[c](marker), color[c](b.name), color.bold(b.median)];
});
rows.unshift(['', 'Score / Metric', 'Median'],['', '-------------', '------']);
options.summary = {out : `${out}\n` + table(rows, tableOpts)};
rows.unshift(
['', 'Score / Metric', 'Median'],
['', '-------------', '------']
);
options.summary = { out: `${out}\n` + table(rows, tableOpts) };
},
renderBriefSummary(dataCollection, context, options) {
let out = getHeader(dataCollection, context, options);
var lines = [],
scores = [],
size = '', reqs = '', rum = '';
getBoxes(dataCollection).map((b) => {
size = '',
reqs = '',
rum = '';
getBoxes(dataCollection).map(b => {
var c = getColor(b.label),
val = b.median,
name;
c = color[c] || noop;
if (/score$/i.test(b.url)) {
name = abbr(b.name.replace('score',''));
name = abbr(b.name.replace('score', ''));
scores.push(c(`${name}:${val}`));
}
else if ('pageSize' === b.url) {
val = val.replace(' ',''); // 10 KB -> 10KB
} else if ('pageSize' === b.url) {
val = val.replace(' ', ''); // 10 KB -> 10KB
size = `${val}`;
}
else if (/total requests/i.test(b.name)) {
} else if (/total requests/i.test(b.name)) {
reqs = `${val} reqs`;
}
else if (b.url === 'rumSpeedIndex') {
} else if (b.url === 'rumSpeedIndex') {
name = abbr(b.name);
rum = color.bold(`${name}: ${val}`);
}

11
lib/plugins/tracestorer/index.js
View File

@ -7,9 +7,14 @@ module.exports = {
processMessage(message) {
switch (message.type) {
case 'browsertime.chrometrace':
case 'webpagetest.chrometrace':
{
return this.storageManager.writeDataForUrl(JSON.stringify(message.data, null, 0), message.name, message.url, '', true);
case 'webpagetest.chrometrace': {
return this.storageManager.writeDataForUrl(
JSON.stringify(message.data, null, 0),
message.name,
message.url,
'',
true
);
}
}
}

72
lib/plugins/webpagetest/aggregator.js
View File

@ -25,25 +25,45 @@ module.exports = {
this.customGroups[group] = {};
}
forEach(wptData.data.runs, (run) => {
forEach(wptData.data.runs, run => {
forEach(run, (viewData, viewName) => {
forEach(metrics, (metric) =>
statsHelpers.pushGroupStats(this.timingStats, this.timingGroups[group], [viewName, metric], viewData[metric]));
forEach(metrics, metric =>
statsHelpers.pushGroupStats(
this.timingStats,
this.timingGroups[group],
[viewName, metric],
viewData[metric]
)
);
forEach(viewData.userTimes, (timingData, timingName) =>
statsHelpers.pushGroupStats(this.timingStats, this.timingGroups[group], [viewName, timingName], timingData));
statsHelpers.pushGroupStats(
this.timingStats,
this.timingGroups[group],
[viewName, timingName],
timingData
)
);
forEach(viewData.breakdown, (contentType, typeName) =>
forEach(['requests', 'bytes'], (property) =>
statsHelpers.pushGroupStats(this.assetStats, this.assetGroups[group], [viewName, typeName, property], contentType[property])));
forEach(['requests', 'bytes'], property =>
statsHelpers.pushGroupStats(
this.assetStats,
this.assetGroups[group],
[viewName, typeName, property],
contentType[property]
)
)
);
forEach(viewData.custom, (metricName) => {
forEach(viewData.custom, metricName => {
if (!isNaN(viewData[metricName])) {
statsHelpers.pushGroupStats(
this.customStats,
this.customGroups[group],
[viewName, 'custom', metricName],
viewData[metricName])
viewData[metricName]
);
}
});
});
@ -62,16 +82,22 @@ module.exports = {
for (let group of Object.keys(this.timingGroups)) {
if (!summary.groups[group]) summary.groups[group] = {};
summary.groups[group].timing = this.summarizePerTimingType(this.timingGroups[group]);
summary.groups[group].timing = this.summarizePerTimingType(
this.timingGroups[group]
);
}
for (let group of Object.keys(this.assetGroups)) {
if (!summary.groups[group]) summary.groups[group] = {};
summary.groups[group].asset = this.summarizePerAssetType(this.assetGroups[group]);
summary.groups[group].asset = this.summarizePerAssetType(
this.assetGroups[group]
);
}
if (this.customGroups) {
for (let group of Object.keys(this.customGroups)) {
if (!summary.groups[group]) summary.groups[group] = {};
summary.groups[group].custom = this.summarizePerCustomType(this.customGroups[group]);
summary.groups[group].custom = this.summarizePerCustomType(
this.customGroups[group]
);
}
}
return summary;
@ -81,14 +107,23 @@ module.exports = {
forEach(type, (view, viewName) =>
forEach(view, (contentType, contentTypeName) =>
forEach(contentType, (stats, propertyName) =>
statsHelpers.setStatsSummary(summary, [viewName, 'breakdown', contentTypeName, propertyName], stats))));
statsHelpers.setStatsSummary(
summary,
[viewName, 'breakdown', contentTypeName, propertyName],
stats
)
)
)
);
return summary;
},
summarizePerTimingType(type) {
const summary = {};
forEach(type, (view, viewName) =>
forEach(view, (stats, name) =>
statsHelpers.setStatsSummary(summary, [viewName, name], stats)));
statsHelpers.setStatsSummary(summary, [viewName, name], stats)
)
);
return summary;
},
summarizePerCustomType(type) {
@ -96,9 +131,14 @@ module.exports = {
forEach(type, (view, viewName) =>
forEach(view, (metricName, name) =>
forEach(metricName, (stats, propertyName) => {
statsHelpers.setStatsSummary(summary, [viewName, name, propertyName], stats)
}
)));
statsHelpers.setStatsSummary(
summary,
[viewName, name, propertyName],
stats
);
})
)
);
return summary;
}
};

95
lib/plugins/webpagetest/analyzer.js
View File

@ -8,13 +8,11 @@ var fs = require('fs'),
WebPageTest = require('webpagetest'),
WPTAPIError = require('webpagetest/lib/helper').WPTAPIError;
Promise.promisifyAll(fs);
Promise.promisifyAll(WebPageTest.prototype);
module.exports = {
analyzeUrl(url, storageManager, wptOptions) {
const wptClient = new WebPageTest(wptOptions.host, wptOptions.key);
wptOptions.firstViewOnly = !wptOptions.includeRepeatView;
let urlOrScript = url;
@ -26,20 +24,31 @@ module.exports = {
// See https://github.com/sitespeedio/sitespeed.io/issues/1367
const options = clone(wptOptions);
return wptClient.runTestAsync(urlOrScript, options)
.then(function(data) {
return wptClient.runTestAsync(urlOrScript, options).then(function(data) {
const id = data.data.id;
log.info('Got %s analysed with id %s from %s', url, id, options.host);
log.verbose('Got JSON from WebPageTest :%:2j', data);
// Something failed with WebPageTest but how should we handle that?
if (data.statusCode !== 200) {
log.error('The test got status code %s from WebPageTest with %s. Checkout %s to try to find the original reason.', data.statusCode, data.statusText, get(data, 'data.summary'));
log.error(
'The test got status code %s from WebPageTest with %s. Checkout %s to try to find the original reason.',
data.statusCode,
data.statusText,
get(data, 'data.summary')
);
}
const promises = [];
let har;
promises.push(wptClient.getHARDataAsync(id, {}).then((theHar => har = theHar)).catch(WPTAPIError, (error) => log.error('Couldnt get HAR data fir id %s %s', id, error)));
promises.push(
wptClient
.getHARDataAsync(id, {})
.then(theHar => (har = theHar))
.catch(WPTAPIError, error =>
log.error('Couldnt get HAR data fir id %s %s', id, error)
)
);
const traces = {};
const views = ['firstView'];
@ -72,34 +81,76 @@ module.exports = {
};
promises.push(
Promise.join(wptClient.getScreenshotImageAsync(id, screenShotOptions), j, view,
(result, index, view) => storageManager.writeDataForUrl(result, 'wpt-' + index + '-' + view + '.png', url,
'screenshots')).catch(WPTAPIError, (error) => log.error('Couldnt get screenshot for id %s %s', id, error))
Promise.join(
wptClient.getScreenshotImageAsync(id, screenShotOptions),
j,
view,
(result, index, view) =>
storageManager.writeDataForUrl(
result,
'wpt-' + index + '-' + view + '.png',
url,
'screenshots'
)
).catch(WPTAPIError, error =>
log.error('Couldnt get screenshot for id %s %s', id, error)
)
);
promises.push(
Promise.join(wptClient.getWaterfallImageAsync(id, waterfallOptions), j, view,
Promise.join(
wptClient.getWaterfallImageAsync(id, waterfallOptions),
j,
view,
(result, index, view) => {
return storageManager.writeDataForUrl(result, 'wpt-waterfall-' + index + '-' + view + '.png', url,
'waterfall').catch(WPTAPIError, (error) => log.error('Couldnt get waterfall %s %s', id, error))
})
return storageManager
.writeDataForUrl(
result,
'wpt-waterfall-' + index + '-' + view + '.png',
url,
'waterfall'
)
.catch(WPTAPIError, error =>
log.error('Couldnt get waterfall %s %s', id, error)
);
}
)
);
promises.push(
Promise.join(wptClient.getWaterfallImageAsync(id, connectionOptions), j, view,
(result, index, view) => storageManager.writeDataForUrl(result, 'wpt-waterfall-connection-' + index + '-' + view +
'.png',
url, 'waterfall')).catch(WPTAPIError, (error) => log.error('Couldnt get watetfall connection for id %s %s', id, error))
Promise.join(
wptClient.getWaterfallImageAsync(id, connectionOptions),
j,
view,
(result, index, view) =>
storageManager.writeDataForUrl(
result,
'wpt-waterfall-connection-' + index + '-' + view + '.png',
url,
'waterfall'
)
).catch(WPTAPIError, error =>
log.error(
'Couldnt get watetfall connection for id %s %s',
id,
error
)
)
);
if (wptOptions.timeline) {
promises.push(
Promise.join(wptClient.getChromeTraceDataAsync(id, timelineOptions), j, view,
Promise.join(
wptClient.getChromeTraceDataAsync(id, timelineOptions),
j,
view,
(result, index, view) => {
traces['trace-' + index + '-wpt-' + view] = result
}).catch(WPTAPIError, (error) => log.error('Couldnt get chrome trace for id %s %s',id, error))
traces['trace-' + index + '-wpt-' + view] = result;
}
).catch(WPTAPIError, error =>
log.error('Couldnt get chrome trace for id %s %s', id, error)
)
);
}
}
});
return Promise.all(promises).then(() => {
@ -109,7 +160,7 @@ module.exports = {
};
myResult.trace = traces;
return myResult;
})
});
});
}
};

86
lib/plugins/webpagetest/index.js
View File

@ -13,7 +13,7 @@ const WebPageTest = require('webpagetest');
const make = messageMaker('webpagetest').make;
const hostRegex = /^(https?:\/\/)?([^\/]*)/i;
const hostRegex = /^(https?:\/\/)?([^/]*)/i;
const defaultWptHost = urlParser.parse(WebPageTest.defaultServer).host;
const DEFAULT_PAGE_SUMMARY_METRICS = [
@ -42,8 +42,11 @@ const DEFAULT_SUMMARY_METRICS = [
function addCustomMetric(result) {
const customMetrics = get(result, 'data.median.firstView.custom');
if (customMetrics) {
for (const customMetric of customMetrics ) {
filterRegistry.addFilterForType('data.median.*.' + customMetric, 'webpagetest.pageSummary');
for (const customMetric of customMetrics) {
filterRegistry.addFilterForType(
'data.median.*.' + customMetric,
'webpagetest.pageSummary'
);
}
}
}
@ -68,66 +71,95 @@ module.exports = {
if (!options.key) {
const host = hostRegex.exec(options.host);
if (host && host[2] === defaultWptHost) {
throw new Error('webpagetest.key needs to be specified when using the public WebPageTest server.');
throw new Error(
'webpagetest.key needs to be specified when using the public WebPageTest server.'
);
}
}
filterRegistry.registerFilterForType(DEFAULT_PAGE_SUMMARY_METRICS , 'webpagetest.pageSummary');
filterRegistry.registerFilterForType(DEFAULT_SUMMARY_METRICS , 'webpagetest.summary');
filterRegistry.registerFilterForType(
DEFAULT_PAGE_SUMMARY_METRICS,
'webpagetest.pageSummary'
);
filterRegistry.registerFilterForType(
DEFAULT_SUMMARY_METRICS,
'webpagetest.summary'
);
},
processMessage(message, queue) {
switch (message.type) {
case 'url':
{
case 'url': {
const url = message.url;
const group = message.group;
return analyzer.analyzeUrl(url, this.storageManager, this.options)
.tap((result) => {
return analyzer
.analyzeUrl(url, this.storageManager, this.options)
.tap(result => {
addCustomMetric(result);
if (result.trace) {
forEach(result.trace, (value, key) => {
queue.postMessage(make('webpagetest.chrometrace', value, {url, group, name: key + '.json'}));
queue.postMessage(
make('webpagetest.chrometrace', value, {
url,
group,
name: key + '.json'
})
);
});
}
queue.postMessage(make('webpagetest.har', result.har, {url, group}));
queue.postMessage(
make('webpagetest.har', result.har, { url, group })
);
forEach(result.data.runs, (run, runKey) =>
queue.postMessage(make('webpagetest.run', run, {
queue.postMessage(
make('webpagetest.run', run, {
url,
group,
runIndex: (parseInt(runKey) - 1)
}))
runIndex: parseInt(runKey) - 1
})
)
);
const location = result.data.location.replace(':', '-').replace(' ', '-').toLowerCase();
const location = result.data.location
.replace(':', '-')
.replace(' ', '-')
.toLowerCase();
// There's no connectivity setup in the default config for WPT, make sure we catch that
const connectivity = get(result, 'data.connectivity', 'native').toLowerCase();
queue.postMessage(make('webpagetest.pageSummary', result, {
const connectivity = get(
result,
'data.connectivity',
'native'
).toLowerCase();
queue.postMessage(
make('webpagetest.pageSummary', result, {
url,
group,
location,
connectivity
}));
})
);
aggregator.addToAggregate(group, result, connectivity, location);
})
.catch((err) => {
.catch(err => {
log.error('Error creating WebPageTest result ', err);
queue.postMessage(make('error', err, {
queue.postMessage(
make('error', err, {
url
}));
})
);
});
}
case 'summarize':
{
case 'summarize': {
let summary = aggregator.summarize();
if (summary && Object.keys(summary.groups).length > 0) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(make('webpagetest.summary', summary.groups[group], {
queue.postMessage(
make('webpagetest.summary', summary.groups[group], {
connectivity: aggregator.connectivity,
location: aggregator.location,
group
}));
})
);
}
}
}

84
lib/sitespeed.js
View File

@ -27,7 +27,7 @@ const budgetResult = {
};
function hasFunctionFilter(functionName) {
return ((obj) => (typeof obj[functionName] === 'function'));
return obj => typeof obj[functionName] === 'function';
}
function allInArray(sampleArray, referenceArray) {
@ -42,7 +42,7 @@ function runOptionalFunction(objects, fN) {
}
return Promise.resolve(objects)
.filter(hasFunctionFilter(fN))
.map((plugin) => Promise.resolve(plugin[fN].apply(plugin, args)));
.map(plugin => Promise.resolve(plugin[fN].apply(plugin, args)));
}
module.exports = {
@ -54,36 +54,64 @@ module.exports = {
timestamp.utc();
}
const {storageManager, resultUrls} = resultsStorage(url, timestamp, options.outputFolder, options.resultBaseURL);
const { storageManager, resultUrls } = resultsStorage(
url,
timestamp,
options.outputFolder,
options.resultBaseURL
);
const dataCollection = new DataCollection();
return storageManager.createDataDir('logs').then((logDir) => {
return storageManager
.createDataDir('logs')
.then(logDir => {
logging.configure(options, logDir);
}).then(() => {
})
.then(() => {
if (log.isEnabledFor(log.VERBOSE)) {
Promise.longStackTraces();
}
log.info('Versions OS: %s nodejs: %s sitespeed.io: %s browsertime: %s coach: %s', os.platform() + ' ' + os.release(), process.version, packageInfo.version, packageInfo.dependencies.browsertime, packageInfo.dependencies.webcoach);
}).then(() => {
return loader.parsePluginNames(options)
}).then((pluginNames) => {
log.info(
'Versions OS: %s nodejs: %s sitespeed.io: %s browsertime: %s coach: %s',
os.platform() + ' ' + os.release(),
process.version,
packageInfo.version,
packageInfo.dependencies.browsertime,
packageInfo.dependencies.webcoach
);
})
.then(() => {
return loader.parsePluginNames(options);
})
.then(pluginNames => {
const plugins = options.plugins;
if (plugins) {
pullAll(pluginNames, toArray(plugins.disable));
pluginNames = union(pluginNames, toArray(plugins.load));
if (plugins.list) {
log.info('The following plugins are enabled: %s', pluginNames.join(', '));
log.info(
'The following plugins are enabled: %s',
pluginNames.join(', ')
);
}
}
// if we run without cli, we still want the default options
// in options to use it in output
if (allInArray(['browsertime'], pluginNames)) {
options.browsertime = merge({}, browsertimeConfig, options.browsertime);
options.browsertime = merge(
{},
browsertimeConfig,
options.browsertime
);
}
if (allInArray(['webpagetest'], pluginNames)) {
options.webpagetest = merge({}, webpagetestConfig, options.webpagetest);
options.webpagetest = merge(
{},
webpagetestConfig,
options.webpagetest
);
}
if (allInArray(['browsertime', 'coach'], pluginNames)) {
options.browsertime = merge({}, options.browsertime, {
@ -97,17 +125,17 @@ module.exports = {
}
return pluginNames;
})
.then((pluginNames) => {
return loader.loadPlugins(pluginNames)
.then((plugins) => {
.then(pluginNames => {
return loader.loadPlugins(pluginNames).then(plugins => {
let urlSources = [urlSource];
const allPlugins = urlSources.concat(plugins),
queueHandler = new QueueHandler(plugins, options);
return runOptionalFunction(allPlugins, 'open', {
return runOptionalFunction(
allPlugins,
'open',
{
storageManager,
resultUrls,
dataCollection,
@ -115,14 +143,20 @@ module.exports = {
budget: budgetResult,
name: url,
log
}, options)
},
options
)
.then(() => runOptionalFunction(allPlugins, 'postOpen', options))
.then(() => queueHandler.run(urlSources))
.tap((errors) => runOptionalFunction(allPlugins, 'close', options, errors))
.tap((errors) => runOptionalFunction(allPlugins, 'postClose', options, errors));
.tap(errors =>
runOptionalFunction(allPlugins, 'close', options, errors)
)
.tap(errors =>
runOptionalFunction(allPlugins, 'postClose', options, errors)
);
});
})
})
.then((errors) => {
.then(errors => {
log.info('Finished analysing %s', url);
if (options.summary && options.summary.out) {
console.log(options.summary.out); // eslint-disable-line no-console
@ -132,9 +166,9 @@ module.exports = {
budgetResult
};
})
.catch((err) => {
.catch(err => {
log.error(err);
throw err;
})
});
}
};

180
lib/support/cli.js
View File

@ -28,7 +28,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('verbose', {
alias: 'v',
describe: 'Verbose mode prints progress messages to the console. Enter up to three times (-vvv)' +
describe:
'Verbose mode prints progress messages to the console. Enter up to three times (-vvv)' +
' to increase the level of detail.',
type: 'count'
})
@ -51,7 +52,16 @@ module.exports.parseCommandLine = function parseCommandLine() {
.option('browsertime.connectivity.profile', {
alias: ['c', 'connectivity'],
default: browsertimeConfig.connectivity.profile,
choices: ['3g', '3gfast', '3gslow', '3gem', '2g', 'cable', 'native', 'custom'],
choices: [
'3g',
'3gfast',
'3gslow',
'3gem',
'2g',
'cable',
'native',
'custom'
],
describe: 'The connectivity profile.',
group: 'Browser'
})
@ -81,20 +91,24 @@ module.exports.parseCommandLine = function parseCommandLine() {
.option('browsertime.connectivity.engine', {
default: browsertimeConfig.connectivity.engine,
choices: ['tc', 'tsproxy', 'external'],
describe: 'The engine for connectivity. TC (Linux Traffic Control) needs tc work but will only setup upload and latency. Use external if you set the connectivity outside of Browsertime. The best way do to this is described in https://github.com/sitespeedio/browsertime#connectivity',
describe:
'The engine for connectivity. TC (Linux Traffic Control) needs tc work but will only setup upload and latency. Use external if you set the connectivity outside of Browsertime. The best way do to this is described in https://github.com/sitespeedio/browsertime#connectivity',
group: 'Browser'
})
.option('browsertime.pageCompleteCheck', {
describe: 'Supply a Javascript that decides when the browser is finished loading the page and can start to collect metrics. The Javascript snippet is repeatedly queried to see if page has completed loading (indicated by the script returning true). Use it to fetch timings happening after the loadEventEnd.',
describe:
'Supply a Javascript that decides when the browser is finished loading the page and can start to collect metrics. The Javascript snippet is repeatedly queried to see if page has completed loading (indicated by the script returning true). Use it to fetch timings happening after the loadEventEnd.',
group: 'Browser'
})
.option('browsertime.script', {
describe: 'Add custom Javascript that collect metrics and run after the page has finished loading. Note that --script can be passed multiple times if you want to collect multiple metrics. The metrics will automatically be pushed to the summary/detailed summary and each individual page + sent to Graphite/InfluxDB.',
describe:
'Add custom Javascript that collect metrics and run after the page has finished loading. Note that --script can be passed multiple times if you want to collect multiple metrics. The metrics will automatically be pushed to the summary/detailed summary and each individual page + sent to Graphite/InfluxDB.',
alias: ['script'],
group: 'Browser'
})
.option('browsertime.selenium.url', {
describe: 'Configure the path to the Selenium server when fetching timings using browsers. If not configured the supplied NodeJS/Selenium version is used.',
describe:
'Configure the path to the Selenium server when fetching timings using browsers. If not configured the supplied NodeJS/Selenium version is used.',
group: 'Browser'
})
.option('browsertime.viewPort', {
@ -103,21 +117,25 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'Browser'
})
.option('browsertime.userAgent', {
describe: 'The full User Agent string, defaults to the User Agent used by the browsertime.browser option.',
describe:
'The full User Agent string, defaults to the User Agent used by the browsertime.browser option.',
group: 'Browser'
})
.option('browsertime.preScript', {
alias: 'preScript',
describe: 'Selenium script(s) to run before you test your URL (use it for login, warm the cache, etc). Note that --preScript can be passed multiple times.',
describe:
'Selenium script(s) to run before you test your URL (use it for login, warm the cache, etc). Note that --preScript can be passed multiple times.',
group: 'Browser'
})
.option('browsertime.postScript', {
alias: 'postScript',
describe: 'Selenium script(s) to run after you test your URL (use it for logout etc). Note that --postScript can be passed multiple times.',
describe:
'Selenium script(s) to run after you test your URL (use it for logout etc). Note that --postScript can be passed multiple times.',
group: 'Browser'
})
.option('browsertime.delay', {
describe: 'Delay between runs, in milliseconds. Use it if your web server needs to rest between runs :)',
describe:
'Delay between runs, in milliseconds. Use it if your web server needs to rest between runs :)',
group: 'Browser'
})
.option('browsertime.speedIndex', {
@ -134,16 +152,19 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('browsertime.preURL', {
alias: 'preURL',
describe: 'A URL that will be accessed first by the browser before the URL that you wanna analyze. Use it to fill the cache.',
describe:
'A URL that will be accessed first by the browser before the URL that you wanna analyze. Use it to fill the cache.',
group: 'Browser'
})
.option('browsertime.userTimingWhitelist', {
alias: 'userTimingWhitelist',
describe: 'This option takes a regex that will whitelist which userTimings to capture in the results. All userTimings are captured by default. T',
describe:
'This option takes a regex that will whitelist which userTimings to capture in the results. All userTimings are captured by default. T',
group: 'Browser'
})
.option('browsertime.firefox.preference', {
describe: 'Extra command line arguments to pass Firefox preferences by the format key:value ' +
describe:
'Extra command line arguments to pass Firefox preferences by the format key:value ' +
'To add multiple preferences, repeat --browsertime.firefox.preference once per argument.',
group: 'Browser'
})
@ -153,7 +174,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'Browser'
})
.option('browsertime.chrome.args', {
describe: 'Extra command line arguments to pass to the Chrome process (e.g. --no-sandbox). ' +
describe:
'Extra command line arguments to pass to the Chrome process (e.g. --no-sandbox). ' +
'To add multiple arguments to Chrome, repeat --browsertime.chrome.args once per argument.',
group: 'Browser'
})
@ -163,11 +185,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'Browser'
})
.option('browsertime.chrome.android.package', {
describe: 'Run Chrome on your Android device. Set to com.android.chrome for default Chrome version. You need to run adb start-server before you start.',
describe:
'Run Chrome on your Android device. Set to com.android.chrome for default Chrome version. You need to run adb start-server before you start.',
group: 'Browser'
})
.option('browsertime.chrome.android.deviceSerial', {
describe: 'Choose which device to use. If you do not set it, the first found device will be used.',
describe:
'Choose which device to use. If you do not set it, the first found device will be used.',
group: 'Browser'
})
// legacy naming of collectTracingEvents
@ -187,15 +211,18 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('browsertime.requestheader', {
alias: 'r',
describe: 'Request header that will be added to the request. Add multiple instances to add multiple request headers. Only Chrome support for now.',
describe:
'Request header that will be added to the request. Add multiple instances to add multiple request headers. Only Chrome support for now.',
group: 'Browser'
})
.option('browsertime.block', {
describe: 'Domain to block. Add multiple instances to add multiple domains that will be blocked. Only Chrome support for now.',
describe:
'Domain to block. Add multiple instances to add multiple domains that will be blocked. Only Chrome support for now.',
group: 'Browser'
})
.option('browsertime.basicAuth', {
describe: 'Use it if your server is behind Basic Auth. Format: username@password (Only Chrome at the moment).',
describe:
'Use it if your server is behind Basic Auth. Format: username@password (Only Chrome at the moment).',
group: 'Browser',
alias: 'basicAuth'
})
@ -214,7 +241,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
*/
.option('crawler.depth', {
alias: 'd',
describe: 'How deep to crawl (1=only one page, 2=include links from first page, etc.)',
describe:
'How deep to crawl (1=only one page, 2=include links from first page, etc.)',
group: 'Crawler'
})
.option('crawler.maxPages', {
@ -235,16 +263,19 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'Graphite'
})
.option('graphite.auth', {
describe: 'The Graphite user and password used for authentication. Format: user:password',
describe:
'The Graphite user and password used for authentication. Format: user:password',
group: 'Graphite'
})
.option('graphite.httpPort', {
describe: 'The Graphite port used to access the user interface and send annotations event',
describe:
'The Graphite port used to access the user interface and send annotations event',
default: 8080,
group: 'Graphite'
})
.option('graphite.webHost', {
describe: 'The graphite-web host. If not specified graphite.host will be used.',
describe:
'The graphite-web host. If not specified graphite.host will be used.',
group: 'Graphite'
})
.option('graphite.namespace', {
@ -254,17 +285,18 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('graphite.includeQueryParams', {
default: graphiteConfig.includeQueryParams,
describe: 'Whether to include query parameters from the URL in the Graphite keys or not',
describe:
'Whether to include query parameters from the URL in the Graphite keys or not',
type: 'boolean',
group: 'Graphite'
})
.option('graphite.arrayTags', {
default: false,
type: 'boolean',
describe: 'Send the tags as array or a string. In Graphite 1.0 the tags is a array.',
describe:
'Send the tags as array or a string. In Graphite 1.0 the tags is a array.',
group: 'Graphite'
})
/** Plugins */
.option('plugins.list', {
describe: 'List all configured plugins in the log.',
@ -273,12 +305,14 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('plugins.disable', {
type: 'array',
describe: 'Disable a plugin. Use it to disable generating html or screenshots.',
describe:
'Disable a plugin. Use it to disable generating html or screenshots.',
group: 'Plugins'
})
.option('plugins.load', {
type: 'array',
describe: 'Extra plugins that you want to run. Relative or absolute path to the plugin.',
describe:
'Extra plugins that you want to run. Relative or absolute path to the plugin.',
group: 'Plugins'
})
/** Budget */
@ -295,7 +329,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
describe: 'The output format of the budget.',
group: 'Budget'
})
/**
InfluxDB cli option
*/
@ -328,12 +361,14 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('influxdb.tags', {
default: influxdbConfig.tags,
describe: 'A comma separated list of tags and values added to each metric',
describe:
'A comma separated list of tags and values added to each metric',
group: 'InfluxDB'
})
.option('influxdb.includeQueryParams', {
default: influxdbConfig.includeQueryParams,
describe: 'Whether to include query parameters from the URL in the InfluxDB keys or not',
describe:
'Whether to include query parameters from the URL in the InfluxDB keys or not',
type: 'boolean',
group: 'InfluxDB'
})
@ -344,17 +379,18 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'Metrics'
})
.option('metrics.filterList', {
describe: 'List all configured filters for metrics in the data folder (configuredMetrics.txt)',
describe:
'List all configured filters for metrics in the data folder (configuredMetrics.txt)',
type: 'boolean',
default: metricsConfig.filterList,
group: 'Metrics'
})
.option('metrics.filter', {
type: 'array',
describe: 'Add/change/remove filters for metrics. If you want to send all metrics, use: *+ . If you want to remove all current metrics and send only the coach score: *- coach.summary.score.*',
describe:
'Add/change/remove filters for metrics. If you want to send all metrics, use: *+ . If you want to remove all current metrics and send only the coach score: *- coach.summary.score.*',
group: 'Metrics'
})
/*
WebPageTest cli options
*/
@ -383,7 +419,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'WebPageTest'
})
.option('webpagetest.custom', {
describe: 'Execute arbitrary Javascript at the end of a test to collect custom metrics.',
describe:
'Execute arbitrary Javascript at the end of a test to collect custom metrics.',
group: 'WebPageTest'
})
.option('webpagetest.file', {
@ -415,7 +452,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
Slack options
*/
.option('slack.hookUrl', {
describe: 'WebHook url for the Slack team (check https://<your team>.slack.com/apps/manage/custom-integrations).',
describe:
'WebHook url for the Slack team (check https://<your team>.slack.com/apps/manage/custom-integrations).',
group: 'Slack'
})
.option('slack.userName', {
@ -424,11 +462,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'Slack'
})
.option('slack.channel', {
describe: 'The slack channel without the # (if something else than the default channel for your hook).',
describe:
'The slack channel without the # (if something else than the default channel for your hook).',
group: 'Slack'
})
.option('slack.type', {
describe: 'Send summary for a run, metrics from all URLs, only on errors or all to Slack.',
describe:
'Send summary for a run, metrics from all URLs, only on errors or all to Slack.',
default: slackConfig.type,
choices: ['summary', 'url', 'error', 'all'],
group: 'Slack'
@ -465,7 +505,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 's3'
})
.option('s3.path', {
describe: 'Override the default folder path in the bucket where the results are uploaded. By default it\'s ' +
describe:
"Override the default folder path in the bucket where the results are uploaded. By default it's " +
'"DOMAIN_OR_FILENAME/TIMESTAMP", or the name of the folder if --outputFolder is specified.',
group: 's3'
})
@ -474,7 +515,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 's3'
})
.option('s3.removeLocalResult', {
describe: 'Remove all the local result files after they have been uploaded to S3',
describe:
'Remove all the local result files after they have been uploaded to S3',
default: false,
type: 'boolean',
group: 's3'
@ -483,25 +525,29 @@ module.exports.parseCommandLine = function parseCommandLine() {
Html options
*/
.option('html.showAllWaterfallSummary', {
describe: 'Set to true to show all waterfalls on page summary HTML report',
describe:
'Set to true to show all waterfalls on page summary HTML report',
default: false,
type: 'boolean',
group: 'HTML'
})
.option('html.fetchHARFiles', {
describe: 'Set to true to load HAR files using fetch instead of including them in the HTML. Turn this on if serve your pages using a server.',
describe:
'Set to true to load HAR files using fetch instead of including them in the HTML. Turn this on if serve your pages using a server.',
default: false,
type: 'boolean',
group: 'HTML'
})
.option('html.logDownloadLink', {
describe: 'Adds a link in the HTML so you easily can download the logs from the sitespeed.io run. If your server is public, be careful so you don\'t log passwords etc',
describe:
"Adds a link in the HTML so you easily can download the logs from the sitespeed.io run. If your server is public, be careful so you don't log passwords etc",
default: false,
type: 'boolean',
group: 'HTML'
})
.option('html.topListSize', {
describe: 'Maximum number of assets to include in each toplist in the toplist tab',
describe:
'Maximum number of assets to include in each toplist in the toplist tab',
default: 10,
group: 'HTML'
})
@ -518,12 +564,14 @@ module.exports.parseCommandLine = function parseCommandLine() {
group: 'text'
})
.option('mobile', {
describe: 'Access pages as mobile a fake mobile device. Set UA and width/height. For Chrome it will use device Apple iPhone 6.',
describe:
'Access pages as mobile a fake mobile device. Set UA and width/height. For Chrome it will use device Apple iPhone 6.',
default: false,
type: 'boolean'
})
.option('resultBaseURL', {
describe: 'The base URL to the server serving the HTML result. In the format of https://result.sitespeed.io'
describe:
'The base URL to the server serving the HTML result. In the format of https://result.sitespeed.io'
})
.option('gzipHAR', {
describe: 'Compress the HAR files with GZIP.',
@ -534,7 +582,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
describe: 'The folder where the result will be stored.'
})
.option('firstParty', {
describe: 'A regex running against each request and categorize it as first vs third party URL. (ex: ".*sitespeed.*")'
describe:
'A regex running against each request and categorize it as first vs third party URL. (ex: ".*sitespeed.*")'
})
.option('utc', {
describe: 'Use Coordinated Universal Time for timestamps',
@ -549,12 +598,14 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (typeof arg === 'object' && !Array.isArray(arg)) {
if (arg.configPath) {
arg.config = JSON.parse(fs.readFileSync(arg.configPath, 'utf8'));
} else if (arg.config){
} else if (arg.config) {
arg.config = JSON.parse(arg.config);
}
return arg;
} else {
throw new Error('[ERROR] Something looks wrong with your budget configuration. Since sitespeed.io 4.4 you should pass the path to your budget file through the --budget.configPath flag instead of directly through the --budget flag.');
throw new Error(
'[ERROR] Something looks wrong with your budget configuration. Since sitespeed.io 4.4 you should pass the path to your budget file through the --budget.configPath flag instead of directly through the --budget flag.'
);
}
})
.coerce('webpagetest', function(arg) {
@ -562,7 +613,9 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (arg.script && fs.existsSync(arg.script)) {
arg.script = fs.readFileSync(path.resolve(arg.script), 'utf8');
/* eslint no-console: off */
console.log('[WARNING] Since sitespeed.io 4.4 you should pass the path to the script file through the --webpagetest.file flag (https://github.com/sitespeedio/sitespeed.io/pull/1445).');
console.log(
'[WARNING] Since sitespeed.io 4.4 you should pass the path to the script file through the --webpagetest.file flag (https://github.com/sitespeedio/sitespeed.io/pull/1445).'
);
return arg;
}
@ -578,27 +631,37 @@ module.exports.parseCommandLine = function parseCommandLine() {
// .describe('browser', 'Specify browser')
.wrap(yargs.terminalWidth())
// .check(validateInput)
.epilog('Read the docs at https://www.sitespeed.io/documentation/sitespeed.io/');
.epilog(
'Read the docs at https://www.sitespeed.io/documentation/sitespeed.io/'
);
const aliases = parsed.getOptions().alias,
argv = parsed.argv;
// aliases are long options -> short option
const aliasLookup = reduce(aliases, (lookup, value, key) => {
const aliasLookup = reduce(
aliases,
(lookup, value, key) => {
lookup.set(value[0], key);
return lookup;
}, new Map());
},
new Map()
);
let explicitOptions = yargs.reset().argv;
explicitOptions = reduce(explicitOptions, (result, value, key) => {
explicitOptions = reduce(
explicitOptions,
(result, value, key) => {
if (aliasLookup.has(key)) {
const fullKey = aliasLookup.get(key);
result = set(result, fullKey, value);
}
result = set(result, key, value);
return result;
}, {});
},
{}
);
if (argv.config) {
const config = require(path.resolve(process.cwd(), argv.config));
@ -606,9 +669,12 @@ module.exports.parseCommandLine = function parseCommandLine() {
}
if (argv.webpagetest.custom) {
argv.webpagetest.custom = fs.readFileSync(path.resolve(argv.webpagetest.custom), {
argv.webpagetest.custom = fs.readFileSync(
path.resolve(argv.webpagetest.custom),
{
encoding: 'utf8'
});
}
);
}
if (argv.summaryDetail) argv.summary = true;

17
lib/support/cliUtil.js
View File

@ -6,7 +6,7 @@ const path = require('path');
module.exports = {
getURLs(urls) {
const allUrls = [];
urls = urls.map((url) => url.trim());
urls = urls.map(url => url.trim());
for (let url of urls) {
if (url.startsWith('http')) {
@ -17,7 +17,7 @@ module.exports = {
const lines = fs.readFileSync(filePath).toString().split('\n');
for (let line of lines) {
if (line.trim().length > 0) {
let lineArray = line.split(" ", 2);
let lineArray = line.split(' ', 2);
let url = lineArray[0].trim();
if (url) {
allUrls.push(url);
@ -36,7 +36,7 @@ module.exports = {
},
getAliases(urls) {
const urlMetaData = {};
urls = urls.map((url) => url.trim());
urls = urls.map(url => url.trim());
for (let url of urls) {
if (url.startsWith('http')) {
@ -46,14 +46,15 @@ module.exports = {
const lines = fs.readFileSync(filePath).toString().split('\n');
for (let line of lines) {
if (line.trim().length > 0) {
let url, alias = null;
let lineArray = line.split(" ", 2);
let url,
alias = null;
let lineArray = line.split(' ', 2);
url = lineArray[0].trim();
if(lineArray[1]) {
if (lineArray[1]) {
alias = lineArray[1].trim();
}
if(url && alias) {
urlMetaData[url] = {'alias' : alias};
if (url && alias) {
urlMetaData[url] = { alias: alias };
}
}
}

17
lib/support/dataCollection.js
View File

@ -19,23 +19,30 @@ class DataCollection {
}
getValidPages() {
return reduce(this.urlPages, (validPages, urlInfo, url) => {
return reduce(
this.urlPages,
(validPages, urlInfo, url) => {
if (Object.keys(urlInfo.data).length > 0) {
validPages[url] = urlInfo;
}
return validPages;
}, {});
},
{}
);
}
getErrorPages() {
return reduce(this.urlPages, (errors, urlInfo, url) => {
return reduce(
this.urlPages,
(errors, urlInfo, url) => {
if (urlInfo.errors) {
errors[url] = urlInfo.errors;
}
return errors;
}, {});
},
{}
);
}
}
module.exports = DataCollection;

5
lib/support/filterRegistry.js
View File

@ -45,7 +45,10 @@ module.exports = {
}
const filteredMessage = clone(message);
filteredMessage.data = metricsFilter.filterMetrics(filteredMessage.data, filterConfig);
filteredMessage.data = metricsFilter.filterMetrics(
filteredMessage.data,
filterConfig
);
return filteredMessage;
}
};

50
lib/support/flattenMessage.js
View File

@ -11,12 +11,12 @@ function toSafeKey(key) {
}
module.exports = {
keypathFromUrl(url, includeQueryParams)
{
keypathFromUrl(url, includeQueryParams) {
function flattenQueryParams(params) {
return Object.keys(params).reduce((result, key) =>
joinNonEmpty([result, key, params[key]], '_'),
'');
return Object.keys(params).reduce(
(result, key) => joinNonEmpty([result, key, params[key]], '_'),
''
);
}
url = urlParser.parse(url, !!includeQueryParams);
@ -24,18 +24,18 @@ module.exports = {
let path = toSafeKey(url.pathname);
if (includeQueryParams) {
path = joinNonEmpty([path, toSafeKey(flattenQueryParams(url.query))], '_');
path = joinNonEmpty(
[path, toSafeKey(flattenQueryParams(url.query))],
'_'
);
}
const keys = [
toSafeKey(url.hostname),
path
];
const keys = [toSafeKey(url.hostname), path];
return joinNonEmpty(keys, '.');
},
flattenMessageData({data, type}) {
flattenMessageData({ data, type }) {
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}
@ -72,18 +72,27 @@ module.exports = {
break;
}
Object.keys(value).forEach((key) => {
Object.keys(value).forEach(key => {
// Hey are you coming to the future from 1980s? Please don't
// look at this code, it's a ugly hack to make sure we can send assets
// to Graphite and don't send them with array position, instead
// use the url to generate the key
if (type === 'pagexray.pageSummary' && keyPrefix === 'assets') {
recursiveFlatten(target, joinNonEmpty([keyPrefix, toSafeKey(value[key].url || key)], '.'), value[key]);
}
else {
recursiveFlatten(target, joinNonEmpty([keyPrefix, toSafeKey(key)], '.'), value[key]);
recursiveFlatten(
target,
joinNonEmpty(
[keyPrefix, toSafeKey(value[key].url || key)],
'.'
),
value[key]
);
} else {
recursiveFlatten(
target,
joinNonEmpty([keyPrefix, toSafeKey(key)], '.'),
value[key]
);
}
});
}
@ -106,7 +115,12 @@ module.exports = {
}
break;
default:
throw new Error('Unhandled value type ' + valueType + ' found when flattening data for prefix ' + keyPrefix);
throw new Error(
'Unhandled value type ' +
valueType +
' found when flattening data for prefix ' +
keyPrefix
);
}
}

4
lib/support/helpers/cap.js
View File

@ -1,5 +1,5 @@
'use strict';
module.exports = function ( word ) {
return word.substr( 0, 1 ).toUpperCase() + word.substr( 1 );
module.exports = function(word) {
return word.substr(0, 1).toUpperCase() + word.substr(1);
};

2
lib/support/helpers/get.js
View File

@ -8,5 +8,5 @@ module.exports = function(object, property, defaultValue) {
if (!object) {
return defaultValue;
}
return get(object, property, defaultValue)
return get(object, property, defaultValue);
};

18
lib/support/helpers/index.js
View File

@ -1,14 +1,14 @@
'use strict';
module.exports = {
size: require( './size' ),
size: require('./size'),
cap: require('./cap'),
time: require( './time' ),
plural: require( './plural' ),
scoreLabel: require( './scoreLabel' ),
label: require( './label' ),
get: require( './get' ),
short: require( './short'),
shortAsset: require( './shortAsset'),
noop: require( './noop' )
time: require('./time'),
plural: require('./plural'),
scoreLabel: require('./scoreLabel'),
label: require('./label'),
get: require('./get'),
short: require('./short'),
shortAsset: require('./shortAsset'),
noop: require('./noop')
};

3
lib/support/helpers/label.js
View File

@ -4,8 +4,7 @@ module.exports = function(value, ok, warning) {
value = value || 0;
if (value > ok) {
return 'ok';
}
else if (value > warning) {
} else if (value > warning) {
return 'warning';
}
return 'error';

4
lib/support/helpers/plural.js
View File

@ -1,7 +1,7 @@
'use strict';
module.exports = function ( number, text ) {
if ( number === 0 || number > 1 ) {
module.exports = function(number, text) {
if (number === 0 || number > 1) {
text += 's';
}
return '' + number + ' ' + text;

3
lib/support/helpers/scoreLabel.js
View File

@ -4,8 +4,7 @@ module.exports = function(value) {
value = value || 0;
if (value > 90) {
return 'ok';
}
else if (value > 80) {
} else if (value > 80) {
return 'warning';
}
return 'error';

6
lib/support/helpers/short.js
View File

@ -1,8 +1,8 @@
'use strict';
module.exports = function ( text, number ) {
if ( text.length > number ) {
return text.slice(0,number) + '...';
module.exports = function(text, number) {
if (text.length > number) {
return text.slice(0, number) + '...';
}
return text;
};

2
lib/support/helpers/shortAsset.js
View File

@ -3,7 +3,7 @@
module.exports = function(url) {
if (url.length > 40) {
let shortUrl = url.replace(/\?.*/, '');
url = (shortUrl.substr(0, 20) + '...' + shortUrl.substr(-17));
url = shortUrl.substr(0, 20) + '...' + shortUrl.substr(-17);
}
return url;
};

9
lib/support/helpers/size.js
View File

@ -1,16 +1,15 @@
'use strict';
const KB = 1024, MB = 1024 * 1024;
const KB = 1024,
MB = 1024 * 1024;
module.exports = {
asKb(bytes) {
if (!bytes || bytes < 0)
return 0;
if (!bytes || bytes < 0) return 0;
return Number(bytes / KB).toFixed(1);
},
format(bytes) {
if (!bytes || bytes < 0)
return 'N/A';
if (!bytes || bytes < 0) return 'N/A';
if (bytes < KB) {
return Number(bytes) + ' B';

2
lib/support/helpers/time.js
View File

@ -13,7 +13,7 @@ module.exports = {
secondsPerDay = 60 * 60 * 24,
secondsPerHour = 60 * 60,
secondsPerMinute = 60,
sign = (seconds < 0) ? '-' : '';
sign = seconds < 0 ? '-' : '';
if (seconds < 0) {
seconds = Math.abs(seconds);

20
lib/support/logging.js
View File

@ -27,23 +27,23 @@ module.exports.configure = function configure(options, logDir) {
if (level === log.INFO) {
log.basicConfig({
'format': '[%(date)s] %(levelname)s: %(message)s',
'level': level
format: '[%(date)s] %(levelname)s: %(message)s',
level: level
});
} else {
log.basicConfig({
'format': '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
'level': level
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
level: level
});
}
log.addHandler(new log.handlers.File(
{
log.addHandler(
new log.handlers.File({
file: logDir + '/sitespeed.io.log',
formatter: new log.Formatter({
'format': '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
'level': level
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
level: level
})
}
));
})
);
};

2
lib/support/messageMaker.js
View File

@ -10,7 +10,7 @@ module.exports = function messageMaker(source) {
const timestamp = moment().format(),
uuid = makeUuid();
return merge({uuid, type, timestamp, source, data}, extras);
return merge({ uuid, type, timestamp, source, data }, extras);
}
};
};

18
lib/support/metricsFilter.js
View File

@ -8,8 +8,7 @@ const toArray = require('./util').toArray,
reduce = require('lodash.reduce');
function normalizePath(path) {
if (path.endsWith('.*'))
return path.slice(0, -2);
if (path.endsWith('.*')) return path.slice(0, -2);
return path;
}
@ -31,8 +30,7 @@ module.exports = {
*/
filterMetrics(json, metricPaths) {
metricPaths = toArray(metricPaths);
if (typeof json !== 'object')
return undefined;
if (typeof json !== 'object') return undefined;
return metricPaths.reduce((result, path) => {
path = normalizePath(path);
@ -44,7 +42,9 @@ module.exports = {
} else if (firstWildcard === 0) {
const leafPath = path.substring(2);
reduce((json), (result, value, key) => {
reduce(
json,
(result, value, key) => {
if (typeof value === 'object') {
const leaf = this.filterMetrics(value, leafPath);
@ -53,12 +53,12 @@ module.exports = {
}
}
return result;
}, result);
},
result
);
} else {
let branchPath = path.substring(0, firstWildcard);
if (branchPath.endsWith('.'))
branchPath = branchPath.slice(0, -1);
if (branchPath.endsWith('.')) branchPath = branchPath.slice(0, -1);
let branch = get(json, branchPath);
const leafPath = path.substring(firstWildcard + 2);

34
lib/support/pluginLoader.js
View File

@ -6,7 +6,20 @@ const Promise = require('bluebird'),
Promise.promisifyAll(fs);
const defaultPlugins = new Set(['browsertime', 'coach', 'datacollector', 'domains', 'assets', 'html', 'screenshot','metrics', 'text', 'harstorer', 'budget', 'tracestorer']);
const defaultPlugins = new Set([
'browsertime',
'coach',
'datacollector',
'domains',
'assets',
'html',
'screenshot',
'metrics',
'text',
'harstorer',
'budget',
'tracestorer'
]);
const pluginsDir = path.join(__dirname, '..', 'plugins');
@ -15,8 +28,10 @@ module.exports = {
// if we don't use the cli, this will work out fine as long
// we configure only what we need
const possibleConfiguredPlugins = options.explicitOptions || options;
const isDefaultOrConfigured = (name) => (defaultPlugins.has(name) || typeof possibleConfiguredPlugins[name] === 'object');
const addMessageLoggerIfDebug = (pluginNames) => {
const isDefaultOrConfigured = name =>
defaultPlugins.has(name) ||
typeof possibleConfiguredPlugins[name] === 'object';
const addMessageLoggerIfDebug = pluginNames => {
if (options.debug) {
// Need to make sure logger is first, so message logs appear
// before messages are handled by other plugins
@ -25,15 +40,16 @@ module.exports = {
return pluginNames;
};
return fs.readdirAsync(pluginsDir)
.map((name) => path.basename(name, '.js'))
.then((builtins) => {
return fs
.readdirAsync(pluginsDir)
.map(name => path.basename(name, '.js'))
.then(builtins => {
let plugins = builtins.filter(isDefaultOrConfigured);
return addMessageLoggerIfDebug(plugins);
});
},
loadPlugins(pluginNames) {
return Promise.resolve(pluginNames).map((name) => {
return Promise.resolve(pluginNames).map(name => {
try {
const plugin = require(path.join(pluginsDir, name));
if (!plugin.name) {
@ -43,8 +59,8 @@ module.exports = {
} catch (err) {
try {
return require(path.resolve(process.cwd(), name));
} catch(error) {
console.error('Couldn\'t load plugin %s: %s', name, err); // eslint-disable-line no-console
} catch (error) {
console.error("Couldn't load plugin %s: %s", name, err); // eslint-disable-line no-console
// if it fails here, let it fail hard
throw error;
}

87
lib/support/queueHandler.js
View File

@ -32,14 +32,23 @@ function validateMessageFormat(message) {
typeDepth = typeParts.length;
if (typeDepth > 2)
throw new Error('Message type has too many dot separated sections: ' + message.type);
throw new Error(
'Message type has too many dot separated sections: ' + message.type
);
const previousDepth = messageTypeDepths[baseType];
if (previousDepth && previousDepth !== typeDepth) {
throw new Error(util.format('All messages of type %s must have the same structure. ' +
throw new Error(
util.format(
'All messages of type %s must have the same structure. ' +
'%s has %d part(s), but earlier messages had %d part(s).',
baseType, message.type, typeDepth, previousDepth));
baseType,
message.type,
typeDepth,
previousDepth
)
);
}
messageTypeDepths[baseType] = typeDepth;
@ -48,11 +57,18 @@ function validateMessageFormat(message) {
function validateSummaryMessages(message) {
const type = message.type;
if (type.endsWith('.summary') && message.url) {
throw new Error(util.format('Summary message (%s) shouldn\'t be url specific, use .pageSummary instead.', type));
throw new Error(
util.format(
"Summary message (%s) shouldn't be url specific, use .pageSummary instead.",
type
)
);
}
if (type.endsWith('.pageSummary') && !message.url) {
throw new Error(util.format('Page summary message (%s) failed to specify a url', type));
throw new Error(
util.format('Page summary message (%s) failed to specify a url', type)
);
}
}
@ -70,26 +86,27 @@ class QueueHandler {
createQueues(plugins) {
this.queues = plugins
.filter((plugin) => plugin.processMessage)
.map((plugin) => {
.filter(plugin => plugin.processMessage)
.map(plugin => {
const concurrency = plugin.concurrency || Infinity;
const queue = cq()
.limit({concurrency});
const queue = cq().limit({ concurrency });
queue.plugin = plugin;
const messageWaitingStart = {},
messageProcessingStart = {};
queue.enqueued((obj) => {
queue.enqueued(obj => {
const message = obj.item;
messageWaitingStart[message.uuid] = process.hrtime();
});
queue.processingStarted((obj) => {
queue.processingStarted(obj => {
const message = obj.item;
const waitingDuration = process.hrtime(messageWaitingStart[message.uuid]),
const waitingDuration = process.hrtime(
messageWaitingStart[message.uuid]
),
waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1];
queueStats.registerQueueTime(message, queue.plugin, waitingNanos);
@ -97,14 +114,16 @@ class QueueHandler {
messageProcessingStart[message.uuid] = process.hrtime();
});
// FIXME handle rejections (i.e. failures while processing messages) properly
queue.processingEnded((obj) => {
queue.processingEnded(obj => {
const message = obj.item;
const err = obj.err;
if (err) {
let rejectionMessage = 'Rejected ' + JSON.stringify(message, shortenData, 2) +
' for plugin: ' + plugin.name();
let rejectionMessage =
'Rejected ' +
JSON.stringify(message, shortenData, 2) +
' for plugin: ' +
plugin.name();
if (message && message.url)
rejectionMessage += ', url: ' + message.url;
@ -115,18 +134,25 @@ class QueueHandler {
this.errors.push(rejectionMessage + '\n' + JSON.stringify(err));
}
const processingDuration = process.hrtime(messageWaitingStart[message.uuid]);
const processingNanos = processingDuration[0] * 1e9 + processingDuration[1];
const processingDuration = process.hrtime(
messageWaitingStart[message.uuid]
);
const processingNanos =
processingDuration[0] * 1e9 + processingDuration[1];
queueStats.registerProcessingTime(message, queue.plugin, processingNanos);
queueStats.registerProcessingTime(
message,
queue.plugin,
processingNanos
);
});
return {plugin, queue};
return { plugin, queue };
});
}
run(sources) {
return Promise.map(sources, (source) => source.findUrls(this))
return Promise.map(sources, source => source.findUrls(this))
.then(() => this.startProcessingQueues())
.then(() => this.drainAllQueues())
.then(() => this.postMessage(make('summarize')))
@ -150,20 +176,25 @@ class QueueHandler {
}
startProcessingQueues() {
return Promise.each(this.queues, (item) => {
const queue = item.queue, plugin = item.plugin;
queue.process((message) => Promise.resolve(plugin.processMessage(message, this)));
return Promise.each(this.queues, item => {
const queue = item.queue,
plugin = item.plugin;
queue.process(message =>
Promise.resolve(plugin.processMessage(message, this))
);
});
}
drainAllQueues() {
const queues = this.queues;
return new Promise((resolve) => {
queues.forEach((item) => item.queue.drained(() => {
if (queues.every((item) => item.queue.isDrained)) {
return new Promise(resolve => {
queues.forEach(item =>
item.queue.drained(() => {
if (queues.every(item => item.queue.isDrained)) {
resolve();
}
}));
})
);
});
}
}

50
lib/support/queueStatistics.js
View File

@ -34,20 +34,48 @@ module.exports = {
includeSum: true
};
const byPluginName = Array.from(pluginNames).reduce((summary, pluginName) => {
set(summary, ['queueTime', pluginName],
stats.summarizeStats(get(queueTimeByPluginName, pluginName), statOptions));
set(summary, ['processingTime', pluginName],
stats.summarizeStats(get(processingTimeByPluginName, pluginName), statOptions));
const byPluginName = Array.from(
pluginNames
).reduce((summary, pluginName) => {
set(
summary,
['queueTime', pluginName],
stats.summarizeStats(
get(queueTimeByPluginName, pluginName),
statOptions
)
);
set(
summary,
['processingTime', pluginName],
stats.summarizeStats(
get(processingTimeByPluginName, pluginName),
statOptions
)
);
return summary;
}, {});
const byMessageType = Array.from(messageTypes).reduce((summary, messageType) => {
set(summary, ['queueTime', messageType],
stats.summarizeStats(get(queueTimeByMessageType, messageType), statOptions));
set(summary, ['processingTime', messageType],
stats.summarizeStats(get(processingTimeByMessageType, messageType), statOptions));
const byMessageType = Array.from(
messageTypes
).reduce((summary, messageType) => {
set(
summary,
['queueTime', messageType],
stats.summarizeStats(
get(queueTimeByMessageType, messageType),
statOptions
)
);
set(
summary,
['processingTime', messageType],
stats.summarizeStats(
get(processingTimeByMessageType, messageType),
statOptions
)
);
return summary;
}, {});
@ -55,6 +83,6 @@ module.exports = {
return {
byPluginName,
byMessageType
}
};
}
};

5
lib/support/resultsStorage/index.js
View File

@ -25,7 +25,10 @@ module.exports = function(input, timestamp, outputFolder, resultBaseURL) {
resultsSubFolders.push(path.basename(outputFolder));
storageBasePath = path.resolve(outputFolder);
} else {
resultsSubFolders.push(getDomainOrFileName(input), timestamp.format('YYYY-MM-DD-HH-mm-ss'));
resultsSubFolders.push(
getDomainOrFileName(input),
timestamp.format('YYYY-MM-DD-HH-mm-ss')
);
storageBasePath = path.resolve('sitespeed-result', ...resultsSubFolders);
}

3
lib/support/resultsStorage/pathToFolder.js
View File

@ -18,6 +18,5 @@ module.exports = function pathFromRootToPageDir(url) {
pathSegments.push('query-' + hash);
}
return pathSegments.join('/')
.concat('/');
return pathSegments.join('/').concat('/');
};

47
lib/support/resultsStorage/storageManager.js
View File

@ -13,20 +13,24 @@ Promise.promisifyAll(zlib);
const mkdirp = Promise.promisify(require('mkdirp'));
function write(dirPath, filename, data, gzip) {
return Promise.join(dirPath, filename, data,
(dirPath, filename, data) => {
return Promise.join(dirPath, filename, data, (dirPath, filename, data) => {
if (gzip) {
const buff = new Buffer(data, 'utf8');
return zlib.gzipAsync(buff, {
return zlib
.gzipAsync(buff, {
level: 1
}).then((buffer) =>
fs.writeFileAsync(path.join(dirPath, filename + '.gz'), buffer, 'utf8')
})
.then(buffer =>
fs.writeFileAsync(
path.join(dirPath, filename + '.gz'),
buffer,
'utf8'
)
);
} else {
return fs.writeFileAsync(path.join(dirPath, filename), data, 'utf8');
}
}
);
});
}
module.exports = function storageManager(baseDir, storagePathPrefix) {
@ -40,13 +44,11 @@ module.exports = function storageManager(baseDir, storagePathPrefix) {
.concat('/');
},
createDataDir(subDir) {
const pathSegments = [
baseDir,
subDir
].filter(Boolean);
const pathSegments = [baseDir, subDir].filter(Boolean);
return Promise.resolve(path.join.apply(null, pathSegments))
.tap((dirPath) => mkdirp(dirPath));
return Promise.resolve(path.join.apply(null, pathSegments)).tap(dirPath =>
mkdirp(dirPath)
);
},
writeData(filename, data) {
return write(this.createDataDir('data'), filename, data);
@ -61,19 +63,16 @@ module.exports = function storageManager(baseDir, storagePathPrefix) {
return storagePathPrefix;
},
copyToResultDir(filename) {
return Promise.join(this.createDataDir(), filename,
(dirPath, filename) =>
fs.copyAsync(filename, dirPath));
return Promise.join(this.createDataDir(), filename, (dirPath, filename) =>
fs.copyAsync(filename, dirPath)
);
},
createDirForUrl(url, subDir) {
const pathSegments = [
baseDir,
pathToFolder(url),
subDir
].filter(Boolean);
const pathSegments = [baseDir, pathToFolder(url), subDir].filter(Boolean);
return Promise.resolve(path.join.apply(null, pathSegments))
.tap((dirPath) => mkdirp(dirPath));
return Promise.resolve(path.join.apply(null, pathSegments)).tap(dirPath =>
mkdirp(dirPath)
);
},
writeDataForUrl(data, filename, url, subDir, gzip) {
const dirPath = ['data', subDir].filter(Boolean).join(path.sep);
@ -82,5 +81,5 @@ module.exports = function storageManager(baseDir, storagePathPrefix) {
writeHtmlForUrl(html, filename, url, gzip) {
return write(this.createDirForUrl(url), filename, html, gzip);
}
}
};
};

116
lib/support/setup/detailed.js
View File

@ -13,7 +13,7 @@ function row(stat, name, metricName, formatter) {
metricName,
node: stat,
h: formatter ? formatter : h.noop
}
};
}
module.exports = function(data) {
@ -34,9 +34,21 @@ module.exports = function(data) {
rows.push(
row(summary.score, 'Coach score', 'overallScore'),
row(summary.performance.score, 'Coach performance score', 'performanceScore'),
row(summary.accessibility.score, 'Accessibility score', 'accessibilityScore'),
row(summary.bestpractice.score, 'Best Practice score', 'bestPracticeScore')
row(
summary.performance.score,
'Coach performance score',
'performanceScore'
),
row(
summary.accessibility.score,
'Accessibility score',
'accessibilityScore'
),
row(
summary.bestpractice.score,
'Best Practice score',
'bestPracticeScore'
)
);
}
@ -45,24 +57,58 @@ module.exports = function(data) {
const contentTypes = summary.contentTypes;
rows.push(
row(contentTypes.image.requests, 'Image requests', 'imageRequestsPerPage'),
row(
contentTypes.image.requests,
'Image requests',
'imageRequestsPerPage'
),
row(contentTypes.css.requests, 'CSS requests', 'cssRequestsPerPage'),
row(contentTypes.javascript.requests, 'Javascript requests', 'jsRequestsPerPage' ),
row(
contentTypes.javascript.requests,
'Javascript requests',
'jsRequestsPerPage'
),
row(contentTypes.font.requests, 'Font requests', 'fontRequestsPerPage'),
row(summary.requests, 'Total requests', 'totalRequestsPerPage')
);
rows.push(
row(contentTypes.image.transferSize, 'Image size', 'imageSizePerPage', h.size.format),
row(contentTypes.html.transferSize, 'HTML size','htmlSizePerPage', h.size.format),
row(contentTypes.css.transferSize, 'CSS size','cssSizePerPage', h.size.format),
row(contentTypes.javascript.transferSize, 'Javascript size', 'jsSizePerPage', h.size.format),
row(contentTypes.font.transferSize, 'Font size', 'fontSizePerPage', h.size.format),
row(summary.transferSize, 'Total size', 'totalSizePerPage', h.size.format));
row(
contentTypes.image.transferSize,
'Image size',
'imageSizePerPage',
h.size.format
),
row(
contentTypes.html.transferSize,
'HTML size',
'htmlSizePerPage',
h.size.format
),
row(
contentTypes.css.transferSize,
'CSS size',
'cssSizePerPage',
h.size.format
),
row(
contentTypes.javascript.transferSize,
'Javascript size',
'jsSizePerPage',
h.size.format
),
row(
contentTypes.font.transferSize,
'Font size',
'fontSizePerPage',
h.size.format
),
row(summary.transferSize, 'Total size', 'totalSizePerPage', h.size.format)
);
const responseCodes = Object.keys(summary.responseCodes);
for (let code of responseCodes) {
rows.push(row(summary.responseCodes[code], code + ' responses'))
rows.push(row(summary.responseCodes[code], code + ' responses'));
}
}
@ -72,26 +118,47 @@ module.exports = function(data) {
rows.push(
row(summary.rumSpeedIndex, 'RUMSpeed Index', 'rumSpeedIndex'),
row(summary.firstPaint, 'First Paint', 'firstPaint'),
row(summary.fullyLoaded, 'Fully loaded', 'fullyLoaded'));
row(summary.fullyLoaded, 'Fully loaded', 'fullyLoaded')
);
const timings = Object.keys(summary.pageTimings);
for (let timing of timings) {
rows.push(row(summary.pageTimings[timing], timing, timing))
rows.push(row(summary.pageTimings[timing], timing, timing));
}
if (summary.custom) {
for (var key of Object.keys(summary.custom)) {
rows.push(row(summary.custom[key],key));
rows.push(row(summary.custom[key], key));
}
}
if (summary.visualMetrics) {
rows.push(
row(summary.visualMetrics.FirstVisualChange, 'First Visual Change', 'FirstVisualChange', h.time.ms),
row(
summary.visualMetrics.FirstVisualChange,
'First Visual Change',
'FirstVisualChange',
h.time.ms
),
row(summary.visualMetrics.SpeedIndex, 'Speed Index', 'SpeedIndex'),
row(summary.visualMetrics.PerceptualSpeedIndex, 'Perceptual Speed Index', 'PerceptualSpeedIndex'),
row(summary.visualMetrics.VisualComplete85, 'Visual Complete 85%', 'VisualComplete85', h.time.ms),
row(summary.visualMetrics.LastVisualChange, 'Last Visual Change', 'LastVisualChange', h.time.ms));
row(
summary.visualMetrics.PerceptualSpeedIndex,
'Perceptual Speed Index',
'PerceptualSpeedIndex'
),
row(
summary.visualMetrics.VisualComplete85,
'Visual Complete 85%',
'VisualComplete85',
h.time.ms
),
row(
summary.visualMetrics.LastVisualChange,
'Last Visual Change',
'LastVisualChange',
h.time.ms
)
);
}
}
@ -101,14 +168,19 @@ module.exports = function(data) {
rows.push(
row(firstView.render, 'WPT render (firstView)', 'render'),
row(firstView.SpeedIndex, 'WPT SpeedIndex (firstView)', 'SpeedIndex'),
row(firstView.fullyLoaded, 'WPT Fully loaded (firstView)', 'fullyLoaded'));
row(
firstView.fullyLoaded,
'WPT Fully loaded (firstView)',
'fullyLoaded'
)
);
}
}
if (gpsi) {
rows.push(
row(gpsi.summary.SPEED.score, 'GPSI Speed Score', 'gpsispeedscore')
)
);
}
return rows.filter(Boolean);

171
lib/support/setup/summaryBoxes.js
View File

@ -8,7 +8,7 @@ function infoBox(stat, name, formatter, url) {
return undefined;
}
return _box(stat, name, 'info', formatter, url)
return _box(stat, name, 'info', formatter, url);
}
function scoreBox(stat, name, url) {
@ -16,7 +16,7 @@ function scoreBox(stat, name, url) {
return undefined;
}
return _box(stat, name, h.scoreLabel(stat.median), h.noop, url)
return _box(stat, name, h.scoreLabel(stat.median), h.noop, url);
}
function metricBox(stat, name, score, formatter, url) {
@ -24,7 +24,7 @@ function metricBox(stat, name, score, formatter, url) {
return undefined;
}
return _box(stat, name, h.scoreLabel(score.median), formatter, url)
return _box(stat, name, h.scoreLabel(score.median), formatter, url);
}
function _box(stat, name, label, formatter, url) {
@ -37,7 +37,7 @@ function _box(stat, name, label, formatter, url) {
median,
p90,
url
}
};
}
module.exports = function(data) {
@ -58,13 +58,42 @@ module.exports = function(data) {
boxes.push(
scoreBox(summary.score, 'Overall score', 'overallScore'),
scoreBox(summary.performance.score, 'Performance score', 'performanceScore'),
scoreBox(summary.accessibility.score, 'Accessibility score', 'accessibilityScore'),
scoreBox(summary.bestpractice.score, 'Best Practice score', 'bestPracticeScore'),
scoreBox(summary.performance.fastRender, 'Fast Render advice', 'fastRender'),
scoreBox(summary.performance.avoidScalingImages, 'Avoid scaling images advice', 'avoidScalingImages'),
scoreBox(summary.performance.compressAssets, 'Compress assets advice', 'compressAssets'),
scoreBox(summary.performance.optimalCssSize, 'Optimal CSS size advice', 'optimalCssSize'));
scoreBox(
summary.performance.score,
'Performance score',
'performanceScore'
),
scoreBox(
summary.accessibility.score,
'Accessibility score',
'accessibilityScore'
),
scoreBox(
summary.bestpractice.score,
'Best Practice score',
'bestPracticeScore'
),
scoreBox(
summary.performance.fastRender,
'Fast Render advice',
'fastRender'
),
scoreBox(
summary.performance.avoidScalingImages,
'Avoid scaling images advice',
'avoidScalingImages'
),
scoreBox(
summary.performance.compressAssets,
'Compress assets advice',
'compressAssets'
),
scoreBox(
summary.performance.optimalCssSize,
'Optimal CSS size advice',
'optimalCssSize'
)
);
}
if (pagexray && coach) {
@ -72,14 +101,35 @@ module.exports = function(data) {
const pxSum = pagexray.summary;
boxes.push(
metricBox(pxSum.transferSize, 'Total size (transfer)',
cSum.performance.pageSize, h.size.format, 'pageSize'),
metricBox(pxSum.contentTypes.image.transferSize, 'Image size (transfer)',
cSum.performance.imageSize, h.size.format, 'imageSize'),
metricBox(pxSum.contentTypes.javascript.transferSize, 'Javascript size (transfer)',
cSum.performance.javascriptSize, h.size.format, 'javascriptSize'),
metricBox(pxSum.contentTypes.css.transferSize, 'CSS size (transfer)', cSum.performance.cssSize, h.size.format, 'cssSize'));
metricBox(
pxSum.transferSize,
'Total size (transfer)',
cSum.performance.pageSize,
h.size.format,
'pageSize'
),
metricBox(
pxSum.contentTypes.image.transferSize,
'Image size (transfer)',
cSum.performance.imageSize,
h.size.format,
'imageSize'
),
metricBox(
pxSum.contentTypes.javascript.transferSize,
'Javascript size (transfer)',
cSum.performance.javascriptSize,
h.size.format,
'javascriptSize'
),
metricBox(
pxSum.contentTypes.css.transferSize,
'CSS size (transfer)',
cSum.performance.cssSize,
h.size.format,
'cssSize'
)
);
}
// no matching rules
@ -98,18 +148,33 @@ module.exports = function(data) {
infoBox(summary.responseCodes['404'], '404 responses'),
infoBox(summary.domains, 'Domains per page'),
infoBox(summary.expireStats, 'Cache time', h.time.duration),
infoBox(summary.lastModifiedStats, 'Time since last modification', h.time.duration));
infoBox(
summary.lastModifiedStats,
'Time since last modification',
h.time.duration
)
);
if (summary.firstParty) {
boxes.push(
infoBox(summary.firstParty.requests, '1st party requests'),
infoBox(summary.firstParty.transferSize, '1st party size', h.size.format));
infoBox(
summary.firstParty.transferSize,
'1st party size',
h.size.format
)
);
}
if (summary.thirdParty) {
boxes.push(
infoBox(summary.thirdParty.requests, '3rd party requests'),
infoBox(summary.thirdParty.transferSize, '3rd party sizes', h.size.format));
infoBox(
summary.thirdParty.transferSize,
'3rd party sizes',
h.size.format
)
);
}
}
@ -117,18 +182,52 @@ module.exports = function(data) {
const summary = browsertime.summary;
boxes.push(
infoBox(summary.rumSpeedIndex, 'RUM Speed Index', h.noop, 'rumSpeedIndex'),
infoBox(
summary.rumSpeedIndex,
'RUM Speed Index',
h.noop,
'rumSpeedIndex'
),
infoBox(summary.firstPaint, 'First Paint', h.time.ms, 'firstPaint'),
infoBox(summary.pageTimings.backEndTime, 'Backend Time', h.time.ms, 'backEndTime'),
infoBox(summary.pageTimings.frontEndTime, 'Frontend Time', h.time.ms, 'frontEndTime'),
infoBox(summary.pageTimings.fullyLoaded, 'Fully Loaded Time', h.time.ms, 'fullyLoaded'));
infoBox(
summary.pageTimings.backEndTime,
'Backend Time',
h.time.ms,
'backEndTime'
),
infoBox(
summary.pageTimings.frontEndTime,
'Frontend Time',
h.time.ms,
'frontEndTime'
),
infoBox(
summary.pageTimings.fullyLoaded,
'Fully Loaded Time',
h.time.ms,
'fullyLoaded'
)
);
if (summary.visualMetrics) {
boxes.push(
infoBox(summary.visualMetrics.FirstVisualChange, 'First Visual Change', h.time.ms),
infoBox(
summary.visualMetrics.FirstVisualChange,
'First Visual Change',
h.time.ms
),
infoBox(summary.visualMetrics.SpeedIndex, 'Speed Index'),
infoBox(summary.visualMetrics.VisualComplete85, 'Visual Complete 85%', h.time.ms),
infoBox(summary.visualMetrics.LastVisualChange, 'Last Visual Change', h.time.ms));
infoBox(
summary.visualMetrics.VisualComplete85,
'Visual Complete 85%',
h.time.ms
),
infoBox(
summary.visualMetrics.LastVisualChange,
'Last Visual Change',
h.time.ms
)
);
}
if (summary.custom) {
@ -143,15 +242,19 @@ module.exports = function(data) {
if (firstView) {
boxes.push(
infoBox(firstView.render, 'WPT render (firstView)'),
infoBox(firstView.SpeedIndex, 'WPT SpeedIndex (firstView)', h.noop, 'SpeedIndex'),
infoBox(firstView.fullyLoaded, 'WPT Fully loaded (firstView)'));
infoBox(
firstView.SpeedIndex,
'WPT SpeedIndex (firstView)',
h.noop,
'SpeedIndex'
),
infoBox(firstView.fullyLoaded, 'WPT Fully loaded (firstView)')
);
}
}
if (gpsi) {
boxes.push(
infoBox(gpsi.summary.SPEED.score, 'GPSI Speed Score')
)
boxes.push(infoBox(gpsi.summary.SPEED.score, 'GPSI Speed Score'));
}
return boxes;

10
lib/support/statsHelpers.js
View File

@ -15,7 +15,6 @@ function percentileName(percentile) {
}
module.exports = {
/**
* Create or update a fast-stats#Stats object in target at path.
*/
@ -52,13 +51,18 @@ module.exports = {
median: parseInt(stats.median().toFixed(decimals)),
mean: parseInt(stats.amean().toFixed(decimals))
};
percentiles.forEach((p) => {
percentiles.forEach(p => {
let name = percentileName(p);
const percentile = stats.percentile(p);
if (Number.isFinite(percentile)) {
data[name] = parseInt(percentile.toFixed(decimals));
} else {
throw new Error('Failed to calculate ' + name + ' for stats: ' + JSON.stringify(stats, null, 2));
throw new Error(
'Failed to calculate ' +
name +
' for stats: ' +
JSON.stringify(stats, null, 2)
);
}
});
if (options.includeSum) {

9
lib/support/tsdbUtil.js
View File

@ -17,9 +17,14 @@ module.exports = {
}
},
getURLAndGroup(options, group, url, includeQueryParams) {
if (group && options.urlsMetaData && options.urlsMetaData[url] && options.urlsMetaData[url].alias) {
if (
group &&
options.urlsMetaData &&
options.urlsMetaData[url] &&
options.urlsMetaData[url].alias
) {
let alias = options.urlsMetaData[url].alias;
return this.toSafeKey(group) + "." + this.toSafeKey(alias);
return this.toSafeKey(group) + '.' + this.toSafeKey(alias);
} else {
return flatten.keypathFromUrl(url, includeQueryParams);
}

4
lib/support/url-source.js
View File

@ -11,7 +11,9 @@ module.exports = {
},
findUrls(queue) {
for (const url of this.options.urls) {
queue.postMessage(make('url', {}, {url: url, group: urlParser.parse(url).hostname}));
queue.postMessage(
make('url', {}, { url: url, group: urlParser.parse(url).hostname })
);
}
}
};

11
lib/support/util.js
View File

@ -13,10 +13,15 @@ module.exports = {
return [arrayLike];
},
throwIfMissing(options, keys, namespace) {
let missingKeys = keys.filter((key) => !options[key]);
let missingKeys = keys.filter(key => !options[key]);
if (missingKeys.length > 0) {
throw new Error(format('Required option(s) %s need to be specified in namespace "%s"',
missingKeys.map((s) => '"' + s + '"'), namespace));
throw new Error(
format(
'Required option(s) %s need to be specified in namespace "%s"',
missingKeys.map(s => '"' + s + '"'),
namespace
)
);
}
}
};

6357
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

12
package.json
View File

@ -35,14 +35,15 @@
"url": "https://github.com/sitespeedio/sitespeed.io/issues"
},
"scripts": {
"lint": "npm run eslint && npm run pug-lint",
"eslint": "eslint .",
"lint": "eslint . && npm run pug-lint",
"lint:fix": "eslint . --fix",
"eslint-check": "eslint --print-config .eslintrc.js | eslint-config-prettier-check",
"eclint": "eclint check * lib/**/* bin/**/* tools/**/* !*.iml",
"eclint:fix": "eclint fix * lib/**/* bin/**/* tools/**/* !*.iml",
"pug-lint": "pug-lint lib/plugins/html/templates",
"test": "mocha",
"check-licenses": "tools/check-licenses.js",
"travis": "npm run lint && npm run test",
"travis": "npm run eslint-check && npm run lint && npm run test",
"build:css": "node-sass lib/plugins/html/src/sass/main.scss > lib/plugins/html/assets/css/index.css && cleancss -o lib/plugins/html/assets/css/index.min.css lib/plugins/html/assets/css/index.css"
},
"engines": {
@ -53,11 +54,14 @@
"chai-as-promised": "^6.0.0",
"clean-css-cli": "^4.0.7",
"eclint": "^1.1.5",
"eslint": "^3.10.2",
"eslint": "^4.2.0",
"eslint-config-prettier": "^2.3.0",
"eslint-plugin-prettier": "^2.1.2",
"jsdoc": "^3.3.3",
"license-checker": "^5.1.2",
"mocha": "^3.1.2",
"node-sass": "^4.5.0",
"prettier": "^1.5.3",
"pug-lint": "^2.3.0",
"pug-lint-config-clock": "^2.0.0"
},

22
test/cliUtilTests.js
View File

@ -6,24 +6,34 @@ const cliUtil = require('../lib/support/cliUtil'),
describe('cliUtil', function() {
describe('getURLs', function() {
it('should extract urls', function() {
let urls = cliUtil.getURLs(["test/fixtures/sitespeed-urls.txt"]);
let urls = cliUtil.getURLs(['test/fixtures/sitespeed-urls.txt']);
expect(urls[0] === 'https://www.sitespeed.io');
expect(urls[3] === 'https://www.sitespeed.io/documentation/faq');
urls = cliUtil.getURLs(["test/fixtures/sitespeed-urls-aliases.txt"]);
urls = cliUtil.getURLs(['test/fixtures/sitespeed-urls-aliases.txt']);
expect(urls[0] === 'https://www.sitespeed.io');
expect(urls[3] === 'https://www.sitespeed.io/documentation/faq');
});
});
describe('getAliases', function() {
it('should extract aliases', function() {
let aliases = cliUtil.getAliases(["test/fixtures/sitespeed-urls.txt"]);
let aliases = cliUtil.getAliases(['test/fixtures/sitespeed-urls.txt']);
expect(aliases['https://www.sitespeed.io']).to.be.empty;
expect(aliases['https://www.sitespeed.io/documentation/sitespeed.io/webpagetest/']).to.be.empty;
expect(
aliases[
'https://www.sitespeed.io/documentation/sitespeed.io/webpagetest/'
]
).to.be.empty;
aliases = cliUtil.getAliases(["test/fixtures/sitespeed-urls-aliases.txt"]);
aliases = cliUtil.getAliases([
'test/fixtures/sitespeed-urls-aliases.txt'
]);
expect(aliases['https://www.sitespeed.io'].alias === 'Home_Page');
expect(aliases['https://www.sitespeed.io/documentation/sitespeed.io/webpagetest/']).to.be.empty;
expect(
aliases[
'https://www.sitespeed.io/documentation/sitespeed.io/webpagetest/'
]
).to.be.empty;
});
});
});

2
test/coachTests.js
View File

@ -11,7 +11,7 @@ const coachRun = JSON.parse(fs.readFileSync(coachRunPath, 'utf8'));
describe('coach', function() {
describe('aggregator', function() {
it('should summarize data', function() {
aggregator.addToAggregate(coachRun,'www.sitespeed.io');
aggregator.addToAggregate(coachRun, 'www.sitespeed.io');
expect(aggregator.summarize()).to.not.be.empty;
});

10
test/domainTests.js
View File

@ -13,10 +13,14 @@ describe('domains', function() {
let har;
beforeEach(function() {
return fs.readFileAsync(path.resolve(__dirname, 'fixtures', 'www-theverge-com.har'), 'utf8')
return fs
.readFileAsync(
path.resolve(__dirname, 'fixtures', 'www-theverge-com.har'),
'utf8'
)
.then(JSON.parse)
.tap((data) => {
har = data
.tap(data => {
har = data;
});
});

75
test/graphiteTests.js
View File

@ -8,24 +8,28 @@ describe('graphite', function() {
describe('dataGenerator', function() {
it('should generate data for gpsi.pageSummary', function() {
const message = {
"type": "gpsi.pageSummary",
"timestamp": "2016-01-08T12:59:06+01:00",
"source": "gpsi",
"data": {
"median": "13",
"mean": "14.42",
"min": "13",
"p10": "13",
"p70": "16",
"p80": "16",
"p90": "16",
"p99": "16",
"max": "16"
type: 'gpsi.pageSummary',
timestamp: '2016-01-08T12:59:06+01:00',
source: 'gpsi',
data: {
median: '13',
mean: '14.42',
min: '13',
p10: '13',
p70: '16',
p80: '16',
p90: '16',
p99: '16',
max: '16'
},
'url': 'http://sub.domain.com/foo/bar'
url: 'http://sub.domain.com/foo/bar'
};
let generator = new DataGenerator('ns', false, {_:['filename'], browser:'chrome', connectivity: 'cable'});
let generator = new DataGenerator('ns', false, {
_: ['filename'],
browser: 'chrome',
connectivity: 'cable'
});
var data = generator.dataFromMessage(message, moment());
expect(data).to.match(/ns.pageSummary.sub_domain_com/);
@ -35,29 +39,34 @@ describe('graphite', function() {
it('should generate data for domains.summary', function() {
const message = {
"type": "domains.summary",
"timestamp": "2016-01-08T12:59:06+01:00",
"source": "domains",
"data": {
"www.sitespeed.io": {
"dns": {
"median": "0",
"mean": "13",
"min": "0",
"p10": "0",
"p90": "40",
"p99": "40",
"max": "40"
type: 'domains.summary',
timestamp: '2016-01-08T12:59:06+01:00',
source: 'domains',
data: {
'www.sitespeed.io': {
dns: {
median: '0',
mean: '13',
min: '0',
p10: '0',
p90: '40',
p99: '40',
max: '40'
}
}
},
'group': 'sub_domain_com'
group: 'sub_domain_com'
};
let generator = new DataGenerator('ns', false, {_:['sub_domain_com'], browser:'chrome', connectivity: 'cable'});
var data = generator.dataFromMessage(message, moment());
expect(data).to.match(/ns.summary.sub_domain_com.chrome.cable.domains.www.sitespeed.io.dns.median/);
let generator = new DataGenerator('ns', false, {
_: ['sub_domain_com'],
browser: 'chrome',
connectivity: 'cable'
});
var data = generator.dataFromMessage(message, moment());
expect(data).to.match(
/ns.summary.sub_domain_com.chrome.cable.domains.www.sitespeed.io.dns.median/
);
});
});
});

571
test/influxdbTests.js
View File

@ -8,344 +8,343 @@ describe('influxdb', function() {
describe('dataGenerator', function() {
it('should generate data for coach.summary', function() {
const message = {
"uuid": "33774328-e781-4152-babe-a367cee27153",
"type": "coach.summary",
"timestamp": "2017-04-04T09:55:59+02:00",
"source": "coach",
"data": {
"score": {
"median": "96",
"mean": "96",
"min": "96",
"p90": "96",
"max": "96"
uuid: '33774328-e781-4152-babe-a367cee27153',
type: 'coach.summary',
timestamp: '2017-04-04T09:55:59+02:00',
source: 'coach',
data: {
score: {
median: '96',
mean: '96',
min: '96',
p90: '96',
max: '96'
},
"accessibility": {
"score": {
"median": "95",
"mean": "95",
"min": "95",
"p90": "95",
"max": "95"
accessibility: {
score: {
median: '95',
mean: '95',
min: '95',
p90: '95',
max: '95'
},
"altImages": {
"median": "80",
"mean": "80",
"min": "80",
"p90": "80",
"max": "80"
altImages: {
median: '80',
mean: '80',
min: '80',
p90: '80',
max: '80'
},
"headings": {
"median": "90",
"mean": "90",
"min": "90",
"p90": "90",
"max": "90"
headings: {
median: '90',
mean: '90',
min: '90',
p90: '90',
max: '90'
},
"labelOnInput": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
labelOnInput: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"landmarks": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
landmarks: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"neverSuppressZoom": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
neverSuppressZoom: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"sections": {
"median": "0",
"mean": "0",
"min": "0",
"p90": "0",
"max": "0"
sections: {
median: '0',
mean: '0',
min: '0',
p90: '0',
max: '0'
},
"table": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
table: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
}
},
"bestpractice": {
"score": {
"median": "85",
"mean": "85",
"min": "85",
"p90": "85",
"max": "85"
bestpractice: {
score: {
median: '85',
mean: '85',
min: '85',
p90: '85',
max: '85'
},
"charset": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
charset: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"doctype": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
doctype: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"https": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
https: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"httpsH2": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
httpsH2: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"language": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
language: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"metaDescription": {
"median": "50",
"mean": "50",
"min": "50",
"p90": "50",
"max": "50"
metaDescription: {
median: '50',
mean: '50',
min: '50',
p90: '50',
max: '50'
},
"optimizely": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
optimizely: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"pageTitle": {
"median": "50",
"mean": "50",
"min": "50",
"p90": "50",
"max": "50"
pageTitle: {
median: '50',
mean: '50',
min: '50',
p90: '50',
max: '50'
},
"spdy": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
spdy: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"url": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
url: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
}
},
"performance": {
"score": {
"median": "98",
"mean": "98",
"min": "98",
"p90": "98",
"max": "98"
performance: {
score: {
median: '98',
mean: '98',
min: '98',
p90: '98',
max: '98'
},
"avoidScalingImages": {
"median": "50",
"mean": "50",
"min": "50",
"p90": "50",
"max": "50"
avoidScalingImages: {
median: '50',
mean: '50',
min: '50',
p90: '50',
max: '50'
},
"cssPrint": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
cssPrint: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"fastRender": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
fastRender: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"inlineCss": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
inlineCss: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"jquery": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
jquery: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"spof": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
spof: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"thirdPartyAsyncJs": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
thirdPartyAsyncJs: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"assetsRedirects": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
assetsRedirects: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"cacheHeaders": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
cacheHeaders: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"cacheHeadersLong": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
cacheHeadersLong: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"compressAssets": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
compressAssets: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"connectionKeepAlive": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
connectionKeepAlive: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"cssSize": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
cssSize: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"documentRedirect": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
documentRedirect: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"favicon": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
favicon: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"fewFonts": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
fewFonts: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"fewRequestsPerDomain": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
fewRequestsPerDomain: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"headerSize": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
headerSize: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"imageSize": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
imageSize: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"javascriptSize": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
javascriptSize: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"mimeTypes": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
mimeTypes: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"optimalCssSize": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
optimalCssSize: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"pageSize": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
pageSize: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"privateAssets": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
privateAssets: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
},
"responseOk": {
"median": "100",
"mean": "100",
"min": "100",
"p90": "100",
"max": "100"
responseOk: {
median: '100',
mean: '100',
min: '100',
p90: '100',
max: '100'
}
}
},
"group": "www.sitespeed.io"
group: 'www.sitespeed.io'
};
let generator = new DataGenerator(false, {
_: ['filename'],
browser: 'chrome',

103
test/metricsFilterTests.js
View File

@ -5,52 +5,74 @@ let metricsFilter = require('../lib/support/metricsFilter'),
path = require('path'),
expect = require('chai').expect;
const wptResultPath = path.resolve(__dirname, '..', 'node_modules',
'webpagetest', 'test', 'fixtures', 'responses', 'testResults.json');
const wptResultPath = path.resolve(
__dirname,
'..',
'node_modules',
'webpagetest',
'test',
'fixtures',
'responses',
'testResults.json'
);
const wptResult = JSON.parse(fs.readFileSync(wptResultPath, 'utf8'));
describe('metricsFilter', () => {
describe('#filterMetrics', () => {
it('should filter a single metric', () => {
const filtered = metricsFilter.filterMetrics(wptResult, 'data.median.firstView.TTFB');
const filtered = metricsFilter.filterMetrics(
wptResult,
'data.median.firstView.TTFB'
);
expect(filtered).to.deep.equal({data: {median: {firstView: {TTFB: 503}}}});
expect(filtered).to.deep.equal({
data: { median: { firstView: { TTFB: 503 } } }
});
});
it('should skip missing metric', () => {
const filtered = metricsFilter.filterMetrics(wptResult, 'data.median.firstView.TTTTTTFB');
const filtered = metricsFilter.filterMetrics(
wptResult,
'data.median.firstView.TTTTTTFB'
);
expect(filtered).to.deep.equal({});
});
it('should filter multiple metrics', () => {
const filtered = metricsFilter.filterMetrics(wptResult,
['data.median.firstView.TTFB', 'data.median.repeatView.TTFB']);
const filtered = metricsFilter.filterMetrics(wptResult, [
'data.median.firstView.TTFB',
'data.median.repeatView.TTFB'
]);
expect(filtered).to.deep.equal({
data: {
median: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
}
}
});
});
it('should filter with ending wildcard', () => {
const filtered = metricsFilter.filterMetrics(wptResult,
['data.median.firstView.rawData.*']);
const filtered = metricsFilter.filterMetrics(wptResult, [
'data.median.firstView.rawData.*'
]);
expect(filtered).to.deep.equal({
data: {
median: {
firstView: {
rawData: {
'headers': 'http:\/\/www.webpagetest.org\/results\/14\/11\/06\/8N\/ZRC\/1_report.txt',
'pageData': 'http:\/\/www.webpagetest.org\/results\/14\/11\/06\/8N\/ZRC\/1_IEWPG.txt',
'requestsData': 'http:\/\/www.webpagetest.org\/results\/14\/11\/06\/8N\/ZRC\/1_IEWTR.txt',
'utilization': 'http:\/\/www.webpagetest.org\/results\/14\/11\/06\/8N\/ZRC\/1_progress.csv'
headers:
'http://www.webpagetest.org/results/14/11/06/8N/ZRC/1_report.txt',
pageData:
'http://www.webpagetest.org/results/14/11/06/8N/ZRC/1_IEWPG.txt',
requestsData:
'http://www.webpagetest.org/results/14/11/06/8N/ZRC/1_IEWTR.txt',
utilization:
'http://www.webpagetest.org/results/14/11/06/8N/ZRC/1_progress.csv'
}
}
}
@ -59,13 +81,16 @@ describe('metricsFilter', () => {
});
it('should filter with wildcard', () => {
const filtered = metricsFilter.filterMetrics(wptResult, 'data.median.*.TTFB');
const filtered = metricsFilter.filterMetrics(
wptResult,
'data.median.*.TTFB'
);
expect(filtered).to.deep.equal({
data: {
median: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
}
}
});
@ -73,8 +98,10 @@ describe('metricsFilter', () => {
// Skip test case for now, since it doesn't work yet
it('should filter for multiple sibling properties using wildcard', () => {
const filtered = metricsFilter.filterMetrics(wptResult, ['data.median.*.TTFB',
'data.median.*.loadTime']);
const filtered = metricsFilter.filterMetrics(wptResult, [
'data.median.*.TTFB',
'data.median.*.loadTime'
]);
expect(filtered).to.deep.equal({
data: {
@ -98,29 +125,32 @@ describe('metricsFilter', () => {
expect(filtered).to.deep.equal({
data: {
average: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
},
standardDeviation: {
firstView: {TTFB: 0},
repeatView: {TTFB: 0}
firstView: { TTFB: 0 },
repeatView: { TTFB: 0 }
},
median: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
}
}
});
});
it('should filter with starting wildcards', () => {
const filtered = metricsFilter.filterMetrics(wptResult, '*.average.*.TTFB');
const filtered = metricsFilter.filterMetrics(
wptResult,
'*.average.*.TTFB'
);
expect(filtered).to.deep.equal({
data: {
average: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
}
}
});
@ -132,16 +162,16 @@ describe('metricsFilter', () => {
expect(filtered).to.deep.equal({
data: {
average: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
},
standardDeviation: {
firstView: {TTFB: 0},
repeatView: {TTFB: 0}
firstView: { TTFB: 0 },
repeatView: { TTFB: 0 }
},
median: {
firstView: {TTFB: 503},
repeatView: {TTFB: 362}
firstView: { TTFB: 503 },
repeatView: { TTFB: 362 }
}
}
});
@ -175,6 +205,5 @@ describe('metricsFilter', () => {
}
});
});
});
});

2
test/prepostscripts/postSample.js
View File

@ -2,4 +2,4 @@ module.exports = {
run(context) {
context.log.info('In posttask!!!');
}
};
};

8
test/prepostscripts/preSample.js
View File

@ -2,10 +2,12 @@ module.exports = {
run(context) {
context.log.info('In pretask!!!');
if (!context.taskData.loadedSitespeed) {
return context.runWithDriver((driver) => {
return driver.get('https://www.sitespeed.io')
return context
.runWithDriver(driver => {
return driver
.get('https://www.sitespeed.io')
.then(() => driver.getTitle())
.then((title) => {
.then(title => {
context.log.info('Loaded page with title: ' + title);
});
})

109
test/resultUrlTests.js
View File

@ -14,65 +14,116 @@ function createResultUrls(url, outputFolder, resultBaseURL) {
describe('resultUrls', function() {
describe('#hasBaseUrl', function() {
it('should be false if base url is missing', function() {
const resultUrls = createResultUrls('http://www.foo.bar', undefined, undefined);
const resultUrls = createResultUrls(
'http://www.foo.bar',
undefined,
undefined
);
expect(resultUrls.hasBaseUrl()).to.be.false;
});
it('should be true if base url is present', function() {
const resultUrls = createResultUrls('http://www.foo.bar', undefined, 'http://results.com');
const resultUrls = createResultUrls(
'http://www.foo.bar',
undefined,
'http://results.com'
);
expect(resultUrls.hasBaseUrl()).to.be.true;
});
});
describe('#reportSummaryUrl', function() {
it('should create url with default output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', undefined, 'http://results.com');
expect(resultUrls.reportSummaryUrl())
.to.equal(`http://results.com/www.foo.bar/${timestampString}`);
const resultUrls = createResultUrls(
'http://www.foo.bar',
undefined,
'http://results.com'
);
expect(resultUrls.reportSummaryUrl()).to.equal(
`http://results.com/www.foo.bar/${timestampString}`
);
});
it('should create url with absolute output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', '/root/leaf', 'http://results.com');
expect(resultUrls.reportSummaryUrl())
.to.equal('http://results.com/leaf');
const resultUrls = createResultUrls(
'http://www.foo.bar',
'/root/leaf',
'http://results.com'
);
expect(resultUrls.reportSummaryUrl()).to.equal('http://results.com/leaf');
});
it('should create url with relative output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', '../leaf', 'http://results.com');
expect(resultUrls.reportSummaryUrl())
.to.equal('http://results.com/leaf');
const resultUrls = createResultUrls(
'http://www.foo.bar',
'../leaf',
'http://results.com'
);
expect(resultUrls.reportSummaryUrl()).to.equal('http://results.com/leaf');
});
});
describe('#absoluteSummaryPageUrl', function() {
it('should create url with default output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', undefined, 'http://results.com');
expect(resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz'))
.to.equal(`http://results.com/www.foo.bar/${timestampString}/pages/www.foo.bar/xyz/`);
const resultUrls = createResultUrls(
'http://www.foo.bar',
undefined,
'http://results.com'
);
expect(
resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal(
`http://results.com/www.foo.bar/${timestampString}/pages/www.foo.bar/xyz/`
);
});
it('should create url with absolute output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', '/root/leaf', 'http://results.com');
expect(resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz'))
.to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
const resultUrls = createResultUrls(
'http://www.foo.bar',
'/root/leaf',
'http://results.com'
);
expect(
resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
});
it('should create url with relative output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', '../leaf', 'http://results.com');
expect(resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz'))
.to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
const resultUrls = createResultUrls(
'http://www.foo.bar',
'../leaf',
'http://results.com'
);
expect(
resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
});
});
describe('#relativeSummaryPageUrl', function() {
it('should create url with default output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', undefined, 'http://results.com');
expect(resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz'))
.to.equal('pages/www.foo.bar/xyz/');
const resultUrls = createResultUrls(
'http://www.foo.bar',
undefined,
'http://results.com'
);
expect(
resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('pages/www.foo.bar/xyz/');
});
it('should create url with absolute output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', '/root/leaf', 'http://results.com');
expect(resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz'))
.to.equal('pages/www.foo.bar/xyz/');
const resultUrls = createResultUrls(
'http://www.foo.bar',
'/root/leaf',
'http://results.com'
);
expect(
resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('pages/www.foo.bar/xyz/');
});
it('should create url with relative output folder', function() {
const resultUrls = createResultUrls('http://www.foo.bar', '../leaf', 'http://results.com');
expect(resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz'))
.to.equal('pages/www.foo.bar/xyz/');
const resultUrls = createResultUrls(
'http://www.foo.bar',
'../leaf',
'http://results.com'
);
expect(
resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('pages/www.foo.bar/xyz/');
});
});
});

8
test/runWithoutCli.js
View File

@ -2,19 +2,19 @@ const sitespeed = require('../lib/sitespeed');
const urls = ['https://www.sitespeed.io/'];
function run() {
sitespeed.run({
sitespeed
.run({
urls,
browsertime: {
iterations: 1
}
})
.then((results) => {
.then(results => {
if (results.error) {
throw new Error(results.error);
}
})
.catch((err) => {
.catch(err => {
/* eslint-disable no-console */
console.error(err);
/* eslint-enable no-console */

26
test/storageManagerTests.js
View File

@ -16,13 +16,17 @@ describe('storageManager', function() {
describe('#rootPathFromUrl', function() {
it('should create path from url', function() {
const storageManager = createManager('http://www.foo.bar');
const path = storageManager.rootPathFromUrl('http://www.foo.bar/x/y/z.html');
const path = storageManager.rootPathFromUrl(
'http://www.foo.bar/x/y/z.html'
);
expect(path).to.equal('../../../../../');
});
it('should create path from url with query string', function() {
const storageManager = createManager('http://www.foo.bar');
const path = storageManager.rootPathFromUrl('http://www.foo.bar/x/y/z?foo=bar');
const path = storageManager.rootPathFromUrl(
'http://www.foo.bar/x/y/z?foo=bar'
);
expect(path).to.equal('../../../../../../');
});
});
@ -30,10 +34,15 @@ describe('storageManager', function() {
describe('#getBaseDir', function() {
it('should create base dir with default output folder', function() {
const storageManager = createManager('http://www.foo.bar');
expect(storageManager.getBaseDir()).to.equal(path.resolve('sitespeed-result', 'www.foo.bar', timestampString));
expect(storageManager.getBaseDir()).to.equal(
path.resolve('sitespeed-result', 'www.foo.bar', timestampString)
);
});
it('should create base dir with custom output folder', function() {
const storageManager = createManager('http://www.foo.bar', '/tmp/sitespeed.io/foo');
const storageManager = createManager(
'http://www.foo.bar',
'/tmp/sitespeed.io/foo'
);
expect(storageManager.getBaseDir()).to.equal('/tmp/sitespeed.io/foo');
});
});
@ -41,10 +50,15 @@ describe('storageManager', function() {
describe('#getStoragePrefix', function() {
it('should create prefix with default output folder', function() {
const storageManager = createManager('http://www.foo.bar');
expect(storageManager.getStoragePrefix()).to.equal(path.join('www.foo.bar', timestampString));
expect(storageManager.getStoragePrefix()).to.equal(
path.join('www.foo.bar', timestampString)
);
});
it('should create prefix with custom output folder', function() {
const storageManager = createManager('http://www.foo.bar', '/tmp/sitespeed.io/foo');
const storageManager = createManager(
'http://www.foo.bar',
'/tmp/sitespeed.io/foo'
);
expect(storageManager.getStoragePrefix()).to.equal('foo');
});
});

6
test/webpagetestTests.js
View File

@ -5,7 +5,11 @@ const aggregator = require('../lib/plugins/webpagetest/aggregator'),
path = require('path'),
expect = require('chai').expect;
const wptResultPath = path.resolve(__dirname, 'fixtures', 'webpagetest.data.json');
const wptResultPath = path.resolve(
__dirname,
'fixtures',
'webpagetest.data.json'
);
const wptResult = JSON.parse(fs.readFileSync(wptResultPath, 'utf8'));
describe('webpagetest', function() {

25
tools/check-licenses.js
View File

@ -7,28 +7,35 @@ const checker = require('license-checker');
const INCOMPATIBLE_LICENCE_REGEX = /GPL/;
checker.init({
checker.init(
{
start: '.'
}, function(json, err) {
},
function(json, err) {
if (err) {
console.error(err.message);
process.exit(1);
} else {
const incompatibleDependencies = Object.keys(json).filter((packageName) => {
const incompatibleDependencies = Object.keys(json).filter(packageName => {
let licenses = json[packageName].licenses;
if (!Array.isArray(licenses))
licenses = [licenses];
if (!Array.isArray(licenses)) licenses = [licenses];
if (licenses.find((license) => license.match(INCOMPATIBLE_LICENCE_REGEX)))
if (licenses.find(license => license.match(INCOMPATIBLE_LICENCE_REGEX)))
return packageName;
});
if (incompatibleDependencies.length > 0) {
console.error('Found packages with incompatible license: ' + JSON.stringify(incompatibleDependencies));
console.error(
'Found packages with incompatible license: ' +
JSON.stringify(incompatibleDependencies)
);
process.exit(1);
} else {
console.log('All is well! No packages with an incompatible license found.');
console.log(
'All is well! No packages with an incompatible license found.'
);
}
}
});
}
);

11
tools/tcp-server.js
View File

@ -3,12 +3,13 @@
var net = require('net');
var server = net.createServer(function(sock) {
var server = net
.createServer(function(sock) {
sock.on('data', function(data) {
console.log(data.toString());
});
}).listen(process.argv[2] || 0, undefined, undefined, () => {
})
.listen(process.argv[2] || 0, undefined, undefined, () => {
var address = server.address();
console.log('Server listening on ' + address.address +':'+ address.port);
});
console.log('Server listening on ' + address.address + ':' + address.port);
});