mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Get CrUx data for a URL (or more) and origin. (#3061) 

* Get CrUx data for a URL (or more) and origin.

Use the CrUx API to get Crux Data

* Correct cli
This commit is contained in:
Peter Hedenskog 2020-06-30 09:09:47 +02:00 committed by GitHub
parent 49ec21783f
commit 0fd6d9edbb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 381 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/linux.yml vendored
View File

@ -48,6 +48,8 @@ jobs:
run: bin/sitespeed.js https://www.sitespeed.io/ -n 1 --graphite.host 127.0.0.1 --xvfb
- name: Run test without a CLI
run: xvfb-run node test/runWithoutCli.js
- name: Run tests with CruX
run: bin/sitespeed.js -b chrome -n 1 --crux.key ${{ secrets.CRUX_KEY }} --xvfb https://www.sitespeed.io
- name: Run tests on WebPageTest
run: bin/sitespeed.js -b chrome -n 2 --summaryDetail --browsertime.chrome.timeline https://www.sitespeed.io/ --webpagetest.key ${{ secrets.WPT_KEY }} --xvfb

5
lib/cli/cli.js
View File

@ -14,6 +14,7 @@ const toArray = require('../support/util').toArray;
const grafanaPlugin = require('../plugins/grafana/index');
const graphitePlugin = require('../plugins/graphite/index');
const influxdbPlugin = require('../plugins/influxdb/index');
const cruxPlugin = require('../plugins/crux/index');
const browsertimeConfig = require('../plugins/browsertime/index').config;
const metricsConfig = require('../plugins/metrics/index').config;
@ -1301,7 +1302,9 @@ module.exports.parseCommandLine = function parseCommandLine() {
describe:
'Instead of using the local copy of the hosting database, you can use the latest version through the Green Web Foundation API. This means sitespeed.io will make HTTP GET to the the hosting info.',
group: 'Sustainable'
})
});
cliUtil.registerPluginOptions(parsed, cruxPlugin);
parsed
.option('mobile', {
describe:
'Access pages as mobile a fake mobile device. Set UA and width/height. For Chrome it will use device Apple iPhone 6.',

15
lib/plugins/crux/cli.js Normal file
View File

@ -0,0 +1,15 @@
module.exports = {
key: {
describe:
'You need to use a key to get data from CrUx. Get the key from https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/getting-started#APIKey',
group: 'CrUx'
},
formFactor: {
default: 'ALL',
type: 'string',
choices: ['ALL', 'DESKTOP', 'PHONE', 'TABLET'],
describe:
'A form factor is the type of device on which a user visits a website.',
group: 'CrUx'
}
};

149
lib/plugins/crux/index.js Normal file
View File

@ -0,0 +1,149 @@
'use strict';
const defaultConfig = {};
const log = require('intel').getLogger('plugin.crux');
const merge = require('lodash.merge');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
const send = require('./send');
const path = require('path');
const pageSummary = require('./pageSummary');
const summary = require('./summary');
const fs = require('fs');
const DEFAULT_METRICS_PAGESUMMARY = [
'loadingExperience.*.FIRST_CONTENTFUL_PAINT_MS.*',
'loadingExperience.*.FIRST_INPUT_DELAY_MS.*',
'loadingExperience.*.LARGEST_CONTENTFUL_PAINT_MS.*',
'loadingExperience.*.CUMULATIVE_LAYOUT_SHIFT_SCORE.*'
];
const DEFAULT_METRICS_SUMMARY = [
'originLoadingExperience.*.FIRST_CONTENTFUL_PAINT_MS.*',
'originLoadingExperience.*.FIRST_INPUT_DELAY_MS.*',
'originLoadingExperience.*.LARGEST_CONTENTFUL_PAINT_MS.*',
'originLoadingExperience.*.CUMULATIVE_LAYOUT_SHIFT_SCORE.*'
];
module.exports = {
name() {
return path.basename(__dirname);
},
open(context, options) {
this.make = context.messageMaker('crux').make;
this.options = merge({}, defaultConfig, options.crux);
this.testedOrigins = {};
throwIfMissing(options.crux, ['key'], 'crux');
this.formFactors = Array.isArray(this.options.formFactor)
? this.options.formFactor
: [this.options.formFactor];
this.pug = fs.readFileSync(
path.resolve(__dirname, 'pug', 'index.pug'),
'utf8'
);
context.filterRegistry.registerFilterForType(
DEFAULT_METRICS_PAGESUMMARY,
'crux.pageSummary'
);
context.filterRegistry.registerFilterForType(
DEFAULT_METRICS_SUMMARY,
'crux.summary'
);
},
async processMessage(message, queue) {
const make = this.make;
switch (message.type) {
case 'sitespeedio.setup': {
queue.postMessage(make('crux.setup'));
// Add the HTML pugs
queue.postMessage(
make('html.pug', {
id: 'crux',
name: 'CrUx',
pug: this.pug,
type: 'pageSummary'
})
);
queue.postMessage(
make('html.pug', {
id: 'crux',
name: 'CrUx',
pug: this.pug,
type: 'run'
})
);
break;
}
case 'url': {
let url = message.url;
let group = message.group;
const originResult = { originLoadingExperience: {} };
if (!this.testedOrigins[group]) {
log.info(`Get CrUx data for domain ${group}`);
for (let formFactor of this.formFactors) {
originResult.originLoadingExperience[formFactor] = await send.get(
url,
this.options.key,
formFactor,
false
);
if (originResult.originLoadingExperience[formFactor].error) {
log.error(
`${
originResult.originLoadingExperience[formFactor].message
} for ${url} using ${formFactor}`
);
} else {
originResult.originLoadingExperience[
formFactor
] = pageSummary.repackage(
originResult.originLoadingExperience[formFactor]
);
}
}
queue.postMessage(make('crux.summary', originResult, { group }));
this.testedOrigins[group] = true;
}
log.info(`Get CrUx data for url ${url}`);
const urlResult = { loadingExperience: {} };
for (let formFactor of this.formFactors) {
urlResult.loadingExperience[formFactor] = await send.get(
url,
this.options.key,
formFactor,
true
);
if (urlResult.loadingExperience[formFactor].error) {
log.error(
`${
urlResult.loadingExperience[formFactor].message
} for ${url} using ${formFactor}`
);
} else {
urlResult.loadingExperience[formFactor] = summary.repackage(
urlResult.loadingExperience[formFactor]
);
}
}
// Attach origin result so we can show it in the HTML
urlResult.originLoadingExperience =
originResult.originLoadingExperience;
queue.postMessage(
make('crux.pageSummary', urlResult, {
url,
group
})
);
}
}
},
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
}
};

48
lib/plugins/crux/pageSummary.js Normal file
View File

@ -0,0 +1,48 @@
'use strict';
module.exports = {
repackage(cruxResult) {
const result = {
FIRST_CONTENTFUL_PAINT_MS: {
p75: cruxResult.record.metrics.first_contentful_paint.percentiles.p75,
fast:
cruxResult.record.metrics.first_contentful_paint.histogram[0].density,
moderate:
cruxResult.record.metrics.first_contentful_paint.histogram[1].density,
slow:
cruxResult.record.metrics.first_contentful_paint.histogram[2].density
},
FIRST_INPUT_DELAY_MS: {
p75: cruxResult.record.metrics.first_input_delay.percentiles.p75,
fast: cruxResult.record.metrics.first_input_delay.histogram[0].density,
moderate:
cruxResult.record.metrics.first_input_delay.histogram[1].density,
slow: cruxResult.record.metrics.first_input_delay.histogram[2].density
},
CUMULATIVE_LAYOUT_SHIFT_SCORE: {
p75: cruxResult.record.metrics.cumulative_layout_shift.percentiles.p75,
fast:
cruxResult.record.metrics.cumulative_layout_shift.histogram[0]
.density,
moderate:
cruxResult.record.metrics.cumulative_layout_shift.histogram[1]
.density,
slow:
cruxResult.record.metrics.cumulative_layout_shift.histogram[2].density
},
LARGEST_CONTENTFUL_PAINT_MS: {
p75: cruxResult.record.metrics.largest_contentful_paint.percentiles.p75,
fast:
cruxResult.record.metrics.largest_contentful_paint.histogram[0]
.density,
moderate:
cruxResult.record.metrics.largest_contentful_paint.histogram[1]
.density,
slow:
cruxResult.record.metrics.largest_contentful_paint.histogram[2].densit
}
};
result.data = cruxResult;
return result;
}
};

64
lib/plugins/crux/pug/index.pug Normal file
View File

@ -0,0 +1,64 @@
mixin rowHeading(items)
thead
tr
each item in items
th= item
mixin numberCell(title, number)
td.number(data-title=title)= number
mixin sizeCell(title, size)
td.number(data-title=title, data-value= size)= h.size.format(size)
a
h2 CrUx
- const crux = pageInfo.data.crux.pageSummary;
- const metrics = {first_contentful_paint:'First Contentful Paint (FCP)', largest_contentful_paint: 'Largest Contentful Paint (LCP)', first_input_delay:'First Input Delay (FID)', cumulative_layout_shift: 'Cumulative Layout Shift'};
- const experiences = ['loadingExperience','originLoadingExperience'];
each experience in experiences
if experience === 'loadingExperience'
p Over the last 30 days, this is the field data for this page from the Chrome User Experience Report.
else
h4 All pages served from this origin
p This is a summary of all pages served from this origin in the Chrome User Experience Report over the last 30 days.
if crux[experience]
each formFactor in Object.keys(crux[experience])
if (crux[experience][formFactor] && crux[experience][formFactor].data)
h3 Form Factor #{formFactor}
table
thead
tr
th Metric
th Value
tbody
each name, key in metrics
tr
td #{name} 75 percentile
td #{crux[experience][formFactor].data.record.metrics[key].percentiles.p75} #{key.indexOf('cumulative') > -1 ? '': 'ms'}
h4 Distribution
table
each name, key in metrics
tr
th #{name}
th Min
th Max
th Users
tr
td Fast
td #{crux[experience][formFactor].data.record.metrics[key].histogram[0].start}
td #{crux[experience][formFactor].data.record.metrics[key].histogram[0].end}
td #{Number(crux[experience][formFactor].data.record.metrics[key].histogram[0].density * 100).toFixed(2)} %
tr
td Moderate
td #{crux[experience][formFactor].data.record.metrics[key].histogram[1].start}
td #{crux[experience][formFactor].data.record.metrics[key].histogram[1].end}
td #{Number(crux[experience][formFactor].data.record.metrics[key].histogram[1].density * 100).toFixed(2)} %
tr
td Slow
td #{crux[experience][formFactor].data.record.metrics[key].histogram[2].start}
td #{crux[experience][formFactor].data.record.metrics[key].histogram[2].end}
td #{Number(crux[experience][formFactor].data.record.metrics[key].histogram[2].density * 100).toFixed(2)} %

47
lib/plugins/crux/send.js Normal file
View File

@ -0,0 +1,47 @@
'use strict';
const https = require('https');
const log = require('intel').getLogger('plugin.crux');
module.exports = {
async get(url, key, formFactor, shouldWeTestThURL) {
let data = shouldWeTestThURL ? { url } : { origin: url };
if (formFactor !== 'ALL') {
data.formFactor = formFactor;
}
data = JSON.stringify(data);
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
const req = https.request(
{
host: 'chromeuxreport.googleapis.com',
port: 443,
path: `/v1/records:queryRecord?key=${key}`,
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(data, 'utf8')
},
method: 'POST'
},
function(res) {
if (res.statusCode < 200 || res.statusCode >= 300) {
log.error(`Got error from CrUx. Error Code: ${res.statusCode}`);
return reject(new Error(`Status Code: ${res.statusCode}`));
}
const data = [];
res.on('data', chunk => {
data.push(chunk);
});
res.on('end', () =>
resolve(JSON.parse(Buffer.concat(data).toString()))
);
}
);
req.write(data);
req.end();
});
}
};

50
lib/plugins/crux/summary.js Normal file
View File

@ -0,0 +1,50 @@
'use strict';
module.exports = {
repackage(cruxResult) {
const result = {
FIRST_CONTENTFUL_PAINT_MS: {
p75: cruxResult.record.metrics.first_contentful_paint.percentiles.p75,
fast:
cruxResult.record.metrics.first_contentful_paint.histogram[0].density,
moderate:
cruxResult.record.metrics.first_contentful_paint.histogram[1].density,
slow:
cruxResult.record.metrics.first_contentful_paint.histogram[2].density
},
FIRST_INPUT_DELAY_MS: {
p75: cruxResult.record.metrics.first_input_delay.percentiles.p75,
fast: cruxResult.record.metrics.first_input_delay.histogram[0].density,
moderate:
cruxResult.record.metrics.first_input_delay.histogram[1].density,
slow: cruxResult.record.metrics.first_input_delay.histogram[2].density
},
CUMULATIVE_LAYOUT_SHIFT_SCORE: {
p75: cruxResult.record.metrics.cumulative_layout_shift.percentiles.p75,
fast:
cruxResult.record.metrics.cumulative_layout_shift.histogram[0]
.density,
moderate:
cruxResult.record.metrics.cumulative_layout_shift.histogram[1]
.density,
slow:
cruxResult.record.metrics.cumulative_layout_shift.histogram[2].density
},
LARGEST_CONTENTFUL_PAINT_MS: {
p75: cruxResult.record.metrics.largest_contentful_paint.percentiles.p75,
fast:
cruxResult.record.metrics.largest_contentful_paint.histogram[0]
.density,
moderate:
cruxResult.record.metrics.largest_contentful_paint.histogram[1]
.density,
slow:
cruxResult.record.metrics.largest_contentful_paint.histogram[2]
.density
}
};
result.data = cruxResult;
return result;
}
};

3
lib/plugins/html/index.js
View File

@ -97,7 +97,8 @@ module.exports = {
'webpagetest.run',
'webpagetest.pageSummary',
'thirdparty.run',
'thirdparty.pageSummary'
'thirdparty.pageSummary',
'crux.pageSummary'
];
},
processMessage(message, queue) {