mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New plugins structure and esmodule (#3769) 

* New plugins structure and esmodule
This commit is contained in:
Peter Hedenskog 2023-02-25 11:16:58 +01:00 committed by GitHub
parent 1dfd8e67a0
commit 631271126f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
153 changed files with 6723 additions and 5927 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintignore
View File

@ -4,7 +4,6 @@ assets/*
sitespeed-result/*
lib/plugins/yslow/scripts/*
lib/plugins/html/assets/js/*
lib/plugins/browsertime/index.js
lib/plugins/browsertime/analyzer.js
bin/browsertimeWebPageReplay.js
test/data/*
test/prepostscripts/*

13
.eslintrc.json
View File

@ -5,10 +5,11 @@
"es6": true
},
"parserOptions": {
"ecmaVersion": 2018
"ecmaVersion": "latest",
"sourceType": "module"
},
"plugins": ["prettier"],
"extends": "eslint:recommended",
"plugins": ["prettier", "unicorn"],
"extends": ["eslint:recommended", "plugin:unicorn/recommended"],
"rules": {
"prettier/prettier": [
"error",
@ -21,6 +22,10 @@
],
"require-atomic-updates": 0,
"no-extra-semi": 0,
"no-mixed-spaces-and-tabs": 0
"no-mixed-spaces-and-tabs": 0,
"unicorn/filename-case": 0,
"unicorn/prevent-abbreviations": 0,
"unicorn/no-array-reduce": 0,
"unicorn/prefer-spread":0
}
}

12
.github/workflows/docker.yml vendored
View File

@ -21,14 +21,14 @@ jobs:
- name: Start local HTTP server
run: (serve test/data/html/ -l 3001&)
- name: Run test on default container for Chrome
run: docker run --rm --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b chrome
run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b chrome
- name: Run test on default container for Firefox
run: docker run --rm --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b firefox
run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b firefox
- name: Run test on default container for Edge
run: docker run --rm --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b edge
run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b edge
- name: Run test on slim container
run: docker run --rm --network=host sitespeedio/sitespeed.io:slim http://127.0.0.1:3001 -n 1 --browsertime.firefox.preference "devtools.netmonitor.persistlog:true"
run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io:slim http://127.0.0.1:3001 -n 1 --browsertime.firefox.preference "devtools.netmonitor.persistlog:true"
- name: Test WebPageReplay with Chrome
run: docker run --cap-add=NET_ADMIN --rm -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b chrome
run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b chrome
- name: Test WebPageReplay with Firefox
run: docker run --cap-add=NET_ADMIN --rm --network=host -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b firefox --browsertime.firefox.acceptInsecureCerts true
run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io --network=host -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b firefox --browsertime.firefox.acceptInsecureCerts true

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM sitespeedio/webbrowsers:chrome-110.0-firefox-109.0-edge-109.0
FROM sitespeedio/webbrowsers:chrome-110.0-firefox-110.0-edge-110.0
ARG TARGETPLATFORM=linux/amd64

32
bin/browsertimeWebPageReplay.js
View File

@ -1,25 +1,29 @@
#!/usr/bin/env node
'use strict';
import { readFileSync } from 'node:fs';
const yargs = require('yargs');
const merge = require('lodash.merge');
const getURLs = require('../lib/cli/util').getURLs;
const get = require('lodash.get');
const set = require('lodash.set');
const findUp = require('find-up');
const fs = require('fs');
const browsertimeConfig = require('../lib/plugins/browsertime/index').config;
import merge from 'lodash.merge';
import set from 'lodash.set';
import get from 'lodash.get';
import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import { findUpSync } from 'find-up';
import { BrowsertimeEngine, configureLogging } from 'browsertime';
import { getURLs } from '../lib/cli/util.js';
import {config as browsertimeConfig} from '../lib/plugins/browsertime/index.js';
const iphone6UserAgent =
'Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 ' +
'(KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25';
const configPath = findUp.sync(['.sitespeed.io.json']);
const configPath = findUpSync(['.sitespeed.io.json']);
let config;
try {
config = configPath ? JSON.parse(fs.readFileSync(configPath)) : {};
config = configPath ? JSON.parse(readFileSync(configPath)) : {};
} catch (e) {
if (e instanceof SyntaxError) {
/* eslint no-console: off */
@ -49,7 +53,8 @@ async function testURLs(engine, urls) {
}
async function runBrowsertime() {
let parsed = yargs
let yargsInstance = yargs(hideBin(process.argv));
let parsed = yargsInstance
.env('SITESPEED_IO')
.require(1, 'urlOrFile')
.option('browsertime.browser', {
@ -139,9 +144,6 @@ async function runBrowsertime() {
}
};
const {BrowsertimeEngine, configureLogging} = await import ('browsertime');
const btOptions = merge({}, parsed.argv.browsertime, defaultConfig);
// hack to keep backward compability to --android
if (parsed.argv.android[0] === true) {

41
bin/sitespeed.js
View File

@ -2,24 +2,30 @@
/*eslint no-console: 0*/
'use strict';
import { writeFileSync } from 'node:fs';
import { execSync } from 'node:child_process';
import { platform } from 'node:os';
import { parseCommandLine } from '../lib/cli/cli.js';
import { run } from '../lib/sitespeed.js';
const fs = require('fs');
const cli = require('../lib/cli/cli');
const sitespeed = require('../lib/sitespeed');
const { execSync } = require('child_process');
const os = require('os');
async function start() {
let parsed = await parseCommandLine();
let budgetFailing = false;
// hack for getting in the unchanged cli options
parsed.options.explicitOptions = parsed.explicitOptions;
parsed.options.urls = parsed.urls;
parsed.options.urlsMetaData = parsed.urlsMetaData;
async function run(options) {
let options = parsed.options;
process.exitCode = 1;
try {
const result = await sitespeed.run(options);
const result = await run(options);
if (options.storeResult) {
if (options.storeResult != 'true') {
fs.writeFileSync(options.storeResult, JSON.stringify(result));
writeFileSync(options.storeResult, JSON.stringify(result));
} else {
fs.writeFileSync('result.json', JSON.stringify(result));
writeFileSync('result.json', JSON.stringify(result));
}
}
@ -27,9 +33,9 @@ async function run(options) {
throw new Error('Errors while running:\n' + result.errors.join('\n'));
}
if ((options.open || options.o) && os.platform() === 'darwin') {
if ((options.open || options.o) && platform() === 'darwin') {
execSync('open ' + result.localPath + '/index.html');
} else if ((options.open || options.o) && os.platform() === 'linux') {
} else if ((options.open || options.o) && platform() === 'linux') {
execSync('xdg-open ' + result.localPath + '/index.html');
}
@ -47,17 +53,12 @@ async function run(options) {
) {
process.exitCode = 0;
}
} catch (e) {
} catch (error) {
process.exitCode = 1;
console.log(error);
} finally {
process.exit();
}
}
let parsed = cli.parseCommandLine();
let budgetFailing = false;
// hack for getting in the unchanged cli options
parsed.options.explicitOptions = parsed.explicitOptions;
parsed.options.urls = parsed.urls;
parsed.options.urlsMetaData = parsed.urlsMetaData;
run(parsed.options);
await start();

74
cz-config.js
View File

@ -1,74 +0,0 @@
module.exports = {
types: [
{ value: 'feat', name: 'feat: A new feature' },
{ value: 'fix', name: 'fix: A bug fix' },
{ value: 'docs', name: 'docs: Documentation only changes' },
{
value: 'style',
name: 'style: Changes that do not affect the meaning of the code\n (white-space, formatting, missing semi-colons, etc)'
},
{
value: 'refactor',
name: 'refactor: A code change that neither fixes a bug nor adds a feature'
},
{
value: 'perf',
name: 'perf: A code change that improves performance'
},
{ value: 'test', name: 'test: Adding missing tests' },
{
value: 'chore',
name: 'chore: Changes to the build process or auxiliary tools\n and libraries such as documentation generation'
},
{ value: 'revert', name: 'revert: Revert to a commit' },
{ value: 'WIP', name: 'WIP: Work in progress' }
],
// scopes: [
// { name: 'accounts' },
// { name: 'admin' },
// { name: 'exampleScope' },
// { name: 'changeMe' }
// ],
allowTicketNumber: false,
isTicketNumberRequired: false,
// ticketNumberPrefix: 'TICKET-',
// ticketNumberRegExp: '\\d{1,5}',
// it needs to match the value for field type. Eg.: 'fix'
/*
scopeOverrides: {
fix: [
{name: 'merge'},
{name: 'style'},
{name: 'e2eTest'},
{name: 'unitTest'}
]
},
*/
// override the messages, defaults are as follows
messages: {
type: "Select the type of change that you're committing:",
// scope: '\nDenote the SCOPE of this change (optional):',
// used if allowCustomScopes is true
// customScope: 'Denote the SCOPE of this change:',
subject: 'Write a SHORT, IMPERATIVE tense description of the change:\n',
body: 'Provide a LONGER description of the change (optional). Use "|" to break new line:\n',
breaking: 'List any BREAKING CHANGES (optional):\n',
footer:
'List any ISSUES CLOSED by this change (optional). E.g.: #31, #34:\n',
confirmCommit: 'Are you sure you want to proceed with the commit above?'
},
// allowCustomScopes: true,
allowBreakingChanges: ['feat', 'fix'],
// skip any questions you want
skipQuestions: ['body'],
// limit subject length
subjectLimit: 100
// breaklineChar: '|', // It is supported for fields body and footer.
// footerPrefix : 'ISSUES CLOSED:'
// askForBreakingChangeFirst : true, // default is false
};

149
docs/documentation/browsertime/configuration/config.md
View File

@ -38,6 +38,18 @@ chrome
--chrome.blockDomainsExcept, --blockDomainsExcept Block all domains except this domain. Use it multiple time to keep multiple domains. You can also wildcard domains like *.sitespeed.io. Use this when you wanna block out all third parties.
--chrome.ignoreCertificateErrors Make Chrome ignore certificate errors. Defaults to true. [boolean] [default: true]
android
--android.powerTesting, --androidPower Enables android power testing - charging must be disabled for this.(You have to disable charging yourself for this - it depends on the phone model). [boolean]
--android.ignoreShutdownFailures, --ignoreShutdownFailures If set, shutdown failures will be ignored on Android. [boolean] [default: false]
--android.rooted, --androidRooted If your phone is rooted you can use this to set it up following Mozillas best practice for stable metrics. [boolean] [default: false]
--android.batteryTemperatureLimit, --androidBatteryTemperatureLimit Do the battery temperature need to be below a specific limit before we start the test?
--android.batteryTemperatureWaitTimeInSeconds, --androidBatteryTemperatureWaitTimeInSeconds How long time to wait (in seconds) if the androidBatteryTemperatureWaitTimeInSeconds is not met before the next try [default: 120]
--android.batteryTemperatureReboot, --androidBatteryTemperatureReboot If your phone does not get the minimum temperature aftet the wait time, reboot the phone. [boolean] [default: false]
--android.pretestPowerPress, --androidPretestPowerPress Press the power button on the phone before a test starts. [boolean] [default: false]
--android.pretestPressHomeButton, --androidPretestPressHomeButton Press the home button on the phone before a test starts. [boolean] [default: false]
--android.verifyNetwork, --androidVerifyNetwork Before a test start, verify that the device has a Internet connection by pinging 8.8.8.8 (or a configurable domain with --androidPingAddress) [boolean] [default: false]
--android.gnirehtet, --gnirehtet Start gnirehtet and reverse tethering the traffic from your Android phone. [boolean] [default: false]
firefox
--firefox.binaryPath Path to custom Firefox binary (e.g. Firefox Nightly). On OS X, the path should be to the binary inside the app bundle, e.g. /Applications/Firefox.app/Contents/MacOS/firefox-bin
--firefox.geckodriverPath Path to custom geckodriver binary. Make sure to use a geckodriver version that's compatible with the version of Firefox (Gecko) you're using
@ -134,76 +146,67 @@ debug
--debug Run Browsertime in debug mode. [boolean] [default: false]
Options:
--cpu Easy way to enable both chrome.timeline for Chrome and geckoProfile for Firefox [boolean]
--androidPower Enables android power testing - charging must be disabled for this.(You have to disable charging yourself for this - it depends on the phone model). [boolean]
--video Record a video and store the video. Set it to false to remove the video that is created by turning on visualMetrics. To remove fully turn off video recordings, make sure to set video and visualMetrics to false. Requires FFMpeg to be installed. [boolean]
--visualMetrics Collect Visual Metrics like First Visual Change, SpeedIndex, Perceptual Speed Index and Last Visual Change. Requires FFMpeg and Python dependencies [boolean]
--visualElements, --visuaElements Collect Visual Metrics from elements. Works only with --visualMetrics turned on. By default you will get visual metrics from the largest image within the view port and the largest h1. You can also configure to pickup your own defined elements with --scriptInput.visualElements [boolean]
--visualMetricsPerceptual Collect Perceptual Speed Index when you run --visualMetrics. [boolean]
--visualMetricsContentful Collect Contentful Speed Index when you run --visualMetrics. [boolean]
--visualMetricsPortable Use the portable visual-metrics processing script (no ImageMagick dependencies). [boolean]
--scriptInput.visualElements Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered.
--scriptInput.longTask, --minLongTaskLength Set the minimum length of a task to be categorised as a CPU Long Task. It can never be smaller than 50. The value is in ms and only works in Chromium browsers at the moment. [number] [default: 50]
-b, --browser Specify browser. Safari only works on OS X/iOS. Edge only work on OS that supports Edge. [choices: "chrome", "firefox", "edge", "safari"] [default: "chrome"]
--ignoreShutdownFailures If set, shutdown failures will be ignored on Android. [boolean] [default: false]
--android Short key to use Android. Defaults to use com.android.chrome unless --browser is specified. [boolean] [default: false]
--androidRooted If your phone is rooted you can use this to set it up following Mozillas best practice for stable metrics. [boolean] [default: false]
--androidBatteryTemperatureLimit Do the battery temperature need to be below a specific limit before we start the test?
--androidBatteryTemperatureWaitTimeInSeconds How long time to wait (in seconds) if the androidBatteryTemperatureWaitTimeInSeconds is not met before the next try [default: 120]
--androidBatteryTemperatureReboot If your phone does not get the minimum temperature aftet the wait time, reboot the phone. [boolean] [default: false]
--androidPretestPowerPress Press the power button on the phone before a test starts. [boolean] [default: false]
--androidPretestPressHomeButton Press the home button on the phone before a test starts. [boolean] [default: false]
--androidVerifyNetwork Before a test start, verify that the device has a Internet connection by pinging 8.8.8.8 (or a configurable domain with --androidPingAddress) [boolean] [default: false]
--processStartTime Capture browser process start time (in milliseconds). Android only for now. [boolean] [default: false]
--pageCompleteCheck Supply a JavaScript (inline or JavaScript file) that decides when the browser is finished loading the page and can start to collect metrics. The JavaScript snippet is repeatedly queried to see if page has completed loading (indicated by the script returning true). Use it to fetch timings happening after the loadEventEnd. By default the tests ends 2 seconds after loadEventEnd. Also checkout --pageCompleteCheckInactivity and --pageCompleteCheckPollTimeout
--pageCompleteWaitTime How long time you want to wait for your pageComplteteCheck to finish, after it is signaled to closed. Extra parameter passed on to your pageCompleteCheck. [default: 8000]
--pageCompleteCheckInactivity Alternative way to choose when to end your test. This will wait for 2 seconds of inactivity that happens after loadEventEnd. [boolean] [default: false]
--pageCompleteCheckPollTimeout The time in ms to wait for running the page complete check the next time. [number] [default: 1500]
--pageCompleteCheckStartWait The time in ms to wait for running the page complete check for the first time. Use this when you have a pageLoadStrategy set to none [number] [default: 5000]
--pageLoadStrategy Set the strategy to waiting for document readiness after a navigation event. After the strategy is ready, your pageCompleteCheck will start runninhg. [string] [choices: "eager", "none", "normal"] [default: "none"]
-n, --iterations Number of times to test the url (restarting the browser between each test) [number] [default: 3]
--prettyPrint Enable to print json/har with spaces and indentation. Larger files, but easier on the eye. [boolean] [default: false]
--delay Delay between runs, in milliseconds [number] [default: 0]
--timeToSettle Extra time added for the browser to settle before starting to test a URL. This delay happens after the browser was opened and before the navigation to the URL [number] [default: 0]
--webdriverPageload Use webdriver.get to initialize the page load instead of window.location. [boolean] [default: false]
-r, --requestheader Request header that will be added to the request. Add multiple instances to add multiple request headers. Works for Firefox and Chrome. Use the following format key:value
--cookie Cookie that will be added to the request. Add multiple instances to add multiple request cookies. Works for Firefox and Chrome. Use the following format cookieName=cookieValue
--injectJs Inject JavaScript into the current page at document_start. Works for Firefox and Chrome. More info: https://developer.mozilla.org/docs/Mozilla/Add-ons/WebExtensions/API/contentScripts
--block Domain to block or URL or URL pattern to block. If you use Chrome you can also use --blockDomainsExcept (that is more performant). Works in Chrome/Edge. For Firefox you can only block domains.
--percentiles The percentile values within the data browsertime will calculate and report. This argument uses Yargs arrays and you you to set them correctly it is recommended to use a configuraration file instead. [array] [default: [0,10,90,99,100]]
--decimals The decimal points browsertime statistics round to. [number] [default: 0]
--iqr Use IQR, or Inter Quartile Range filtering filters data based on the spread of the data. See https://en.wikipedia.org/wiki/Interquartile_range. In some cases, IQR filtering may not filter out anything. This can happen if the acceptable range is wider than the bounds of your dataset. [boolean] [default: false]
--cacheClearRaw Use internal browser functionality to clear browser cache between runs instead of only using Selenium. [boolean] [default: false]
--basicAuth Use it if your server is behind Basic Auth. Format: username@password (Only Chrome and Firefox at the moment).
--preScript, --setUp Selenium script(s) to run before you test your URL/script. They will run outside of the analyse phase. Note that --preScript can be passed multiple times.
--postScript, --tearDown Selenium script(s) to run after you test your URL. They will run outside of the analyse phase. Note that --postScript can be passed multiple times.
--script Add custom Javascript to run after the page has finished loading to collect metrics. If a single js file is specified, it will be included in the category named "custom" in the output json. Pass a folder to include all .js scripts in the folder, and have the folder name be the category. Note that --script can be passed multiple times.
--userAgent Override user agent
--appendToUserAgent Append a String to the user agent. Works in Chrome/Edge and Firefox.
-q, --silent Only output info in the logs, not to the console. Enter twice to suppress summary line. [count]
-o, --output Specify file name for Browsertime data (ex: 'browsertime'). Unless specified, file will be named browsertime.json
--har Specify file name for .har file (ex: 'browsertime'). Unless specified, file will be named browsertime.har
--skipHar Pass --skipHar to not collect a HAR file. [boolean]
--gzipHar Pass --gzipHar to gzip the HAR file [boolean]
--config Path to JSON config file. You can also use a .browsertime.json file that will automatically be found by Browsertime using find-up.
--viewPort Size of browser window WIDTHxHEIGHT or "maximize". Note that "maximize" is ignored for xvfb.
--resultDir Set result directory for the files produced by Browsertime
--useSameDir Store all files in the same structure and do not use the path structure released in 4.0. Use this only if you are testing ONE URL.
--xvfb Start xvfb before the browser is started [boolean] [default: false]
--xvfbParams.display The display used for xvfb [default: 99]
--tcpdump Collect a tcpdump for each tested URL. [boolean] [default: false]
--tcpdumpPacketBuffered Use together with --tcpdump to save each packet directly to the file, instead of buffering. [boolean] [default: false]
--urlAlias Use an alias for the URL. You need to pass on the same amount of alias as URLs. The alias is used as the name of the URL and used for filepath. Pass on multiple --urlAlias for multiple alias/URLs. You can also add alias direct in your script. [string]
--preURL, --warmLoad A URL that will be accessed first by the browser before the URL that you wanna analyze. Use it to fill the browser cache.
--preURLDelay, --warmLoadDealy Delay between preURL and the URL you want to test (in milliseconds) [default: 1500]
--userTimingWhitelist All userTimings are captured by default this option takes a regex that will whitelist which userTimings to capture in the results.
--headless Run the browser in headless mode. Works for Firefox and Chrome. [boolean] [default: false]
--gnirehtet Start gnirehtet and reverse tethering the traffic from your Android phone. [boolean] [default: false]
--flushDNS Flush DNS between runs, works on Mac OS and Linux. Your user needs sudo rights to be able to flush the DNS. [boolean] [default: false]
--extension Path to a WebExtension to be installed in the browser. Note that --extension can be passed multiple times.
--spa Convenient parameter to use if you test a SPA application: will automatically wait for X seconds after last network activity and use hash in file names. Read more: https://www.sitespeed.io/documentation/sitespeed.io/spa/ [boolean] [default: false]
--browserRestartTries If the browser fails to start, you can retry to start it this amount of times. [number] [default: 3]
--preWarmServer Do pre test requests to the URL(s) that you want to test that is not measured. Do that to make sure your web server is ready to serve. The pre test requests is done with another browser instance that is closed after pre testing is done. [boolean] [default: false]
--preWarmServerWaitTime The wait time before you start the real testing after your pre-cache request. [number] [default: 5000]
-h, --help Show help [boolean]
-V, --version Show version number [boolean]
--cpu Easy way to enable both chrome.timeline for Chrome and geckoProfile for Firefox [boolean]
--video Record a video and store the video. Set it to false to remove the video that is created by turning on visualMetrics. To remove fully turn off video recordings, make sure to set video and visualMetrics to false. Requires FFMpeg to be installed. [boolean]
--visualMetrics Collect Visual Metrics like First Visual Change, SpeedIndex, Perceptual Speed Index and Last Visual Change. Requires FFMpeg and Python dependencies [boolean]
--visualElements, --visuaElements Collect Visual Metrics from elements. Works only with --visualMetrics turned on. By default you will get visual metrics from the largest image within the view port and the largest h1. You can also configure to pickup your own defined elements with --scriptInput.visualElements [boolean]
--visualMetricsPerceptual Collect Perceptual Speed Index when you run --visualMetrics. [boolean]
--visualMetricsContentful Collect Contentful Speed Index when you run --visualMetrics. [boolean]
--visualMetricsPortable Use the portable visual-metrics processing script (no ImageMagick dependencies). [boolean] [default: true]
--scriptInput.visualElements Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered.
--scriptInput.longTask, --minLongTaskLength Set the minimum length of a task to be categorised as a CPU Long Task. It can never be smaller than 50. The value is in ms and only works in Chromium browsers at the moment. [number] [default: 50]
-b, --browser Specify browser. Safari only works on OS X/iOS. Edge only work on OS that supports Edge. [choices: "chrome", "firefox", "edge", "safari"] [default: "chrome"]
--android Short key to use Android. Defaults to use com.android.chrome unless --browser is specified. [boolean] [default: false]
--processStartTime Capture browser process start time (in milliseconds). Android only for now. [boolean] [default: false]
--pageCompleteCheck Supply a JavaScript (inline or JavaScript file) that decides when the browser is finished loading the page and can start to collect metrics. The JavaScript snippet is repeatedly queried to see if page has completed loading (indicated by the script returning true). Use it to fetch timings happening after the loadEventEnd. By default the tests ends 2 seconds after loadEventEnd. Also checkout --pageCompleteCheckInactivity and --pageCompleteCheckPollTimeout
--pageCompleteWaitTime How long time you want to wait for your pageComplteteCheck to finish, after it is signaled to closed. Extra parameter passed on to your pageCompleteCheck. [default: 8000]
--pageCompleteCheckInactivity Alternative way to choose when to end your test. This will wait for 2 seconds of inactivity that happens after loadEventEnd. [boolean] [default: false]
--pageCompleteCheckPollTimeout The time in ms to wait for running the page complete check the next time. [number] [default: 1500]
--pageCompleteCheckStartWait The time in ms to wait for running the page complete check for the first time. Use this when you have a pageLoadStrategy set to none [number] [default: 5000]
--pageLoadStrategy Set the strategy to waiting for document readiness after a navigation event. After the strategy is ready, your pageCompleteCheck will start runninhg. [string] [choices: "eager", "none", "normal"] [default: "none"]
-n, --iterations Number of times to test the url (restarting the browser between each test) [number] [default: 3]
--prettyPrint Enable to print json/har with spaces and indentation. Larger files, but easier on the eye. [boolean] [default: false]
--delay Delay between runs, in milliseconds [number] [default: 0]
--timeToSettle Extra time added for the browser to settle before starting to test a URL. This delay happens after the browser was opened and before the navigation to the URL [number] [default: 0]
--webdriverPageload Use webdriver.get to initialize the page load instead of window.location. [boolean] [default: false]
-r, --requestheader Request header that will be added to the request. Add multiple instances to add multiple request headers. Works for Firefox and Chrome. Use the following format key:value
--cookie Cookie that will be added to the request. Add multiple instances to add multiple request cookies. Works for Firefox and Chrome. Use the following format cookieName=cookieValue
--injectJs Inject JavaScript into the current page at document_start. Works for Firefox and Chrome. More info: https://developer.mozilla.org/docs/Mozilla/Add-ons/WebExtensions/API/contentScripts
--block Domain to block or URL or URL pattern to block. If you use Chrome you can also use --blockDomainsExcept (that is more performant). Works in Chrome/Edge. For Firefox you can only block domains.
--percentiles The percentile values within the data browsertime will calculate and report. This argument uses Yargs arrays and you you to set them correctly it is recommended to use a configuraration file instead. [array] [default: [0,10,90,99,100]]
--decimals The decimal points browsertime statistics round to. [number] [default: 0]
--iqr Use IQR, or Inter Quartile Range filtering filters data based on the spread of the data. See https://en.wikipedia.org/wiki/Interquartile_range. In some cases, IQR filtering may not filter out anything. This can happen if the acceptable range is wider than the bounds of your dataset. [boolean] [default: false]
--cacheClearRaw Use internal browser functionality to clear browser cache between runs instead of only using Selenium. [boolean] [default: false]
--basicAuth Use it if your server is behind Basic Auth. Format: username@password (Only Chrome and Firefox at the moment).
--preScript, --setUp Selenium script(s) to run before you test your URL/script. They will run outside of the analyse phase. Note that --preScript can be passed multiple times.
--postScript, --tearDown Selenium script(s) to run after you test your URL. They will run outside of the analyse phase. Note that --postScript can be passed multiple times.
--script Add custom Javascript to run after the page has finished loading to collect metrics. If a single js file is specified, it will be included in the category named "custom" in the output json. Pass a folder to include all .js scripts in the folder, and have the folder name be the category. Note that --script can be passed multiple times.
--userAgent Override user agent
--appendToUserAgent Append a String to the user agent. Works in Chrome/Edge and Firefox.
-q, --silent Only output info in the logs, not to the console. Enter twice to suppress summary line. [count]
-o, --output Specify file name for Browsertime data (ex: 'browsertime'). Unless specified, file will be named browsertime.json
--har Specify file name for .har file (ex: 'browsertime'). Unless specified, file will be named browsertime.har
--skipHar Pass --skipHar to not collect a HAR file. [boolean]
--gzipHar Pass --gzipHar to gzip the HAR file [boolean]
--config Path to JSON config file. You can also use a .browsertime.json file that will automatically be found by Browsertime using find-up.
--viewPort Size of browser window WIDTHxHEIGHT or "maximize". Note that "maximize" is ignored for xvfb.
--resultDir Set result directory for the files produced by Browsertime
--useSameDir Store all files in the same structure and do not use the path structure released in 4.0. Use this only if you are testing ONE URL.
--xvfb Start xvfb before the browser is started [boolean] [default: false]
--xvfbParams.display The display used for xvfb [default: 99]
--tcpdump Collect a tcpdump for each tested URL. [boolean] [default: false]
--tcpdumpPacketBuffered Use together with --tcpdump to save each packet directly to the file, instead of buffering. [boolean] [default: false]
--urlAlias Use an alias for the URL. You need to pass on the same amount of alias as URLs. The alias is used as the name of the URL and used for filepath. Pass on multiple --urlAlias for multiple alias/URLs. You can also add alias direct in your script. [string]
--preURL, --warmLoad A URL that will be accessed first by the browser before the URL that you wanna analyze. Use it to fill the browser cache.
--preURLDelay, --warmLoadDealy Delay between preURL and the URL you want to test (in milliseconds) [default: 1500]
--userTimingWhitelist All userTimings are captured by default this option takes a regex that will whitelist which userTimings to capture in the results.
--headless Run the browser in headless mode. Works for Firefox and Chrome. [boolean] [default: false]
--flushDNS Flush DNS between runs, works on Mac OS and Linux. Your user needs sudo rights to be able to flush the DNS. [boolean] [default: false]
--extension Path to a WebExtension to be installed in the browser. Note that --extension can be passed multiple times.
--spa Convenient parameter to use if you test a SPA application: will automatically wait for X seconds after last network activity and use hash in file names. Read more: https://www.sitespeed.io/documentation/sitespeed.io/spa/ [boolean] [default: false]
--cjs Load scripting files that ends with .js as common js. Default (false) loads files as esmodules. [boolean] [default: false]
--browserRestartTries If the browser fails to start, you can retry to start it this amount of times. [number] [default: 3]
--preWarmServer Do pre test requests to the URL(s) that you want to test that is not measured. Do that to make sure your web server is ready to serve. The pre test requests is done with another browser instance that is closed after pre testing is done. [boolean] [default: false]
--preWarmServerWaitTime The wait time before you start the real testing after your pre-cache request. [number] [default: 5000]
-h, --help Show help [boolean]
-V, --version Show version number [boolean]

479
lib/cli/cli.js
View File

@ -1,29 +1,24 @@
'use strict';
import { resolve } from 'node:path';
import { platform } from 'node:os';
import { readFileSync, statSync, existsSync } from 'node:fs';
const yargs = require('yargs');
const path = require('path');
const merge = require('lodash.merge');
const reduce = require('lodash.reduce');
const cliUtil = require('./util');
const fs = require('fs');
const set = require('lodash.set');
const get = require('lodash.get');
const findUp = require('find-up');
const os = require('os');
const toArray = require('../support/util').toArray;
import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import merge from 'lodash.merge';
import reduce from 'lodash.reduce';
import set from 'lodash.set';
import get from 'lodash.get';
import { findUpSync } from 'find-up';
const grafanaPlugin = require('../plugins/grafana/index');
const graphitePlugin = require('../plugins/graphite/index');
const influxdbPlugin = require('../plugins/influxdb/index');
const cruxPlugin = require('../plugins/crux/index');
const matrixPlugin = require('../plugins/matrix/index');
import { getURLs, getAliases } from './util.js';
import { toArray } from '../support/util.js';
import friendlynames from '../support/friendlynames.js';
import { config as browsertimeConfig } from '../plugins/browsertime/index.js';
import { config as metricsConfig } from '../plugins/metrics/index.js';
import { config as slackConfig } from '../plugins/slack/index.js';
import { config as htmlConfig } from '../plugins/html/index.js';
import { messageTypes as matrixMessageTypes } from '../plugins/matrix/index.js';
const browsertimeConfig = require('../plugins/browsertime/index').config;
const metricsConfig = require('../plugins/metrics/index').config;
const slackConfig = require('../plugins/slack/index').config;
const htmlConfig = require('../plugins/html/index').config;
const friendlynames = require('../support/friendlynames');
const metricList = Object.keys(friendlynames);
const configFiles = ['.sitespeed.io.json'];
@ -33,20 +28,20 @@ if (process.argv.includes('--config')) {
configFiles.unshift(process.argv[index + 1]);
}
const configPath = findUp.sync(configFiles);
const configPath = findUpSync(configFiles);
let config;
try {
config = configPath ? JSON.parse(fs.readFileSync(configPath)) : undefined;
} catch (e) {
if (e instanceof SyntaxError) {
config = configPath ? JSON.parse(readFileSync(configPath)) : undefined;
} catch (error) {
if (error instanceof SyntaxError) {
console.error(
'Error: Could not parse the config JSON file ' +
configPath +
'. Is the file really valid JSON?'
);
}
throw e;
throw error;
}
function validateInput(argv) {
@ -80,7 +75,7 @@ function validateInput(argv) {
}
if (argv.slug) {
const characters = /[^A-Za-z_\-0-9]/g;
const characters = /[^\w-]/g;
if (characters.test(argv.slug)) {
return 'The slug can only use characters A-Z a-z 0-9 and -_.';
}
@ -100,7 +95,7 @@ function validateInput(argv) {
if (
argv.urlAlias &&
argv._ &&
cliUtil.getURLs(argv._).length !== toArray(argv.urlAlias).length
getURLs(argv._).length !== toArray(argv.urlAlias).length
) {
return 'Error: You have a miss match between number of alias and URLs.';
}
@ -108,21 +103,18 @@ function validateInput(argv) {
if (
argv.groupAlias &&
argv._ &&
cliUtil.getURLs(argv._).length !== toArray(argv.groupAlias).length
getURLs(argv._).length !== toArray(argv.groupAlias).length
) {
return 'Error: You have a miss match between number of alias for groups and URLs.';
}
if (
argv.browsertime.connectivity &&
argv.browsertime.connectivity.engine === 'humble'
argv.browsertime.connectivity.engine === 'humble' &&
(!argv.browsertime.connectivity.humble ||
!argv.browsertime.connectivity.humble.url)
) {
if (
!argv.browsertime.connectivity.humble ||
!argv.browsertime.connectivity.humble.url
) {
return 'You need to specify the URL to Humble by using the --browsertime.connectivity.humble.url option.';
}
return 'You need to specify the URL to Humble by using the --browsertime.connectivity.humble.url option.';
}
if (
@ -159,8 +151,8 @@ function validateInput(argv) {
if (!urlOrFile.startsWith('http')) {
// is existing file?
try {
fs.statSync(urlOrFile);
} catch (e) {
statSync(urlOrFile);
} catch {
return (
'Error: ' +
urlOrFile +
@ -182,23 +174,23 @@ function validateInput(argv) {
}
for (let metric of toArray(argv.html.summaryBoxes)) {
if (htmlConfig.html.summaryBoxes.indexOf(metric) === -1) {
if (!htmlConfig.html.summaryBoxes.includes(metric)) {
return `Error: ${metric} is not part of summary box metric.`;
}
}
if (argv.html && argv.html.summaryBoxesThresholds) {
try {
const box = fs.readFileSync(
path.resolve(argv.html.summaryBoxesThresholds),
{
encoding: 'utf8'
}
);
const box = readFileSync(resolve(argv.html.summaryBoxesThresholds), {
encoding: 'utf8'
});
argv.html.summaryBoxesThresholds = JSON.parse(box);
} catch (e) {
} catch (error) {
return (
'Error: Could not read ' + argv.html.summaryBoxesThresholds + ' ' + e
'Error: Could not read ' +
argv.html.summaryBoxesThresholds +
' ' +
error
);
}
}
@ -206,8 +198,9 @@ function validateInput(argv) {
return true;
}
module.exports.parseCommandLine = function parseCommandLine() {
let parsed = yargs
export async function parseCommandLine() {
let yargsInstance = yargs(hideBin(process.argv));
let parsed = yargsInstance
.parserConfiguration({ 'deep-merge-config': true })
.env('SITESPEED_IO')
.usage('$0 [options] <url>/<file>')
@ -344,7 +337,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
})
.option('browsertime.timeouts.pageCompleteCheck', {
alias: 'maxLoadTime',
default: 120000,
default: 120_000,
type: 'number',
describe:
'The max load time to wait for a page to finish loading (in milliseconds).',
@ -693,7 +686,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
.option('browsertime.firefox.geckoProfilerParams.bufferSize', {
alias: 'firefox.geckoProfilerParams.bufferSize',
describe: 'Buffer size in elements. Default is ~90MB.',
default: 1000000,
default: 1_000_000,
type: 'number',
group: 'Firefox'
})
@ -1142,14 +1135,203 @@ module.exports.parseCommandLine = function parseCommandLine() {
describe:
'Remove the files locally when the files has been copied to the other server.',
group: 'scp'
})
.option('grafana.host', {
describe: 'The Grafana host used when sending annotations.',
group: 'Grafana'
})
.option('grafana.port', {
default: 80,
describe: 'The Grafana port used when sending annotations to Grafana.',
group: 'Grafana'
})
.option('grafana.auth', {
describe:
'The Grafana auth/bearer value used when sending annotations to Grafana. If you do not set Bearer/Auth, Bearer is automatically set. See http://docs.grafana.org/http_api/auth/#authentication-api',
group: 'Grafana'
})
.option('grafana.annotationTitle', {
describe: 'Add a title to the annotation sent for a run.',
group: 'Grafana'
})
.option('grafana.annotationMessage', {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Grafana'
})
.option('grafana.annotationTag', {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --grafana.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Grafana'
})
.option('grafana.annotationScreenshot', {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Grafana'
})
.option('graphite.host', {
describe: 'The Graphite host used to store captured metrics.',
group: 'Graphite'
})
.option('graphite.port', {
default: 2003,
describe: 'The Graphite port used to store captured metrics.',
group: 'Graphite'
})
.option('graphite.auth', {
describe:
'The Graphite user and password used for authentication. Format: user:password',
group: 'Graphite'
})
.option('graphite.httpPort', {
describe:
'The Graphite port used to access the user interface and send annotations event',
default: 8080,
group: 'Graphite'
})
.option('graphite.webHost', {
describe:
'The graphite-web host. If not specified graphite.host will be used.',
group: 'Graphite'
})
.option('graphite.namespace', {
default: 'sitespeed_io.default',
describe: 'The namespace key added to all captured metrics.',
group: 'Graphite'
})
.option('graphite.includeQueryParams', {
default: false,
describe:
'Whether to include query parameters from the URL in the Graphite keys or not',
type: 'boolean',
group: 'Graphite'
})
.option('graphite.arrayTags', {
default: true,
type: 'boolean',
describe:
'Send the tags as Array or a String. In Graphite 1.0 the tags is a array. Before a String',
group: 'Graphite'
})
.option('graphite.annotationTitle', {
describe: 'Add a title to the annotation sent for a run.',
group: 'Graphite'
})
.option('graphite.annotationMessage', {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Graphite'
})
.option('graphite.annotationScreenshot', {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Graphite'
})
.option('graphite.sendAnnotation', {
default: true,
type: 'boolean',
describe:
'Send annotations when a run is finished. You need to specify a --resultBaseURL for this to work. However if you for example use a Prometheus exporter, you may want to make sure annotations are not sent, then set it to false.',
group: 'Graphite'
})
.option('graphite.annotationRetentionMinutes', {
type: 'number',
describe:
'The retention in minutes, to make annotation match the retention in Graphite.',
group: 'Graphite'
})
.option('graphite.statsd', {
default: false,
type: 'boolean',
describe: 'Uses the StatsD interface',
group: 'Graphite'
})
.option('graphite.annotationTag', {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --graphite.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Graphite'
})
.option('graphite.addSlugToKey', {
default: true,
type: 'boolean',
describe:
'Add the slug (name of the test) as an extra key in the namespace.',
group: 'Graphite'
})
.option('graphite.bulkSize', {
default: undefined,
type: 'number',
describe: 'Break up number of metrics to send with each request.',
group: 'Graphite'
})
.option('graphite.messages', {
default: ['pageSummary', 'summary'],
options: ['pageSummary', 'summary', 'run'],
group: 'Graphite'
})
.option('influxdb.protocol', {
describe: 'The protocol used to store connect to the InfluxDB host.',
default: 'http',
group: 'InfluxDB'
})
.option('influxdb.host', {
describe: 'The InfluxDB host used to store captured metrics.',
group: 'InfluxDB'
})
.option('influxdb.port', {
default: 8086,
describe: 'The InfluxDB port used to store captured metrics.',
group: 'InfluxDB'
})
.option('influxdb.username', {
describe: 'The InfluxDB username for your InfluxDB instance.',
group: 'InfluxDB'
})
.option('influxdb.password', {
describe: 'The InfluxDB password for your InfluxDB instance.',
group: 'InfluxDB'
})
.option('influxdb.database', {
default: 'sitespeed',
describe: 'The database name used to store captured metrics.',
group: 'InfluxDB'
})
.option('influxdb.tags', {
default: 'category=default',
describe:
'A comma separated list of tags and values added to each metric',
group: 'InfluxDB'
})
.option('influxdb.includeQueryParams', {
default: false,
describe:
'Whether to include query parameters from the URL in the InfluxDB keys or not',
type: 'boolean',
group: 'InfluxDB'
})
.option('influxdb.groupSeparator', {
default: '_',
describe:
'Choose which character that will separate a group/domain. Default is underscore, set it to a dot if you wanna keep the original domain name.',
group: 'InfluxDB'
})
.option('influxdb.annotationScreenshot', {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'InfluxDB'
});
// Grafana CLI options
cliUtil.registerPluginOptions(parsed, grafanaPlugin);
// Graphite CLI options
cliUtil.registerPluginOptions(parsed, graphitePlugin);
parsed
/** Plugins */
.option('plugins.list', {
@ -1244,7 +1426,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
/**
InfluxDB cli option
*/
cliUtil.registerPluginOptions(parsed, influxdbPlugin);
parsed
// Metrics
@ -1267,6 +1448,36 @@ module.exports.parseCommandLine = function parseCommandLine() {
'Add/change/remove filters for metrics. If you want to send all metrics, use: *+ . If you want to remove all current metrics and send only the coach score: *- coach.summary.score.*',
group: 'Metrics'
})
.option('matrix.host', {
describe: 'The Matrix host.',
group: 'Matrix'
})
.option('matrix.accessToken', {
describe: 'The Matrix access token.',
group: 'Matrix'
})
.option('matrix.room', {
describe:
'The default Matrix room. It is alsways used. You can override the room per message type using --matrix.rooms',
group: 'Matrix'
})
.option('matrix.messages', {
describe:
'Choose what type of message to send to Matrix. There are two types of messages: Error messages and budget messages. Errors are errors that happens through the tests (failures like strarting a test) and budget is test failing against your budget.',
choices: matrixMessageTypes(),
default: matrixMessageTypes(),
group: 'Matrix'
})
.option('matrix.rooms', {
describe:
'Send messages to different rooms. Current message types are [' +
matrixMessageTypes +
']. If you want to send error messages to a specific room use --matrix.rooms.error ROOM',
group: 'Matrix'
})
/**
Slack options
*/
@ -1411,6 +1622,34 @@ module.exports.parseCommandLine = function parseCommandLine() {
type: 'boolean',
group: 'GoogleCloudStorage'
})
.option('crux.key', {
describe:
'You need to use a key to get data from CrUx. Get the key from https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/getting-started#APIKey',
group: 'CrUx'
})
.option('crux.enable', {
default: true,
describe:
'Enable the CrUx plugin. This is on by defauly but you also need the Crux key. If you chose to disable it with this key, set this to false and you can still use the CrUx key in your configuration.',
group: 'CrUx'
})
.option('crux.formFactor', {
default: 'ALL',
type: 'string',
choices: ['ALL', 'DESKTOP', 'PHONE', 'TABLET'],
describe:
'A form factor is the type of device on which a user visits a website.',
group: 'CrUx'
})
.option('crux.collect', {
default: 'ALL',
type: 'string',
choices: ['ALL', 'URL', 'ORIGIN'],
describe:
'Choose what data to collect. URL is data for a specific URL, ORIGIN for the domain and ALL for both of them',
group: 'CrUx'
})
/**
Html options
*/
@ -1517,8 +1756,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
'Instead of using the local copy of the hosting database, you can use the latest version through the Green Web Foundation API. This means sitespeed.io will make HTTP GET to the the hosting info.',
group: 'Sustainable'
});
cliUtil.registerPluginOptions(parsed, cruxPlugin);
cliUtil.registerPluginOptions(parsed, matrixPlugin);
parsed
.option('mobile', {
describe:
@ -1600,17 +1837,19 @@ module.exports.parseCommandLine = function parseCommandLine() {
.alias('help', 'h')
.config(config)
.alias('version', 'V')
.coerce('budget', function (arg) {
if (arg) {
if (typeof arg === 'object' && !Array.isArray(arg)) {
if (arg.configPath) {
arg.config = JSON.parse(fs.readFileSync(arg.configPath, 'utf8'));
} else if (arg.config) {
arg.config = JSON.parse(arg.config);
.coerce('budget', function (argument) {
if (argument) {
if (typeof argument === 'object' && !Array.isArray(argument)) {
if (argument.configPath) {
argument.config = JSON.parse(
readFileSync(argument.configPath, 'utf8')
);
} else if (argument.config) {
argument.config = JSON.parse(argument.config);
}
return arg;
return argument;
} else {
throw new Error(
throw new TypeError(
'[ERROR] Something looks wrong with your budget configuration. Since sitespeed.io 4.4 you should pass the path to your budget file through the --budget.configPath flag instead of directly through the --budget flag.'
);
}
@ -1646,30 +1885,30 @@ module.exports.parseCommandLine = function parseCommandLine() {
return plugins;
}
})
.coerce('webpagetest', function (arg) {
if (arg) {
.coerce('webpagetest', function (argument) {
if (argument) {
// for backwards compatible reasons we check if the passed parameters is a path to a script, if so just us it (PR #1445)
if (arg.script && fs.existsSync(arg.script)) {
arg.script = fs.readFileSync(path.resolve(arg.script), 'utf8');
if (argument.script && existsSync(argument.script)) {
argument.script = readFileSync(resolve(argument.script), 'utf8');
/* eslint no-console: off */
console.log(
'[WARNING] Since sitespeed.io 4.4 you should pass the path to the script file through the --webpagetest.file flag (https://github.com/sitespeedio/sitespeed.io/pull/1445).'
);
return arg;
return argument;
}
if (arg.file) {
arg.script = fs.readFileSync(path.resolve(arg.file), 'utf8');
} else if (arg.script) {
if (argument.file) {
argument.script = readFileSync(resolve(argument.file), 'utf8');
} else if (argument.script) {
// because the escaped characters are passed re-escaped from the console
arg.script = arg.script.split('\\t').join('\t');
arg.script = arg.script.split('\\n').join('\n');
argument.script = argument.script.split('\\t').join('\t');
argument.script = argument.script.split('\\n').join('\n');
}
return arg;
return argument;
}
})
// .describe('browser', 'Specify browser')
.wrap(yargs.terminalWidth())
.wrap(yargsInstance.terminalWidth())
// .check(validateInput)
.epilog(
'Read the docs at https://www.sitespeed.io/documentation/sitespeed.io/'
@ -1693,8 +1932,12 @@ module.exports.parseCommandLine = function parseCommandLine() {
new Map()
);
let explicitOptions = require('yargs/yargs')(process.argv.slice(2)).argv;
explicitOptions = merge(explicitOptions, yargs.getOptions().configObjects[0]);
let explicitOptions = yargs(hideBin(process.argv)).argv;
explicitOptions = merge(
explicitOptions,
yargsInstance.getOptions().configObjects[0]
);
explicitOptions = reduce(
explicitOptions,
@ -1710,17 +1953,14 @@ module.exports.parseCommandLine = function parseCommandLine() {
);
if (argv.config) {
const config = require(path.resolve(process.cwd(), argv.config));
const config = await import(resolve(process.cwd(), argv.config));
explicitOptions = merge(explicitOptions, config);
}
if (argv.webpagetest && argv.webpagetest.custom) {
argv.webpagetest.custom = fs.readFileSync(
path.resolve(argv.webpagetest.custom),
{
encoding: 'utf8'
}
);
argv.webpagetest.custom = readFileSync(resolve(argv.webpagetest.custom), {
encoding: 'utf8'
});
}
if (argv.summaryDetail) argv.summary = true;
@ -1750,15 +1990,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (argv.ios) {
set(argv, 'safari.ios', true);
} else if (argv.android) {
if (argv.browser === 'chrome') {
// Default to Chrome Android.
set(
argv,
'browsertime.chrome.android.package',
get(argv, 'browsertime.chrome.android.package', 'com.android.chrome')
);
}
} else if (argv.android && argv.browser === 'chrome') {
// Default to Chrome Android.
set(
argv,
'browsertime.chrome.android.package',
get(argv, 'browsertime.chrome.android.package', 'com.android.chrome')
);
}
// Always use hash by default when you configure spa
@ -1767,20 +2005,19 @@ module.exports.parseCommandLine = function parseCommandLine() {
set(argv, 'browsertime.useHash', true);
}
if (argv.cpu) {
if (
argv.browsertime.browser === 'chrome' ||
argv.browsertime.browser === 'edge'
) {
set(argv, 'browsertime.chrome.collectLongTasks', true);
set(argv, 'browsertime.chrome.timeline', true);
}
// Enable when we know how much overhead it will add
/*
if (
argv.cpu &&
(argv.browsertime.browser === 'chrome' ||
argv.browsertime.browser === 'edge')
) {
set(argv, 'browsertime.chrome.collectLongTasks', true);
set(argv, 'browsertime.chrome.timeline', true);
}
// Enable when we know how much overhead it will add
/*
else if (argv.browsertime.browser === 'firefox') {
set(argv, 'browsertime.firefox.geckoProfiler', true);
}*/
}
// we missed to populate this to Browsertime in the cli
// so to stay backward compatible we do it manually
@ -1800,7 +2037,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (argv.browsertime.safari && argv.browsertime.safari.useSimulator) {
set(argv, 'browsertime.connectivity.engine', 'throttle');
} else if (
(os.platform() === 'darwin' || os.platform() === 'linux') &&
(platform() === 'darwin' || platform() === 'linux') &&
!argv.browsertime.android &&
!argv.browsertime.safari.ios &&
!argv.browsertime.docker &&
@ -1823,7 +2060,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
);
}
let urlsMetaData = cliUtil.getAliases(argv._, argv.urlAlias, argv.groupAlias);
let urlsMetaData = getAliases(argv._, argv.urlAlias, argv.groupAlias);
// Copy the alias so it is also used by Browsertime
if (argv.urlAlias) {
// Browsertime has it own way of handling alias
@ -1833,8 +2070,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (!Array.isArray(argv.urlAlias)) argv.urlAlias = [argv.urlAlias];
for (let i = 0; i < urls.length; i++) {
meta[urls[i]] = argv.urlAlias[i];
for (const [index, url] of urls.entries()) {
meta[url] = argv.urlAlias[index];
}
set(argv, 'browsertime.urlMetaData', meta);
} else if (Object.keys(urlsMetaData).length > 0) {
@ -1847,15 +2084,15 @@ module.exports.parseCommandLine = function parseCommandLine() {
// Set the timeouts to a maximum while debugging
if (argv.debug) {
set(argv, 'browsertime.timeouts.pageload', 2147483647);
set(argv, 'browsertime.timeouts.script', 2147483647);
set(argv, 'browsertime.timeouts.pageCompleteCheck', 2147483647);
set(argv, 'browsertime.timeouts.pageload', 2_147_483_647);
set(argv, 'browsertime.timeouts.script', 2_147_483_647);
set(argv, 'browsertime.timeouts.pageCompleteCheck', 2_147_483_647);
}
return {
urls: argv.multi ? argv._ : cliUtil.getURLs(argv._),
urls: argv.multi ? argv._ : getURLs(argv._),
urlsMetaData,
options: argv,
explicitOptions: explicitOptions
};
};
}

233
lib/cli/util.js
View File

@ -1,11 +1,10 @@
'use strict';
/*eslint no-console: 0*/
const fs = require('fs');
const path = require('path');
const toArray = require('../support/util').toArray;
const format = require('util').format;
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { format } from 'node:util';
import { toArray } from '../support/util.js';
/**
*
@ -18,7 +17,7 @@ function sanitizePluginOptions(options) {
const isValidType =
typeof cliOptions === 'object' &&
cliOptions !== null &&
cliOptions !== undefined &&
cliOptions.constructor === Object;
if (!isValidType) {
@ -29,130 +28,112 @@ function sanitizePluginOptions(options) {
return cliOptions;
}
module.exports = {
getURLs(urls) {
const allUrls = [];
urls = urls.map(url => url.trim());
export function getURLs(urls) {
const allUrls = [];
urls = urls.map(url => url.trim());
for (let url of urls) {
if (url.startsWith('http')) {
allUrls.push(url);
} else {
const filePath = path.resolve(url);
try {
const lines = fs.readFileSync(filePath).toString().split('\n');
for (let line of lines) {
if (line.trim().length > 0) {
let lineArray = line.split(' ', 2);
let url = lineArray[0].trim();
if (url) {
if (url.startsWith('http')) {
allUrls.push(url);
} else if (url.startsWith('module.exports')) {
// This looks like someone is trying to run a script without adding the --multi parameter
// For now just write to the log and in the future we can maybe automatically fix it
console.error(
'Please use --multi if you want to run scripts. See https://www.sitespeed.io/documentation/sitespeed.io/scripting/#run'
);
} else {
// We use skip adding it
}
for (let url of urls) {
if (url.startsWith('http')) {
allUrls.push(url);
} else {
const filePath = resolve(url);
try {
const lines = readFileSync(filePath).toString().split('\n');
for (let line of lines) {
if (line.trim().length > 0) {
let lineArray = line.split(' ', 2);
let url = lineArray[0].trim();
if (url) {
if (url.startsWith('http')) {
allUrls.push(url);
} else if (url.startsWith('module.exports')) {
// This looks like someone is trying to run a script without adding the --multi parameter
// For now just write to the log and in the future we can maybe automatically fix it
console.error(
'Please use --multi if you want to run scripts. See https://www.sitespeed.io/documentation/sitespeed.io/scripting/#run'
);
} else {
// do nada
}
}
}
} catch (e) {
if (e.code === 'ENOENT') {
throw new Error(`Couldn't find url file at ${filePath}`);
}
throw e;
}
} catch (error) {
if (error.code === 'ENOENT') {
throw new Error(`Couldn't find url file at ${filePath}`);
}
throw error;
}
}
return allUrls;
},
getAliases(urls, alias, groupAlias) {
const urlMetaData = {};
urls = urls.map(url => url.trim());
let al = toArray(alias);
let allGroupAlias = toArray(groupAlias);
let pos = 0;
for (let url of urls) {
if (url.startsWith('http')) {
if (al.length > 0 && al[pos]) {
urlMetaData[url] = { urlAlias: al[pos] };
}
if (allGroupAlias.length > 0 && allGroupAlias[pos]) {
urlMetaData[url] = { groupAlias: allGroupAlias[pos] };
}
pos += 1;
} else {
const filePath = url;
const lines = fs.readFileSync(filePath).toString().split('\n');
for (let line of lines) {
if (line.trim().length > 0) {
let url,
alias,
groupAlias = null;
let lineArray = line.split(' ', 3);
url = lineArray[0].trim();
if (lineArray[1]) {
alias = lineArray[1].trim();
}
if (lineArray[2]) {
groupAlias = lineArray[2].trim();
}
if (url && alias) {
urlMetaData[url] = { urlAlias: alias };
}
if (url && groupAlias) {
urlMetaData[url].groupAlias = groupAlias;
}
}
}
}
}
return urlMetaData;
},
/**
* Retrieve a mapping of option names and their corressponding default values based on a plugin CLI configuration
*
* @param {Object<string, require('yargs').Options>} cliOptions a map of option names: yargs option
*/
pluginDefaults(cliOptions) {
let config = {};
try {
Object.entries(sanitizePluginOptions(cliOptions)).forEach(values => {
const [key, options] = values;
if (typeof options.default !== 'undefined') {
config[key] = options.default;
}
});
} catch (err) {
// In case of invalid values, just assume an empty object.
config = {};
}
return config;
},
/**
* Configure yargs options for a given plugin defining a `cliOptions` method or property. The names
* of configuration options are namespaced with the plugin name.
*
* @param {require('yargs').Argv} parsed yargs instance
* @param {object} plugin a sitespeed plugin instance
*/
registerPluginOptions(parsed, plugin) {
if (typeof plugin.name !== 'function' || !plugin.name()) {
throw new Error(
'Missing name() method for plugin registering CLI options'
);
}
const cliOptions = sanitizePluginOptions(plugin.cliOptions);
Object.entries(cliOptions).forEach(value => {
const [key, yargsOptions] = value;
parsed.option(`${plugin.name()}.${key}`, yargsOptions);
});
}
};
return allUrls;
}
export function getAliases(urls, alias, groupAlias) {
const urlMetaData = {};
urls = urls.map(url => url.trim());
let al = toArray(alias);
let allGroupAlias = toArray(groupAlias);
let pos = 0;
for (let url of urls) {
if (url.startsWith('http')) {
if (al.length > 0 && al[pos]) {
urlMetaData[url] = { urlAlias: al[pos] };
}
if (allGroupAlias.length > 0 && allGroupAlias[pos]) {
urlMetaData[url] = { groupAlias: allGroupAlias[pos] };
}
pos += 1;
} else {
const filePath = url;
const lines = readFileSync(filePath).toString().split('\n');
for (let line of lines) {
if (line.trim().length > 0) {
let url, alias, groupAlias;
let lineArray = line.split(' ', 3);
url = lineArray[0].trim();
if (lineArray[1]) {
alias = lineArray[1].trim();
}
if (lineArray[2]) {
groupAlias = lineArray[2].trim();
}
if (url && alias) {
urlMetaData[url] = { urlAlias: alias };
}
if (url && groupAlias) {
urlMetaData[url].groupAlias = groupAlias;
}
}
}
}
}
return urlMetaData;
}
export function pluginDefaults(cliOptions) {
let config = {};
try {
for (const values of Object.entries(sanitizePluginOptions(cliOptions))) {
const [key, options] = values;
if (typeof options.default !== 'undefined') {
config[key] = options.default;
}
}
} catch {
// In case of invalid values, just assume an empty object.
config = {};
}
return config;
}
export function registerPluginOptions(parsed, plugin) {
if (typeof plugin.name !== 'function' || !plugin.getName()) {
throw new Error(
'Missing getName() method for plugin registering CLI options'
);
}
const cliOptions = sanitizePluginOptions(plugin.cliOptions);
for (const value of Object.entries(cliOptions)) {
const [key, yargsOptions] = value;
parsed.option(`${plugin.getName()}.${key}`, yargsOptions);
}
}

52
lib/core/logging.js
View File

@ -1,51 +1,65 @@
'use strict';
import intel from 'intel';
let log = require('intel');
const {
INFO,
DEBUG,
VERBOSE,
TRACE,
NONE,
basicConfig,
addHandler,
handlers,
Formatter
} = intel;
module.exports.configure = function configure(options, logDir) {
export function configure(options, logDir) {
options = options || {};
let level = log.INFO;
let level = INFO;
switch (options.verbose) {
case 1:
level = log.DEBUG;
case 1: {
level = DEBUG;
break;
case 2:
level = log.VERBOSE;
}
case 2: {
level = VERBOSE;
break;
case 3:
level = log.TRACE;
}
case 3: {
level = TRACE;
break;
default:
}
default: {
break;
}
}
if (options.silent) {
level = log.NONE;
level = NONE;
}
if (level === log.INFO) {
log.basicConfig({
if (level === INFO) {
basicConfig({
format: '[%(date)s] %(levelname)s: %(message)s',
level: level
});
} else {
log.basicConfig({
basicConfig({
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
level: level
});
}
if (options.logToFile) {
log.addHandler(
new log.handlers.File({
addHandler(
new handlers.File({
file: logDir + '/sitespeed.io.log',
formatter: new log.Formatter({
formatter: new Formatter({
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
level: level
})
})
);
}
};
}

110
lib/core/pluginLoader.js
View File

@ -1,9 +1,11 @@
'use strict';
import { join, basename, resolve, dirname } from 'node:path';
import { readdir as _readdir } from 'node:fs';
import { promisify } from 'node:util';
const path = require('path');
const fs = require('fs');
const { promisify } = require('util');
const readdir = promisify(fs.readdir);
import pkg from 'import-global';
const { silent } = pkg;
const readdir = promisify(_readdir);
const __dirname = dirname(import.meta.url);
const defaultPlugins = new Set([
'browsertime',
@ -22,60 +24,72 @@ const defaultPlugins = new Set([
'remove'
]);
const pluginsDir = path.join(__dirname, '..', 'plugins');
const pluginsDir = join(__dirname, '..', 'plugins');
module.exports = {
async parsePluginNames(options) {
// There's a problem with Safari on iOS runninhg a big blob
// of JavaScript
// https://github.com/sitespeedio/browsertime/issues/1275
if (options.safari && options.safari.ios) {
defaultPlugins.delete('coach');
export async function parsePluginNames(options) {
// There's a problem with Safari on iOS runninhg a big blob
// of JavaScript
// https://github.com/sitespeedio/browsertime/issues/1275
if (options.safari && options.safari.ios) {
defaultPlugins.delete('coach');
}
// if we don't use the cli, this will work out fine as long
// we configure only what we need
const possibleConfiguredPlugins = options.explicitOptions || options;
const isDefaultOrConfigured = name =>
defaultPlugins.has(name) ||
typeof possibleConfiguredPlugins[name] === 'object';
const addMessageLoggerIfDebug = pluginNames => {
if (options.debugMessages) {
// Need to make sure logger is first, so message logs appear
// before messages are handled by other plugins
pluginNames = ['messagelogger'].concat(pluginNames);
}
return pluginNames;
};
// if we don't use the cli, this will work out fine as long
// we configure only what we need
const possibleConfiguredPlugins = options.explicitOptions || options;
const isDefaultOrConfigured = name =>
defaultPlugins.has(name) ||
typeof possibleConfiguredPlugins[name] === 'object';
const addMessageLoggerIfDebug = pluginNames => {
if (options.debugMessages) {
// Need to make sure logger is first, so message logs appear
// before messages are handled by other plugins
pluginNames = ['messagelogger'].concat(pluginNames);
}
return pluginNames;
};
const files = await readdir(new URL(pluginsDir));
const files = await readdir(pluginsDir);
const builtins = files.map(name => path.basename(name, '.js'));
const plugins = builtins.filter(isDefaultOrConfigured);
return addMessageLoggerIfDebug(plugins);
},
async loadPlugins(pluginNames) {
const plugins = [];
for (let name of pluginNames) {
const builtins = files.map(name => basename(name, '.js'));
// eslint-disable-next-line unicorn/no-array-callback-reference
const plugins = builtins.filter(isDefaultOrConfigured);
return addMessageLoggerIfDebug(plugins);
}
export async function loadPlugins(pluginNames, options, context, queue) {
const plugins = [];
for (let name of pluginNames) {
try {
let { default: plugin } = await import(
join(pluginsDir, name, 'index.js')
);
let p = new plugin(options, context, queue);
plugins.push(p);
} catch (error_) {
try {
const plugin = require(path.join(pluginsDir, name));
if (!plugin.name) {
plugin.name = () => name;
}
plugins.push(plugin);
} catch (err) {
let { default: plugin } = await import(resolve(process.cwd(), name));
let p = new plugin(options, context, queue);
plugins.push(p);
} catch {
try {
plugins.push(require(path.resolve(process.cwd(), name)));
let { default: plugin } = await import(name);
let p = new plugin(options, context, queue);
plugins.push(p);
} catch (error) {
try {
plugins.push(require(name));
} catch (error) {
console.error("Couldn't load plugin %s: %s", name, err); // eslint-disable-line no-console
// try global
let plugin = silent(name);
if (plugin) {
let p = new plugin(options, context, queue);
plugins.push(p);
} else {
console.error("Couldn't load plugin %s: %s", name, error_); // eslint-disable-line no-console
// if it fails here, let it fail hard
throw error;
}
}
}
}
return plugins;
}
};
return plugins;
}

175
lib/core/queueHandler.js
View File

@ -1,13 +1,17 @@
'use strict';
/* eslint no-console:0 */
const cq = require('concurrent-queue');
const log = require('intel').getLogger('sitespeedio.queuehandler');
const messageMaker = require('../support/messageMaker');
const queueStats = require('./queueStatistics');
import cq from 'concurrent-queue';
import intel from 'intel';
import { messageMaker } from '../support/messageMaker.js';
import {
registerQueueTime,
registerProcessingTime,
generateStatistics
} from './queueStatistics.js';
const make = messageMaker('queueHandler').make;
const log = intel.getLogger('sitespeedio.queuehandler');
function shortenData(key, value) {
if (key === 'data') {
@ -16,6 +20,60 @@ function shortenData(key, value) {
return value;
}
function validatePageSummary(message) {
const type = message.type;
if (!type.endsWith('.pageSummary')) return;
if (!message.url)
throw new Error(`Page summary message (${type}) didn't specify a url`);
if (!message.group)
throw new Error(`Page summary message (${type}) didn't specify a group.`);
}
function validateTypeStructure(message) {
const typeParts = message.type.split('.'),
baseType = typeParts[0],
typeDepth = typeParts.length;
if (typeDepth > 2)
throw new Error(
'Message type has too many dot separated sections: ' + message.type
);
const previousDepth = messageTypeDepths[baseType];
if (previousDepth && previousDepth !== typeDepth) {
throw new Error(
`All messages of type ${baseType} must have the same structure. ` +
`${message.type} has ${typeDepth} part(s), but earlier messages had ${previousDepth} part(s).`
);
}
messageTypeDepths[baseType] = typeDepth;
}
function validateSummaryMessage(message) {
const type = message.type;
if (!type.endsWith('.summary')) return;
if (message.url)
throw new Error(
`Summary message (${type}) shouldn't be url specific, use .pageSummary instead.`
);
if (!message.group)
throw new Error(`Summary message (${type}) didn't specify a group.`);
const groups = groupsPerSummaryType[type] || [];
if (groups.includes(message.group)) {
throw new Error(
`Multiple summary messages of type ${type} and group ${message.group}`
);
}
groupsPerSummaryType[type] = groups.concat(message.group);
}
const messageTypeDepths = {};
const groupsPerSummaryType = {};
@ -25,70 +83,18 @@ const groupsPerSummaryType = {};
* @param message the message to check
*/
function validateMessageFormat(message) {
function validateTypeStructure(message) {
const typeParts = message.type.split('.'),
baseType = typeParts[0],
typeDepth = typeParts.length;
if (typeDepth > 2)
throw new Error(
'Message type has too many dot separated sections: ' + message.type
);
const previousDepth = messageTypeDepths[baseType];
if (previousDepth && previousDepth !== typeDepth) {
throw new Error(
`All messages of type ${baseType} must have the same structure. ` +
`${message.type} has ${typeDepth} part(s), but earlier messages had ${previousDepth} part(s).`
);
}
messageTypeDepths[baseType] = typeDepth;
}
function validatePageSummary(message) {
const type = message.type;
if (!type.endsWith('.pageSummary')) return;
if (!message.url)
throw new Error(`Page summary message (${type}) didn't specify a url`);
if (!message.group)
throw new Error(`Page summary message (${type}) didn't specify a group.`);
}
function validateSummaryMessage(message) {
const type = message.type;
if (!type.endsWith('.summary')) return;
if (message.url)
throw new Error(
`Summary message (${type}) shouldn't be url specific, use .pageSummary instead.`
);
if (!message.group)
throw new Error(`Summary message (${type}) didn't specify a group.`);
const groups = groupsPerSummaryType[type] || [];
if (groups.includes(message.group)) {
throw new Error(
`Multiple summary messages of type ${type} and group ${message.group}`
);
}
groupsPerSummaryType[type] = groups.concat(message.group);
}
validateTypeStructure(message);
validatePageSummary(message);
validateSummaryMessage(message);
}
class QueueHandler {
constructor(plugins, options) {
export class QueueHandler {
constructor(options) {
this.options = options;
this.errors = [];
}
setup(plugins) {
this.createQueues(plugins);
}
@ -96,7 +102,7 @@ class QueueHandler {
this.queues = plugins
.filter(plugin => plugin.processMessage)
.map(plugin => {
const concurrency = plugin.concurrency || Infinity;
const concurrency = plugin.concurrency || Number.POSITIVE_INFINITY;
const queue = cq().limit({ concurrency });
queue.plugin = plugin;
@ -104,42 +110,42 @@ class QueueHandler {
const messageWaitingStart = {},
messageProcessingStart = {};
queue.enqueued(obj => {
const message = obj.item;
queue.enqueued(object => {
const message = object.item;
messageWaitingStart[message.uuid] = process.hrtime();
});
queue.processingStarted(obj => {
const message = obj.item;
queue.processingStarted(object => {
const message = object.item;
const waitingDuration = process.hrtime(
messageWaitingStart[message.uuid]
),
waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1];
queueStats.registerQueueTime(message, queue.plugin, waitingNanos);
registerQueueTime(message, queue.plugin, waitingNanos);
messageProcessingStart[message.uuid] = process.hrtime();
});
// FIXME handle rejections (i.e. failures while processing messages) properly
queue.processingEnded(obj => {
const message = obj.item;
const err = obj.err;
if (err) {
queue.processingEnded(object => {
const message = object.item;
const error = object.err;
if (error) {
let rejectionMessage =
'Rejected ' +
JSON.stringify(message, shortenData, 2) +
' for plugin: ' +
plugin.name();
plugin.getName();
if (message && message.url)
rejectionMessage += ', url: ' + message.url;
if (err.stack) {
log.error(err.stack);
if (error.stack) {
log.error(error.stack);
}
this.errors.push(rejectionMessage + '\n' + JSON.stringify(err));
this.errors.push(rejectionMessage + '\n' + JSON.stringify(error));
}
const processingDuration = process.hrtime(
@ -148,11 +154,7 @@ class QueueHandler {
const processingNanos =
processingDuration[0] * 1e9 + processingDuration[1];
queueStats.registerProcessingTime(
message,
queue.plugin,
processingNanos
);
registerProcessingTime(message, queue.plugin, processingNanos);
});
return { plugin, queue };
@ -172,7 +174,7 @@ class QueueHandler {
.then(() => this.drainAllQueues())
.then(async () => {
for (let source of sources) {
await source.findUrls(this);
await source.findUrls(this, this.options);
}
})
.then(() => this.drainAllQueues())
@ -184,7 +186,7 @@ class QueueHandler {
.then(() => this.drainAllQueues())
.then(() => {
if (this.options.queueStats) {
log.info(JSON.stringify(queueStats.generateStatistics(), null, 2));
log.info(JSON.stringify(generateStatistics(), undefined, 2));
}
return this.errors;
});
@ -218,15 +220,12 @@ class QueueHandler {
async drainAllQueues() {
const queues = this.queues;
return new Promise(resolve => {
queues.forEach(item =>
for (const item of queues)
item.queue.drained(() => {
if (queues.every(item => item.queue.isDrained)) {
resolve();
}
})
);
});
});
}
}
module.exports = QueueHandler;

123
lib/core/queueStatistics.js
View File

@ -1,8 +1,7 @@
'use strict';
import get from 'lodash.get';
import set from 'lodash.set';
const stats = require('../support/statsHelpers'),
get = require('lodash.get'),
set = require('lodash.set');
import { pushStats, summarizeStats } from '../support/statsHelpers.js';
const queueTimeByPluginName = {},
queueTimeByMessageType = {},
@ -11,80 +10,58 @@ const queueTimeByPluginName = {},
messageTypes = new Set(),
pluginNames = new Set();
module.exports = {
registerQueueTime(message, plugin, nanos) {
messageTypes.add(message.type);
pluginNames.add(plugin.name());
export function registerQueueTime(message, plugin, nanos) {
messageTypes.add(message.type);
pluginNames.add(plugin.getName());
stats.pushStats(queueTimeByMessageType, message.type, nanos / 1000000);
stats.pushStats(queueTimeByPluginName, plugin.name(), nanos / 1000000);
},
pushStats(queueTimeByMessageType, message.type, nanos / 1_000_000);
pushStats(queueTimeByPluginName, plugin.getName(), nanos / 1_000_000);
}
export function registerProcessingTime(message, plugin, nanos) {
messageTypes.add(message.type);
pluginNames.add(plugin.getName());
registerProcessingTime(message, plugin, nanos) {
messageTypes.add(message.type);
pluginNames.add(plugin.name());
pushStats(processingTimeByMessageType, message.type, nanos / 1_000_000);
pushStats(processingTimeByPluginName, plugin.getName(), nanos / 1_000_000);
}
export function generateStatistics() {
const statOptions = {
percentiles: [0, 100],
includeSum: true
};
stats.pushStats(processingTimeByMessageType, message.type, nanos / 1000000);
stats.pushStats(processingTimeByPluginName, plugin.name(), nanos / 1000000);
},
generateStatistics() {
const statOptions = {
percentiles: [0, 100],
includeSum: true
};
const byPluginName = Array.from(pluginNames).reduce(
(summary, pluginName) => {
set(
summary,
['queueTime', pluginName],
stats.summarizeStats(
get(queueTimeByPluginName, pluginName),
statOptions
)
);
set(
summary,
['processingTime', pluginName],
stats.summarizeStats(
get(processingTimeByPluginName, pluginName),
statOptions
)
);
return summary;
},
{}
const byPluginName = [...pluginNames].reduce((summary, pluginName) => {
set(
summary,
['queueTime', pluginName],
summarizeStats(get(queueTimeByPluginName, pluginName), statOptions)
);
set(
summary,
['processingTime', pluginName],
summarizeStats(get(processingTimeByPluginName, pluginName), statOptions)
);
const byMessageType = Array.from(messageTypes).reduce(
(summary, messageType) => {
set(
summary,
['queueTime', messageType],
stats.summarizeStats(
get(queueTimeByMessageType, messageType),
statOptions
)
);
set(
summary,
['processingTime', messageType],
stats.summarizeStats(
get(processingTimeByMessageType, messageType),
statOptions
)
);
return summary;
}, {});
return summary;
},
{}
const byMessageType = [...messageTypes].reduce((summary, messageType) => {
set(
summary,
['queueTime', messageType],
summarizeStats(get(queueTimeByMessageType, messageType), statOptions)
);
set(
summary,
['processingTime', messageType],
summarizeStats(get(processingTimeByMessageType, messageType), statOptions)
);
return {
byPluginName,
byMessageType
};
}
};
return summary;
}, {});
return {
byPluginName,
byMessageType
};
}

38
lib/core/resultsStorage/index.js
View File

@ -1,54 +1,50 @@
'use strict';
import { parse, format } from 'node:url';
import { basename, resolve, join } from 'node:path';
const urlParser = require('url');
const path = require('path');
const resultUrls = require('./resultUrls');
const storageManager = require('./storageManager');
import { resultUrls } from './resultUrls.js';
import { storageManager } from './storageManager.js';
function getDomainOrFileName(input) {
let domainOrFile = input;
if (domainOrFile.startsWith('http')) {
domainOrFile = urlParser.parse(domainOrFile).hostname;
} else {
domainOrFile = path.basename(domainOrFile).replace(/\./g, '_');
}
domainOrFile = domainOrFile.startsWith('http')
? parse(domainOrFile).hostname
: basename(domainOrFile).replace(/\./g, '_');
return domainOrFile;
}
module.exports = function (input, timestamp, options) {
export function resultsStorage(input, timestamp, options) {
const outputFolder = options.outputFolder;
const resultBaseURL = options.resultBaseURL;
const resultsSubFolders = [];
let storageBasePath;
let storagePathPrefix;
let resultUrl = undefined;
let resultUrl;
if (outputFolder) {
resultsSubFolders.push(path.basename(outputFolder));
storageBasePath = path.resolve(outputFolder);
resultsSubFolders.push(basename(outputFolder));
storageBasePath = resolve(outputFolder);
} else {
resultsSubFolders.push(
options.slug || getDomainOrFileName(input),
timestamp.format('YYYY-MM-DD-HH-mm-ss')
);
storageBasePath = path.resolve('sitespeed-result', ...resultsSubFolders);
storageBasePath = resolve('sitespeed-result', ...resultsSubFolders);
}
// backfill the slug
options.slug = options.slug || getDomainOrFileName(input).replace(/\./g, '_');
storagePathPrefix = path.join(...resultsSubFolders);
storagePathPrefix = join(...resultsSubFolders);
if (resultBaseURL) {
const url = urlParser.parse(resultBaseURL);
resultsSubFolders.unshift(url.pathname.substr(1));
const url = parse(resultBaseURL);
resultsSubFolders.unshift(url.pathname.slice(1));
url.pathname = resultsSubFolders.join('/');
resultUrl = urlParser.format(url);
resultUrl = format(url);
}
return {
storageManager: storageManager(storageBasePath, storagePathPrefix, options),
resultUrls: resultUrls(resultUrl, options)
};
};
}

38
lib/core/resultsStorage/pathToFolder.js
View File

@ -1,23 +1,23 @@
'use strict';
import { parse } from 'node:url';
import { createHash } from 'node:crypto';
const isEmpty = require('lodash.isempty');
const crypto = require('crypto');
const log = require('intel').getLogger('sitespeedio.file');
const urlParser = require('url');
import isEmpty from 'lodash.isempty';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.file');
function toSafeKey(key) {
// U+2013 : EN DASH – as used on https://en.wikipedia.org/wiki/2019–20_coronavirus_pandemic
return key.replace(/[.~ /+|,:?&%–)(]|%7C/g, '-');
return key.replace(/[ %&()+,./:?|~–]|%7C/g, '-');
}
module.exports = function pathFromRootToPageDir(url, options, alias) {
export function pathToFolder(url, options, alias) {
const useHash = options.useHash;
const parsedUrl = urlParser.parse(decodeURIComponent(url));
const parsedUrl = parse(decodeURIComponent(url));
const pathSegments = [];
const urlSegments = [];
pathSegments.push('pages');
pathSegments.push(parsedUrl.hostname.split('.').join('_'));
pathSegments.push('pages', parsedUrl.hostname.split('.').join('_'));
if (options.urlMetaData && options.urlMetaData[url]) {
pathSegments.push(options.urlMetaData[url]);
@ -29,14 +29,14 @@ module.exports = function pathFromRootToPageDir(url, options, alias) {
}
if (useHash && !isEmpty(parsedUrl.hash)) {
const md5 = crypto.createHash('md5'),
hash = md5.update(parsedUrl.hash).digest('hex').substring(0, 8);
const md5 = createHash('md5'),
hash = md5.update(parsedUrl.hash).digest('hex').slice(0, 8);
urlSegments.push('hash-' + hash);
}
if (!isEmpty(parsedUrl.search)) {
const md5 = crypto.createHash('md5'),
hash = md5.update(parsedUrl.search).digest('hex').substring(0, 8);
const md5 = createHash('md5'),
hash = md5.update(parsedUrl.search).digest('hex').slice(0, 8);
urlSegments.push('query-' + hash);
}
@ -49,7 +49,7 @@ module.exports = function pathFromRootToPageDir(url, options, alias) {
log.info(
`The URL ${url} hit the 255 character limit used when stored on disk, you may want to give your URL an alias to make sure it will not collide with other URLs.`
);
pathSegments.push(folder.substr(0, 254));
pathSegments.push(folder.slice(0, 254));
} else {
pathSegments.push(folder);
}
@ -58,11 +58,11 @@ module.exports = function pathFromRootToPageDir(url, options, alias) {
// pathSegments.push('data');
pathSegments.forEach(function (segment, index) {
for (const [index, segment] of pathSegments.entries()) {
if (segment) {
pathSegments[index] = segment.replace(/[^-a-z0-9_.\u0621-\u064A]/gi, '-');
pathSegments[index] = segment.replace(/[^\w.\u0621-\u064A-]/gi, '-');
}
});
}
return pathSegments.join('/').concat('/');
};
}

13
lib/core/resultsStorage/resultUrls.js
View File

@ -1,17 +1,16 @@
'use strict';
import { parse, format } from 'node:url';
const urlParser = require('url');
const pathToFolder = require('./pathToFolder');
import { pathToFolder } from './pathToFolder.js';
function getPageUrl({ url, resultBaseUrl, options, alias }) {
const pageUrl = urlParser.parse(resultBaseUrl);
const pageUrl = parse(resultBaseUrl);
pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, options, alias)].join(
'/'
);
return urlParser.format(pageUrl);
return format(pageUrl);
}
module.exports = function resultUrls(resultBaseUrl, options) {
export function resultUrls(resultBaseUrl, options) {
return {
hasBaseUrl() {
return !!resultBaseUrl;
@ -30,4 +29,4 @@ module.exports = function resultUrls(resultBaseUrl, options) {
return pathToFolder(url, options, alias);
}
};
};
}

73
lib/core/resultsStorage/storageManager.js
View File

@ -1,38 +1,51 @@
'use strict';
import { join } from 'node:path';
import { promisify } from 'node:util';
import {
rmdir as _rmdir,
mkdir as _mkdir,
lstat as _lstat,
readdir as _readdir,
unlink as _unlink,
writeFile as _writeFile
} from 'node:fs';
const fs = require('fs-extra');
const path = require('path');
const log = require('intel').getLogger('sitespeedio.storageManager');
const { promisify } = require('util');
const mkdir = promisify(fs.mkdir);
const readdir = promisify(fs.readdir);
const lstat = promisify(fs.lstat);
const unlink = promisify(fs.unlink);
const rmdir = promisify(fs.rmdir);
const pathToFolder = require('./pathToFolder');
import { copy } from 'fs-extra/esm';
import intel from 'intel';
import { pathToFolder } from './pathToFolder.js';
const log = intel.getLogger('sitespeedio.storageManager');
const mkdir = promisify(_mkdir);
const readdir = promisify(_readdir);
const lstat = promisify(_lstat);
const unlink = promisify(_unlink);
const rmdir = promisify(_rmdir);
const writeFile = promisify(_writeFile);
function write(dirPath, filename, data) {
return fs.writeFile(path.join(dirPath, filename), data);
return writeFile(join(dirPath, filename), data);
}
function isValidDirectoryName(name) {
return name !== undefined && name !== '';
}
module.exports = function storageManager(baseDir, storagePathPrefix, options) {
export function storageManager(baseDir, storagePathPrefix, options) {
return {
rootPathFromUrl(url, alias) {
return pathToFolder(url, options, alias)
.split('/')
.filter(isValidDirectoryName)
.filter(element => isValidDirectoryName(element))
.map(() => '..')
.join('/')
.concat('/');
},
createDirectory(...subDirs) {
const pathSegments = [baseDir, ...subDirs].filter(isValidDirectoryName);
createDirectory(...subDirectories) {
const pathSegments = [baseDir, ...subDirectories].filter(element =>
isValidDirectoryName(element)
);
const dirPath = path.join.apply(null, pathSegments);
const dirPath = join.apply(undefined, pathSegments);
return mkdir(dirPath, { recursive: true }).then(() => dirPath);
},
writeData(data, filename) {
@ -50,41 +63,37 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
return baseDir;
},
getFullPathToURLDir(url, alias) {
return path.join(baseDir, pathToFolder(url, options, alias));
return join(baseDir, pathToFolder(url, options, alias));
},
getStoragePrefix() {
return storagePathPrefix;
},
copyToResultDir(filename) {
return this.createDirectory().then(dir => fs.copy(filename, dir));
return this.createDirectory().then(dir => copy(filename, dir));
},
copyFileToDir(filename, dir) {
return fs.copy(filename, dir);
return copy(filename, dir);
},
// TODO is missing alias
removeDataForUrl(url) {
const dirName = path.join(baseDir, pathToFolder(url, options));
const dirName = join(baseDir, pathToFolder(url, options));
const removeDir = async dir => {
try {
const files = await readdir(dir);
await Promise.all(
files.map(async file => {
try {
const p = path.join(dir, file);
const p = join(dir, file);
const stat = await lstat(p);
if (stat.isDirectory()) {
await removeDir(p);
} else {
await unlink(p);
}
} catch (err) {
log.error('Could not remove file:' + file, err);
await (stat.isDirectory() ? removeDir(p) : unlink(p));
} catch (error) {
log.error('Could not remove file:' + file, error);
}
})
);
await rmdir(dir);
} catch (err) {
log.error('Could not remove dir:' + dir, err);
} catch (error) {
log.error('Could not remove dir:' + dir, error);
}
};
return removeDir(dirName);
@ -105,4 +114,4 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
);
}
};
};
}

18
lib/core/script-source.js
View File

@ -1,15 +1,9 @@
'use strict';
const messageMaker = require('../support/messageMaker');
import { messageMaker } from '../support/messageMaker.js';
const make = messageMaker('script-reader').make;
module.exports = {
open(context, options) {
this.options = options;
},
findUrls(queue) {
queue.postMessage(
make('browsertime.navigationScripts', {}, { url: this.options.urls })
);
}
};
export function findUrls(queue, options) {
queue.postMessage(
make('browsertime.navigationScripts', {}, { url: options.urls })
);
}

47
lib/core/url-source.js
View File

@ -1,30 +1,23 @@
'use strict';
const urlParser = require('url');
const messageMaker = require('../support/messageMaker');
import { parse } from 'node:url';
import { messageMaker } from '../support/messageMaker.js';
const make = messageMaker('url-reader').make;
module.exports = {
open(context, options) {
this.options = options;
},
findUrls(queue) {
for (const url of this.options.urls) {
queue.postMessage(
make(
'url',
{},
{
url: url,
group:
this.options.urlsMetaData &&
this.options.urlsMetaData[url] &&
this.options.urlsMetaData[url].groupAlias
? this.options.urlsMetaData[url].groupAlias
: urlParser.parse(url).hostname
}
)
);
}
export function findUrls(queue, options) {
for (const url of options.urls) {
queue.postMessage(
make(
'url',
{},
{
url: url,
group:
options.urlsMetaData &&
options.urlsMetaData[url] &&
options.urlsMetaData[url].groupAlias
? options.urlsMetaData[url].groupAlias
: parse(url).hostname
}
)
);
}
};
}

80
lib/plugins/analysisstorer/index.js
View File

@ -1,48 +1,50 @@
'use strict';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
function shouldIgnoreMessage(message) {
return (
[
'url',
'browsertime.navigationScripts',
'error',
'sitespeedio.summarize',
'sitespeedio.prepareToRender',
'sitespeedio.render',
'html.finished',
'axe.setup',
'browsertime.har',
'browsertime.config',
'browsertime.setup',
'browsertime.scripts',
'browsertime.asyncscripts',
'sitespeedio.setup',
'webpagetest.har',
'webpagetest.setup',
'aggregateassets.summary',
'slowestassets.summary',
'largestassets.summary',
'budget.addMessageType',
'html.css',
'html.pug',
's3.finished',
'scp.finished',
'gcs.finished',
'ftp.finished',
'graphite.setup',
'influxdb.setup',
'grafana.setup',
'sustainable.setup',
'scp.setup'
].indexOf(message.type) >= 0
);
return [
'url',
'browsertime.navigationScripts',
'error',
'sitespeedio.summarize',
'sitespeedio.prepareToRender',
'sitespeedio.render',
'html.finished',
'axe.setup',
'browsertime.har',
'browsertime.config',
'browsertime.setup',
'browsertime.scripts',
'browsertime.asyncscripts',
'sitespeedio.setup',
'webpagetest.har',
'webpagetest.setup',
'aggregateassets.summary',
'slowestassets.summary',
'largestassets.summary',
'budget.addMessageType',
'html.css',
'html.pug',
's3.finished',
'scp.finished',
'gcs.finished',
'ftp.finished',
'graphite.setup',
'influxdb.setup',
'grafana.setup',
'sustainable.setup',
'scp.setup'
].includes(message.type);
}
module.exports = {
export default class AnalysisstorerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'analysisstorer', options, context, queue });
}
open(context) {
this.storageManager = context.storageManager;
this.alias = {};
},
}
processMessage(message) {
if (shouldIgnoreMessage(message)) {
return;
@ -75,4 +77,4 @@ module.exports = {
return this.storageManager.writeData(jsonData, fileName);
}
}
};
}

42
lib/plugins/assets/aggregator.js
View File

@ -1,17 +1,18 @@
'use strict';
import get from 'lodash.get';
import { AssetsBySize } from './assetsBySize.js';
import { AssetsBySpeed } from './assetsBySpeed.js';
let AssetsBySize = require('./assetsBySize'),
get = require('lodash.get'),
AssetsBySpeed = require('./assetsBySpeed');
export class AssetsAggregator {
constructor() {
this.assets = {};
this.groups = {};
this.largestAssets = {};
this.largestAssetsByGroup = {};
this.slowestAssetsByGroup = {};
this.largestThirdPartyAssetsByGroup = {};
this.slowestThirdPartyAssetsByGroup = {};
}
module.exports = {
assets: {},
groups: {},
largestAssets: {},
largestAssetsByGroup: {},
slowestAssetsByGroup: {},
largestThirdPartyAssetsByGroup: {},
slowestThirdPartyAssetsByGroup: {},
addToAggregate(data, group, url, resultUrls, runIndex, options, alias) {
const maxSize = get(options, 'html.topListSize', 10);
let page = resultUrls.relativeSummaryPageUrl(url, alias[url]);
@ -53,13 +54,11 @@ module.exports = {
const url = asset.url;
if (options.firstParty) {
if (!url.match(options.firstParty)) {
this.slowestAssetsThirdParty.add(asset, page, runPage);
this.largestAssetsThirdParty.add(asset, page, runPage);
this.slowestThirdPartyAssetsByGroup[group].add(asset, page, runPage);
this.largestThirdPartyAssetsByGroup[group].add(asset, page, runPage);
}
if (options.firstParty && !options.firstParty.test(url)) {
this.slowestAssetsThirdParty.add(asset, page, runPage);
this.largestAssetsThirdParty.add(asset, page, runPage);
this.slowestThirdPartyAssetsByGroup[group].add(asset, page, runPage);
this.largestThirdPartyAssetsByGroup[group].add(asset, page, runPage);
}
const urlInfo = this.assets[url] || {
@ -91,8 +90,7 @@ module.exports = {
urlInfoGroup.requestCount++;
this.groups[group][url] = urlInfoGroup;
}
},
}
summarize() {
const summary = {
groups: {
@ -134,4 +132,4 @@ module.exports = {
return summary;
}
};
}

6
lib/plugins/assets/assetsBySize.js
View File

@ -1,6 +1,4 @@
'use strict';
class AssetsBySize {
export class AssetsBySize {
constructor(maxSize) {
this.maxSize = maxSize;
this.items = [];
@ -42,5 +40,3 @@ class AssetsBySize {
return this.items;
}
}
module.exports = AssetsBySize;

6
lib/plugins/assets/assetsBySpeed.js
View File

@ -1,6 +1,4 @@
'use strict';
class AssetsBySpeed {
export class AssetsBySpeed {
constructor(maxSize) {
this.maxSize = maxSize;
this.items = [];
@ -52,5 +50,3 @@ class AssetsBySpeed {
return this.items;
}
}
module.exports = AssetsBySpeed;

22
lib/plugins/assets/index.js
View File

@ -1,15 +1,19 @@
'use strict';
const isEmpty = require('lodash.isempty');
const aggregator = require('./aggregator');
import isEmpty from 'lodash.isempty';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { AssetsAggregator } from './aggregator.js';
const DEFAULT_METRICS_LARGEST_ASSETS = ['image.0.transferSize'];
module.exports = {
export default class AssetsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'assets', options, context, queue });
}
open(context, options) {
this.make = context.messageMaker('assets').make;
this.options = options;
this.alias = {};
this.resultUrls = context.resultUrls;
this.assetsAggregator = new AssetsAggregator();
context.filterRegistry.registerFilterForType(
DEFAULT_METRICS_LARGEST_ASSETS,
'largestassets.summary'
@ -24,12 +28,12 @@ module.exports = {
[],
'largestthirdpartyassets.summary'
);
},
}
processMessage(message, queue) {
const make = this.make;
switch (message.type) {
case 'pagexray.run': {
aggregator.addToAggregate(
this.assetsAggregator.addToAggregate(
message.data,
message.group,
message.url,
@ -46,7 +50,7 @@ module.exports = {
break;
}
case 'sitespeedio.summarize': {
const summary = aggregator.summarize();
const summary = this.assetsAggregator.summarize();
if (!isEmpty(summary)) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(
@ -87,4 +91,4 @@ module.exports = {
}
}
}
};
}

5
lib/plugins/axe/axePostScript.cjs
View File

@ -36,10 +36,11 @@ module.exports = async function (context) {
);
// Use the extras field in Browsertime and pass on the result
context.result[context.result.length - 1].extras.axe = result;
} catch (e) {
} catch (error) {
context.log.error(
'Could not run the AXE script, no AXE information collected',
e
error
);
}
};

29
lib/plugins/axe/index.js
View File

@ -1,25 +1,30 @@
'use strict';
const path = require('path');
const fs = require('fs');
const log = require('intel').getLogger('sitespeedio.plugin.axe');
import { resolve } from 'node:path';
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = intel.getLogger('sitespeedio.plugin.axe');
const __dirname = fileURLToPath(new URL('.', import.meta.url));
export default class AxePlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'axe', options, context, queue });
}
module.exports = {
open(context, options) {
this.options = options;
this.make = context.messageMaker('axe').make;
this.pug = fs.readFileSync(
path.resolve(__dirname, 'pug', 'index.pug'),
'utf8'
);
this.pug = readFileSync(resolve(__dirname, 'pug', 'index.pug'), 'utf8');
log.info('Axe plugin activated');
},
}
processMessage(message, queue) {
const make = this.make;
switch (message.type) {
case 'browsertime.setup': {
queue.postMessage(
make('browsertime.config', {
postURLScript: path.resolve(__dirname, 'axePostScript.cjs')
postURLScript: resolve(__dirname, 'axePostScript.cjs')
})
);
break;
@ -53,4 +58,4 @@ module.exports = {
}
}
}
};
}

166
lib/plugins/browsertime/analyzer.js
View File

@ -1,12 +1,12 @@
'use strict';
const merge = require('lodash.merge');
const forEach = require('lodash.foreach');
const path = require('path');
const set = require('lodash.set');
const get = require('lodash.get');
const coach = require('coach-core');
const log = require('intel').getLogger('plugin.browsertime');
import { resolve } from 'node:path';
import merge from 'lodash.merge';
import forEach from 'lodash.foreach';
import set from 'lodash.set';
import get from 'lodash.get';
import coach from 'coach-core';
const { getDomAdvice } = coach;
import intel from 'intel';
const log = intel.getLogger('plugin.browsertime');
const defaultBrowsertimeOptions = {
statistics: true
@ -52,28 +52,26 @@ async function preWarmServer(urls, options, scriptOrMultiple) {
}
}
const {BrowsertimeEngine} = await import ('browsertime');
const engine = new BrowsertimEngine(preWarmOptions);
const { BrowsertimeEngine } = await import('browsertime');
const engine = new BrowsertimeEngine(preWarmOptions);
await engine.start();
log.info('Start pre-testing/warming' + urls);
if (scriptOrMultiple) {
await engine.runMultiple(urls, {});
} else {
await engine.run(urls, {});
}
await (scriptOrMultiple
? engine.runMultiple(urls, {})
: engine.run(urls, {}));
await engine.stop();
log.info('Pre-testing done, closed the browser.');
return delay(options.preWarmServerWaitTime || 5000);
}
async function parseUserScripts(scripts) {
const {browserScripts} = await import ('browsertime');
const { browserScripts } = await import('browsertime');
if (!Array.isArray(scripts)) scripts = [scripts];
const allUserScripts = {};
for (let script of scripts) {
let myScript = await browserScripts.findAndParseScripts(
path.resolve(script),
resolve(script),
'custom'
);
if (!myScript['custom']) {
@ -85,7 +83,7 @@ async function parseUserScripts(scripts) {
}
async function addCoachScripts(scripts) {
const coachAdvice = await coach.getDomAdvice();
const coachAdvice = await getDomAdvice();
scripts.coach = {
coachAdvice: coachAdvice
};
@ -115,74 +113,68 @@ function setupAsynScripts(asyncScripts) {
return allAsyncScripts;
}
module.exports = {
async analyzeUrl(
url,
scriptOrMultiple,
pluginScripts,
pluginAsyncScripts,
options
) {
const btOptions = merge({}, defaultBrowsertimeOptions, options);
export async function analyzeUrl(
url,
scriptOrMultiple,
pluginScripts,
pluginAsyncScripts,
options
) {
const btOptions = merge({}, defaultBrowsertimeOptions, options);
// set mobile options
if (options.mobile) {
btOptions.viewPort = '360x640';
if (btOptions.browser === 'chrome' || btOptions.browser === 'edge') {
const emulation = get(
btOptions,
'chrome.mobileEmulation.deviceName',
'Moto G4'
);
btOptions.chrome.mobileEmulation = {
deviceName: emulation
};
} else {
btOptions.userAgent = iphone6UserAgent;
}
}
const {BrowsertimeEngine, browserScripts } = await import ('browsertime');
const scriptCategories = await browserScripts.allScriptCategories();
let scriptsByCategory = await browserScripts.getScriptsForCategories(
scriptCategories
);
if (btOptions.script) {
const userScripts = await parseUserScripts(btOptions.script);
scriptsByCategory = merge(scriptsByCategory, userScripts);
}
if (btOptions.coach) {
scriptsByCategory = addCoachScripts(scriptsByCategory);
}
scriptsByCategory = await addExtraScripts(scriptsByCategory, pluginScripts);
if (btOptions.preWarmServer) {
await preWarmServer(url, btOptions, scriptOrMultiple);
}
const engine = new BrowsertimeEngine(btOptions);
const asyncScript =
pluginAsyncScripts.length > 0
? await setupAsynScripts(pluginAsyncScripts)
: undefined;
try {
await engine.start();
if (scriptOrMultiple) {
const res = await engine.runMultiple(
url,
scriptsByCategory,
asyncScript
);
return res;
} else {
const res = await engine.run(url, scriptsByCategory, asyncScript);
return res;
}
} finally {
await engine.stop();
// set mobile options
if (options.mobile) {
btOptions.viewPort = '360x640';
if (btOptions.browser === 'chrome' || btOptions.browser === 'edge') {
const emulation = get(
btOptions,
'chrome.mobileEmulation.deviceName',
'Moto G4'
);
btOptions.chrome.mobileEmulation = {
deviceName: emulation
};
} else {
btOptions.userAgent = iphone6UserAgent;
}
}
};
const { BrowsertimeEngine, browserScripts } = await import('browsertime');
const scriptCategories = await browserScripts.allScriptCategories();
let scriptsByCategory = await browserScripts.getScriptsForCategories(
scriptCategories
);
if (btOptions.script) {
const userScripts = await parseUserScripts(btOptions.script);
scriptsByCategory = merge(scriptsByCategory, userScripts);
}
if (btOptions.coach) {
scriptsByCategory = addCoachScripts(scriptsByCategory);
}
scriptsByCategory = await addExtraScripts(scriptsByCategory, pluginScripts);
if (btOptions.preWarmServer) {
await preWarmServer(url, btOptions, scriptOrMultiple);
}
const engine = new BrowsertimeEngine(btOptions);
const asyncScript =
pluginAsyncScripts.length > 0
? await setupAsynScripts(pluginAsyncScripts)
: undefined;
try {
await engine.start();
if (scriptOrMultiple) {
const res = await engine.runMultiple(url, scriptsByCategory, asyncScript);
return res;
} else {
const res = await engine.run(url, scriptsByCategory, asyncScript);
return res;
}
} finally {
await engine.stop();
}
}

17
lib/plugins/browsertime/axeAggregator.js
View File

@ -1,7 +1,6 @@
'use strict';
const Stats = require('fast-stats').Stats;
const statsHelpers = require('../../support/statsHelpers');
class AxeAggregator {
import { Stats } from 'fast-stats';
import { summarizeStats as _summarizeStats } from '../../support/statsHelpers.js';
export class AxeAggregator {
constructor(options) {
this.options = options;
this.axeViolations = {
@ -33,13 +32,11 @@ class AxeAggregator {
summarizeStats() {
return {
violations: {
critical: statsHelpers.summarizeStats(this.axeViolations.critical),
serious: statsHelpers.summarizeStats(this.axeViolations.serious),
minor: statsHelpers.summarizeStats(this.axeViolations.minor),
moderate: statsHelpers.summarizeStats(this.axeViolations.moderate)
critical: _summarizeStats(this.axeViolations.critical),
serious: _summarizeStats(this.axeViolations.serious),
minor: _summarizeStats(this.axeViolations.minor),
moderate: _summarizeStats(this.axeViolations.moderate)
}
};
}
}
module.exports = AxeAggregator;

99
lib/plugins/browsertime/aggregator.js → lib/plugins/browsertime/browsertimeAggregator.js
View File

@ -1,13 +1,13 @@
'use strict';
const forEach = require('lodash.foreach'),
statsHelpers = require('../../support/statsHelpers');
import forEach from 'lodash.foreach';
import { pushGroupStats, setStatsSummary } from '../../support/statsHelpers.js';
const timings = ['firstPaint', 'timeToDomContentFlushed'];
module.exports = {
statsPerType: {},
groups: {},
export class BrowsertimeAggregator {
constructor() {
this.statsPerType = {};
this.groups = {};
}
addToAggregate(browsertimeRunData, group) {
if (this.groups[group] === undefined) {
@ -15,7 +15,7 @@ module.exports = {
}
if (browsertimeRunData.fullyLoaded) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['timings', 'fullyLoaded'],
@ -24,7 +24,7 @@ module.exports = {
}
if (browsertimeRunData.memory) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['memory'],
@ -34,7 +34,7 @@ module.exports = {
if (browsertimeRunData.googleWebVitals) {
for (let metric of Object.keys(browsertimeRunData.googleWebVitals)) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['googleWebVitals', metric],
@ -44,7 +44,7 @@ module.exports = {
}
if (browsertimeRunData.timings.largestContentfulPaint) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['timings', 'largestContentfulPaint'],
@ -53,7 +53,7 @@ module.exports = {
}
if (browsertimeRunData.timings.interactionToNextPaint) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['timings', 'interactionToNextPaint'],
@ -62,7 +62,7 @@ module.exports = {
}
if (browsertimeRunData.pageinfo.cumulativeLayoutShift) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['pageinfo', 'cumulativeLayoutShift'],
@ -72,7 +72,7 @@ module.exports = {
forEach(timings, timing => {
if (browsertimeRunData.timings[timing]) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
timing,
@ -83,7 +83,7 @@ module.exports = {
forEach(browsertimeRunData.timings.navigationTiming, (value, name) => {
if (value) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['navigationTiming', name],
@ -94,7 +94,7 @@ module.exports = {
// pick up one level of custom metrics
forEach(browsertimeRunData.custom, (value, name) => {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['custom', name],
@ -103,7 +103,7 @@ module.exports = {
});
forEach(browsertimeRunData.timings.pageTimings, (value, name) => {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['pageTimings', name],
@ -112,7 +112,7 @@ module.exports = {
});
forEach(browsertimeRunData.timings.paintTiming, (value, name) => {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['paintTiming', name],
@ -121,7 +121,7 @@ module.exports = {
});
forEach(browsertimeRunData.timings.userTimings.marks, timing => {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['userTimings', 'marks', timing.name],
@ -130,7 +130,7 @@ module.exports = {
});
forEach(browsertimeRunData.timings.userTimings.measures, timing => {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['userTimings', 'measures', timing.name],
@ -141,8 +141,8 @@ module.exports = {
forEach(browsertimeRunData.visualMetrics, (value, name) => {
// Sometimes visual elements fails and gives us null values
// And skip VisualProgress, ContentfulSpeedIndexProgress and others
if (name.indexOf('Progress') === -1 && value !== null) {
statsHelpers.pushGroupStats(
if (!name.includes('Progress') && value !== null) {
pushGroupStats(
this.statsPerType,
this.groups[group],
['visualMetrics', name],
@ -153,28 +153,28 @@ module.exports = {
if (browsertimeRunData.cpu) {
if (browsertimeRunData.cpu.longTasks) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['cpu', 'longTasks', 'tasks'],
browsertimeRunData.cpu.longTasks.tasks
);
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['cpu', 'longTasks', 'totalDuration'],
browsertimeRunData.cpu.longTasks.totalDuration
);
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['cpu', 'longTasks', 'totalBlockingTime'],
browsertimeRunData.cpu.longTasks.totalBlockingTime
);
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['cpu', 'longTasks', 'maxPotentialFid'],
@ -185,7 +185,7 @@ module.exports = {
for (let categoryName of Object.keys(
browsertimeRunData.cpu.categories
)) {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerType,
this.groups[group],
['cpu', 'categories', categoryName],
@ -194,10 +194,11 @@ module.exports = {
}
}
}
},
}
summarize() {
if (Object.keys(this.statsPerType).length === 0) {
return undefined;
return;
}
const summary = {
@ -210,51 +211,51 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]);
}
return summary;
},
}
summarizePerObject(obj) {
return Object.keys(obj).reduce((summary, name) => {
if (timings.indexOf(name) > -1) {
statsHelpers.setStatsSummary(summary, name, obj[name]);
} else if ('userTimings'.indexOf(name) > -1) {
summarizePerObject(object) {
return Object.keys(object).reduce((summary, name) => {
if (timings.includes(name)) {
setStatsSummary(summary, name, object[name]);
} else if ('userTimings'.includes(name)) {
summary.userTimings = {};
const marksData = {},
measuresData = {};
forEach(obj.userTimings.marks, (stats, timingName) => {
statsHelpers.setStatsSummary(marksData, timingName, stats);
forEach(object.userTimings.marks, (stats, timingName) => {
setStatsSummary(marksData, timingName, stats);
});
forEach(obj.userTimings.measures, (stats, timingName) => {
statsHelpers.setStatsSummary(measuresData, timingName, stats);
forEach(object.userTimings.measures, (stats, timingName) => {
setStatsSummary(measuresData, timingName, stats);
});
summary.userTimings.marks = marksData;
summary.userTimings.measures = measuresData;
} else if ('cpu'.indexOf(name) > -1) {
} else if ('cpu'.includes(name)) {
const longTasks = {};
const categories = {};
summary.cpu = {};
forEach(obj.cpu.longTasks, (stats, name) => {
statsHelpers.setStatsSummary(longTasks, name, stats);
forEach(object.cpu.longTasks, (stats, name) => {
setStatsSummary(longTasks, name, stats);
});
forEach(obj.cpu.categories, (stats, name) => {
statsHelpers.setStatsSummary(categories, name, stats);
forEach(object.cpu.categories, (stats, name) => {
setStatsSummary(categories, name, stats);
});
summary.cpu.longTasks = longTasks;
summary.cpu.categories = categories;
} else if ('memory'.indexOf(name) > -1) {
} else if ('memory'.includes(name)) {
const memory = {};
statsHelpers.setStatsSummary(memory, 'memory', obj[name]);
setStatsSummary(memory, 'memory', object[name]);
summary.memory = memory.memory;
} else {
const categoryData = {};
forEach(obj[name], (stats, timingName) => {
statsHelpers.setStatsSummary(categoryData, timingName, stats);
forEach(object[name], (stats, timingName) => {
setStatsSummary(categoryData, timingName, stats);
});
summary[name] = categoryData;
}
return summary;
}, {});
}
};
}

15
lib/plugins/browsertime/consoleLogAggregator.js
View File

@ -1,8 +1,7 @@
'use strict';
const Stats = require('fast-stats').Stats;
const statsHelpers = require('../../support/statsHelpers');
const { getGzippedFileAsJson } = require('./reader.js');
class ConsoleLogAggregator {
import { Stats } from 'fast-stats';
import { summarizeStats as _summarizeStats } from '../../support/statsHelpers.js';
import { getGzippedFileAsJson } from './reader.js';
export class ConsoleLogAggregator {
constructor(options) {
this.options = options;
this.logs = { SEVERE: new Stats(), WARNING: new Stats() };
@ -35,10 +34,8 @@ class ConsoleLogAggregator {
summarizeStats() {
return {
error: statsHelpers.summarizeStats(this.logs.SEVERE),
warning: statsHelpers.summarizeStats(this.logs.WARNING)
error: _summarizeStats(this.logs.SEVERE),
warning: _summarizeStats(this.logs.WARNING)
};
}
}
module.exports = ConsoleLogAggregator;

4
lib/plugins/browsertime/default/config.js
View File

@ -1,6 +1,4 @@
'use strict';
module.exports = {
export const browsertimeDefaultSettings = {
browser: 'chrome',
iterations: 3,
connectivity: {

4
lib/plugins/browsertime/default/metricsPageSummary.js
View File

@ -1,6 +1,4 @@
'use strict';
module.exports = [
export const metricsPageSummary = [
'statistics.timings.pageTimings',
'statistics.timings.fullyLoaded',
'statistics.timings.firstPaint',

2
lib/plugins/browsertime/default/metricsRun.js
View File

@ -1,4 +1,4 @@
module.exports = [
export const metricsRun = [
'fullyLoaded',
'timings.ttfb',
'timings.firstPaint',

2
lib/plugins/browsertime/default/metricsRunLimited.js
View File

@ -1,4 +1,4 @@
module.exports = [
export const metricsRunLimited = [
'fullyLoaded',
'timings.ttfb',
'timings.paintTiming.first-contentful-paint',

4
lib/plugins/browsertime/default/metricsSummary.js
View File

@ -1,6 +1,4 @@
'use strict';
module.exports = [
export const metricsSummary = [
'firstPaint',
'timeToDomContentFlushed',
'fullyLoaded',

228
lib/plugins/browsertime/filmstrip.js
View File

@ -1,10 +1,9 @@
'use strict';
const fs = require('fs');
const { promisify } = require('util');
const readdir = promisify(fs.readdir);
const path = require('path');
const log = require('intel').getLogger('sitespeedio.plugin.browsertime');
import { readdir as _readdir } from 'node:fs';
import { promisify } from 'node:util';
import { join } from 'node:path';
const readdir = promisify(_readdir);
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.browsertime');
function findFrame(videoFrames, time) {
let frame = videoFrames[0];
@ -29,7 +28,7 @@ function getMetricsFromBrowsertime(data) {
for (let mark of data.timings.userTimings.marks) {
metrics.push({
metric: mark.name,
value: mark.startTime.toFixed()
value: mark.startTime.toFixed(0)
});
userTimings++;
if (userTimings > maxUserTimings) {
@ -56,16 +55,18 @@ function getMetricsFromBrowsertime(data) {
});
if (data.timings.pageTimings) {
metrics.push({
metric: 'domContentLoadedTime',
name: 'DOM Content Loaded Time',
value: data.timings.pageTimings.domContentLoadedTime
});
metrics.push({
metric: 'pageLoadTime',
name: 'Page Load Time',
value: data.timings.pageTimings.pageLoadTime
});
metrics.push(
{
metric: 'domContentLoadedTime',
name: 'DOM Content Loaded Time',
value: data.timings.pageTimings.domContentLoadedTime
},
{
metric: 'pageLoadTime',
name: 'Page Load Time',
value: data.timings.pageTimings.pageLoadTime
}
);
}
if (
@ -84,20 +85,17 @@ function getMetricsFromBrowsertime(data) {
data.timings.largestContentfulPaint.renderTime
) {
let name = 'LCP';
if (data.timings.largestContentfulPaint.tagName === 'IMG') {
name +=
' <a href="' +
data.timings.largestContentfulPaint.url +
'">&#60;IMG&#62;</a>';
} else {
name +=
' &#60;' +
data.timings.largestContentfulPaint.tagName +
'&#62;' +
(data.timings.largestContentfulPaint.id !== ''
? ' ' + data.timings.largestContentfulPaint.id
: '');
}
name +=
data.timings.largestContentfulPaint.tagName === 'IMG'
? ' <a href="' +
data.timings.largestContentfulPaint.url +
'">&#60;IMG&#62;</a>'
: ' &#60;' +
data.timings.largestContentfulPaint.tagName +
'&#62;' +
(data.timings.largestContentfulPaint.id !== ''
? ' ' + data.timings.largestContentfulPaint.id
: '');
metrics.push({
metric: 'largestContentfulPaint',
name,
@ -106,31 +104,33 @@ function getMetricsFromBrowsertime(data) {
}
if (data.visualMetrics) {
metrics.push({
metric: 'FirstVisualChange',
name: 'First Visual Change',
value: data.visualMetrics.FirstVisualChange
});
metrics.push({
metric: 'LastVisualChange',
name: 'Last Visual Change',
value: data.visualMetrics.LastVisualChange
});
metrics.push({
metric: 'VisualComplete85',
name: 'Visual Complete 85%',
value: data.visualMetrics.VisualComplete85
});
metrics.push({
metric: 'VisualComplete95',
name: 'Visual Complete 95%',
value: data.visualMetrics.VisualComplete95
});
metrics.push({
metric: 'VisualComplete99',
name: 'Visual Complete 99%',
value: data.visualMetrics.VisualComplete99
});
metrics.push(
{
metric: 'FirstVisualChange',
name: 'First Visual Change',
value: data.visualMetrics.FirstVisualChange
},
{
metric: 'LastVisualChange',
name: 'Last Visual Change',
value: data.visualMetrics.LastVisualChange
},
{
metric: 'VisualComplete85',
name: 'Visual Complete 85%',
value: data.visualMetrics.VisualComplete85
},
{
metric: 'VisualComplete95',
name: 'Visual Complete 95%',
value: data.visualMetrics.VisualComplete95
},
{
metric: 'VisualComplete99',
name: 'Visual Complete 99%',
value: data.visualMetrics.VisualComplete99
}
);
if (data.visualMetrics.LargestImage) {
metrics.push({
metric: 'LargestImage',
@ -169,59 +169,65 @@ function getMetricsFromBrowsertime(data) {
function findTimings(timings, start, end) {
return timings.filter(timing => timing.value > start && timing.value <= end);
}
module.exports = {
async getFilmstrip(browsertimeData, run, dir, options, fullPath) {
let doWeHaveFilmstrip =
options.browsertime.visualMetrics === true &&
options.browsertime.videoParams.createFilmstrip === true;
export async function getFilmstrip(
browsertimeData,
run,
dir,
options,
fullPath
) {
let doWeHaveFilmstrip =
options.browsertime.visualMetrics === true &&
options.browsertime.videoParams.createFilmstrip === true;
if (doWeHaveFilmstrip === false) {
return [];
}
const toTheFront = [];
try {
let metrics = [];
if (browsertimeData) {
metrics = getMetricsFromBrowsertime(browsertimeData);
}
const files = await readdir(
path.join(dir, 'data', 'filmstrip', run + '')
);
const timings = [];
for (let file of files) {
timings.push({ time: file.replace(/\D/g, ''), file });
}
const maxTiming = timings.slice(-1)[0].time;
// We step 100 ms each step ... but if you wanna show all and the last change is late
// use 200 ms
const step =
maxTiming > 10000 && options.filmstrip && options.filmstrip.showAll
? 200
: 100;
let fileName = '';
for (let i = 0; i <= Number(maxTiming) + step; i = i + step) {
const entry = findFrame(timings, i);
const timingMetrics = findTimings(metrics, i - step, i);
if (
entry.file !== fileName ||
timingMetrics.length > 0 ||
(options.filmstrip && options.filmstrip.showAll)
) {
toTheFront.push({
time: i / 1000,
file: fullPath ? fullPath + entry.file : entry.file,
timings: timingMetrics
});
}
fileName = entry.file;
}
} catch (e) {
log.info('Could not read filmstrip dir', e);
}
return toTheFront;
if (doWeHaveFilmstrip === false) {
return [];
}
};
const toTheFront = [];
try {
let metrics = [];
if (browsertimeData) {
metrics = getMetricsFromBrowsertime(browsertimeData);
}
const files = await readdir(join(dir, 'data', 'filmstrip', run + ''));
const timings = [];
for (let file of files) {
timings.push({ time: file.replace(/\D/g, ''), file });
}
const maxTiming = timings.slice(-1)[0].time;
// We step 100 ms each step ... but if you wanna show all and the last change is late
// use 200 ms
const step =
maxTiming > 10_000 && options.filmstrip && options.filmstrip.showAll
? 200
: 100;
let fileName = '';
for (
let index = 0;
index <= Number(maxTiming) + step;
index = index + step
) {
const entry = findFrame(timings, index);
const timingMetrics = findTimings(metrics, index - step, index);
if (
entry.file !== fileName ||
timingMetrics.length > 0 ||
(options.filmstrip && options.filmstrip.showAll)
) {
toTheFront.push({
time: index / 1000,
file: fullPath ? fullPath + entry.file : entry.file,
timings: timingMetrics
});
}
fileName = entry.file;
}
} catch (error) {
log.info('Could not read filmstrip dir', error);
}
return toTheFront;
}

341
lib/plugins/browsertime/index.js
View File

@ -1,35 +1,46 @@
'use strict';
import { parse } from 'node:url';
import { default as _merge } from 'lodash.merge';
import intel from 'intel';
const log = intel.getLogger('plugin.browsertime');
import dayjs from 'dayjs';
import isEmpty from 'lodash.isempty';
import get from 'lodash.get';
import { Stats } from 'fast-stats';
import coach from 'coach-core';
const { pickAPage, analyseHar, merge } = coach;
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { summarizeStats } from '../../support/statsHelpers.js';
import { analyzeUrl } from './analyzer.js';
import { BrowsertimeAggregator } from './browsertimeAggregator.js';
import { metricsPageSummary as DEFAULT_METRICS_PAGE_SUMMARY } from './default/metricsPageSummary.js';
import { metricsSummary as DEFAULT_METRICS_SUMMARY } from './default/metricsSummary.js';
import { metricsRun as DEFAULT_METRICS_RUN } from './default/metricsRun.js';
import { metricsRunLimited as DEFAULT_METRICS_RUN_LIMITED } from './default/metricsRunLimited.js';
import { ConsoleLogAggregator } from './consoleLogAggregator.js';
import { AxeAggregator } from './axeAggregator.js';
import { getFilmstrip } from './filmstrip.js';
import { getGzippedFileAsJson } from './reader.js';
import { browsertimeDefaultSettings as defaultConfig } from './default/config.js';
const aggregator = require('./aggregator');
const api = require('coach-core');
const log = require('intel').getLogger('plugin.browsertime');
const merge = require('lodash.merge');
const analyzer = require('./analyzer');
const dayjs = require('dayjs');
const isEmpty = require('lodash.isempty');
const get = require('lodash.get');
const defaultConfig = require('./default/config');
const urlParser = require('url');
const Stats = require('fast-stats').Stats;
const statsHelpers = require('../../support/statsHelpers');
const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
const DEFAULT_METRICS_PAGE_SUMMARY = require('./default/metricsPageSummary');
const DEFAULT_METRICS_SUMMARY = require('./default/metricsSummary');
const DEFAULT_METRICS_RUN = require('./default/metricsRun');
const DEFAULT_METRICS_RUN_LIMITED = require('./default/metricsRunLimited');
const ConsoleLogAggregator = require('./consoleLogAggregator');
const AxeAggregator = require('./axeAggregator');
const filmstrip = require('./filmstrip');
const { getGzippedFileAsJson } = require('./reader.js');
export default class BrowsertimePlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'browsertime', options, context, queue });
}
concurrency = 1;
module.exports = {
concurrency: 1,
open(context, options) {
this.make = context.messageMaker('browsertime').make;
// this.make = context.messageMaker('browsertime').make;
this.useAxe = options.axe && options.axe.enable;
this.options = merge({}, defaultConfig, options.browsertime);
this.options = _merge({}, defaultConfig, options.browsertime);
this.allOptions = options;
merge(this.options, { verbose: options.verbose, axe: options.axe });
_merge(this.options, { verbose: options.verbose, axe: options.axe });
this.firstParty = options.firstParty;
this.options.mobile = options.mobile;
this.storageManager = context.storageManager;
@ -42,8 +53,10 @@ module.exports = {
'browsertime.screenshotParams.type',
defaultConfig.screenshotParams.type
);
this.scriptOrMultiple = options.multi;
this.allAlias = {};
this.browsertimeAggregator = new BrowsertimeAggregator();
// hack for disabling viewport on Android that's not supported
if (
@ -69,11 +82,10 @@ module.exports = {
'browsertime.run'
);
this.axeAggregatorTotal = new AxeAggregator(this.options);
},
async processMessage(message, queue) {
const {configureLogging} = await import ('browsertime');
}
async processMessage(message) {
const { configureLogging } = await import('browsertime');
configureLogging(this.options);
const make = this.make;
const options = this.options;
switch (message.type) {
// When sistespeed.io starts, a setup messages is posted on the queue
@ -81,19 +93,17 @@ module.exports = {
// to receive configuration
case 'sitespeedio.setup': {
// Let other plugins know that the browsertime plugin is alive
queue.postMessage(make('browsertime.setup'));
super.sendMessage('browsertime.setup');
// Unfify alias setup
if (this.options.urlMetaData) {
for (let url of Object.keys(this.options.urlMetaData)) {
const alias = this.options.urlMetaData[url];
const group = urlParser.parse(url).hostname;
const group = parse(url).hostname;
this.allAlias[alias] = url;
queue.postMessage(
make('browsertime.alias', alias, {
url,
group
})
);
super.sendMessage('browsertime.alias', alias, {
url,
group
});
}
}
@ -101,18 +111,16 @@ module.exports = {
// what type of images that are used (so for exmaple the HTML pluin can create
// correct links).
if (options.screenshot) {
queue.postMessage(
make('browsertime.config', {
screenshot: true,
screenshotType: this.screenshotType
})
);
super.sendMessage('browsertime.config', {
screenshot: true,
screenshotType: this.screenshotType
});
}
break;
}
// Another plugin sent configuration options to Browsertime
case 'browsertime.config': {
merge(options, message.data);
_merge(options, message.data);
break;
}
// Andother plugin got JavaScript that they want to run in Browsertime
@ -151,7 +159,7 @@ module.exports = {
// it's used in BT when we record a video
options.resultDir = await this.storageManager.getBaseDir();
const consoleLogAggregator = new ConsoleLogAggregator(options);
const result = await analyzer.analyzeUrl(
const result = await analyzeUrl(
url,
this.scriptOrMultiple,
this.pluginScripts,
@ -159,28 +167,22 @@ module.exports = {
options
);
// We need to check for alias first, since when we send the HAR (incliude all runs)
// We need to check for alias first, since when we send the HAR (include all runs)
// need to know if alias exists, else we will end up with things like
// https://github.com/sitespeedio/sitespeed.io/issues/2341
for (
let resultIndex = 0;
resultIndex < result.length;
resultIndex++
) {
for (const element of result) {
// Browsertime supports alias for URLS in a script
const alias = result[resultIndex].info.alias;
const alias = element.info.alias;
if (alias) {
if (this.scriptOrMultiple) {
url = result[resultIndex].info.url;
group = urlParser.parse(url).hostname;
url = element.info.url;
group = parse(url).hostname;
}
this.allAlias[url] = alias;
queue.postMessage(
make('browsertime.alias', alias, {
url,
group
})
);
super.sendMessage('browsertime.alias', alias, {
url,
group
});
}
}
@ -199,7 +201,7 @@ module.exports = {
// multiple/scripting lets do it like this for now
if (this.scriptOrMultiple) {
url = result[resultIndex].info.url;
group = urlParser.parse(url).hostname;
group = parse(url).hostname;
}
let runIndex = 0;
for (let run of result[resultIndex].browserScripts) {
@ -223,7 +225,7 @@ module.exports = {
if (result.har.log._android) {
testInfo.android = result.har.log._android;
}
queue.postMessage(make('browsertime.browser', testInfo));
super.sendMessage('browsertime.browser', testInfo);
// Add meta data to be used when we compare multiple HARs
// the meta field is added in Browsertime
if (result.har.log.pages[harIndex]) {
@ -244,7 +246,7 @@ module.exports = {
_meta.result = `${base}${runIndex + 1}.html`;
if (options.video) {
_meta.video = `${base}data/video/${runIndex + 1}.mp4`;
_meta.filmstrip = await filmstrip.getFilmstrip(
_meta.filmstrip = await getFilmstrip(
run,
`${runIndex + 1}`,
`${
@ -265,7 +267,7 @@ module.exports = {
url
);
}
run.har = api.pickAPage(result.har, harIndex);
run.har = pickAPage(result.har, harIndex);
} else {
// If we do not have a HAR, use browser info from the result
if (result.length > 0) {
@ -275,39 +277,42 @@ module.exports = {
version: result[0].info.browser.version
}
};
queue.postMessage(make('browsertime.browser', testInfo));
super.sendMessage('browsertime.browser', testInfo);
}
}
// Hack to get axe data. In the future we can make this more generic
if (result[resultIndex].extras.length > 0) {
if (result[resultIndex].extras[runIndex].axe) {
const order = ['critical', 'serious', 'moderate', 'minor'];
result[resultIndex].extras[runIndex].axe.violations.sort(
(a, b) => order.indexOf(a.impact) > order.indexOf(b.impact)
);
if (
result[resultIndex].extras.length > 0 &&
result[resultIndex].extras[runIndex].axe
) {
const order = ['critical', 'serious', 'moderate', 'minor'];
result[resultIndex].extras[runIndex].axe.violations.sort(
(a, b) => order.indexOf(a.impact) > order.indexOf(b.impact)
);
axeAggregatorPerURL.addStats(
result[resultIndex].extras[runIndex].axe
);
axeAggregatorPerURL.addStats(
result[resultIndex].extras[runIndex].axe
);
this.axeAggregatorTotal.addStats(
result[resultIndex].extras[runIndex].axe
);
this.axeAggregatorTotal.addStats(
result[resultIndex].extras[runIndex].axe
);
queue.postMessage(
make('axe.run', result[resultIndex].extras[runIndex].axe, {
url,
group,
runIndex,
iteration: runIndex + 1
})
);
// Another hack: Browsertime automatically creates statistics for alla data in extras
// but we don't really need that for AXE.
delete result[resultIndex].extras[runIndex].axe;
delete result[resultIndex].statistics.extras.axe;
}
super.sendMessage(
'axe.run',
result[resultIndex].extras[runIndex].axe,
{
url,
group,
runIndex,
iteration: runIndex + 1
}
);
// Another hack: Browsertime automatically creates statistics for alla data in extras
// but we don't really need that for AXE.
delete result[resultIndex].extras[runIndex].axe;
delete result[resultIndex].statistics.extras.axe;
}
if (result[resultIndex].cpu) {
run.cpu = result[resultIndex].cpu[runIndex];
@ -360,7 +365,6 @@ module.exports = {
// The packaging of screenshots from browsertime
// Is not optimal, the same array of screenshots hold all
// screenshots from one run (the automatic ones and user generated)
// If we only test one page per run, take all screenshots (user generated etc)
if (result.length === 1) {
run.screenshots =
@ -372,12 +376,12 @@ module.exports = {
runIndex
]) {
if (
screenshot.indexOf(
screenshot.includes(
`${this.resultUrls.relativeSummaryPageUrl(
url,
this.allAlias[url]
)}data`
) > -1
)
) {
run.screenshots.push(screenshot);
}
@ -391,15 +395,13 @@ module.exports = {
errorStats.push(error.length);
}
queue.postMessage(
make('browsertime.run', run, {
url,
group,
runIndex,
runTime: run.timestamp,
iteration: runIndex + 1
})
);
super.sendMessage('browsertime.run', run, {
url,
group,
runIndex,
runTime: run.timestamp,
iteration: runIndex + 1
});
if (
options.chrome &&
@ -412,15 +414,13 @@ module.exports = {
result[resultIndex].files.consoleLog[runIndex]
);
queue.postMessage(
make('browsertime.console', consoleData, {
url,
group,
runIndex,
iteration: runIndex + 1
})
);
} catch (e) {
super.sendMessage('browsertime.console', consoleData, {
url,
group,
runIndex,
iteration: runIndex + 1
});
} catch {
// This could happen if the run failed somehow
log.error('Could not fetch the console log');
}
@ -438,14 +438,12 @@ module.exports = {
options.resultDir,
`trace-${runIndex + 1}.json.gz`
);
queue.postMessage(
make('browsertime.chrometrace', traceData, {
url,
group,
name: `trace-${runIndex + 1}.json`, // backward compatible to 2.x
runIndex
})
);
super.sendMessage('browsertime.chrometrace', traceData, {
url,
group,
name: `trace-${runIndex + 1}.json`,
runIndex
});
}
// If the coach is turned on, collect the coach result
@ -458,17 +456,15 @@ module.exports = {
url,
coachAdvice.errors
);
queue.postMessage(
make(
'error',
'The coach got the following errors: ' +
JSON.stringify(coachAdvice.errors),
{
url,
runIndex,
iteration: runIndex + 1
}
)
super.sendMessage(
'error',
'The coach got the following errors: ' +
JSON.stringify(coachAdvice.errors),
{
url,
runIndex,
iteration: runIndex + 1
}
);
}
@ -476,26 +472,25 @@ module.exports = {
// If we run without HAR
if (result.har) {
// make sure to get the right run in the HAR
const myHar = api.pickAPage(result.har, harIndex);
const harResult = await api.analyseHar(
const myHar = pickAPage(result.har, harIndex);
const harResult = await analyseHar(
myHar,
undefined,
coachAdvice,
options
);
advice = api.merge(coachAdvice, harResult);
advice = merge(coachAdvice, harResult);
}
queue.postMessage(
make('coach.run', advice, {
url,
group,
runIndex,
iteration: runIndex + 1
})
);
super.sendMessage('coach.run', advice, {
url,
group,
runIndex,
iteration: runIndex + 1
});
}
aggregator.addToAggregate(run, group);
this.browsertimeAggregator.addToAggregate(run, group);
runIndex++;
}
@ -509,34 +504,31 @@ module.exports = {
consoleLogAggregator.summarizeStats();
}
result[resultIndex].statistics.errors =
statsHelpers.summarizeStats(errorStats);
result[resultIndex].statistics.errors = summarizeStats(errorStats);
// Post the result on the queue so other plugins can use it
queue.postMessage(
make('browsertime.pageSummary', result[resultIndex], {
url,
group,
runTime: result.timestamp
})
);
super.sendMessage('browsertime.pageSummary', result[resultIndex], {
url,
group,
runTime: result.timestamp
});
// Post the HAR on the queue so other plugins can use it
if (result.har) {
queue.postMessage(
make('browsertime.har', result.har, {
url,
group
})
);
super.sendMessage('browsertime.har', result.har, {
url,
group
});
}
// Post the result on the queue so other plugins can use it
if (this.useAxe) {
queue.postMessage(
make('axe.pageSummary', axeAggregatorPerURL.summarizeStats(), {
super.sendMessage(
'axe.pageSummary',
axeAggregatorPerURL.summarizeStats(),
{
url,
group
})
}
);
}
@ -544,13 +536,13 @@ module.exports = {
// [[],[],[]] where one iteration can have multiple errors
for (let errorsForOneIteration of result[resultIndex].errors) {
for (let error of errorsForOneIteration) {
queue.postMessage(make('error', error, merge({ url })));
super.sendMessage('error', error, _merge({ url }));
}
}
}
break;
} catch (error) {
queue.postMessage(make('error', error, merge({ url })));
super.sendMessage('error', error, _merge({ url }));
log.error('Caught error from Browsertime', error);
break;
}
@ -559,26 +551,29 @@ module.exports = {
// and post the summary on the queue
case 'sitespeedio.summarize': {
log.debug('Generate summary metrics from Browsertime');
const summary = aggregator.summarize();
const summary = this.browsertimeAggregator.summarize();
if (summary) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(
make('browsertime.summary', summary.groups[group], { group })
);
super.sendMessage('browsertime.summary', summary.groups[group], {
group
});
}
}
if (this.useAxe) {
queue.postMessage(
make('axe.summary', this.axeAggregatorTotal.summarizeStats(), {
super.sendMessage(
'axe.summary',
this.axeAggregatorTotal.summarizeStats(),
{
group: 'total'
})
}
);
}
break;
}
}
},
config: defaultConfig
};
}
}
export { browsertimeDefaultSettings as config } from './default/config.js';

25
lib/plugins/browsertime/reader.js
View File

@ -1,9 +1,8 @@
'use strict';
const fs = require('fs');
const path = require('path');
const zlib = require('zlib');
const { promisify } = require('util');
const gunzip = promisify(zlib.gunzip);
import { createReadStream } from 'node:fs';
import { join } from 'node:path';
import { gunzip as _gunzip } from 'node:zlib';
import { promisify } from 'node:util';
const gunzip = promisify(_gunzip);
async function streamToString(stream) {
return new Promise((resolve, reject) => {
@ -14,11 +13,9 @@ async function streamToString(stream) {
});
}
module.exports = {
async getGzippedFileAsJson(dir, file) {
const readStream = fs.createReadStream(path.join(dir, file));
const text = await streamToString(readStream);
const unzipped = await gunzip(text);
return JSON.parse(unzipped.toString());
}
};
export async function getGzippedFileAsJson(dir, file) {
const readStream = createReadStream(join(dir, file));
const text = await streamToString(readStream);
const unzipped = await gunzip(text);
return JSON.parse(unzipped.toString());
}

109
lib/plugins/budget/deprecatedVerify.js
View File

@ -1,12 +1,10 @@
'use strict';
/**
* The old verificatuion and budget.json was deprecated in 8.0
*/
const get = require('lodash.get');
const noop = require('../../support/helpers').noop;
const size = require('../../support/helpers').size.format;
const log = require('intel').getLogger('sitespeedio.plugin.budget');
import get from 'lodash.get';
import { noop, size } from '../../support/helpers/index.js';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
function getItem(url, type, metric, value, limit, limitType) {
return {
@ -20,71 +18,66 @@ function getItem(url, type, metric, value, limit, limitType) {
}
function getHelperFunction(metric) {
if (
metric.indexOf('transferSize') > -1 ||
metric.indexOf('contentSize') > -1
) {
return size;
} else if (metric.indexOf('timings') > -1) {
if (metric.includes('transferSize') || metric.includes('contentSize')) {
return size.format;
} else if (metric.includes('timings')) {
return function (time) {
return time + ' ms';
};
} else return noop;
}
module.exports = {
verify(message, result, budgets) {
const failing = [];
const working = [];
// do we have an entry in the budget for this kind of message?
if (budgets[message.type]) {
for (var budget of budgets[message.type]) {
let value = get(message.data, budget.metric);
export function verify(message, result, budgets) {
const failing = [];
const working = [];
// do we have an entry in the budget for this kind of message?
if (budgets[message.type]) {
for (var budget of budgets[message.type]) {
let value = get(message.data, budget.metric);
if (value !== undefined) {
const format = getHelperFunction(budget.metric);
if (value !== undefined) {
const format = getHelperFunction(budget.metric);
const item = getItem(
message.url,
message.type,
budget.metric,
format(value),
budget.max !== undefined ? format(budget.max) : format(budget.min),
budget.max !== undefined ? 'max' : 'min'
);
const item = getItem(
message.url,
message.type,
budget.metric,
format(value),
budget.max !== undefined ? format(budget.max) : format(budget.min),
budget.max !== undefined ? 'max' : 'min'
);
if (budget.max !== undefined) {
if (value <= budget.max) {
working.push(item);
} else {
item.status = 'failing';
failing.push(item);
}
if (budget.max !== undefined) {
if (value <= budget.max) {
working.push(item);
} else {
if (value >= budget.min) {
working.push(item);
} else {
item.status = 'failing';
failing.push(item);
}
item.status = 'failing';
failing.push(item);
}
} else {
log.debug('Missing data for budget metric ' + budget.metric);
if (value >= budget.min) {
working.push(item);
} else {
item.status = 'failing';
failing.push(item);
}
}
} else {
log.debug('Missing data for budget metric ' + budget.metric);
}
}
// group working/failing per URL
if (failing.length > 0) {
result.failing[message.url] = result.failing[message.url]
? result.failing[message.url].concat(failing)
: failing;
}
if (working.length > 0) {
result.working[message.url] = result.working[message.url]
? result.working[message.url].concat(working)
: working;
}
}
};
// group working/failing per URL
if (failing.length > 0) {
result.failing[message.url] = result.failing[message.url]
? [...result.failing[message.url], ...failing]
: failing;
}
if (working.length > 0) {
result.working[message.url] = result.working[message.url]
? [...result.working[message.url], ...working]
: working;
}
}

57
lib/plugins/budget/index.js
View File

@ -1,13 +1,18 @@
'use strict';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { verify as deprecatedVerify } from './deprecatedVerify.js';
import { verify } from './verify.js';
import { writeTap } from './tap.js';
import { writeJunit } from './junit.js';
import { writeJson } from './json.js';
const deprecatedVerify = require('./deprecatedVerify').verify;
const verify = require('./verify').verify;
const tap = require('./tap');
const junit = require('./junit');
const json = require('./json');
const log = require('intel').getLogger('sitespeedio.plugin.budget');
const log = intel.getLogger('sitespeedio.plugin.budget');
export default class BudgetPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'budget', options, context, queue });
}
module.exports = {
open(context, options) {
this.options = options;
this.budgetOptions = options.budget || {};
@ -22,7 +27,7 @@ module.exports = {
'coach.pageSummary',
'axe.pageSummary'
];
},
}
processMessage(message, queue) {
// if there's no configured budget do nothing
if (!this.options.budget) {
@ -35,7 +40,7 @@ module.exports = {
}
const budget = this.options.budget.config;
if (this.budgetTypes.indexOf(message.type) > -1) {
if (this.budgetTypes.includes(message.type)) {
// If it doesn't have the new structure of a budget file
// use the old verdion
if (!budget.budget) {
@ -102,20 +107,30 @@ module.exports = {
case 'sitespeedio.render': {
if (this.options.budget) {
if (this.options.budget.output === 'json') {
json.writeJson(this.result, this.storageManager.getBaseDir());
} else if (this.options.budget.output === 'tap') {
tap.writeTap(this.result, this.storageManager.getBaseDir());
} else if (this.options.budget.output === 'junit') {
junit.writeJunit(
this.result,
this.storageManager.getBaseDir(),
this.options
);
switch (this.options.budget.output) {
case 'json': {
writeJson(this.result, this.storageManager.getBaseDir());
break;
}
case 'tap': {
writeTap(this.result, this.storageManager.getBaseDir());
break;
}
case 'junit': {
writeJunit(
this.result,
this.storageManager.getBaseDir(),
this.options
);
break;
}
}
}
}
}
}
}
};
}

18
lib/plugins/budget/json.js
View File

@ -1,11 +1,11 @@
'use strict';
import { writeFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
const fs = require('fs'),
log = require('intel').getLogger('sitespeedio.plugin.budget'),
path = require('path');
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
exports.writeJson = function (results, dir) {
const file = path.join(dir, 'budgetResult.json');
log.info('Write budget to %s', path.resolve(file));
fs.writeFileSync(file, JSON.stringify(results, null, 2));
};
export function writeJson(results, dir) {
const file = join(dir, 'budgetResult.json');
log.info('Write budget to %s', resolve(file));
writeFileSync(file, JSON.stringify(results, undefined, 2));
}

34
lib/plugins/budget/junit.js
View File

@ -1,12 +1,14 @@
'use strict';
import { join, resolve } from 'node:path';
import { parse } from 'node:url';
const builder = require('junit-report-builder'),
urlParser = require('url'),
log = require('intel').getLogger('sitespeedio.plugin.budget'),
path = require('path'),
merge = require('lodash.merge');
import jrp from 'junit-report-builder';
exports.writeJunit = function (results, dir, options) {
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
import merge from 'lodash.merge';
export function writeJunit(results, dir, options) {
// lets have one suite per URL
const urls = Object.keys(merge({}, results.failing, results.working));
@ -14,20 +16,16 @@ exports.writeJunit = function (results, dir, options) {
// The URL can be an alias
let name = url;
if (url.startsWith('http')) {
const parsedUrl = urlParser.parse(url);
const parsedUrl = parse(url);
name = url.startsWith('http') ? url : url;
parsedUrl.hostname.replace(/\./g, '_') +
'.' +
parsedUrl.path.replace(/\./g, '_').replace(/\//g, '_');
}
const suite = builder
const suite = jrp
.testSuite()
.name(
options.budget.friendlyName
? options.budget.friendlyName
: 'sitespeed.io' + '.' + name
);
.name(options.budget.friendlyName || 'sitespeed.io' + '.' + name);
if (results.failing[url]) {
for (const result of results.failing[url]) {
@ -65,7 +63,7 @@ exports.writeJunit = function (results, dir, options) {
}
}
}
const file = path.join(dir, 'junit.xml');
log.info('Write junit budget to %s', path.resolve(file));
builder.writeTo(file);
};
const file = join(dir, 'junit.xml');
log.info('Write junit budget to %s', resolve(file));
jrp.writeTo(file);
}

17
lib/plugins/budget/tap.js
View File

@ -1,12 +1,11 @@
'use strict';
import path from 'node:path';
import fs from 'node:fs';
import { EOL } from 'node:os';
import tap from 'tape';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
const tap = require('tape'),
fs = require('fs'),
log = require('intel').getLogger('sitespeedio.plugin.budget'),
path = require('path'),
EOL = require('os').EOL;
exports.writeTap = function (results, dir) {
export function writeTap(results, dir) {
const file = path.join(dir, 'budget.tap');
log.info('Write budget to %s', path.resolve(file));
const tapOutput = fs.createWriteStream(file);
@ -34,4 +33,4 @@ exports.writeTap = function (results, dir) {
}
}
}
};
}

286
lib/plugins/budget/verify.js
View File

@ -1,13 +1,12 @@
'use strict';
/**
* The old verificatuion and budget.json was deprecated in 8.0
*/
const get = require('lodash.get');
const merge = require('lodash.merge');
const log = require('intel').getLogger('sitespeedio.plugin.budget');
const friendlyNames = require('../../support/friendlynames');
const time = require('../../support/helpers/time');
import get from 'lodash.get';
import merge from 'lodash.merge';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
import friendlyNames from '../../support/friendlynames.js';
import { time } from '../../support/helpers/time.js';
function getItem(friendlyName, type, metric, value, limit, limitType) {
return {
@ -22,163 +21,156 @@ function getItem(friendlyName, type, metric, value, limit, limitType) {
};
}
module.exports = {
verify(message, result, budgets, alias) {
// Let us merge the specific configuration for this URL together
// with the generic one. In the future we can fine tune this, since
// we keep all metrics within a specific URL
export function verify(message, result, budgets, alias) {
// Let us merge the specific configuration for this URL together
// with the generic one. In the future we can fine tune this, since
// we keep all metrics within a specific URL
// If we have a match for the alias, use that first, if not use the URL and
// then the specific one
let budgetSetupSpecificForThisURL;
if (alias) {
budgetSetupSpecificForThisURL = get(budgets.budget, alias);
}
if (!budgetSetupSpecificForThisURL) {
budgetSetupSpecificForThisURL = get(budgets.budget, message.url);
}
const budgetForThisURL = {};
merge(budgetForThisURL, budgets.budget, budgetSetupSpecificForThisURL);
// If we have a match for the alias, use that first, if not use the URL and
// then the specific one
let budgetSetupSpecificForThisURL;
if (alias) {
budgetSetupSpecificForThisURL = get(budgets.budget, alias);
}
if (!budgetSetupSpecificForThisURL) {
budgetSetupSpecificForThisURL = get(budgets.budget, message.url);
}
const budgetForThisURL = {};
merge(budgetForThisURL, budgets.budget, budgetSetupSpecificForThisURL);
// Keep failing/working metrics here
const failing = [];
const working = [];
const url = message.url;
log.verbose('Applying budget to url %s', url);
const tool = message.type.split('.')[0];
// Go through all metrics that are configured
// timing, request, coach etc
for (let metricType of Object.keys(budgetForThisURL)) {
for (let metric of Object.keys(budgetForThisURL[metricType])) {
if (friendlyNames[tool][metricType]) {
if (!friendlyNames[tool][metricType][metric]) {
log.error(
`It seems like you configure a metric ${metric} that we do not have a friendly name. Please check the docs if it is right.`
// Keep failing/working metrics here
const failing = [];
const working = [];
const url = message.url;
log.verbose('Applying budget to url %s', url);
const tool = message.type.split('.')[0];
// Go through all metrics that are configured
// timing, request, coach etc
for (let metricType of Object.keys(budgetForThisURL)) {
for (let metric of Object.keys(budgetForThisURL[metricType])) {
if (friendlyNames[tool][metricType]) {
if (!friendlyNames[tool][metricType][metric]) {
log.error(
`It seems like you configure a metric ${metric} that we do not have a friendly name. Please check the docs if it is right.`
);
}
const fullPath = friendlyNames[tool][metricType][metric].path;
let value = get(message.data, fullPath);
if (value && message.type === 'lighthouse.pageSummary') {
value = value * 100; // The score in Lighthouse is 0-1
} else if (
value &&
message.type === 'pagexray.pageSummary' &&
metricType === 'httpErrors'
) {
// count number of http server error
value = Object.keys(value).filter(code => code > 399).length;
}
// We got a matching metric
if (value !== undefined) {
const budgetValue = budgetForThisURL[metricType][metric];
const item = getItem(
friendlyNames[tool][metricType][metric],
metricType,
metric,
value,
budgetValue,
tool === 'coach' || tool === 'lighthouse' || tool === 'gpsi'
? 'min'
: 'max'
);
if (tool === 'coach' || tool === 'lighthouse' || tool === 'gpsi') {
if (value < budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
working.push(item);
}
} else {
if (value > budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
working.push(item);
}
}
} else {
log.debug('Missing data for budget metric ' + metric);
}
} else {
if (metricType === 'usertimings' && tool === 'browsertime') {
const budgetValue = budgetForThisURL[metricType][metric];
let value =
get(
message.data,
'statistics.timings.userTimings.marks.' + metric + '.median'
) ||
get(
message.data,
'statistics.timings.userTimings.measures.' + metric + '.median'
);
}
const fullPath = friendlyNames[tool][metricType][metric].path;
let value = get(message.data, fullPath);
if (value && message.type === 'lighthouse.pageSummary') {
value = value * 100; // The score in Lighthouse is 0-1
} else if (
value &&
message.type === 'pagexray.pageSummary' &&
metricType === 'httpErrors'
) {
// count number of http server error
value = Object.keys(value).filter(code => code > 399).length;
}
// We got a matching metric
if (value !== undefined) {
const budgetValue = budgetForThisURL[metricType][metric];
if (value) {
const item = getItem(
friendlyNames[tool][metricType][metric],
{ name: metric, format: time.ms },
metricType,
metric,
value,
budgetValue,
tool === 'coach' || tool === 'lighthouse' || tool === 'gpsi'
? 'min'
: 'max'
'max'
);
if (tool === 'coach' || tool === 'lighthouse' || tool === 'gpsi') {
if (value < budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
working.push(item);
}
if (value > budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
if (value > budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
working.push(item);
}
working.push(item);
}
continue;
} else {
log.debug('Missing data for budget metric ' + metric);
log.error(`Could not find the user timing ${metric}`);
continue;
}
} else {
if (metricType === 'usertimings' && tool === 'browsertime') {
const budgetValue = budgetForThisURL[metricType][metric];
let value =
get(
message.data,
'statistics.timings.userTimings.marks.' + metric + '.median'
) ||
get(
message.data,
'statistics.timings.userTimings.measures.' + metric + '.median'
);
if (value) {
const item = getItem(
{ name: metric, format: time.ms },
metricType,
metric,
value,
budgetValue,
'max'
);
if (value > budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
working.push(item);
}
continue;
} else {
log.error(`Could not find the user timing ${metric}`);
continue;
}
} else if (
metricType === 'scriptingmetrics' &&
tool === 'browsertime'
) {
const budgetValue = budgetForThisURL[metricType][metric];
const value = get(
message.data,
'statistics.extras.' + metric + '.median'
} else if (
metricType === 'scriptingmetrics' &&
tool === 'browsertime'
) {
const budgetValue = budgetForThisURL[metricType][metric];
const value = get(
message.data,
'statistics.extras.' + metric + '.median'
);
if (value) {
const item = getItem(
{ name: metric, format: time.ms },
metricType,
metric,
value,
budgetValue,
'max'
);
if (value) {
const item = getItem(
{ name: metric, format: time.ms },
metricType,
metric,
value,
budgetValue,
'max'
);
if (value > budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
working.push(item);
}
continue;
if (value > budgetValue) {
item.status = 'failing';
failing.push(item);
} else {
log.error(`Could not find the scripting metric ${metric}`);
continue;
working.push(item);
}
continue;
} else {
log.error(`Could not find the scripting metric ${metric}`);
continue;
}
}
}
}
// group working/failing per URL
if (failing.length > 0) {
result.failing[alias || message.url] = result.failing[
alias || message.url
]
? result.failing[alias || message.url].concat(failing)
: failing;
}
if (working.length > 0) {
result.working[alias || message.url] = result.working[
alias || message.url
]
? result.working[alias || message.url].concat(working)
: working;
}
}
};
// group working/failing per URL
if (failing.length > 0) {
result.failing[alias || message.url] = result.failing[alias || message.url]
? [...result.failing[alias || message.url], ...failing]
: failing;
}
if (working.length > 0) {
result.working[alias || message.url] = result.working[alias || message.url]
? [...result.working[alias || message.url], ...working]
: working;
}
}

32
lib/plugins/coach/aggregator.js
View File

@ -1,11 +1,11 @@
'use strict';
import forEach from 'lodash.foreach';
import { pushGroupStats, setStatsSummary } from '../../support/statsHelpers.js';
const forEach = require('lodash.foreach'),
statsHelpers = require('../../support/statsHelpers');
module.exports = {
statsPerCategory: {},
groups: {},
export class CoachAggregator {
constructor() {
this.statsPerCategory = {};
this.groups = {};
}
addToAggregate(coachData, group) {
if (this.groups[group] === undefined) {
@ -13,7 +13,7 @@ module.exports = {
}
// push the total score
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerCategory,
this.groups[group],
'score',
@ -25,7 +25,7 @@ module.exports = {
return;
}
// Push the score per category
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerCategory,
this.groups[group],
[categoryName, 'score'],
@ -33,7 +33,7 @@ module.exports = {
);
forEach(category.adviceList, (advice, adviceName) => {
statsHelpers.pushGroupStats(
pushGroupStats(
this.statsPerCategory,
this.groups[group],
[categoryName, adviceName],
@ -41,10 +41,10 @@ module.exports = {
);
});
});
},
}
summarize() {
if (Object.keys(this.statsPerCategory).length === 0) {
return undefined;
return;
}
const summary = {
@ -57,16 +57,16 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]);
}
return summary;
},
}
summarizePerObject(type) {
return Object.keys(type).reduce((summary, categoryName) => {
if (categoryName === 'score') {
statsHelpers.setStatsSummary(summary, 'score', type[categoryName]);
setStatsSummary(summary, 'score', type[categoryName]);
} else {
const categoryData = {};
forEach(type[categoryName], (stats, name) => {
statsHelpers.setStatsSummary(categoryData, name, stats);
setStatsSummary(categoryData, name, stats);
});
summary[categoryName] = categoryData;
@ -75,4 +75,4 @@ module.exports = {
return summary;
}, {});
}
};
}

22
lib/plugins/coach/index.js
View File

@ -1,6 +1,7 @@
'use strict';
const aggregator = require('./aggregator');
const log = require('intel').getLogger('plugin.coach');
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { CoachAggregator } from './aggregator.js';
import intel from 'intel';
const log = intel.getLogger('plugin.coach');
const DEFAULT_METRICS_RUN = [];
@ -24,10 +25,15 @@ const DEFAULT_METRICS_PAGESUMMARY = [
'advice.info.localStorageSize'
];
module.exports = {
export default class CoachPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'coach', options, context, queue });
}
open(context, options) {
this.options = options;
this.make = context.messageMaker('coach').make;
this.coachAggregator = new CoachAggregator();
context.filterRegistry.registerFilterForType(
DEFAULT_METRICS_SUMMARY,
@ -41,7 +47,7 @@ module.exports = {
DEFAULT_METRICS_PAGESUMMARY,
'coach.pageSummary'
);
},
}
processMessage(message, queue) {
const make = this.make;
switch (message.type) {
@ -60,13 +66,13 @@ module.exports = {
);
}
aggregator.addToAggregate(message.data, message.group);
this.coachAggregator.addToAggregate(message.data, message.group);
break;
}
case 'sitespeedio.summarize': {
log.debug('Generate summary metrics from the Coach');
let summary = aggregator.summarize();
let summary = this.coachAggregator.summarize();
if (summary) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(
@ -78,4 +84,4 @@ module.exports = {
}
}
}
};
}

41
lib/plugins/crawler/index.js
View File

@ -1,17 +1,22 @@
'use strict';
import { extname } from 'node:path';
import merge from 'lodash.merge';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const path = require('path');
const merge = require('lodash.merge');
const log = require('intel').getLogger('sitespeedio.plugin.crawler');
const Crawler = require('simplecrawler');
const throwIfMissing = require('../../support/util').throwIfMissing;
const toArray = require('../../support/util').toArray;
const log = intel.getLogger('sitespeedio.plugin.crawler');
import Crawler from 'simplecrawler';
import { throwIfMissing } from '../../support/util';
import { toArray } from '../../support/util';
const defaultOptions = {
depth: 3
};
module.exports = {
export default class CrawlerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'crawler', options, context, queue });
}
open(context, options) {
throwIfMissing(options.crawler, ['depth'], 'crawler');
this.options = merge({}, defaultOptions, options.crawler);
@ -22,10 +27,9 @@ module.exports = {
this.basicAuth = options.browsertime
? options.browsertime.basicAuth
: undefined;
this.cookie = options.browsertime.cookie
? options.browsertime.cookie
: undefined;
},
this.cookie = options.browsertime.cookie || undefined;
}
processMessage(message, queue) {
const make = this.make;
if (message.type === 'url' && message.source !== 'crawler') {
@ -66,9 +70,9 @@ module.exports = {
}
crawler.addFetchCondition(queueItem => {
const extension = path.extname(queueItem.path);
const extension = extname(queueItem.path);
// Don't try to download these, based on file name.
if (['png', 'jpg', 'gif', 'pdf'].indexOf(extension) !== -1) {
if (['png', 'jpg', 'gif', 'pdf'].includes(extension)) {
return false;
}
@ -112,8 +116,11 @@ module.exports = {
crawler.userAgent = this.userAgent;
}
crawler.on('fetchconditionerror', (queueItem, err) => {
log.warn('An error occurred in the fetchCondition callback: %s', err);
crawler.on('fetchconditionerror', (queueItem, error) => {
log.warn(
'An error occurred in the fetchCondition callback: %s',
error
);
});
crawler.on('fetchredirect', (queueItem, parsedURL, response) => {
@ -157,4 +164,4 @@ module.exports = {
});
}
}
};
}

29
lib/plugins/crux/cli.js
View File

@ -1,29 +0,0 @@
module.exports = {
key: {
describe:
'You need to use a key to get data from CrUx. Get the key from https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/getting-started#APIKey',
group: 'CrUx'
},
enable: {
default: true,
describe:
'Enable the CrUx plugin. This is on by defauly but you also need the Crux key. If you chose to disable it with this key, set this to false and you can still use the CrUx key in your configuration.',
group: 'CrUx'
},
formFactor: {
default: 'ALL',
type: 'string',
choices: ['ALL', 'DESKTOP', 'PHONE', 'TABLET'],
describe:
'A form factor is the type of device on which a user visits a website.',
group: 'CrUx'
},
collect: {
default: 'ALL',
type: 'string',
choices: ['ALL', 'URL', 'ORIGIN'],
describe:
'Choose what data to collect. URL is data for a specific URL, ORIGIN for the domain and ALL for both of them',
group: 'CrUx'
}
};

55
lib/plugins/crux/index.js
View File

@ -1,14 +1,19 @@
'use strict';
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import intel from 'intel';
import merge from 'lodash.merge';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { throwIfMissing } from '../../support/util.js';
import { repackage } from './repackage.js';
import { send } from './send.js';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const log = intel.getLogger('plugin.crux');
const defaultConfig = {};
const log = require('intel').getLogger('plugin.crux');
const merge = require('lodash.merge');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
const send = require('./send');
const path = require('path');
const repackage = require('./repackage');
const fs = require('fs');
const DEFAULT_METRICS_PAGESUMMARY = [
'loadingExperience.*.FIRST_CONTENTFUL_PAINT_MS.*',
@ -33,10 +38,11 @@ function wait(ms) {
const CRUX_WAIT_TIME = 300;
module.exports = {
name() {
return path.basename(__dirname);
},
export default class CruxPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'crux', options, context, queue });
}
open(context, options) {
this.make = context.messageMaker('crux').make;
this.options = merge({}, defaultConfig, options.crux);
@ -46,10 +52,7 @@ module.exports = {
this.formFactors = Array.isArray(this.options.formFactor)
? this.options.formFactor
: [this.options.formFactor];
this.pug = fs.readFileSync(
path.resolve(__dirname, 'pug', 'index.pug'),
'utf8'
);
this.pug = readFileSync(resolve(__dirname, 'pug', 'index.pug'), 'utf8');
if (this.options.collect === 'ALL' || this.options.collect === 'URL') {
context.filterRegistry.registerFilterForType(
@ -63,7 +66,7 @@ module.exports = {
'crux.summary'
);
}
},
}
async processMessage(message, queue) {
if (this.options.enable === true) {
const make = this.make;
@ -101,7 +104,7 @@ module.exports = {
this.testedOrigins[group] = true;
log.info(`Get CrUx data for domain ${group}`);
for (let formFactor of this.formFactors) {
originResult.originLoadingExperience[formFactor] = await send.get(
originResult.originLoadingExperience[formFactor] = await send(
url,
this.options.key,
formFactor,
@ -127,7 +130,7 @@ module.exports = {
originResult.originLoadingExperience[formFactor] = repackage(
originResult.originLoadingExperience[formFactor]
);
} catch (e) {
} catch {
log.error(
'Could not repackage the JSON for origin from CrUx, is it broken? %j',
originResult.originLoadingExperience[formFactor]
@ -146,7 +149,7 @@ module.exports = {
log.info(`Get CrUx data for url ${url}`);
const urlResult = { loadingExperience: {} };
for (let formFactor of this.formFactors) {
urlResult.loadingExperience[formFactor] = await send.get(
urlResult.loadingExperience[formFactor] = await send(
url,
this.options.key,
formFactor,
@ -172,7 +175,7 @@ module.exports = {
urlResult.loadingExperience[formFactor] = repackage(
urlResult.loadingExperience[formFactor]
);
} catch (e) {
} catch {
log.error(
'Could not repackage the JSON from CrUx, is it broken? %j',
urlResult.loadingExperience[formFactor]
@ -208,11 +211,5 @@ module.exports = {
}
}
}
},
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
}
};
}

6
lib/plugins/crux/repackage.js
View File

@ -1,6 +1,4 @@
'use strict';
module.exports = function (cruxResult) {
export function repackage(cruxResult) {
const result = {};
if (cruxResult.record.metrics.first_contentful_paint) {
result.FIRST_CONTENTFUL_PAINT_MS = {
@ -77,4 +75,4 @@ module.exports = function (cruxResult) {
result.data = cruxResult;
return result;
};
}

91
lib/plugins/crux/send.js
View File

@ -1,49 +1,46 @@
'use strict';
import { request as _request } from 'node:https';
import intel from 'intel';
const log = intel.getLogger('plugin.crux');
const https = require('https');
const log = require('intel').getLogger('plugin.crux');
module.exports = {
async get(url, key, formFactor, shouldWeTestTheURL) {
let data = shouldWeTestTheURL ? { url } : { origin: url };
if (formFactor !== 'ALL') {
data.formFactor = formFactor;
}
data = JSON.stringify(data);
// Return new promise
return new Promise(function (resolve, reject) {
// Do async job
const req = https.request(
{
host: 'chromeuxreport.googleapis.com',
port: 443,
path: `/v1/records:queryRecord?key=${key}`,
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(data, 'utf8')
},
method: 'POST'
},
function (res) {
if (res.statusCode >= 499) {
log.error(
`Got error from CrUx. Error Code: ${res.statusCode} Message: ${res.statusMessage}`
);
return reject(new Error(`Status Code: ${res.statusCode}`));
}
const data = [];
res.on('data', chunk => {
data.push(chunk);
});
res.on('end', () =>
resolve(JSON.parse(Buffer.concat(data).toString()))
);
}
);
req.write(data);
req.end();
});
export async function send(url, key, formFactor, shouldWeTestTheURL) {
let data = shouldWeTestTheURL ? { url } : { origin: url };
if (formFactor !== 'ALL') {
data.formFactor = formFactor;
}
};
data = JSON.stringify(data);
// Return new promise
return new Promise(function (resolve, reject) {
// Do async job
const request = _request(
{
host: 'chromeuxreport.googleapis.com',
port: 443,
path: `/v1/records:queryRecord?key=${key}`,
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(data, 'utf8')
},
method: 'POST'
},
function (res) {
if (res.statusCode >= 499) {
log.error(
`Got error from CrUx. Error Code: ${res.statusCode} Message: ${res.statusMessage}`
);
return reject(new Error(`Status Code: ${res.statusCode}`));
}
const data = [];
res.on('data', chunk => {
data.push(chunk);
});
res.on('end', () =>
resolve(JSON.parse(Buffer.concat(data).toString()))
);
}
);
request.write(data);
request.end();
});
}

51
lib/plugins/domains/aggregator.js
View File

@ -1,11 +1,13 @@
'use strict';
import { parse } from 'node:url';
const Stats = require('fast-stats').Stats,
urlParser = require('url'),
log = require('intel').getLogger('sitespeedio.plugin.domains'),
statsHelpers = require('../../support/statsHelpers'),
isEmpty = require('lodash.isempty'),
reduce = require('lodash.reduce');
import { Stats } from 'fast-stats';
import intel from 'intel';
import isEmpty from 'lodash.isempty';
import reduce from 'lodash.reduce';
import { summarizeStats } from '../../support/statsHelpers.js';
const log = intel.getLogger('sitespeedio.plugin.domains');
const timingNames = [
'blocked',
@ -18,7 +20,7 @@ const timingNames = [
];
function parseDomainName(url) {
return urlParser.parse(url).hostname;
return parse(url).hostname;
}
function getDomain(domainName) {
@ -45,16 +47,16 @@ function calc(domains) {
domainName
};
const stats = statsHelpers.summarizeStats(domainStats.totalTime);
const stats = summarizeStats(domainStats.totalTime);
if (!isEmpty(stats)) {
domainSummary.totalTime = stats;
}
timingNames.forEach(name => {
const stats = statsHelpers.summarizeStats(domainStats[name]);
for (const name of timingNames) {
const stats = summarizeStats(domainStats[name]);
if (!isEmpty(stats)) {
domainSummary[name] = stats;
}
});
}
summary[domainName] = domainSummary;
return summary;
@ -66,12 +68,15 @@ function calc(domains) {
function isValidTiming(timing) {
// The HAR format uses -1 to indicate invalid/missing timings
// isNan see https://github.com/sitespeedio/sitespeed.io/issues/2159
return typeof timing === 'number' && timing !== -1 && !isNaN(timing);
return typeof timing === 'number' && timing !== -1 && !Number.isNaN(timing);
}
module.exports = {
groups: {},
domains: {},
export class DomainsAggregator {
constructor() {
this.domains = {};
this.groups = {};
}
addToAggregate(har, url) {
const mainDomain = parseDomainName(url);
if (this.groups[mainDomain] === undefined) {
@ -79,10 +84,10 @@ module.exports = {
}
const firstPageId = har.log.pages[0].id;
har.log.entries.forEach(entry => {
for (const entry of har.log.entries) {
if (entry.pageref !== firstPageId) {
// Only pick the first request out of multiple runs.
return;
continue;
}
const domainName = parseDomainName(entry.request.url),
@ -101,18 +106,18 @@ module.exports = {
log.debug('Missing time from har entry for url: ' + entry.request.url);
}
timingNames.forEach(name => {
for (const name of timingNames) {
const timing = entry.timings[name];
if (isValidTiming(timing)) {
domain[name].push(timing);
groupDomain[name].push(timing);
}
});
}
this.domains[domainName] = domain;
this.groups[mainDomain][domainName] = groupDomain;
});
},
}
}
summarize() {
const summary = {
groups: {
@ -126,4 +131,4 @@ module.exports = {
return summary;
}
};
}

23
lib/plugins/domains/index.js
View File

@ -1,14 +1,19 @@
'use strict';
const isEmpty = require('lodash.isempty');
const aggregator = require('./aggregator');
import isEmpty from 'lodash.isempty';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { DomainsAggregator } from './aggregator.js';
export default class DomainsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'domains', options, context, queue });
}
module.exports = {
open(context) {
this.make = context.messageMaker('domains').make;
// '*.requestCounts, 'domains.summary'
context.filterRegistry.registerFilterForType([], 'domains.summary');
this.browsertime = false;
},
this.domainsAggregator = new DomainsAggregator();
}
processMessage(message, queue) {
const make = this.make;
switch (message.type) {
@ -18,20 +23,20 @@ module.exports = {
}
case 'browsertime.har': {
aggregator.addToAggregate(message.data, message.url);
this.domainsAggregator.addToAggregate(message.data, message.url);
break;
}
case 'webpagetest.har': {
// Only collect WebPageTest data if we don't run Browsertime
if (this.browsertime === false) {
aggregator.addToAggregate(message.data, message.url);
this.domainsAggregator.addToAggregate(message.data, message.url);
}
break;
}
case 'sitespeedio.summarize': {
const summary = aggregator.summarize();
const summary = this.domainsAggregator.summarize();
if (!isEmpty(summary)) {
for (let group of Object.keys(summary.groups)) {
queue.postMessage(
@ -43,4 +48,4 @@ module.exports = {
}
}
}
};
}

63
lib/plugins/gcs/index.js
View File

@ -1,26 +1,26 @@
'use strict';
const fs = require('fs-extra');
const path = require('path');
const readdir = require('recursive-readdir');
import { relative, join, resolve, sep } from 'node:path';
import { statSync, remove } from 'fs-extra';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import readdir from 'recursive-readdir';
// Documentation of @google-cloud/storage: https://cloud.google.com/nodejs/docs/reference/storage/2.3.x/Bucket#upload
const { Storage } = require('@google-cloud/storage');
import { Storage } from '@google-cloud/storage';
import intel from 'intel';
import { throwIfMissing } from '../../support/util';
const log = require('intel').getLogger('sitespeedio.plugin.gcs');
const throwIfMissing = require('../../support/util').throwIfMissing;
const log = intel.getLogger('sitespeedio.plugin.gcs');
function ignoreDirectories(file, stats) {
return stats.isDirectory();
}
async function uploadLatestFiles(dir, gcsOptions, prefix) {
function ignoreDirs(file, stats) {
return stats.isDirectory();
}
const storage = new Storage({
projectId: gcsOptions.projectId,
keyFilename: gcsOptions.key
});
const bucket = storage.bucket(gcsOptions.bucketname);
const files = await readdir(dir, [ignoreDirs]);
const files = await readdir(dir, [ignoreDirectories]);
const promises = [];
for (let file of files) {
@ -41,7 +41,7 @@ async function upload(dir, gcsOptions, prefix) {
const bucket = storage.bucket(gcsOptions.bucketname);
for (let file of files) {
const stats = fs.statSync(file);
const stats = statSync(file);
if (stats.isFile()) {
promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir));
@ -60,10 +60,10 @@ async function uploadFile(
baseDir,
noCacheTime
) {
const subPath = path.relative(baseDir, file);
const fileName = path.join(gcsOptions.path || prefix, subPath);
const subPath = relative(baseDir, file);
const fileName = join(gcsOptions.path || prefix, subPath);
const params = {
const parameters = {
public: !!gcsOptions.public,
destination: fileName,
resumable: false,
@ -71,23 +71,26 @@ async function uploadFile(
gzip: !!gcsOptions.gzip,
metadata: {
metadata: {
cacheControl: 'public, max-age=' + noCacheTime ? 0 : 31536000
cacheControl: 'public, max-age=' + noCacheTime ? 0 : 31_536_000
}
}
};
return bucket.upload(file, params);
return bucket.upload(file, parameters);
}
module.exports = {
export default class GcsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'gcs', options, context, queue });
}
open(context, options) {
this.gcsOptions = options.gcs;
this.options = options;
this.make = context.messageMaker('gcs').make;
throwIfMissing(this.gcsOptions, ['bucketname'], 'gcs');
this.storageManager = context.storageManager;
},
}
async processMessage(message, queue) {
if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the GCS plugin is alive
@ -108,9 +111,9 @@ module.exports = {
this.storageManager.getStoragePrefix()
);
if (this.options.copyLatestFilesToBase) {
const rootPath = path.resolve(baseDir, '..');
const dirsAsArray = rootPath.split(path.sep);
const rootName = dirsAsArray.slice(-1)[0];
const rootPath = resolve(baseDir, '..');
const directoriesAsArray = rootPath.split(sep);
const rootName = directoriesAsArray.slice(-1)[0];
await uploadLatestFiles(rootPath, gcsOptions, rootName);
}
log.info('Finished upload to Google Cloud Storage');
@ -120,18 +123,18 @@ module.exports = {
);
}
if (gcsOptions.removeLocalResult) {
await fs.remove(baseDir);
await remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`);
} else {
log.debug(
`Local result files and directories are stored in ${baseDir}`
);
}
} catch (e) {
queue.postMessage(make('error', e));
log.error('Could not upload to Google Cloud Storage', e);
} catch (error) {
queue.postMessage(make('error', error));
log.error('Could not upload to Google Cloud Storage', error);
}
queue.postMessage(make('gcs.finished'));
}
}
};
}

37
lib/plugins/grafana/cli.js
View File

@ -1,37 +0,0 @@
module.exports = {
host: {
describe: 'The Grafana host used when sending annotations.',
group: 'Grafana'
},
port: {
default: 80,
describe: 'The Grafana port used when sending annotations to Grafana.',
group: 'Grafana'
},
auth: {
describe:
'The Grafana auth/bearer value used when sending annotations to Grafana. If you do not set Bearer/Auth, Bearer is automatically set. See http://docs.grafana.org/http_api/auth/#authentication-api',
group: 'Grafana'
},
annotationTitle: {
describe: 'Add a title to the annotation sent for a run.',
group: 'Grafana'
},
annotationMessage: {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Grafana'
},
annotationTag: {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --grafana.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Grafana'
},
annotationScreenshot: {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Grafana'
}
};

164
lib/plugins/grafana/index.js
View File

@ -1,24 +1,13 @@
'use strict';
const path = require('path');
const sendAnnotations = require('./send-annotation');
const tsdbUtil = require('../../support/tsdbUtil');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
module.exports = {
name() {
return path.basename(__dirname);
},
import { send } from './send-annotation.js';
import { toSafeKey } from '../../support/tsdbUtil.js';
import { throwIfMissing } from '../../support/util.js';
/**
* Define `yargs` options with their respective default values. When displayed by the CLI help message
* all options are namespaced by its plugin name.
*
* @return {Object<string, require('yargs').Options} an object mapping the name of the option and its yargs configuration
*/
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
export default class GrafanaPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'grafana', options, context, queue });
}
open(context, options) {
throwIfMissing(options.grafana, ['host', 'port'], 'grafana');
@ -31,7 +20,7 @@ module.exports = {
this.make = context.messageMaker('grafana').make;
this.alias = {};
this.wptExtras = {};
},
}
processMessage(message, queue) {
if (message.type === 'webpagetest.pageSummary') {
@ -39,9 +28,7 @@ module.exports = {
this.wptExtras[message.url].webPageTestResultURL =
message.data.data.summary;
this.wptExtras[message.url].connectivity = message.connectivity;
this.wptExtras[message.url].location = tsdbUtil.toSafeKey(
message.location
);
this.wptExtras[message.url].location = toSafeKey(message.location);
}
if (this.messageTypesToFireAnnotations.includes(message.type)) {
this.receivedTypesThatFireAnnotations[message.url]
@ -50,66 +37,75 @@ module.exports = {
}
// First catch if we are running Browsertime and/or WebPageTest
if (message.type === 'browsertime.setup') {
this.usingBrowsertime = true;
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
} else if (message.type === 'webpagetest.setup') {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
} else if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the Grafana plugin is alive
queue.postMessage(this.make('grafana.setup'));
} else if (message.type === 'influxdb.setup') {
// Default we use Graphite config, else use influxdb
this.tsdbType = 'influxdb';
} else if (message.type === 'browsertime.config') {
if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType;
}
} else if (message.type === 'browsertime.browser') {
this.browser = message.data.browser;
} else if (
message.type === 'webpagetest.browser' &&
!this.usingBrowsertime
) {
// We are only interested in WebPageTest browser if we run it standalone
this.browser = message.data.browser;
} else if (message.type === 'browsertime.alias') {
this.alias[message.url] = message.data;
} else if (
this.receivedTypesThatFireAnnotations[message.url] ===
this.messageTypesToFireAnnotations.length &&
this.resultUrls.hasBaseUrl()
) {
const absolutePagePath = this.resultUrls.absoluteSummaryPageUrl(
message.url,
this.alias[message.url]
);
this.receivedTypesThatFireAnnotations[message.url] = 0;
return sendAnnotations.send(
message.url,
message.group,
absolutePagePath,
this.useScreenshots,
this.screenshotType,
this.timestamp,
this.tsdbType,
this.alias,
this.wptExtras[message.url],
this.usingBrowsertime,
this.browser,
this.options
);
}
},
switch (message.type) {
case 'browsertime.setup': {
this.usingBrowsertime = true;
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
/**
* At the time of this writing, this property's usage could be verified only by the CLI portion of the codebase.
* Instead of introducting a breaking change in the plugin interface, this is kept.
*
* @todo Inspect the code base and plugin dependencies to ensure this property can be removed (if necessary)
*/
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
break;
}
case 'webpagetest.setup': {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
break;
}
case 'sitespeedio.setup': {
// Let other plugins know that the Grafana plugin is alive
queue.postMessage(this.make('grafana.setup'));
break;
}
case 'influxdb.setup': {
// Default we use Graphite config, else use influxdb
this.tsdbType = 'influxdb';
break;
}
case 'browsertime.config': {
if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType;
}
break;
}
case 'browsertime.browser': {
this.browser = message.data.browser;
break;
}
default: {
if (message.type === 'webpagetest.browser' && !this.usingBrowsertime) {
// We are only interested in WebPageTest browser if we run it standalone
this.browser = message.data.browser;
} else if (message.type === 'browsertime.alias') {
this.alias[message.url] = message.data;
} else if (
this.receivedTypesThatFireAnnotations[message.url] ===
this.messageTypesToFireAnnotations.length &&
this.resultUrls.hasBaseUrl()
) {
const absolutePagePath = this.resultUrls.absoluteSummaryPageUrl(
message.url,
this.alias[message.url]
);
this.receivedTypesThatFireAnnotations[message.url] = 0;
return send(
message.url,
message.group,
absolutePagePath,
this.useScreenshots,
this.screenshotType,
this.timestamp,
this.tsdbType,
this.alias,
this.wptExtras[message.url],
this.usingBrowsertime,
this.browser,
this.options
);
}
}
}
}
};
}

266
lib/plugins/grafana/send-annotation.js
View File

@ -1,141 +1,141 @@
'use strict';
const http = require('http');
const https = require('https');
const log = require('intel').getLogger('sitespeedio.plugin.grafana');
const tsdbUtil = require('../../support/tsdbUtil');
const annotationsHelper = require('../../support/annotationsHelper');
const util = require('../../support/util');
const packageInfo = require('../../../package');
import http from 'node:http';
import https from 'node:https';
import { createRequire } from 'node:module';
module.exports = {
send(
url,
import intel from 'intel';
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
import {
getTagsAsArray,
getAnnotationMessage
} from '../../support/annotationsHelper.js';
import { toArray } from '../../support/util.js';
const require = createRequire(import.meta.url);
const version = require('../../../package.json').version;
const log = intel.getLogger('sitespeedio.plugin.grafana');
export function send(
url,
group,
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
time,
tsdbType,
alias,
webPageTestExtraData,
usingBrowsertime,
browserNameAndVersion,
options
) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
// variables in Grafana.
const connectivity = getConnectivity(options);
const browser = options.browser;
// Hmm, here we have hardcoded Graphite ...
const namespace = options.graphite.namespace.split('.');
const urlAndGroup = getURLAndGroup(
options,
group,
url,
tsdbType === 'graphite'
? options.graphite.includeQueryParams
: options.influxdb.includeQueryParams,
alias
).split('.');
const tags = [
connectivity,
browser,
namespace[0],
namespace[1],
urlAndGroup[0],
urlAndGroup[1]
];
// Avoid having the same annotations twice https://github.com/sitespeedio/sitespeed.io/issues/3277#
if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug);
}
const extraTags = toArray(options.grafana.annotationTag);
// We got some extra tag(s) from the user, let us add them to the annotation
if (extraTags.length > 0) {
tags.push(...extraTags);
}
if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity, webPageTestExtraData.location);
}
const tagsArray = getTagsAsArray(tags);
const message = getAnnotationMessage(
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
time,
tsdbType,
alias,
webPageTestExtraData,
webPageTestExtraData
? webPageTestExtraData.webPageTestResultURL
: undefined,
usingBrowsertime,
browserNameAndVersion,
options
) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
// variables in Grafana.
const connectivity = tsdbUtil.getConnectivity(options);
const browser = options.browser;
// Hmm, here we have hardcoded Graphite ...
const namespace = options.graphite.namespace.split('.');
const urlAndGroup = tsdbUtil
.getURLAndGroup(
options,
group,
url,
tsdbType === 'graphite'
? options.graphite.includeQueryParams
: options.influxdb.includeQueryParams,
alias
)
.split('.');
const tags = [
connectivity,
browser,
namespace[0],
namespace[1],
urlAndGroup[0],
urlAndGroup[1]
];
// Avoid having the same annotations twice https://github.com/sitespeedio/sitespeed.io/issues/3277#
if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug);
}
const extraTags = util.toArray(options.grafana.annotationTag);
// We got some extra tag(s) from the user, let us add them to the annotation
if (extraTags.length > 0) {
tags.push(...extraTags);
}
if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity);
tags.push(webPageTestExtraData.location);
}
const tagsArray = annotationsHelper.getTagsAsArray(tags);
const message = annotationsHelper.getAnnotationMessage(
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
webPageTestExtraData
? webPageTestExtraData.webPageTestResultURL
: undefined,
usingBrowsertime,
options
);
let what =
packageInfo.version +
(browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : '');
if (options.grafana.annotationTitle) {
what = options.grafana.annotationTitle;
}
const timestamp = Math.round(time.valueOf() / 1000);
const postData = `{"what": "${what}", "tags": ${tagsArray}, "data": "${message}", "when": ${timestamp}}`;
const postOptions = {
hostname: options.grafana.host,
port: options.grafana.port,
path: '/api/annotations/graphite',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(postData)
}
};
// If Grafana is behind auth, use it!
if (options.grafana.auth) {
log.debug('Using auth for Grafana');
if (
options.grafana.auth.startsWith('Bearer') ||
options.grafana.auth.startsWith('Basic')
) {
postOptions.headers.Authorization = options.grafana.auth;
} else {
postOptions.headers.Authorization = 'Bearer ' + options.grafana.auth;
}
}
log.verbose('Send annotation to Grafana: %j', postData);
return new Promise((resolve, reject) => {
// not perfect but maybe work for us
const lib = options.grafana.port === 443 ? https : http;
const req = lib.request(postOptions, res => {
if (res.statusCode !== 200) {
const e = new Error(
`Got ${res.statusCode} from Grafana when sending annotation`
);
if (res.statusCode === 403) {
log.warn('Authentication required.', e.message);
} else if (res.statusCode === 401) {
log.warn('No valid authentication.', e.message);
} else {
log.warn(e.message);
}
reject(e);
} else {
res.setEncoding('utf8');
log.debug('Sent annotation to Grafana');
resolve();
}
});
req.on('error', err => {
log.error('Got error from Grafana when sending annotation', err);
reject(err);
});
req.write(postData);
req.end();
});
);
let what =
version +
(browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : '');
if (options.grafana.annotationTitle) {
what = options.grafana.annotationTitle;
}
};
const timestamp = Math.round(time.valueOf() / 1000);
const postData = `{"what": "${what}", "tags": ${tagsArray}, "data": "${message}", "when": ${timestamp}}`;
const postOptions = {
hostname: options.grafana.host,
port: options.grafana.port,
path: '/api/annotations/graphite',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(postData)
}
};
// If Grafana is behind auth, use it!
if (options.grafana.auth) {
log.debug('Using auth for Grafana');
postOptions.headers.Authorization =
options.grafana.auth.startsWith('Bearer') ||
options.grafana.auth.startsWith('Basic')
? options.grafana.auth
: 'Bearer ' + options.grafana.auth;
}
log.verbose('Send annotation to Grafana: %j', postData);
return new Promise((resolve, reject) => {
// not perfect but maybe work for us
const library = options.grafana.port === 443 ? https : http;
const request = library.request(postOptions, res => {
if (res.statusCode !== 200) {
const e = new Error(
`Got ${res.statusCode} from Grafana when sending annotation`
);
if (res.statusCode === 403) {
log.warn('Authentication required.', e.message);
} else if (res.statusCode === 401) {
log.warn('No valid authentication.', e.message);
} else {
log.warn(e.message);
}
reject(e);
} else {
res.setEncoding('utf8');
log.debug('Sent annotation to Grafana');
resolve();
}
});
request.on('error', error => {
log.error('Got error from Grafana when sending annotation', error);
reject(error);
});
request.write(postData);
request.end();
});
}

104
lib/plugins/graphite/cli.js
View File

@ -1,104 +0,0 @@
module.exports = {
host: {
describe: 'The Graphite host used to store captured metrics.',
group: 'Graphite'
},
port: {
default: 2003,
describe: 'The Graphite port used to store captured metrics.',
group: 'Graphite'
},
auth: {
describe:
'The Graphite user and password used for authentication. Format: user:password',
group: 'Graphite'
},
httpPort: {
describe:
'The Graphite port used to access the user interface and send annotations event',
default: 8080,
group: 'Graphite'
},
webHost: {
describe:
'The graphite-web host. If not specified graphite.host will be used.',
group: 'Graphite'
},
namespace: {
default: 'sitespeed_io.default',
describe: 'The namespace key added to all captured metrics.',
group: 'Graphite'
},
includeQueryParams: {
default: false,
describe:
'Whether to include query parameters from the URL in the Graphite keys or not',
type: 'boolean',
group: 'Graphite'
},
arrayTags: {
default: true,
type: 'boolean',
describe:
'Send the tags as Array or a String. In Graphite 1.0 the tags is a array. Before a String',
group: 'Graphite'
},
annotationTitle: {
describe: 'Add a title to the annotation sent for a run.',
group: 'Graphite'
},
annotationMessage: {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Graphite'
},
annotationScreenshot: {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Graphite'
},
sendAnnotation: {
default: true,
type: 'boolean',
describe:
'Send annotations when a run is finished. You need to specify a --resultBaseURL for this to work. However if you for example use a Prometheus exporter, you may want to make sure annotations are not sent, then set it to false.',
group: 'Graphite'
},
annotationRetentionMinutes: {
type: 'number',
describe:
'The retention in minutes, to make annotation match the retention in Graphite.',
group: 'Graphite'
},
statsd: {
default: false,
type: 'boolean',
describe: 'Uses the StatsD interface',
group: 'Graphite'
},
annotationTag: {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --graphite.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Graphite'
},
addSlugToKey: {
default: true,
type: 'boolean',
describe:
'Add the slug (name of the test) as an extra key in the namespace.',
group: 'Graphite'
},
bulkSize: {
default: null,
type: 'number',
describe: 'Break up number of metrics to send with each request.',
group: 'Graphite'
},
messages: {
default: ['pageSummary', 'summary'],
options: ['pageSummary', 'summary', 'run'],
group: 'Graphite'
}
};

89
lib/plugins/graphite/data-generator.js
View File

@ -1,38 +1,40 @@
'use strict';
const flatten = require('../../support/flattenMessage'),
util = require('util'),
graphiteUtil = require('../../support/tsdbUtil'),
reduce = require('lodash.reduce'),
formatEntry = require('./helpers/format-entry'),
isStatsd = require('./helpers/is-statsd');
import util, { format } from 'node:util';
import reduce from 'lodash.reduce';
import {
getConnectivity,
toSafeKey,
getURLAndGroup
} from '../../support/tsdbUtil.js';
import { flattenMessageData } from '../../support/flattenMessage.js';
import { formatEntry } from './helpers/format-entry.js';
import { isStatsD } from './helpers/is-statsd.js';
const STATSD = 'statsd';
const GRAPHITE = 'graphite';
function keyPathFromMessage(message, options, includeQueryParams, alias) {
function keyPathFromMessage(message, options, includeQueryParameters, alias) {
let typeParts = message.type.split('.');
typeParts.push(typeParts.shift());
// always have browser and connectivity in Browsertime and related tools
if (
message.type.match(
/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains|^thirdparty|^axe|^sustainable)/
/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains|^thirdparty|^axe|^sustainable)/.test(
message.type
)
) {
// if we have a friendly name for your connectivity, use that!
let connectivity = graphiteUtil.getConnectivity(options);
let connectivity = getConnectivity(options);
typeParts.splice(1, 0, connectivity);
typeParts.splice(1, 0, options.browser);
} else if (message.type.match(/(^webpagetest)/)) {
} else if (/(^webpagetest)/.test(message.type)) {
if (message.connectivity) {
typeParts.splice(2, 0, message.connectivity);
}
if (message.location) {
typeParts.splice(2, 0, graphiteUtil.toSafeKey(message.location));
typeParts.splice(2, 0, toSafeKey(message.location));
}
} else if (message.type.match(/(^gpsi)/)) {
} else if (/(^gpsi)/.test(message.type)) {
typeParts.splice(2, 0, options.mobile ? 'mobile' : 'desktop');
}
@ -41,17 +43,17 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
typeParts.splice(
1,
0,
graphiteUtil.getURLAndGroup(
getURLAndGroup(
options,
message.group,
message.url,
includeQueryParams,
includeQueryParameters,
alias
)
);
} else if (message.group) {
// add the group of the summary message
typeParts.splice(1, 0, graphiteUtil.toSafeKey(message.group));
typeParts.splice(1, 0, toSafeKey(message.group));
}
if (options.graphite && options.graphite.addSlugToKey) {
@ -60,13 +62,12 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
return typeParts.join('.');
}
class GraphiteDataGenerator {
constructor(namespace, includeQueryParams, options) {
export class GraphiteDataGenerator {
constructor(namespace, includeQueryParameters, options) {
this.namespace = namespace;
this.includeQueryParams = !!includeQueryParams;
this.includeQueryParams = !!includeQueryParameters;
this.options = options;
this.entryFormat = isStatsd(options.graphite) ? STATSD : GRAPHITE;
this.entryFormat = isStatsD(options.graphite) ? STATSD : GRAPHITE;
}
dataFromMessage(message, time, alias) {
@ -80,50 +81,46 @@ class GraphiteDataGenerator {
);
return reduce(
flatten.flattenMessageData(message),
flattenMessageData(message),
(entries, value, key) => {
if (message.type === 'browsertime.run') {
if (key.includes('timings') && key.includes('marks')) {
key = key.replace(/marks\.(\d+)/, function (match, idx) {
key = key.replace(/marks\.(\d+)/, function (match, index) {
return (
'marks.' + message.data.timings.userTimings.marks[idx].name
'marks.' + message.data.timings.userTimings.marks[index].name
);
});
}
if (key.includes('timings') && key.includes('measures')) {
key = key.replace(/measures\.(\d+)/, function (match, idx) {
key = key.replace(/measures\.(\d+)/, function (match, index) {
return (
'measures.' +
message.data.timings.userTimings.measures[idx].name
message.data.timings.userTimings.measures[index].name
);
});
}
}
if (message.type === 'pagexray.run') {
if (key.includes('assets')) {
key = key.replace(
/assets\.(\d+)/,
function (match, idx) {
let url = new URL(message.data.assets[idx].url);
url.search = '';
return 'assets.' + graphiteUtil.toSafeKey(url.toString());
},
{}
);
}
if (message.type === 'pagexray.run' && key.includes('assets')) {
key = key.replace(
/assets\.(\d+)/,
function (match, index) {
let url = new URL(message.data.assets[index].url);
url.search = '';
return 'assets.' + toSafeKey(url.toString());
},
{}
);
}
const fullKey = util.format('%s.%s.%s', this.namespace, keypath, key);
const args = [formatEntry(this.entryFormat), fullKey, value];
this.entryFormat === GRAPHITE && args.push(timestamp);
const fullKey = format('%s.%s.%s', this.namespace, keypath, key);
const arguments_ = [formatEntry(this.entryFormat), fullKey, value];
this.entryFormat === GRAPHITE && arguments_.push(timestamp);
entries.push(util.format.apply(util, args));
entries.push(format.apply(util, arguments_));
return entries;
},
[]
);
}
}
module.exports = GraphiteDataGenerator;

12
lib/plugins/graphite/graphite-sender.js
View File

@ -1,9 +1,7 @@
'use strict';
import { connect } from 'node:net';
import { Sender } from './sender.js';
const net = require('net');
const Sender = require('./sender');
class GraphiteSender extends Sender {
export class GraphiteSender extends Sender {
get facility() {
return 'Graphite';
}
@ -12,7 +10,7 @@ class GraphiteSender extends Sender {
this.log(data);
return new Promise((resolve, reject) => {
const socket = net.connect(this.port, this.host, () => {
const socket = connect(this.port, this.host, () => {
socket.write(data);
socket.end();
resolve();
@ -21,5 +19,3 @@ class GraphiteSender extends Sender {
});
}
}
module.exports = GraphiteSender;

11
lib/plugins/graphite/helpers/format-entry.js
View File

@ -3,12 +3,13 @@
* @param {string} [type='graphite'] ['statsd', 'graphite']
* @return {string} The string template
*/
module.exports = type => {
export function formatEntry(type) {
switch (type) {
case 'statsd':
case 'statsd': {
return '%s:%s|ms';
case 'graphite':
default:
}
default: {
return '%s %s %s';
}
}
};
}

4
lib/plugins/graphite/helpers/is-statsd.js
View File

@ -3,4 +3,6 @@
* @param {Object} opts graphite options
* @return {boolean}
*/
module.exports = (opts = {}) => opts.statsd === true;
export function isStatsD(options = {}) {
return options.statsd === true;
}

154
lib/plugins/graphite/index.js
View File

@ -1,34 +1,25 @@
'use strict';
const path = require('path');
const isEmpty = require('lodash.isempty');
const GraphiteSender = require('./graphite-sender');
const StatsDSender = require('./statsd-sender');
const merge = require('lodash.merge');
const get = require('lodash.get');
const dayjs = require('dayjs');
const log = require('intel').getLogger('sitespeedio.plugin.graphite');
const sendAnnotations = require('./send-annotation');
const DataGenerator = require('./data-generator');
const graphiteUtil = require('../../support/tsdbUtil');
const isStatsd = require('./helpers/is-statsd');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
const toArray = require('../../support/util').toArray;
import isEmpty from 'lodash.isempty';
import merge from 'lodash.merge';
import get from 'lodash.get';
import dayjs from 'dayjs';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
module.exports = {
name() {
return path.basename(__dirname);
},
import { send } from './send-annotation.js';
import { GraphiteDataGenerator as DataGenerator } from './data-generator.js';
import { toSafeKey } from '../../support/tsdbUtil.js';
import { isStatsD } from './helpers/is-statsd.js';
import { throwIfMissing } from '../../support/util.js';
import { toArray } from '../../support/util.js';
import { GraphiteSender } from './graphite-sender.js';
import { StatsDSender } from './statsd-sender.js';
/**
* Define `yargs` options with their respective default values. When displayed by the CLI help message
* all options are namespaced by its plugin name.
*
* @return {Object<string, require('yargs').Options} an object mapping the name of the option and its yargs configuration
*/
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
const log = intel.getLogger('sitespeedio.plugin.graphite');
export default class GraphitePlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'graphite', options, context, queue });
}
open(context, options) {
throwIfMissing(options.graphite, ['host'], 'graphite');
@ -39,61 +30,86 @@ module.exports = {
);
}
const opts = merge({}, this.config, options.graphite);
const options_ = merge({}, this.config, options.graphite);
this.options = options;
this.perIteration = get(opts, 'perIteration', false);
const SenderConstructor = isStatsd(opts) ? StatsDSender : GraphiteSender;
this.perIteration = get(options_, 'perIteration', false);
const SenderConstructor = isStatsD(options_)
? StatsDSender
: GraphiteSender;
this.filterRegistry = context.filterRegistry;
this.sender = new SenderConstructor(opts.host, opts.port, opts.bulkSize);
this.sender = new SenderConstructor(
options_.host,
options_.port,
options_.bulkSize
);
this.dataGenerator = new DataGenerator(
opts.namespace,
opts.includeQueryParams,
options_.namespace,
options_.includeQueryParams,
options
);
log.debug(
'Setting up Graphite %s:%s for namespace %s',
opts.host,
opts.port,
opts.namespace
options_.host,
options_.port,
options_.namespace
);
this.timestamp = context.timestamp;
this.resultUrls = context.resultUrls;
this.messageTypesToFireAnnotations = [];
this.receivedTypesThatFireAnnotations = {};
this.make = context.messageMaker('graphite').make;
this.sendAnnotation = opts.sendAnnotation;
this.sendAnnotation = options_.sendAnnotation;
this.alias = {};
this.wptExtras = {};
this.usingBrowsertime = false;
this.types = toArray(options.graphite.messages);
},
}
processMessage(message, queue) {
// First catch if we are running Browsertime and/or WebPageTest
if (message.type === 'browsertime.setup') {
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
this.usingBrowsertime = true;
} else if (message.type === 'webpagetest.setup') {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
} else if (message.type === 'browsertime.config') {
if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType;
switch (message.type) {
case 'browsertime.setup': {
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
this.usingBrowsertime = true;
break;
}
case 'webpagetest.setup': {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
break;
}
case 'browsertime.config': {
if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType;
}
break;
}
case 'sitespeedio.setup': {
// Let other plugins know that the Graphite plugin is alive
queue.postMessage(this.make('graphite.setup'));
break;
}
case 'grafana.setup': {
this.sendAnnotation = false;
break;
}
case 'browsertime.browser': {
this.browser = message.data.browser;
break;
}
default: {
if (message.type === 'webpagetest.browser' && !this.usingBrowsertime) {
// We are only interested in WebPageTest browser if we run it standalone
this.browser = message.data.browser;
}
}
} else if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the Graphite plugin is alive
queue.postMessage(this.make('graphite.setup'));
} else if (message.type === 'grafana.setup') {
this.sendAnnotation = false;
} else if (message.type === 'browsertime.browser') {
this.browser = message.data.browser;
} else if (
message.type === 'webpagetest.browser' &&
!this.usingBrowsertime
) {
// We are only interested in WebPageTest browser if we run it standalone
this.browser = message.data.browser;
}
if (message.type === 'browsertime.alias') {
@ -118,9 +134,7 @@ module.exports = {
this.wptExtras[message.url].webPageTestResultURL =
message.data.data.summary;
this.wptExtras[message.url].connectivity = message.connectivity;
this.wptExtras[message.url].location = graphiteUtil.toSafeKey(
message.location
);
this.wptExtras[message.url].location = toSafeKey(message.location);
}
// we only sends individual groups to Graphite, not the
@ -162,7 +176,7 @@ module.exports = {
message.url,
this.alias[message.url]
);
return sendAnnotations.send(
return send(
message.url,
message.group,
absolutePagePath,
@ -181,13 +195,9 @@ module.exports = {
return Promise.reject(
new Error(
'No data to send to graphite for message:\n' +
JSON.stringify(message, null, 2)
JSON.stringify(message, undefined, 2)
)
);
}
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
}
};
}

249
lib/plugins/graphite/send-annotation.js
View File

@ -1,133 +1,136 @@
'use strict';
const http = require('http');
const https = require('https');
const log = require('intel').getLogger('sitespeedio.plugin.graphite');
const graphiteUtil = require('../../support/tsdbUtil');
const annotationsHelper = require('../../support/annotationsHelper');
const util = require('../../support/util');
const packageInfo = require('../../../package');
import http from 'node:http';
import https from 'node:https';
import { createRequire } from 'node:module';
import intel from 'intel';
module.exports = {
send(
url,
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
import {
getTagsAsArray,
getTagsAsString,
getAnnotationMessage
} from '../../support/annotationsHelper.js';
import { toArray } from '../../support/util.js';
const require = createRequire(import.meta.url);
const version = require('../../../package.json').version;
const log = intel.getLogger('sitespeedio.plugin.graphite');
export function send(
url,
group,
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
time,
alias,
webPageTestExtraData,
usingBrowsertime,
browserNameAndVersion,
options
) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
// variables in Grafana.
const connectivity = getConnectivity(options);
const browser = options.browser;
const namespace = options.graphite.namespace.split('.');
const urlAndGroup = getURLAndGroup(
options,
group,
url,
options.graphite.includeQueryParams,
alias
).split('.');
const tags = [
connectivity,
browser,
namespace[0],
namespace[1],
urlAndGroup[0],
urlAndGroup[1]
];
// See https://github.com/sitespeedio/sitespeed.io/issues/3277
if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug);
}
const extraTags = toArray(options.graphite.annotationTag);
// We got some extra tag(s) from the user, let us add them to the annotation
if (extraTags.length > 0) {
tags.push(...extraTags);
}
if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity, webPageTestExtraData.location);
}
const theTags = options.graphite.arrayTags
? getTagsAsArray(tags)
: getTagsAsString(tags);
const message = getAnnotationMessage(
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
time,
alias,
webPageTestExtraData,
webPageTestExtraData
? webPageTestExtraData.webPageTestResultURL
: undefined,
usingBrowsertime,
browserNameAndVersion,
options
) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
// variables in Grafana.
const connectivity = graphiteUtil.getConnectivity(options);
const browser = options.browser;
const namespace = options.graphite.namespace.split('.');
const urlAndGroup = graphiteUtil
.getURLAndGroup(
options,
group,
url,
options.graphite.includeQueryParams,
alias
)
.split('.');
const tags = [
connectivity,
browser,
namespace[0],
namespace[1],
urlAndGroup[0],
urlAndGroup[1]
];
// See https://github.com/sitespeedio/sitespeed.io/issues/3277
if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug);
}
const extraTags = util.toArray(options.graphite.annotationTag);
// We got some extra tag(s) from the user, let us add them to the annotation
if (extraTags.length > 0) {
tags.push(...extraTags);
}
if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity);
tags.push(webPageTestExtraData.location);
}
const theTags = options.graphite.arrayTags
? annotationsHelper.getTagsAsArray(tags)
: annotationsHelper.getTagsAsString(tags);
);
const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo;
const roundDownToMinutes = roundDownTo(
1000 * 60 * options.graphite.annotationRetentionMinutes
);
const message = annotationsHelper.getAnnotationMessage(
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
webPageTestExtraData
? webPageTestExtraData.webPageTestResultURL
: undefined,
usingBrowsertime,
options
);
const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo;
const roundDownToMinutes = roundDownTo(
1000 * 60 * options.graphite.annotationRetentionMinutes
);
const timestamp = options.graphite.annotationRetentionMinutes
? Math.round(roundDownToMinutes(time.valueOf()) / 1000)
: Math.round(time.valueOf() / 1000);
const timestamp = options.graphite.annotationRetentionMinutes
? Math.round(roundDownToMinutes(time.valueOf()) / 1000)
: Math.round(time.valueOf() / 1000);
let what =
packageInfo.version +
(browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : '');
if (options.graphite.annotationTitle) {
what = options.graphite.annotationTitle;
}
const postData = `{"what": "${what}", "tags": ${theTags}, "data": "${message}", "when": ${timestamp}}`;
const postOptions = {
hostname: options.graphite.webHost || options.graphite.host,
port: options.graphite.httpPort || 8080,
path: '/events/',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(postData)
}
};
// If Graphite is behind auth, use it!
if (options.graphite.auth) {
log.debug('Using auth for Graphite');
postOptions.auth = options.graphite.auth;
}
return new Promise((resolve, reject) => {
log.verbose('Send annotation to Graphite: %j', postData);
// not perfect but maybe work for us
const lib = options.graphite.httpPort === 443 ? https : http;
const req = lib.request(postOptions, res => {
if (res.statusCode !== 200) {
const e = new Error(
`Got ${res.statusCode} from Graphite when sending annotation`
);
log.warn(e.message);
reject(e);
} else {
res.setEncoding('utf8');
log.debug('Sent annotation to Graphite');
resolve();
}
});
req.on('error', err => {
log.error('Got error from Graphite when sending annotation', err);
reject(err);
});
req.write(postData);
req.end();
});
let what =
version +
(browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : '');
if (options.graphite.annotationTitle) {
what = options.graphite.annotationTitle;
}
};
const postData = `{"what": "${what}", "tags": ${theTags}, "data": "${message}", "when": ${timestamp}}`;
const postOptions = {
hostname: options.graphite.webHost || options.graphite.host,
port: options.graphite.httpPort || 8080,
path: '/events/',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(postData)
}
};
// If Graphite is behind auth, use it!
if (options.graphite.auth) {
log.debug('Using auth for Graphite');
postOptions.auth = options.graphite.auth;
}
return new Promise((resolve, reject) => {
log.verbose('Send annotation to Graphite: %j', postData);
// not perfect but maybe work for us
const library = options.graphite.httpPort === 443 ? https : http;
const request = library.request(postOptions, res => {
if (res.statusCode !== 200) {
const e = new Error(
`Got ${res.statusCode} from Graphite when sending annotation`
);
log.warn(e.message);
reject(e);
} else {
res.setEncoding('utf8');
log.debug('Sent annotation to Graphite');
resolve();
}
});
request.on('error', error => {
log.error('Got error from Graphite when sending annotation', error);
reject(error);
});
request.write(postData);
request.end();
});
}

13
lib/plugins/graphite/sender.js
View File

@ -1,8 +1,7 @@
'use strict';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.graphite');
const log = require('intel').getLogger('sitespeedio.plugin.graphite');
class Sender {
export class Sender {
constructor(host, port, bulkSize) {
this.host = host;
this.port = port;
@ -29,14 +28,12 @@ class Sender {
bulks(data) {
const lines = data.split('\n');
const promises = [];
const bulkSize = this.bulkSize || lines.length;
const bulkSize = this.bulkSize || lines.length > 0;
while (lines.length) {
while (lines.length > 0) {
promises.push(this.bulk(lines.splice(-bulkSize).join('\n')));
}
return Promise.all(promises);
}
}
module.exports = Sender;

12
lib/plugins/graphite/statsd-sender.js
View File

@ -1,9 +1,7 @@
'use strict';
import { createSocket } from 'node:dgram';
import { Sender } from './sender.js';
const dgram = require('dgram');
const Sender = require('./sender');
class StatsDSender extends Sender {
export class StatsDSender extends Sender {
get facility() {
return 'StatsD';
}
@ -12,7 +10,7 @@ class StatsDSender extends Sender {
this.log(data);
return new Promise((resolve, reject) => {
const client = dgram.createSocket('udp4');
const client = createSocket('udp4');
client.send(data, 0, data.length, this.port, this.host, error =>
client.close() && error ? reject(error) : resolve()
@ -20,5 +18,3 @@ class StatsDSender extends Sender {
});
}
}
module.exports = StatsDSender;

54
lib/plugins/harstorer/index.js
View File

@ -1,15 +1,19 @@
'use strict';
import { gzip as _gzip } from 'node:zlib';
import { promisify } from 'node:util';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const gzip = promisify(_gzip);
const zlib = require('zlib');
const { promisify } = require('util');
const gzip = promisify(zlib.gzip);
export default class HarstorerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'harstorer', options, context, queue });
}
module.exports = {
open(context, options) {
this.storageManager = context.storageManager;
this.gzipHAR = options.gzipHAR;
this.alias = {};
},
}
processMessage(message) {
switch (message.type) {
case 'browsertime.alias': {
@ -20,29 +24,27 @@ module.exports = {
case 'webpagetest.har': {
const json = JSON.stringify(message.data);
if (this.gzipHAR) {
return gzip(Buffer.from(json), {
level: 1
}).then(gziped =>
this.storageManager.writeDataForUrl(
gziped,
`${message.type}.gz`,
return this.gzipHAR
? gzip(Buffer.from(json), {
level: 1
}).then(gziped =>
this.storageManager.writeDataForUrl(
gziped,
`${message.type}.gz`,
message.url,
undefined,
this.alias[message.url]
)
)
: this.storageManager.writeDataForUrl(
json,
message.type,
message.url,
undefined,
this.alias[message.url]
)
);
} else {
return this.storageManager.writeDataForUrl(
json,
message.type,
message.url,
undefined,
this.alias[message.url]
);
}
);
}
}
}
};
}

14
lib/plugins/html/dataCollector.js
View File

@ -1,11 +1,9 @@
'use strict';
import merge from 'lodash.merge';
import get from 'lodash.get';
import reduce from 'lodash.reduce';
import set from 'lodash.set';
const merge = require('lodash.merge'),
get = require('lodash.get'),
reduce = require('lodash.reduce'),
set = require('lodash.set');
class DataCollector {
export class DataCollector {
constructor(resultUrls) {
this.resultUrls = resultUrls;
this.urlRunPages = {};
@ -138,5 +136,3 @@ class DataCollector {
return this.budget;
}
}
module.exports = DataCollector;

4
lib/plugins/html/defaultConfig.js
View File

@ -1,6 +1,4 @@
'use strict';
module.exports = {
export default {
html: {
showAllWaterfallSummary: false,
pageSummaryMetrics: [

11
lib/plugins/html/getScripts.js
View File

@ -1,9 +1,8 @@
'use strict';
const fs = require('fs');
const { promisify } = require('util');
const readFile = promisify(fs.readFile);
import { readFile as _readFile } from 'node:fs';
import { promisify } from 'node:util';
const readFile = promisify(_readFile);
module.exports = async options => {
export default async options => {
const scripts = [];
for (let file of options._) {
// We could promise all these in the future
@ -11,7 +10,7 @@ module.exports = async options => {
try {
const code = await readFile(file);
scripts.push({ name: file, code: code });
} catch (e) {
} catch {
// do nada
}
}

122
lib/plugins/html/htmlBuilder.js
View File

@ -1,34 +1,43 @@
'use strict';
import { join } from 'node:path';
import osName from 'os-name';
import { promisify } from 'node:util';
import { platform } from 'node:os';
import { createRequire } from 'node:module';
import { fileURLToPath } from 'node:url';
import dayjs from 'dayjs';
import getos from 'getos';
import intel from 'intel';
import { markdown } from 'markdown';
import merge from 'lodash.merge';
import get from 'lodash.get';
import isEmpty from 'lodash.isempty';
import chunk from 'lodash.chunk';
const helpers = require('../../support/helpers');
const path = require('path');
const osName = require('os-name');
const getos = require('getos');
const { promisify } = require('util');
const getOS = promisify(getos);
const os = require('os');
const merge = require('lodash.merge');
const get = require('lodash.get');
const log = require('intel').getLogger('sitespeedio.plugin.html');
const chunk = require('lodash.chunk');
const packageInfo = require('../../../package');
const renderer = require('./renderer');
const metricHelper = require('./metricHelper');
const markdown = require('markdown').markdown;
const isEmpty = require('lodash.isempty');
const dayjs = require('dayjs');
const defaultConfigHTML = require('./defaultConfig');
const summaryBoxesSetup = require('./setup/summaryBoxes');
const detailedSetup = require('./setup/detailed');
const log = intel.getLogger('sitespeedio.plugin.html');
const require = createRequire(import.meta.url);
const { dependencies, version } = require('../../../package.json');
import { renderTemplate } from './renderer.js';
import {
pickMedianRun,
getMetricsFromPageSummary,
getMetricsFromRun
} from './metricHelper.js';
const filmstrip = require('../browsertime/filmstrip');
const getScripts = require('./getScripts');
const friendlyNames = require('../../support/friendlynames');
const toArray = require('../../support/util').toArray;
import * as helpers from '../../support/helpers/index.js';
import * as _html from './defaultConfig.js';
import summaryBoxesSetup from './setup/summaryBoxes.js';
import detailedSetup from './setup/detailed.js';
import { getFilmstrip } from '../browsertime/filmstrip.js';
import getScripts from './getScripts.js';
import friendlyNames from '../../support/friendlynames.js';
import { toArray } from '../../support/util.js';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
class HTMLBuilder {
export class HTMLBuilder {
constructor(context, options) {
this.storageManager = context.storageManager;
this.timestamp = context.timestamp.format(TIME_FORMAT);
@ -55,8 +64,9 @@ class HTMLBuilder {
this.summaries.push({ id, name });
break;
}
default:
default: {
log.info('Got a undefined page type ' + type);
}
}
}
@ -197,7 +207,7 @@ class HTMLBuilder {
for (let url of Object.keys(validPages)) {
const pageInfo = validPages[url];
const runPages = dataCollector.getURLRuns(url);
const medianRun = metricHelper.pickMedianRun(runPages, pageInfo);
const medianRun = pickMedianRun(runPages, pageInfo);
// If we have multiple URLs in the same HAR the median run must be converted
// to the right run in the HAR
const harIndex = pageNumber + (medianRun.runIndex - 1) * testedPages;
@ -272,7 +282,7 @@ class HTMLBuilder {
const medianPageInfo = runPages[medianRun.runIndex - 1];
let filmstripData =
medianPageInfo && medianPageInfo.data && medianPageInfo.data.browsertime
? await filmstrip.getFilmstrip(
? await getFilmstrip(
medianPageInfo.data.browsertime.run,
medianRun.runIndex,
this.storageManager.getFullPathToURLDir(url, daurlAlias),
@ -310,18 +320,18 @@ class HTMLBuilder {
this.options.browsertime.iterations,
'run'
)} ${url} at ${summaryTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromPageSummary(
pageDescription: `${getMetricsFromPageSummary(
pageInfo
)} collected by sitespeed.io ${packageInfo.version}`,
)} collected by sitespeed.io ${version}`,
headers: this.summary,
version: packageInfo.version,
version: version,
timestamp: summaryTimestamp,
context: this.context,
pageSummaries
};
for (const summary of pageSummaries) {
pugs[summary.id] = renderer.renderTemplate(summary.id, data);
pugs[summary.id] = renderTemplate(summary.id, data);
}
data.pugs = pugs;
@ -339,7 +349,7 @@ class HTMLBuilder {
);
const filmstripData = pageInfo.data.browsertime
? await filmstrip.getFilmstrip(
? await getFilmstrip(
pageInfo.data.browsertime.run,
iteration,
this.storageManager.getFullPathToURLDir(url, daurlAlias),
@ -379,13 +389,13 @@ class HTMLBuilder {
assetsPath: assetsBaseURL || rootPath,
menu: 'pages',
pageTitle: `Run ${
parseInt(runIndex) + 1
Number.parseInt(runIndex) + 1
} for ${url} at ${runTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromRun(
pageDescription: `${getMetricsFromRun(
pageInfo
)} collected by sitespeed.io ${packageInfo.version}`,
)} collected by sitespeed.io ${version}`,
headers: this.summary,
version: packageInfo.version,
version: version,
timestamp: runTimestamp,
friendlyNames,
context: this.context,
@ -393,16 +403,21 @@ class HTMLBuilder {
};
// Add pugs for extra plugins
for (const run of pageRuns) {
pugs[run.id] = renderer.renderTemplate(run.id, data);
pugs[run.id] = renderTemplate(run.id, data);
}
data.pugs = pugs;
urlPageRenders.push(
this._renderUrlRunPage(url, parseInt(runIndex) + 1, data, daurlAlias)
this._renderUrlRunPage(
url,
Number.parseInt(runIndex) + 1,
data,
daurlAlias
)
);
// Do only once per URL
if (parseInt(runIndex) === 0) {
if (Number.parseInt(runIndex) === 0) {
data.mySummary = mySummary;
urlPageRenders.push(
this._renderMetricSummaryPage(url, 'metrics', data, daurlAlias)
@ -414,11 +429,10 @@ class HTMLBuilder {
// Kind of clumsy way to decide if the user changed HTML summaries,
// so we in the pug can automatically add visual metrics
const hasPageSummaryMetricInput =
options.html.pageSummaryMetrics !==
defaultConfigHTML.html.pageSummaryMetrics;
options.html.pageSummaryMetrics !== _html.pageSummaryMetrics;
let osInfo = osName();
if (os.platform() === 'linux') {
if (platform() === 'linux') {
const linux = await getOS();
osInfo = `${linux.dist} ${linux.release}`;
}
@ -451,8 +465,8 @@ class HTMLBuilder {
usingBrowsertime,
usingWebPageTest,
headers: this.summary,
version: packageInfo.version,
browsertimeVersion: packageInfo.dependencies.browsertime,
version: version,
browsertimeVersion: dependencies.browsertime,
timestamp: this.timestamp,
context: this.context,
get,
@ -469,11 +483,9 @@ class HTMLBuilder {
);
let res;
if (this.options.html.assetsBaseURL) {
res = Promise.resolve();
} else {
res = this.storageManager.copyToResultDir(path.join(__dirname, 'assets'));
}
res = this.options.html.assetsBaseURL
? Promise.resolve()
: this.storageManager.copyToResultDir(join(__dirname, 'assets'));
return res.then(() =>
Promise.allSettled(summaryRenders)
@ -487,7 +499,7 @@ class HTMLBuilder {
async _renderUrlPage(url, name, locals, alias) {
log.debug('Render URL page %s', name);
return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/summary/' + name, locals),
renderTemplate('url/summary/' + name, locals),
name + '.html',
url,
alias
@ -497,7 +509,7 @@ class HTMLBuilder {
async _renderUrlRunPage(url, name, locals, alias) {
log.debug('Render URL run page %s', name);
return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/iteration/index', locals),
renderTemplate('url/iteration/index', locals),
name + '.html',
url,
alias
@ -507,7 +519,7 @@ class HTMLBuilder {
async _renderMetricSummaryPage(url, name, locals, alias) {
log.debug('Render URL metric page %s', name);
return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/summary/metrics/index', locals),
renderTemplate('url/summary/metrics/index', locals),
name + '.html',
url,
alias
@ -518,10 +530,8 @@ class HTMLBuilder {
log.debug('Render summary page %s', name);
return this.storageManager.writeHtml(
renderer.renderTemplate(name, locals),
renderTemplate(name, locals),
name + '.html'
);
}
}
module.exports = HTMLBuilder;

36
lib/plugins/html/index.js
View File

@ -1,17 +1,20 @@
'use strict';
const HTMLBuilder = require('./htmlBuilder');
const get = require('lodash.get');
const set = require('lodash.set');
const reduce = require('lodash.reduce');
const DataCollector = require('./dataCollector');
const renderer = require('./renderer');
import get from 'lodash.get';
import set from 'lodash.set';
import reduce from 'lodash.reduce';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { DataCollector } from './dataCollector.js';
import { HTMLBuilder } from './htmlBuilder.js';
import { addTemplate } from './renderer.js';
// lets keep this in the HTML context, since we need things from the
// regular options object in the output
const defaultConfig = require('./defaultConfig');
import defaultConfig from './defaultConfig.js';
export default class HTMLPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'html', options, context, queue });
}
module.exports = {
open(context, options) {
this.make = context.messageMaker('html').make;
// we have to overwrite the default summary metrics, if given
@ -37,13 +40,13 @@ module.exports = {
'thirdparty.pageSummary',
'crux.pageSummary'
];
},
}
processMessage(message, queue) {
const dataCollector = this.dataCollector;
const make = this.make;
// If this type is registered
if (this.collectDataFrom.indexOf(message.type) > -1) {
if (this.collectDataFrom.includes(message.type)) {
dataCollector.addDataForUrl(
message.url,
message.type,
@ -81,7 +84,7 @@ module.exports = {
case 'html.pug': {
// we got a pug from plugins, let compile and cache them
renderer.addTemplate(message.data.id, message.data.pug);
addTemplate(message.data.id, message.data.pug);
// and also keep the types so we can render them
this.HTMLBuilder.addType(
message.data.id,
@ -230,6 +233,7 @@ module.exports = {
}
}
}
},
config: defaultConfig
};
}
}
export { default as config } from './defaultConfig.js';

156
lib/plugins/html/metricHelper.js
View File

@ -1,90 +1,86 @@
const get = require('lodash.get');
/* eslint-disable unicorn/no-nested-ternary */
import get from 'lodash.get';
module.exports = {
pickMedianRun(runs, pageInfo) {
// Choose the median run. Early first version, in the future we can make
// this configurable through the CLI
// If we have SpeedIndex use that else backup with loadEventEnd
const speedIndexMedian = get(
pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics.SpeedIndex.median'
);
const loadEventEndMedian = get(
pageInfo,
'data.browsertime.pageSummary.statistics.timings.loadEventEnd.median'
);
if (speedIndexMedian) {
for (let run of runs) {
if (
// https://github.com/sitespeedio/sitespeed.io/issues/3618
run.data.browsertime.run.visualMetrics &&
speedIndexMedian === run.data.browsertime.run.visualMetrics.SpeedIndex
) {
return {
name: 'SpeedIndex',
runIndex: run.runIndex + 1
};
}
}
} else if (loadEventEndMedian) {
for (let rumRuns of runs) {
// make sure we run Browsertime for that run = 3 runs WPT and 2 runs BT
if (
rumRuns.data.browsertime &&
loadEventEndMedian ===
rumRuns.data.browsertime.run.timings.loadEventEnd
) {
return {
name: 'LoadEventEnd',
runIndex: rumRuns.runIndex + 1
};
}
export function pickMedianRun(runs, pageInfo) {
// Choose the median run. Early first version, in the future we can make
// this configurable through the CLI
// If we have SpeedIndex use that else backup with loadEventEnd
const speedIndexMedian = get(
pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics.SpeedIndex.median'
);
const loadEventEndMedian = get(
pageInfo,
'data.browsertime.pageSummary.statistics.timings.loadEventEnd.median'
);
if (speedIndexMedian) {
for (let run of runs) {
if (
// https://github.com/sitespeedio/sitespeed.io/issues/3618
run.data.browsertime.run.visualMetrics &&
speedIndexMedian === run.data.browsertime.run.visualMetrics.SpeedIndex
) {
return {
name: 'SpeedIndex',
runIndex: run.runIndex + 1
};
}
}
return {
name: '',
runIndex: 1,
default: true
};
},
// Get metrics from a run as a String to use in description
getMetricsFromRun(pageInfo) {
const visualMetrics = get(pageInfo, 'data.browsertime.run.visualMetrics');
const timings = get(pageInfo, 'data.browsertime.run.timings');
if (visualMetrics) {
return `First Visual Change: ${visualMetrics.FirstVisualChange},
} else if (loadEventEndMedian) {
for (let rumRuns of runs) {
// make sure we run Browsertime for that run = 3 runs WPT and 2 runs BT
if (
rumRuns.data.browsertime &&
loadEventEndMedian === rumRuns.data.browsertime.run.timings.loadEventEnd
) {
return {
name: 'LoadEventEnd',
runIndex: rumRuns.runIndex + 1
};
}
}
}
return {
name: '',
runIndex: 1,
default: true
};
}
export function getMetricsFromRun(pageInfo) {
const visualMetrics = get(pageInfo, 'data.browsertime.run.visualMetrics');
const timings = get(pageInfo, 'data.browsertime.run.timings');
if (visualMetrics) {
return `First Visual Change: ${visualMetrics.FirstVisualChange},
Speed Index: ${visualMetrics.SpeedIndex},
Visual Complete 85%: ${visualMetrics.VisualComplete85},
Last Visual Change: ${visualMetrics.LastVisualChange}`;
} else if (timings) {
return `Load Event End: ${timings.loadEventEnd}`;
} else {
return '';
}
},
getMetricsFromPageSummary(pageInfo) {
const visualMetrics = get(
pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics'
);
const timings = get(
pageInfo,
'data.browsertime.pageSummary.statistics.timings'
);
if (visualMetrics) {
return `Median First Visual Change: ${visualMetrics.FirstVisualChange.median},
} else if (timings) {
return `Load Event End: ${timings.loadEventEnd}`;
} else {
return '';
}
}
export function getMetricsFromPageSummary(pageInfo) {
const visualMetrics = get(
pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics'
);
const timings = get(
pageInfo,
'data.browsertime.pageSummary.statistics.timings'
);
if (visualMetrics) {
return `Median First Visual Change: ${visualMetrics.FirstVisualChange.median},
Median Speed Index: ${visualMetrics.SpeedIndex.median},
Median Visual Complete 85%: ${visualMetrics.VisualComplete85.median},
Median Last Visual Change: ${visualMetrics.LastVisualChange.median}`;
} else if (timings) {
return timings.loadEventEnd
? `Median LoadEventEnd: ${timings.loadEventEnd.median}`
: '' + timings.fullyLoaded
? `Median Fully loaded: ${timings.fullyLoaded.median}`
: '';
} else {
return '';
}
} else if (timings) {
return timings.loadEventEnd
? `Median LoadEventEnd: ${timings.loadEventEnd.median}`
: '' + timings.fullyLoaded
? `Median Fully loaded: ${timings.fullyLoaded.median}`
: '';
} else {
return '';
}
};
}

39
lib/plugins/html/renderer.js
View File

@ -1,9 +1,12 @@
'use strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const pug = require('pug');
const path = require('path');
const log = require('intel').getLogger('sitespeedio.plugin.html');
const basePath = path.resolve(__dirname, 'templates');
import { compileFile, compile } from 'pug';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.html');
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const basePath = resolve(__dirname, 'templates');
const templateCache = {};
@ -15,23 +18,21 @@ function getTemplate(templateName) {
return template;
}
const filename = path.resolve(basePath, templateName);
const renderedTemplate = pug.compileFile(filename);
const filename = resolve(basePath, templateName);
const renderedTemplate = compileFile(filename);
templateCache[templateName] = renderedTemplate;
return renderedTemplate;
}
module.exports = {
renderTemplate(templateName, locals) {
try {
return getTemplate(templateName)(locals);
} catch (e) {
log.error('Could not generate %s, %s', templateName, e.message);
}
},
addTemplate(templateName, templateString) {
const compiledTemplate = pug.compile(templateString);
templateCache[templateName + '.pug'] = compiledTemplate;
export function renderTemplate(templateName, locals) {
try {
return getTemplate(templateName)(locals);
} catch (error) {
log.error('Could not generate %s, %s', templateName, error.message);
}
};
}
export function addTemplate(templateName, templateString) {
const compiledTemplate = compile(templateString);
templateCache[templateName + '.pug'] = compiledTemplate;
}

94
lib/plugins/html/setup/detailed.js
View File

@ -1,22 +1,20 @@
'use strict';
const h = require('../../../support/helpers');
const get = require('lodash.get');
import { noop, size, time } from '../../../support/helpers/index.js';
import get from 'lodash.get';
function row(stat, name, metricName, formatter) {
if (typeof stat === 'undefined') {
return undefined;
return;
}
return {
name,
metricName,
node: stat,
h: formatter ? formatter : h.noop
h: formatter ?? noop
};
}
module.exports = function (data) {
export default function (data) {
if (!data) {
return [];
}
@ -65,41 +63,38 @@ module.exports = function (data) {
'jsRequestsPerPage'
),
row(contentTypes.font.requests, 'Font requests', 'fontRequestsPerPage'),
row(summary.requests, 'Total requests', 'totalRequestsPerPage')
);
rows.push(
row(summary.requests, 'Total requests', 'totalRequestsPerPage'),
row(
contentTypes.image.transferSize,
'Image size',
'imageSizePerPage',
h.size.format
size.format
),
row(
contentTypes.html.transferSize,
'HTML size',
'htmlSizePerPage',
h.size.format
size.format
),
row(
contentTypes.css.transferSize,
'CSS size',
'cssSizePerPage',
h.size.format
size.format
),
row(
contentTypes.javascript.transferSize,
'Javascript size',
'jsSizePerPage',
h.size.format
size.format
),
row(
contentTypes.font.transferSize,
'Font size',
'fontSizePerPage',
h.size.format
size.format
),
row(summary.transferSize, 'Total size', 'totalSizePerPage', h.size.format)
row(summary.transferSize, 'Total size', 'totalSizePerPage', size.format)
);
const responseCodes = Object.keys(summary.responseCodes);
@ -110,16 +105,11 @@ module.exports = function (data) {
if (browsertime) {
const summary = browsertime.summary;
rows.push(row(summary.firstPaint, 'First Paint', 'firstPaint', h.time.ms));
rows.push(row(summary.firstPaint, 'First Paint', 'firstPaint', time.ms));
if (summary.timings) {
rows.push(
row(
summary.timings.fullyLoaded,
'Fully Loaded',
'fullyLoaded',
h.time.ms
)
row(summary.timings.fullyLoaded, 'Fully Loaded', 'fullyLoaded', time.ms)
);
}
@ -129,7 +119,7 @@ module.exports = function (data) {
summary.timeToDomContentFlushed,
'DOMContentFlushed',
'timeToDomContentFlushed',
h.time.ms
time.ms
)
);
}
@ -140,13 +130,13 @@ module.exports = function (data) {
summary.timings.largestContentfulPaint,
'Largest Contentful Paint',
'largestContentfulPaint',
h.time.ms
time.ms
)
);
}
if (summary.memory) {
rows.push(row(summary.memory, 'Memory usage', 'memory', h.size.format));
rows.push(row(summary.memory, 'Memory usage', 'memory', size.format));
}
if (summary.paintTiming) {
@ -156,13 +146,13 @@ module.exports = function (data) {
'first-contentful-paint': 'First Contentful Paint'
};
for (let pt of paintTimings) {
rows.push(row(summary.paintTiming[pt], lookup[pt], pt, h.time.ms));
rows.push(row(summary.paintTiming[pt], lookup[pt], pt, time.ms));
}
}
const timings = Object.keys(summary.pageTimings);
for (let timing of timings) {
rows.push(row(summary.pageTimings[timing], timing, timing, h.time.ms));
rows.push(row(summary.pageTimings[timing], timing, timing, time.ms));
}
if (summary.custom) {
@ -187,49 +177,49 @@ module.exports = function (data) {
summary.visualMetrics.FirstVisualChange,
'First Visual Change',
'FirstVisualChange',
h.time.ms
time.ms
),
row(
summary.visualMetrics.SpeedIndex,
'Speed Index',
'SpeedIndex',
h.time.ms
time.ms
),
row(
summary.visualMetrics.PerceptualSpeedIndex,
'Perceptual Speed Index',
'PerceptualSpeedIndex',
h.time.ms
time.ms
),
row(
summary.visualMetrics.ContentfulSpeedIndex,
'Contentful Speed Index',
'ContentfulSpeedIndex',
h.time.ms
time.ms
),
row(
summary.visualMetrics.VisualComplete85,
'Visual Complete 85%',
'VisualComplete85',
h.time.ms
time.ms
),
row(
summary.visualMetrics.VisualComplete95,
'Visual Complete 95%',
'VisualComplete95',
h.time.ms
time.ms
),
row(
summary.visualMetrics.VisualComplete99,
'Visual Complete 99%',
'VisualComplete99',
h.time.ms
time.ms
),
row(
summary.visualMetrics.LastVisualChange,
'Last Visual Change',
'LastVisualChange',
h.time.ms
time.ms
)
);
@ -239,17 +229,17 @@ module.exports = function (data) {
summary.visualMetrics.LargestImage,
'Largest Image',
'LargestImage',
h.time.ms
time.ms
)
);
}
if (summary.visualMetrics.Heading) {
rows.push(
row(summary.visualMetrics.Heading, 'Heading', 'Heading', h.time.ms)
row(summary.visualMetrics.Heading, 'Heading', 'Heading', time.ms)
);
}
if (summary.visualMetrics.Logo) {
rows.push(row(summary.visualMetrics.Logo, 'Logo', 'Logo', h.time.ms));
rows.push(row(summary.visualMetrics.Logo, 'Logo', 'Logo', time.ms));
}
}
@ -265,19 +255,19 @@ module.exports = function (data) {
summary.cpu.longTasks.totalDuration,
'CPU Long Tasks total duration',
'cpuLongTasksTotalDurationPerPage',
h.time.ms
time.ms
),
row(
summary.cpu.longTasks.totalBlockingTime,
'Total Blocking Time',
'totalBlockingTime',
h.time.ms
time.ms
),
row(
summary.cpu.longTasks.maxPotentialFid,
'Max Potential First Input Delay',
'maxPotentialFirstInputDelay',
h.time.ms
time.ms
)
);
}
@ -288,31 +278,31 @@ module.exports = function (data) {
summary.cpu.categories.parseHTML,
'CPU Parse HTML',
'parseHTMLPerPage',
h.time.ms
time.ms
),
row(
summary.cpu.categories.styleLayout,
'CPU Style Layout',
'styleLayoutPerPage',
h.time.ms
time.ms
),
row(
summary.cpu.categories.paintCompositeRender,
'CPU Paint Composite Render',
'paintCompositeRenderPerPage',
h.time.ms
time.ms
),
row(
summary.cpu.categories.scriptParseCompile,
'CPU Script Parse Compile',
'scriptParseCompilePerPage',
h.time.ms
time.ms
),
row(
summary.cpu.categories.scriptEvaluation,
'CPU Script Evaluation',
'scriptEvaluationPerPage',
h.time.ms
time.ms
)
);
}
@ -323,18 +313,18 @@ module.exports = function (data) {
const firstView = get(webpagetest, 'summary.timing.firstView');
if (firstView) {
rows.push(
row(firstView.render, 'WPT render (firstView)', 'render', h.time.ms),
row(firstView.render, 'WPT render (firstView)', 'render', time.ms),
row(
firstView.SpeedIndex,
'WPT SpeedIndex (firstView)',
'SpeedIndex',
h.time.ms
time.ms
),
row(
firstView.fullyLoaded,
'WPT Fully loaded (firstView)',
'fullyLoaded',
h.time.ms
time.ms
)
);
}
@ -383,4 +373,4 @@ module.exports = function (data) {
}
return rows.filter(Boolean);
};
}

44
lib/plugins/html/setup/summaryBoxes.js
View File

@ -1,15 +1,14 @@
'use strict';
const log = require('intel').getLogger('sitespeedio.plugin.html');
const toArray = require('../../../support/util').toArray;
const friendlyNames = require('../../../support/friendlynames');
const get = require('lodash.get');
const defaultLimits = require('./summaryBoxesDefaultLimits');
const merge = require('lodash.merge');
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.html');
import { toArray } from '../../../support/util.js';
import friendlyNames from '../../../support/friendlynames.js';
import get from 'lodash.get';
import defaultLimits from './summaryBoxesDefaultLimits.js';
import merge from 'lodash.merge';
function infoBox(stat, name, formatter) {
if (typeof stat === 'undefined') {
return undefined;
return;
}
return _box(stat, name, 'info', formatter, name.replace(/\s/g, ''));
@ -17,7 +16,7 @@ function infoBox(stat, name, formatter) {
function scoreBox(stat, name, formatter, box, limits) {
if (typeof stat === 'undefined') {
return undefined;
return;
}
let label = 'info';
@ -36,7 +35,7 @@ function scoreBox(stat, name, formatter, box, limits) {
function timingBox(stat, name, formatter, box, limits) {
if (typeof stat === 'undefined') {
return undefined;
return;
}
let label = 'info';
@ -56,7 +55,7 @@ function timingBox(stat, name, formatter, box, limits) {
function pagexrayBox(stat, name, formatter, box, limits) {
if (typeof stat === 'undefined') {
return undefined;
return;
}
let label = 'info';
@ -75,7 +74,7 @@ function pagexrayBox(stat, name, formatter, box, limits) {
function axeBox(stat, name, formatter, url, limits) {
if (typeof stat === 'undefined') {
return undefined;
return;
}
let label = 'info';
@ -106,7 +105,7 @@ function _box(stat, name, label, formatter, url) {
};
}
module.exports = function (data, html) {
export default function (data, html) {
if (!data) {
return [];
}
@ -122,20 +121,25 @@ module.exports = function (data, html) {
if (friendly) {
let boxType;
switch (tool) {
case 'coach':
case 'coach': {
boxType = scoreBox;
break;
case 'axe':
}
case 'axe': {
boxType = axeBox;
break;
case 'pagexray':
}
case 'pagexray': {
boxType = pagexrayBox;
break;
case 'browsertime':
}
case 'browsertime': {
boxType = timingBox;
break;
default:
}
default: {
boxType = infoBox;
}
}
const stats = get(data, tool + '.summary.' + friendly.summaryPath);
@ -158,4 +162,4 @@ module.exports = function (data, html) {
}
}
return boxes;
};
}

9
lib/plugins/html/setup/summaryBoxesDefaultLimits.js
View File

@ -1,5 +1,4 @@
'use strict';
module.exports = {
export default {
score: {
score: {
green: 90,
@ -22,7 +21,6 @@ module.exports = {
yellow: 80
}
},
// All timings are in ms
timings: {
firstPaint: { green: 1000, yellow: 2000 },
firstContentfulPaint: { green: 2000, yellow: 4000 },
@ -45,16 +43,15 @@ module.exports = {
css: {},
image: {}
},
// Size in byte
transferSize: {
total: { green: 1000000, yellow: 1500000 },
total: { green: 1_000_000, yellow: 1_500_000 },
html: {},
css: {},
image: {},
javascript: {}
},
contentSize: {
javascript: { green: 100000, yellow: 150000 }
javascript: { green: 100_000, yellow: 150_000 }
},
thirdParty: {
requests: {},

16
lib/plugins/html/templates/url/includes/tabScripts.js
View File

@ -2,8 +2,8 @@ window.addEventListener('DOMContentLoaded', function () {
let tabsRoot = document.querySelector('#tabs');
let navigationLinks = document.querySelectorAll('#pageNavigation a');
for (let i = 0; i < navigationLinks.length; ++i) {
navigationLinks[i].addEventListener('click', event => {
for (const navigationLink of navigationLinks) {
navigationLink.addEventListener('click', event => {
if (!location.hash) return;
event.preventDefault();
location.href = `${event.target.href}${location.hash}_ref`;
@ -22,8 +22,8 @@ window.addEventListener('DOMContentLoaded', function () {
if (!currentTab) currentTab = tabsRoot.querySelector('a');
let sections = document.querySelectorAll('#tabSections section');
for (let i = 0; i < sections.length; i++) {
sections[i].style.display = 'none';
for (const section of sections) {
section.style.display = 'none';
}
selectTab(currentTab, false);
});
@ -47,14 +47,14 @@ function selectTab(newSelection, updateUrlFragment) {
section.style.display = 'block';
let charts = section.querySelectorAll('.ct-chart');
for (let i = 0; i < charts.length; i++) {
if (charts[i].__chartist__) {
charts[i].__chartist__.update();
for (const chart of charts) {
if (chart.__chartist__) {
chart.__chartist__.update();
}
}
if (updateUrlFragment && history.replaceState) {
history.replaceState(null, null, '#' + newSelection.id);
history.replaceState(undefined, undefined, '#' + newSelection.id);
}
return false;

54
lib/plugins/influxdb/cli.js
View File

@ -1,54 +0,0 @@
module.exports = {
protocol: {
describe: 'The protocol used to store connect to the InfluxDB host.',
default: 'http',
group: 'InfluxDB'
},
host: {
describe: 'The InfluxDB host used to store captured metrics.',
group: 'InfluxDB'
},
port: {
default: 8086,
describe: 'The InfluxDB port used to store captured metrics.',
group: 'InfluxDB'
},
username: {
describe: 'The InfluxDB username for your InfluxDB instance.',
group: 'InfluxDB'
},
password: {
describe: 'The InfluxDB password for your InfluxDB instance.',
group: 'InfluxDB'
},
database: {
default: 'sitespeed',
describe: 'The database name used to store captured metrics.',
group: 'InfluxDB'
},
tags: {
default: 'category=default',
describe: 'A comma separated list of tags and values added to each metric',
group: 'InfluxDB'
},
includeQueryParams: {
default: false,
describe:
'Whether to include query parameters from the URL in the InfluxDB keys or not',
type: 'boolean',
group: 'InfluxDB'
},
groupSeparator: {
default: '_',
describe:
'Choose which character that will separate a group/domain. Default is underscore, set it to a dot if you wanna keep the original domain name.',
group: 'InfluxDB'
},
annotationScreenshot: {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'InfluxDB'
}
};

319
lib/plugins/influxdb/data-generator.js
View File

@ -1,115 +1,28 @@
'use strict';
import merge from 'lodash.merge';
import reduce from 'lodash.reduce';
const flatten = require('../../support/flattenMessage'),
merge = require('lodash.merge'),
util = require('../../support/tsdbUtil'),
reduce = require('lodash.reduce');
import { flattenMessageData } from '../../support/flattenMessage.js';
import {
getConnectivity,
getURLAndGroup,
toSafeKey
} from '../../support/tsdbUtil.js';
class InfluxDBDataGenerator {
constructor(includeQueryParams, options) {
this.includeQueryParams = !!includeQueryParams;
this.options = options;
this.defaultTags = {};
for (let row of options.influxdb.tags.split(',')) {
const keyAndValue = row.split('=');
this.defaultTags[keyAndValue[0]] = keyAndValue[1];
}
}
dataFromMessage(message, time, alias) {
function getTagsFromMessage(
message,
includeQueryParams,
options,
defaultTags
) {
const tags = merge({}, defaultTags);
let typeParts = message.type.split('.');
tags.origin = typeParts[0];
typeParts.push(typeParts.shift());
tags.summaryType = typeParts[0];
// always have browser and connectivity in Browsertime and related tools
if (
message.type.match(
/(^pagexray|^coach|^browsertime|^thirdparty|^axe|^sustainable)/
)
) {
// if we have a friendly name for your connectivity, use that!
let connectivity = util.getConnectivity(options);
tags.connectivity = connectivity;
tags.browser = options.browser;
} else if (message.type.match(/(^webpagetest)/)) {
if (message.connectivity) {
tags.connectivity = message.connectivity;
}
if (message.location) {
tags.location = message.location;
}
} else if (message.type.match(/(^gpsi)/)) {
tags.strategy = options.mobile ? 'mobile' : 'desktop';
}
// if we get a URL type, add the URL
if (message.url) {
const urlAndGroup = util
.getURLAndGroup(
options,
message.group,
message.url,
includeQueryParams,
alias
)
.split('.');
tags.page = urlAndGroup[1];
tags.group = urlAndGroup[0];
} else if (message.group) {
// add the group of the summary message
tags.group = util.toSafeKey(
message.group,
options.influxdb.groupSeparator
);
}
tags.testName = options.slug;
return tags;
}
function getFieldAndSeriesName(key) {
const functions = [
'min',
'p10',
'median',
'mean',
'avg',
'max',
'p90',
'p99',
'mdev',
'stddev'
];
const keyArray = key.split('.');
const end = keyArray.pop();
if (functions.indexOf(end) > -1) {
return { field: end, seriesName: keyArray.pop() };
}
return { field: 'value', seriesName: end };
}
function getAdditionalTags(key, type) {
let tags = {};
const keyArray = key.split('.');
if (key.match(/(^contentTypes)/)) {
// contentTypes.favicon.requests.mean
// contentTypes.favicon.requests
// contentTypes.css.transferSize
tags.contentType = keyArray[1];
} else if (key.match(/(^pageTimings|^visualMetrics)/)) {
// pageTimings.serverResponseTime.max
// visualMetrics.SpeedIndex.median
tags.timings = keyArray[0];
} else if (type === 'browsertime.pageSummary') {
function getAdditionalTags(key, type) {
let tags = {};
const keyArray = key.split('.');
if (/(^contentTypes)/.test(key)) {
// contentTypes.favicon.requests.mean
// contentTypes.favicon.requests
// contentTypes.css.transferSize
tags.contentType = keyArray[1];
} else if (/(^pageTimings|^visualMetrics)/.test(key)) {
// pageTimings.serverResponseTime.max
// visualMetrics.SpeedIndex.median
tags.timings = keyArray[0];
} else
switch (type) {
case 'browsertime.pageSummary': {
// statistics.timings.pageTimings.backEndTime.median
// statistics.timings.userTimings.marks.logoTime.median
// statistics.visualMetrics.SpeedIndex.median
@ -117,20 +30,26 @@ class InfluxDBDataGenerator {
if (keyArray.length >= 5) {
tags[keyArray[2]] = keyArray[3];
}
if (key.indexOf('cpu.categories') > -1) {
if (key.includes('cpu.categories')) {
tags.cpu = 'category';
} else if (key.indexOf('cpu.events') > -1) {
} else if (key.includes('cpu.events')) {
tags.cpu = 'event';
} else if (key.indexOf('cpu.longTasks') > -1) {
} else if (key.includes('cpu.longTasks')) {
tags.cpu = 'longTask';
}
} else if (type === 'browsertime.summary') {
break;
}
case 'browsertime.summary': {
// firstPaint.median
// userTimings.marks.logoTime.median
if (key.indexOf('userTimings') > -1) {
if (key.includes('userTimings')) {
tags[keyArray[0]] = keyArray[1];
}
} else if (type === 'coach.pageSummary') {
break;
}
case 'coach.pageSummary': {
// advice.score
// advice.performance.score
if (keyArray.length > 2) {
@ -142,89 +61,205 @@ class InfluxDBDataGenerator {
if (keyArray.length > 4) {
tags.adviceName = keyArray[3];
}
} else if (type === 'coach.summary') {
break;
}
case 'coach.summary': {
// score.max
// performance.score.median
if (keyArray.length === 3) {
tags.advice = keyArray[0];
}
} else if (type === 'webpagetest.pageSummary') {
break;
}
case 'webpagetest.pageSummary': {
// data.median.firstView.SpeedIndex webpagetest.pageSummary
tags.view = keyArray[2];
// data.median.firstView.breakdown.html.requests
// data.median.firstView.breakdown.html.bytes
if (key.indexOf('breakdown') > -1) {
if (key.includes('breakdown')) {
tags.contentType = keyArray[4];
}
} else if (type === 'webpagetest.summary') {
break;
}
case 'webpagetest.summary': {
// timing.firstView.SpeedIndex.median
tags.view = keyArray[1];
// asset.firstView.breakdown.html.requests.median
if (key.indexOf('breakdown') > -1) {
if (key.includes('breakdown')) {
tags.contentType = keyArray[4];
}
} else if (type === 'pagexray.summary') {
break;
}
case 'pagexray.summary': {
// firstParty.requests.min pagexray.summary
// requests.median
// responseCodes.307.max pagexray.summary
// requests.min pagexray.summary
if (key.indexOf('responseCodes') > -1) {
if (key.includes('responseCodes')) {
tags.responseCodes = 'response';
}
if (key.indexOf('firstParty') > -1 || key.indexOf('thirdParty') > -1) {
if (key.includes('firstParty') || key.includes('thirdParty')) {
tags.party = keyArray[0];
}
} else if (type === 'pagexray.pageSummary') {
break;
}
case 'pagexray.pageSummary': {
// thirdParty.contentTypes.json.requests pagexray.pageSummary
// thirdParty.requests pagexray.pageSummary
// firstParty.cookieStats.max pagexray.pageSummary
// responseCodes.200 pagexray.pageSummary
// expireStats.max pagexray.pageSummary
// totalDomains pagexray.pageSummary
if (key.indexOf('firstParty') > -1 || key.indexOf('thirdParty') > -1) {
if (key.includes('firstParty') || key.includes('thirdParty')) {
tags.party = keyArray[0];
}
if (key.indexOf('responseCodes') > -1) {
if (key.includes('responseCodes')) {
tags.responseCodes = 'response';
}
if (key.indexOf('contentTypes') > -1) {
if (key.includes('contentTypes')) {
tags.contentType = keyArray[2];
}
} else if (type === 'thirdparty.pageSummary') {
break;
}
case 'thirdparty.pageSummary': {
tags.thirdPartyCategory = keyArray[1];
tags.thirdPartyType = keyArray[2];
} else if (type === 'lighthouse.pageSummary') {
break;
}
case 'lighthouse.pageSummary': {
// categories.seo.score
// categories.performance.score
if (key.indexOf('score') > -1) {
if (key.includes('score')) {
tags.audit = keyArray[1];
}
if (key.indexOf('audits') > -1) {
if (key.includes('audits')) {
tags.audit = keyArray[1];
}
} else if (type === 'crux.pageSummary') {
break;
}
case 'crux.pageSummary': {
tags.experience = keyArray[0];
tags.formFactor = keyArray[1];
tags.metric = keyArray[2];
} else if (type === 'gpsi.pageSummary') {
if (key.indexOf('googleWebVitals') > -1) {
break;
}
case 'gpsi.pageSummary': {
if (key.includes('googleWebVitals')) {
tags.testType = 'googleWebVitals';
} else if (key.indexOf('score') > -1) {
} else if (key.includes('score')) {
tags.testType = 'score';
} else if (key.indexOf('loadingExperience') > -1) {
} else if (key.includes('loadingExperience')) {
tags.experience = keyArray[0];
tags.metric = keyArray[1];
tags.testType = 'crux';
}
} else {
// console.log('Missed added tags to ' + key + ' ' + type);
break;
}
default:
// console.log('Missed added tags to ' + key + ' ' + type);
}
return tags;
}
function getFieldAndSeriesName(key) {
const functions = [
'min',
'p10',
'median',
'mean',
'avg',
'max',
'p90',
'p99',
'mdev',
'stddev'
];
const keyArray = key.split('.');
const end = keyArray.pop();
if (functions.includes(end)) {
return { field: end, seriesName: keyArray.pop() };
}
return { field: 'value', seriesName: end };
}
export class InfluxDBDataGenerator {
constructor(includeQueryParameters, options) {
this.includeQueryParams = !!includeQueryParameters;
this.options = options;
this.defaultTags = {};
for (let row of options.influxdb.tags.split(',')) {
const keyAndValue = row.split('=');
this.defaultTags[keyAndValue[0]] = keyAndValue[1];
}
}
dataFromMessage(message, time, alias) {
console.log('GET DATA');
function getTagsFromMessage(
message,
includeQueryParameters,
options,
defaultTags
) {
const tags = merge({}, defaultTags);
let typeParts = message.type.split('.');
tags.origin = typeParts[0];
typeParts.push(typeParts.shift());
tags.summaryType = typeParts[0];
// always have browser and connectivity in Browsertime and related tools
if (
/(^pagexray|^coach|^browsertime|^thirdparty|^axe|^sustainable)/.test(
message.type
)
) {
// if we have a friendly name for your connectivity, use that!
let connectivity = getConnectivity(options);
tags.connectivity = connectivity;
tags.browser = options.browser;
} else if (/(^webpagetest)/.test(message.type)) {
if (message.connectivity) {
tags.connectivity = message.connectivity;
}
if (message.location) {
tags.location = message.location;
}
} else if (/(^gpsi)/.test(message.type)) {
tags.strategy = options.mobile ? 'mobile' : 'desktop';
}
// if we get a URL type, add the URL
if (message.url) {
const urlAndGroup = getURLAndGroup(
options,
message.group,
message.url,
includeQueryParameters,
alias
).split('.');
tags.page = urlAndGroup[1];
tags.group = urlAndGroup[0];
} else if (message.group) {
// add the group of the summary message
tags.group = toSafeKey(message.group, options.influxdb.groupSeparator);
}
tags.testName = options.slug;
return tags;
}
return reduce(
flatten.flattenMessageData(message),
flattenMessageData(message),
(entries, value, key) => {
const fieldAndSeriesName = getFieldAndSeriesName(key);
let tags = getTagsFromMessage(
@ -249,5 +284,3 @@ class InfluxDBDataGenerator {
);
}
}
module.exports = InfluxDBDataGenerator;

112
lib/plugins/influxdb/index.js
View File

@ -1,29 +1,17 @@
'use strict';
import isEmpty from 'lodash.isempty';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { InfluxDBSender as Sender } from './sender.js';
import { toSafeKey } from '../../support/tsdbUtil.js';
import { send } from './send-annotation.js';
import { InfluxDBDataGenerator as DataGenerator } from './data-generator.js';
import { throwIfMissing } from '../../support/util.js';
const throwIfMissing = require('../../support/util').throwIfMissing;
const isEmpty = require('lodash.isempty');
const log = require('intel').getLogger('sitespeedio.plugin.influxdb');
const Sender = require('./sender');
const tsdbUtil = require('../../support/tsdbUtil');
const sendAnnotations = require('./send-annotation');
const DataGenerator = require('./data-generator');
const path = require('path');
const cliUtil = require('../../cli/util');
module.exports = {
name() {
return path.basename(__dirname);
},
/**
* Define `yargs` options with their respective default values. When displayed by the CLI help message
* all options are namespaced by its plugin name.
*
* @return {Object<string, require('yargs').Options} an object mapping the name of the option and its yargs configuration
*/
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
const log = intel.getLogger('sitespeedio.plugin.influxdb');
export default class InfluxDBPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'influxdb', options, context, queue });
}
open(context, options) {
throwIfMissing(options.influxdb, ['host', 'database'], 'influxdb');
@ -34,38 +22,59 @@ module.exports = {
options.influxdb.database
);
const opts = options.influxdb;
const options_ = options.influxdb;
this.options = options;
this.sender = new Sender(opts);
this.sender = new Sender(options_);
this.timestamp = context.timestamp;
this.resultUrls = context.resultUrls;
this.dataGenerator = new DataGenerator(opts.includeQueryParams, options);
this.dataGenerator = new DataGenerator(
options_.includeQueryParams,
options
);
this.messageTypesToFireAnnotations = [];
this.receivedTypesThatFireAnnotations = {};
this.make = context.messageMaker('influxdb').make;
this.sendAnnotation = true;
this.alias = {};
this.wptExtras = {};
},
}
processMessage(message, queue) {
const filterRegistry = this.filterRegistry;
// First catch if we are running Browsertime and/or WebPageTest
if (message.type === 'browsertime.setup') {
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
this.usingBrowsertime = true;
} else if (message.type === 'webpagetest.setup') {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
} else if (message.type === 'browsertime.config') {
if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType;
switch (message.type) {
case 'browsertime.setup': {
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
this.usingBrowsertime = true;
break;
}
} else if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the InfluxDB plugin is alive
queue.postMessage(this.make('influxdb.setup'));
} else if (message.type === 'grafana.setup') {
this.sendAnnotation = false;
case 'webpagetest.setup': {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
break;
}
case 'browsertime.config': {
if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType;
}
break;
}
case 'sitespeedio.setup': {
// Let other plugins know that the InfluxDB plugin is alive
queue.postMessage(this.make('influxdb.setup'));
break;
}
case 'grafana.setup': {
this.sendAnnotation = false;
break;
}
// No default
}
if (message.type === 'browsertime.alias') {
@ -92,15 +101,13 @@ module.exports = {
this.wptExtras[message.url].webPageTestResultURL =
message.data.data.summary;
this.wptExtras[message.url].connectivity = message.connectivity;
this.wptExtras[message.url].location = tsdbUtil.toSafeKey(
message.location
);
this.wptExtras[message.url].location = toSafeKey(message.location);
}
// Let us skip this for a while and concentrate on the real deal
if (
message.type.match(
/(^largestassets|^slowestassets|^aggregateassets|^domains)/
/(^largestassets|^slowestassets|^aggregateassets|^domains)/.test(
message.type
)
)
return;
@ -139,7 +146,7 @@ module.exports = {
);
this.receivedTypesThatFireAnnotations[message.url] = 0;
return sendAnnotations.send(
return send(
message.url,
message.group,
absolutePagePath,
@ -158,12 +165,9 @@ module.exports = {
return Promise.reject(
new Error(
'No data to send to influxdb for message:\n' +
JSON.stringify(message, null, 2)
JSON.stringify(message, undefined, 2)
)
);
}
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
}
};
}

219
lib/plugins/influxdb/send-annotation.js
View File

@ -1,118 +1,119 @@
'use strict';
const http = require('http');
const https = require('https');
const log = require('intel').getLogger('sitespeedio.plugin.influxdb');
const querystring = require('querystring');
const tsdbUtil = require('../../support/tsdbUtil');
const annotationsHelper = require('../../support/annotationsHelper');
const dayjs = require('dayjs');
import http from 'node:http';
import https from 'node:https';
import { stringify } from 'node:querystring';
module.exports = {
send(
url,
import intel from 'intel';
import dayjs from 'dayjs';
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
import {
getAnnotationMessage,
getTagsAsString
} from '../../support/annotationsHelper.js';
const log = intel.getLogger('sitespeedio.plugin.influxdb');
export function send(
url,
group,
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
runTime,
alias,
webPageTestExtraData,
usingBrowsertime,
options
) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
// variables in Grafana.
const connectivity = getConnectivity(options);
const browser = options.browser;
const urlAndGroup = getURLAndGroup(
options,
group,
url,
options.influxdb.includeQueryParams,
alias
).split('.');
let tags = [connectivity, browser, urlAndGroup[0], urlAndGroup[1]];
// See https://github.com/sitespeedio/sitespeed.io/issues/3277
if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug);
}
if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity, webPageTestExtraData.location);
}
const message = getAnnotationMessage(
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
runTime,
alias,
webPageTestExtraData,
webPageTestExtraData
? webPageTestExtraData.webPageTestResultURL
: undefined,
usingBrowsertime,
options
) {
// The tags make it possible for the dashboard to use the
// templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template
// variables in Grafana.
const connectivity = tsdbUtil.getConnectivity(options);
const browser = options.browser;
const urlAndGroup = tsdbUtil
.getURLAndGroup(
options,
group,
url,
options.influxdb.includeQueryParams,
alias
)
.split('.');
let tags = [connectivity, browser, urlAndGroup[0], urlAndGroup[1]];
// See https://github.com/sitespeedio/sitespeed.io/issues/3277
if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug);
);
const timestamp = runTime
? Math.round(dayjs(runTime) / 1000)
: Math.round(dayjs() / 1000);
// if we have a category, let us send that category too
if (options.influxdb.tags) {
for (let row of options.influxdb.tags.split(',')) {
const keyAndValue = row.split('=');
tags.push(keyAndValue[1]);
}
if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity);
tags.push(webPageTestExtraData.location);
}
const message = annotationsHelper.getAnnotationMessage(
absolutePagePath,
screenShotsEnabledInBrowsertime,
screenshotType,
webPageTestExtraData
? webPageTestExtraData.webPageTestResultURL
: undefined,
usingBrowsertime,
options
);
const timestamp = runTime
? Math.round(dayjs(runTime) / 1000)
: Math.round(dayjs() / 1000);
// if we have a category, let us send that category too
if (options.influxdb.tags) {
for (let row of options.influxdb.tags.split(',')) {
const keyAndValue = row.split('=');
tags.push(keyAndValue[1]);
}
}
const influxDBTags = annotationsHelper.getTagsAsString(tags);
const postData = `events title="Sitespeed.io",text="${message}",tags=${influxDBTags} ${timestamp}`;
const postOptions = {
hostname: options.influxdb.host,
port: options.influxdb.port,
path: '/write?db=' + options.influxdb.database + '&precision=s',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(postData)
}
};
if (options.influxdb.username) {
postOptions.path =
postOptions.path +
'&' +
querystring.stringify({
u: options.influxdb.username,
p: options.influxdb.password
});
}
return new Promise((resolve, reject) => {
log.debug('Send annotation to Influx: %j', postData);
// not perfect but maybe work for us
const lib = options.influxdb.protocol === 'https' ? https : http;
const req = lib.request(postOptions, res => {
if (res.statusCode !== 204) {
const e = new Error(
`Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
);
log.warn(e.message);
reject(e);
} else {
res.setEncoding('utf-8');
log.debug('Sent annotation to InfluxDB');
resolve();
}
});
req.on('error', err => {
log.error('Got error from InfluxDB when sending annotation', err);
reject(err);
});
req.write(postData);
req.end();
});
}
};
const influxDBTags = getTagsAsString(tags);
const postData = `events title="Sitespeed.io",text="${message}",tags=${influxDBTags} ${timestamp}`;
const postOptions = {
hostname: options.influxdb.host,
port: options.influxdb.port,
path: '/write?db=' + options.influxdb.database + '&precision=s',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(postData)
}
};
if (options.influxdb.username) {
postOptions.path =
postOptions.path +
'&' +
stringify({
u: options.influxdb.username,
p: options.influxdb.password
});
}
return new Promise((resolve, reject) => {
log.debug('Send annotation to Influx: %j', postData);
// not perfect but maybe work for us
const library = options.influxdb.protocol === 'https' ? https : http;
const request = library.request(postOptions, res => {
if (res.statusCode !== 204) {
const e = new Error(
`Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
);
log.warn(e.message);
reject(e);
} else {
res.setEncoding('utf8');
log.debug('Sent annotation to InfluxDB');
resolve();
}
});
request.on('error', error => {
log.error('Got error from InfluxDB when sending annotation', error);
reject(error);
});
request.write(postData);
request.end();
});
}

10
lib/plugins/influxdb/sender.js
View File

@ -1,10 +1,8 @@
'use strict';
import { InfluxDB } from 'influx';
const Influx = require('influx');
class InfluxDBSender {
export class InfluxDBSender {
constructor({ protocol, host, port, database, username, password }) {
this.client = new Influx.InfluxDB({
this.client = new InfluxDB({
protocol,
host,
port,
@ -26,5 +24,3 @@ class InfluxDBSender {
return this.client.writePoints(points);
}
}
module.exports = InfluxDBSender;

55
lib/plugins/lateststorer/index.js
View File

@ -1,22 +1,27 @@
'use strict';
import { resolve, join } from 'node:path';
import { platform } from 'node:os';
import { promisify } from 'node:util';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import osName from 'os-name';
import getos from 'getos';
import get from 'lodash.get';
import { getURLAndGroup, getConnectivity } from '../../support/tsdbUtil.js';
import { cap, plural } from '../../support/helpers/index.js';
const path = require('path');
const osName = require('os-name');
const getos = require('getos');
const { promisify } = require('util');
const getOS = promisify(getos);
const os = require('os');
const get = require('lodash.get');
const graphiteUtil = require('../../support/tsdbUtil');
const helpers = require('../../support/helpers');
export default class LatestStorerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'lateststorer', options, context, queue });
}
module.exports = {
open(context, options) {
this.storageManager = context.storageManager;
this.alias = {};
this.options = options;
this.context = context;
},
}
async processMessage(message) {
switch (message.type) {
// Collect alias so we can use it
@ -47,7 +52,7 @@ module.exports = {
const browserData = this.browserData;
const baseDir = this.storageManager.getBaseDir();
// Hack to get out of the date dir
const newPath = path.resolve(baseDir, '..');
const newPath = resolve(baseDir, '..');
// This is a hack to get the same name as in Grafana, meaning we can
// generate the path to the URL there
@ -55,14 +60,14 @@ module.exports = {
(options.copyLatestFilesToBaseGraphiteNamespace
? `${options.graphite.namespace}.`
: '') +
graphiteUtil.getURLAndGroup(
getURLAndGroup(
options,
message.group,
message.url,
this.options.graphite.includeQueryParams,
this.alias
);
const connectivity = graphiteUtil.getConnectivity(options);
const connectivity = getConnectivity(options);
if (this.useScreenshots) {
let imagePath = '';
@ -74,13 +79,13 @@ module.exports = {
screenshot
)
) {
const type = screenshot.substring(
const type = screenshot.slice(
screenshot.lastIndexOf('/') + 1,
screenshot.lastIndexOf('.')
);
imagePath = screenshot;
const imageFullPath = path.join(baseDir, imagePath);
const imageFullPath = join(baseDir, imagePath);
await this.storageManager.copyFileToDir(
imageFullPath,
newPath +
@ -95,12 +100,12 @@ module.exports = {
this.screenshotType
);
} else {
// This is a user generated screenshot, we do not copy that
// do nada
}
}
}
if (options.browsertime && options.browsertime.video) {
const videoFullPath = path.join(baseDir, message.data.video);
const videoFullPath = join(baseDir, message.data.video);
await this.storageManager.copyFileToDir(
videoFullPath,
@ -147,14 +152,12 @@ module.exports = {
}
json.browser = {};
json.browser.name = helpers.cap(
get(browserData, 'browser.name', 'unknown')
);
json.browser.name = cap(get(browserData, 'browser.name', 'unknown'));
json.browser.version = get(browserData, 'browser.version', 'unknown');
json.friendlyHTML = `<b><a href="${message.url}">${
json.alias ? json.alias : message.url
}</a></b> ${helpers.plural(
json.alias ?? message.url
}</a></b> ${plural(
options.browsertime.iterations,
'iteration'
)} at <i>${json.timestamp}</i> using ${json.browser.name} ${
@ -189,7 +192,7 @@ module.exports = {
} else {
// We are testing on desktop
let osInfo = osName();
if (os.platform() === 'linux') {
if (platform() === 'linux') {
const linux = await getOS();
osInfo = `${linux.dist} ${linux.release}`;
}
@ -211,7 +214,7 @@ module.exports = {
json.result = resultURL;
}
const data = JSON.stringify(json, null, 0);
const data = JSON.stringify(json, undefined, 0);
return this.storageManager.writeDataToDir(
data,
name + '.' + options.browser + '.' + connectivity + '.json',
@ -221,4 +224,4 @@ module.exports = {
}
}
}
};
}

31
lib/plugins/matrix/cli.js
View File

@ -1,31 +0,0 @@
const { messageTypes } = require('.');
module.exports = {
host: {
describe: 'The Matrix host.',
group: 'Matrix'
},
accessToken: {
describe: 'The Matrix access token.',
group: 'Matrix'
},
room: {
describe:
'The default Matrix room. It is alsways used. You can override the room per message type using --matrix.rooms',
group: 'Matrix'
},
messages: {
describe:
'Choose what type of message to send to Matrix. There are two types of messages: Error messages and budget messages. Errors are errors that happens through the tests (failures like strarting a test) and budget is test failing against your budget.',
choices: messageTypes,
default: messageTypes,
group: 'Matrix'
},
rooms: {
describe:
'Send messages to different rooms. Current message types are [' +
messageTypes +
']. If you want to send error messages to a specific room use --matrix.rooms.error ROOM',
group: 'Matrix'
}
};

98
lib/plugins/matrix/index.js
View File

@ -1,33 +1,31 @@
'use strict';
import intel from 'intel';
import get from 'lodash.get';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const throwIfMissing = require('../../support/util').throwIfMissing;
const log = require('intel').getLogger('sitespeedio.plugin.matrix');
const path = require('path');
const get = require('lodash.get');
const cliUtil = require('../../cli/util');
const send = require('./send');
import send from './send.js';
import { throwIfMissing } from '../../support/util.js';
const log = intel.getLogger('sitespeedio.plugin.matrix');
function getBrowserData(data) {
if (data && data.browser) {
return `${data.browser.name} ${data.browser.version} ${get(
data,
'android.model',
''
)} ${get(data, 'android.androidVersion', '')} ${get(
data,
'android.id',
''
)} `;
} else return '';
return data && data.browser
? `${data.browser.name} ${data.browser.version} ${get(
data,
'android.model',
''
)} ${get(data, 'android.androidVersion', '')} ${get(
data,
'android.id',
''
)} `
: '';
}
module.exports = {
name() {
return path.basename(__dirname);
},
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
export default class MatrixPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'matrix', options, context, queue });
}
open(context, options = {}) {
this.matrixOptions = options.matrix || {};
this.options = options;
@ -37,7 +35,8 @@ module.exports = {
this.errorTexts = '';
this.waitForUpload = false;
this.alias = {};
},
}
async processMessage(message) {
const options = this.matrixOptions;
switch (message.type) {
@ -67,7 +66,7 @@ module.exports = {
this.errorTexts
);
log.debug('Got %j from the matrix server', answer);
} catch (e) {
} catch {
// TODO what todo?
}
}
@ -82,17 +81,15 @@ module.exports = {
case 'error': {
// We can send too many messages to Matrix and get 429 so instead
// we bulk send them all one time
if (options.messages.indexOf('error') > -1) {
if (options.messages.includes('error')) {
this.errorTexts += `&#9888;&#65039; Error from <b>${
message.source
}</b> testing ${message.url ? message.url : ''} <pre>${
message.data
}</pre>`;
}</b> testing ${message.url || ''} <pre>${message.data}</pre>`;
}
break;
}
case 'budget.result': {
if (options.messages.indexOf('budget') > -1) {
if (options.messages.includes('budget')) {
let text = '';
// We have failing URLs in the budget
if (Object.keys(message.data.failing).length > 0) {
@ -103,19 +100,17 @@ module.exports = {
}${getBrowserData(this.browserData)}</p>`;
for (let url of failingURLs) {
text += `<h5>&#10060; ${url}`;
if (this.resultUrls.hasBaseUrl()) {
text += ` (<a href="${this.resultUrls.absoluteSummaryPagePath(
url,
this.alias[url]
)}index.html">result</a> - <a href="${this.resultUrls.absoluteSummaryPagePath(
url,
this.alias[url]
)}data/screenshots/1/afterPageCompleteCheck.${
this.screenshotType
}">screenshot</a>)</h5>`;
} else {
text += '</h5>';
}
text += this.resultUrls.hasBaseUrl()
? ` (<a href="${this.resultUrls.absoluteSummaryPagePath(
url,
this.alias[url]
)}index.html">result</a> - <a href="${this.resultUrls.absoluteSummaryPagePath(
url,
this.alias[url]
)}data/screenshots/1/afterPageCompleteCheck.${
this.screenshotType
}">screenshot</a>)</h5>`
: '</h5>';
text += '<ul>';
for (let failing of message.data.failing[url]) {
text += `<li>${failing.metric} : ${failing.friendlyValue} (${failing.friendlyLimit})</li>`;
@ -155,18 +150,15 @@ module.exports = {
case 'scp.finished':
case 'ftp.finished':
case 's3.finished': {
if (this.waitForUpload && options.messages.indexOf('budget') > -1) {
if (this.waitForUpload && options.messages.includes('budget')) {
const room = get(options, 'rooms.budget', options.room);
await send(options.host, room, options.accessToken, this.budgetText);
}
break;
}
}
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
},
get messageTypes() {
return ['error', 'budget'];
}
};
}
export function messageTypes() {
return ['error', 'budget'];
}

25
lib/plugins/matrix/send.js
View File

@ -1,7 +1,6 @@
'use strict';
const https = require('https');
const log = require('intel').getLogger('sitespeedio.plugin.matrix');
import { request as _request } from 'node:https';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.matrix');
function send(
host,
@ -12,9 +11,9 @@ function send(
retries = 3,
backoff = 5000
) {
const retryCodes = [408, 429, 500, 503];
const retryCodes = new Set([408, 429, 500, 503]);
return new Promise((resolve, reject) => {
const req = https.request(
const request = _request(
{
host,
port: 443,
@ -26,9 +25,9 @@ function send(
method: 'POST'
},
res => {
const { statusCode } = res;
const { statusCode, statusMessage } = res;
if (statusCode < 200 || statusCode > 299) {
if (retries > 0 && retryCodes.includes(statusCode)) {
if (retries > 0 && retryCodes.has(statusCode)) {
setTimeout(() => {
return send(
host,
@ -42,9 +41,9 @@ function send(
}, backoff);
} else {
log.error(
`Got error from Matrix. Error Code: ${res.statusCode} Message: ${res.statusMessage}`
`Got error from Matrix. Error Code: ${statusCode} Message: ${statusMessage}`
);
reject(new Error(`Status Code: ${res.statusCode}`));
reject(new Error(`Status Code: ${statusCode}`));
}
} else {
const data = [];
@ -57,12 +56,12 @@ function send(
}
}
);
req.write(JSON.stringify(data));
req.end();
request.write(JSON.stringify(data));
request.end();
});
}
module.exports = async (host, room, accessToken, message) => {
export default async (host, room, accessToken, message) => {
const data = {
msgtype: 'm.notice',
body: '',

38
lib/plugins/messagelogger/index.js
View File

@ -1,18 +1,20 @@
'use strict';
/* eslint no-console:0 */
const log = require('intel').getLogger('sitespeedio.plugin.messagelogger');
const isEmpty = require('lodash.isempty');
import intel from 'intel';
import isEmpty from 'lodash.isempty';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = intel.getLogger('sitespeedio.plugin.messagelogger');
function shortenData(key, value) {
if (key === 'data' && !isEmpty(value)) {
switch (typeof value) {
case 'object':
case 'object': {
return Array.isArray(value) ? '[...]' : '{...}';
}
case 'string': {
if (value.length > 100) {
return value.substring(0, 97) + '...';
return value.slice(0, 97) + '...';
}
}
}
@ -20,26 +22,32 @@ function shortenData(key, value) {
return value;
}
module.exports = {
export default class MessageLoggerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'messagelogger', options, context, queue });
}
open(context, options) {
this.verbose = Number(options.verbose || 0);
},
}
processMessage(message) {
let replacerFunc;
let replacerFunction;
switch (message.type) {
case 'browsertime.har':
case 'browsertime.run':
case 'domains.summary':
case 'webpagetest.pageSummary':
case 'browsertime.screenshot':
replacerFunc = this.verbose > 1 ? null : shortenData;
case 'browsertime.screenshot': {
replacerFunction = this.verbose > 1 ? undefined : shortenData;
break;
}
default:
replacerFunc = this.verbose > 0 ? null : shortenData;
default: {
replacerFunction = this.verbose > 0 ? undefined : shortenData;
}
}
log.info(JSON.stringify(message, replacerFunc, 2));
log.info(JSON.stringify(message, replacerFunction, 2));
}
};
}

86
lib/plugins/metrics/index.js
View File

@ -1,20 +1,24 @@
'use strict';
const flatten = require('../../support/flattenMessage');
const merge = require('lodash.merge');
import merge from 'lodash.merge';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { flattenMessageData } from '../../support/flattenMessage.js';
const defaultConfig = {
list: false,
filterList: false
};
module.exports = {
export default class MetricsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'metrics', options, context, queue });
}
open(context, options) {
this.options = merge({}, defaultConfig, options.metrics);
this.metrics = {};
this.storageManager = context.storageManager;
this.filterRegistry = context.filterRegistry;
},
}
processMessage(message) {
const filterRegistry = this.filterRegistry;
@ -38,40 +42,38 @@ module.exports = {
}
// only dance if we all wants to
if (this.options.filter) {
if (message.type === 'sitespeedio.setup') {
const filters = Array.isArray(this.options.filter)
? this.options.filter
: [this.options.filter];
if (this.options.filter && message.type === 'sitespeedio.setup') {
const filters = Array.isArray(this.options.filter)
? this.options.filter
: [this.options.filter];
for (let metric of filters) {
// for all filters
// cleaning all filters means (right now) that all
// metrics are sent
if (metric === '*+') {
filterRegistry.clearAll();
} else if (metric === '*-') {
// all registered types will be set as unmatching,
// use it if you want to have a clean filter where
// all types are removed and then you can add your own
let types = filterRegistry.getTypes();
filterRegistry.clearAll();
for (let type of types) {
filterRegistry.registerFilterForType('-', type);
}
} else {
let parts = metric.split('.');
// the type is "always" the first two
let type = parts.shift() + '.' + parts.shift();
let filter = parts.join('.');
let oldFilter = filterRegistry.getFilterForType(type);
if (oldFilter && typeof oldFilter === 'object') {
oldFilter.push(filter);
} else {
oldFilter = [filter];
}
filterRegistry.registerFilterForType(oldFilter, type);
for (let metric of filters) {
// for all filters
// cleaning all filters means (right now) that all
// metrics are sent
if (metric === '*+') {
filterRegistry.clearAll();
} else if (metric === '*-') {
// all registered types will be set as unmatching,
// use it if you want to have a clean filter where
// all types are removed and then you can add your own
let types = filterRegistry.getTypes();
filterRegistry.clearAll();
for (let type of types) {
filterRegistry.registerFilterForType('-', type);
}
} else {
let parts = metric.split('.');
// the type is "always" the first two
let type = parts.shift() + '.' + parts.shift();
let filter = parts.join('.');
let oldFilter = filterRegistry.getFilterForType(type);
if (oldFilter && typeof oldFilter === 'object') {
oldFilter.push(filter);
} else {
oldFilter = [filter];
}
filterRegistry.registerFilterForType(oldFilter, type);
}
}
}
@ -86,11 +88,11 @@ module.exports = {
) {
return;
}
let flattenMess = flatten.flattenMessageData(message);
let flattenMess = flattenMessageData(message);
for (let key of Object.keys(flattenMess)) {
this.metrics[message.type + '.' + key] = 1;
}
}
},
config: defaultConfig
};
}
}
export const config = defaultConfig;

42
lib/plugins/pagexray/index.js
View File

@ -1,9 +1,15 @@
'use strict';
const pagexrayAggregator = require('./pagexrayAggregator');
const pagexray = require('coach-core').getPageXray();
const urlParser = require('url');
const log = require('intel').getLogger('plugin.pagexray');
const h = require('../../support/helpers');
import { parse } from 'node:url';
import intel from 'intel';
import coach from 'coach-core';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { PageXrayAggregator } from './pagexrayAggregator.js';
import { short } from '../../support/helpers/index.js';
const { getPageXray } = coach;
const pagexray = getPageXray();
const log = intel.getLogger('plugin.pagexray');
const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = [
'contentTypes',
'transferSize',
@ -33,11 +39,17 @@ const DEFAULT_PAGEXRAY_SUMMARY_METRICS = [
const DEFAULT_PAGEXRAY_RUN_METRICS = [];
module.exports = {
export default class PageXrayPlugin extends SitespeedioPlugin {
constructor(options, context) {
super({ name: 'pagexray', options, context });
}
open(context, options) {
this.options = options;
this.make = context.messageMaker('pagexray').make;
this.pageXrayAggregator = new PageXrayAggregator();
context.filterRegistry.registerFilterForType(
DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS,
'pagexray.pageSummary'
@ -54,7 +66,7 @@ module.exports = {
this.usingWebpagetest = false;
this.usingBrowsertime = false;
this.multi = options.multi;
},
}
processMessage(message, queue) {
const make = this.make;
switch (message.type) {
@ -75,9 +87,7 @@ module.exports = {
const group = message.group;
let config = {
includeAssets: true,
firstParty: this.options.firstParty
? this.options.firstParty
: undefined
firstParty: this.options.firstParty ?? undefined
};
const pageSummary = pagexray.convert(message.data, config);
//check and print any http server error > 399
@ -88,20 +98,20 @@ module.exports = {
log.info(
`The server responded with a ${
asset.status
} status code for ${h.short(asset.url, 60)}`
} status code for ${short(asset.url, 60)}`
);
}
}
}
}
pagexrayAggregator.addToAggregate(pageSummary, group);
this.pageXrayAggregator.addToAggregate(pageSummary, group);
if (this.multi) {
// The HAR file can have multiple URLs
const sentURL = {};
for (let summary of pageSummary) {
// The group can be different so take it per url
const myGroup = urlParser.parse(summary.url).hostname;
const myGroup = parse(summary.url).hostname;
if (!sentURL[summary.url]) {
sentURL[summary.url] = 1;
queue.postMessage(
@ -149,7 +159,7 @@ module.exports = {
case 'sitespeedio.summarize': {
log.debug('Generate summary metrics from PageXray');
let pagexraySummary = pagexrayAggregator.summarize();
let pagexraySummary = this.pageXrayAggregator.summarize();
if (pagexraySummary) {
for (let group of Object.keys(pagexraySummary.groups)) {
queue.postMessage(
@ -161,4 +171,4 @@ module.exports = {
}
}
}
};
}

113
lib/plugins/pagexray/pagexrayAggregator.js
View File

@ -1,13 +1,15 @@
'use strict';
import forEach from 'lodash.foreach';
const statsHelpers = require('../../support/statsHelpers'),
forEach = require('lodash.foreach');
import { pushGroupStats, setStatsSummary } from '../../support/statsHelpers.js';
const METRIC_NAMES = ['transferSize', 'contentSize', 'requests'];
module.exports = {
stats: {},
groups: {},
export class PageXrayAggregator {
constructor() {
this.stats = {};
this.groups = {};
}
addToAggregate(pageSummary, group) {
if (this.groups[group] === undefined) {
this.groups[group] = {};
@ -16,52 +18,56 @@ module.exports = {
let stats = this.stats;
let groups = this.groups;
pageSummary.forEach(function (summary) {
for (const summary of pageSummary) {
// stats for the whole page
METRIC_NAMES.forEach(function (metric) {
for (const metric of METRIC_NAMES) {
// There's a bug in Firefox/https://github.com/devtools-html/har-export-trigger
// that sometimes generate content size that is null, see
// https://github.com/sitespeedio/sitespeed.io/issues/2090
if (!isNaN(summary[metric])) {
statsHelpers.pushGroupStats(
stats,
groups[group],
metric,
summary[metric]
);
if (!Number.isNaN(summary[metric])) {
pushGroupStats(stats, groups[group], metric, summary[metric]);
}
});
}
Object.keys(summary.contentTypes).forEach(function (contentType) {
METRIC_NAMES.forEach(function (metric) {
for (const contentType of Object.keys(summary.contentTypes)) {
for (const metric of METRIC_NAMES) {
// There's a bug in Firefox/https://github.com/devtools-html/har-export-trigger
// that sometimes generate content size that is null, see
// https://github.com/sitespeedio/sitespeed.io/issues/2090
if (!isNaN(summary.contentTypes[contentType][metric])) {
statsHelpers.pushGroupStats(
if (!Number.isNaN(summary.contentTypes[contentType][metric])) {
pushGroupStats(
stats,
groups[group],
'contentTypes.' + contentType + '.' + metric,
summary.contentTypes[contentType][metric]
);
}
});
});
}
}
Object.keys(summary.responseCodes).forEach(function (responseCode) {
statsHelpers.pushGroupStats(
for (const responseCode of Object.keys(summary.responseCodes)) {
pushGroupStats(
stats,
groups[group],
'responseCodes.' + responseCode,
summary.responseCodes[responseCode]
);
});
}
/*
for (const responseCode of Object.keys(summary.responseCodes)) {
pushGroupStats(
stats,
groups[group],
'responseCodes.' + responseCode,
summary.responseCodes[responseCode]
);
}*/
// extras for firstParty vs third
if (summary.firstParty.requests) {
METRIC_NAMES.forEach(function (metric) {
for (const metric of METRIC_NAMES) {
if (summary.firstParty[metric] !== undefined) {
statsHelpers.pushGroupStats(
pushGroupStats(
stats,
groups[group],
'firstParty' + '.' + metric,
@ -69,18 +75,18 @@ module.exports = {
);
}
if (summary.thirdParty[metric] !== undefined) {
statsHelpers.pushGroupStats(
pushGroupStats(
stats,
groups[group],
'thirdParty' + '.' + metric,
summary.thirdParty[metric]
);
}
});
}
}
// Add the total amount of domains on this page
statsHelpers.pushGroupStats(
pushGroupStats(
stats,
groups[group],
'domains',
@ -88,32 +94,22 @@ module.exports = {
);
// And the total amounts of cookies
statsHelpers.pushGroupStats(
stats,
groups[group],
'cookies',
summary.cookies
);
pushGroupStats(stats, groups[group], 'cookies', summary.cookies);
forEach(summary.assets, asset => {
statsHelpers.pushGroupStats(
stats,
groups[group],
'expireStats',
asset.expires
);
statsHelpers.pushGroupStats(
pushGroupStats(stats, groups[group], 'expireStats', asset.expires);
pushGroupStats(
stats,
groups[group],
'lastModifiedStats',
asset.timeSinceLastModified
);
});
});
},
}
}
summarize() {
if (Object.keys(this.stats).length === 0) {
return undefined;
return;
}
const total = this.summarizePerObject(this.stats);
@ -128,37 +124,34 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]);
}
return summary;
},
}
summarizePerObject(type) {
return Object.keys(type).reduce((summary, name) => {
if (
METRIC_NAMES.indexOf(name) > -1 ||
name.match(/(^domains|^expireStats|^lastModifiedStats|^cookies)/)
METRIC_NAMES.includes(name) ||
/(^domains|^expireStats|^lastModifiedStats|^cookies)/.test(name)
) {
statsHelpers.setStatsSummary(summary, name, type[name]);
setStatsSummary(summary, name, type[name]);
} else {
if (name === 'contentTypes') {
const contentTypeData = {};
forEach(Object.keys(type[name]), contentType => {
forEach(type[name][contentType], (stats, metric) => {
statsHelpers.setStatsSummary(
contentTypeData,
[contentType, metric],
stats
);
setStatsSummary(contentTypeData, [contentType, metric], stats);
});
});
summary[name] = contentTypeData;
} else if (name === 'responseCodes') {
const responseCodeData = {};
type.responseCodes.forEach((stats, metric) => {
statsHelpers.setStatsSummary(responseCodeData, metric, stats);
});
for (const [metric, stats] of type.responseCodes.entries()) {
if (stats != undefined)
setStatsSummary(responseCodeData, metric, stats);
}
summary[name] = responseCodeData;
} else {
const data = {};
forEach(type[name], (stats, metric) => {
statsHelpers.setStatsSummary(data, metric, stats);
setStatsSummary(data, metric, stats);
});
summary[name] = data;
}
@ -166,4 +159,4 @@ module.exports = {
return summary;
}, {});
}
};
}

14
lib/plugins/remove/index.js
View File

@ -1,12 +1,16 @@
'use strict';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = require('intel').getLogger('sitespeedio.plugin.remove');
const log = intel.getLogger('sitespeedio.plugin.remove');
module.exports = {
export default class RemovePlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'remove', options, context, queue });
}
open(context, options) {
this.storageManager = context.storageManager;
this.options = options;
},
}
async processMessage(message) {
switch (message.type) {
case 'remove.url': {
@ -16,4 +20,4 @@ module.exports = {
}
}
}
};
}

12
lib/plugins/s3/contentType.js
View File

@ -1,5 +1,3 @@
'use strict';
const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
const types = {
@ -19,12 +17,10 @@ const types = {
log: 'text/plain'
};
function getExt(filename) {
function getExtension(filename) {
return filename.split('.').pop();
}
module.exports = {
getContentType(file) {
return types[getExt(file)] || DEFAULT_CONTENT_TYPE;
}
};
export function getContentType(file) {
return types[getExtension(file)] || DEFAULT_CONTENT_TYPE;
}

78
lib/plugins/s3/index.js
View File

@ -1,19 +1,25 @@
'use strict';
import { relative, join, resolve as _resolve, sep } from 'node:path';
const fs = require('fs-extra');
const path = require('path');
const AWS = require('aws-sdk');
const readdir = require('recursive-readdir');
const pLimit = require('p-limit');
import { createReadStream, remove } from 'fs-extra';
import { Endpoint, S3 } from 'aws-sdk';
import readdir from 'recursive-readdir';
import pLimit from 'p-limit';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = require('intel').getLogger('sitespeedio.plugin.s3');
const throwIfMissing = require('../../support/util').throwIfMissing;
const { getContentType } = require('./contentType');
import { throwIfMissing } from '../../support/util';
import { getContentType } from './contentType';
const log = intel.getLogger('sitespeedio.plugin.s3');
function ignoreDirectories(file, stats) {
return stats.isDirectory();
}
function createS3(s3Options) {
let endpoint = s3Options.endpoint || 's3.amazonaws.com';
const options = {
endpoint: new AWS.Endpoint(endpoint),
endpoint: new Endpoint(endpoint),
accessKeyId: s3Options.key,
secretAccessKey: s3Options.secret,
signatureVersion: 'v4'
@ -21,7 +27,7 @@ function createS3(s3Options) {
// You can also set some extra options see
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
Object.assign(options, s3Options.options);
return new AWS.S3(options);
return new S3(options);
}
async function upload(dir, s3Options, prefix) {
@ -38,12 +44,8 @@ async function upload(dir, s3Options, prefix) {
}
async function uploadLatestFiles(dir, s3Options, prefix) {
function ignoreDirs(file, stats) {
return stats.isDirectory();
}
const s3 = createS3(s3Options);
const files = await readdir(dir, [ignoreDirs]);
const files = await readdir(dir, [ignoreDirectories]);
// Backward compability naming for old S3 plugin
const limit = pLimit(s3Options.maxAsyncS3 || 20);
const promises = [];
@ -55,39 +57,42 @@ async function uploadLatestFiles(dir, s3Options, prefix) {
}
async function uploadFile(file, s3, s3Options, prefix, baseDir) {
const stream = fs.createReadStream(file);
const stream = createReadStream(file);
const contentType = getContentType(file);
return new Promise((resolve, reject) => {
const onUpload = err => {
if (err) {
reject(err);
const onUpload = error => {
if (error) {
reject(error);
} else {
resolve();
}
};
const options = { partSize: 10 * 1024 * 1024, queueSize: 1 };
// See https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
const subPath = path.relative(baseDir, file);
const params = {
const subPath = relative(baseDir, file);
const parameters = {
Body: stream,
Bucket: s3Options.bucketname,
ContentType: contentType,
Key: path.join(s3Options.path || prefix, subPath),
Key: join(s3Options.path || prefix, subPath),
StorageClass: s3Options.storageClass || 'STANDARD'
};
if (s3Options.acl) {
params.ACL = s3Options.acl;
parameters.ACL = s3Options.acl;
}
// Override/set all the extra options you need
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
Object.assign(params, s3Options.params);
Object.assign(parameters, s3Options.params);
s3.upload(params, options, onUpload);
s3.upload(parameters, options, onUpload);
});
}
module.exports = {
export default class S3Plugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 's3', options, context, queue });
}
open(context, options) {
this.s3Options = options.s3;
this.options = options;
@ -97,8 +102,7 @@ module.exports = {
throwIfMissing(this.s3Options, ['key', 'secret'], 's3');
}
this.storageManager = context.storageManager;
},
}
async processMessage(message, queue) {
if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the s3 plugin is alive
@ -119,21 +123,21 @@ module.exports = {
this.storageManager.getStoragePrefix()
);
if (this.options.copyLatestFilesToBase) {
const rootPath = path.resolve(baseDir, '..');
const dirsAsArray = rootPath.split(path.sep);
const rootName = dirsAsArray.slice(-1)[0];
const rootPath = _resolve(baseDir, '..');
const directoriesAsArray = rootPath.split(sep);
const rootName = directoriesAsArray.slice(-1)[0];
await uploadLatestFiles(rootPath, s3Options, rootName);
}
log.info('Finished upload to s3');
if (s3Options.removeLocalResult) {
await fs.remove(baseDir);
await remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`);
}
} catch (e) {
queue.postMessage(make('error', e));
log.error('Could not upload to S3', e);
} catch (error) {
queue.postMessage(make('error', error));
log.error('Could not upload to S3', error);
}
queue.postMessage(make('s3.finished'));
}
}
};
}

71
lib/plugins/scp/index.js
View File

@ -1,11 +1,13 @@
'use strict';
import { join, basename, resolve } from 'node:path';
const fs = require('fs-extra');
const path = require('path');
const { Client } = require('node-scp');
const readdir = require('recursive-readdir');
const log = require('intel').getLogger('sitespeedio.plugin.scp');
const throwIfMissing = require('../../support/util').throwIfMissing;
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { readFileSync, remove } from 'fs-extra';
import { Client } from 'node-scp';
import readdir from 'recursive-readdir';
import intel from 'intel';
import { throwIfMissing } from '../../support/util';
const log = intel.getLogger('sitespeedio.plugin.scp');
async function getClient(scpOptions) {
const options = {
@ -19,7 +21,7 @@ async function getClient(scpOptions) {
options.password = scpOptions.password;
}
if (scpOptions.privateKey) {
options.privateKey = fs.readFileSync(scpOptions.privateKey);
options.privateKey = readFileSync(scpOptions.privateKey);
}
if (scpOptions.passphrase) {
options.passphrase = scpOptions.passphrase;
@ -31,21 +33,21 @@ async function upload(dir, scpOptions, prefix) {
let client;
try {
client = await getClient(scpOptions);
const dirs = prefix.split('/');
const directories = prefix.split('/');
let fullPath = '';
for (let dir of dirs) {
for (let dir of directories) {
fullPath += dir + '/';
const doThePathExist = await client.exists(
path.join(scpOptions.destinationPath, fullPath)
join(scpOptions.destinationPath, fullPath)
);
if (!doThePathExist) {
await client.mkdir(path.join(scpOptions.destinationPath, fullPath));
await client.mkdir(join(scpOptions.destinationPath, fullPath));
}
}
await client.uploadDir(dir, path.join(scpOptions.destinationPath, prefix));
} catch (e) {
log.error(e);
throw e;
await client.uploadDir(dir, join(scpOptions.destinationPath, prefix));
} catch (error) {
log.error(error);
throw error;
} finally {
if (client) {
client.close();
@ -60,12 +62,12 @@ async function uploadFiles(files, scpOptions, prefix) {
for (let file of files) {
await client.uploadFile(
file,
path.join(scpOptions.destinationPath, prefix, path.basename(file))
join(scpOptions.destinationPath, prefix, basename(file))
);
}
} catch (e) {
log.error(e);
throw e;
} catch (error) {
log.error(error);
throw error;
} finally {
if (client) {
client.close();
@ -73,16 +75,20 @@ async function uploadFiles(files, scpOptions, prefix) {
}
}
function ignoreDirectories(file, stats) {
return stats.isDirectory();
}
async function uploadLatestFiles(dir, scpOptions, prefix) {
function ignoreDirs(file, stats) {
return stats.isDirectory();
}
const files = await readdir(dir, [ignoreDirs]);
const files = await readdir(dir, [ignoreDirectories]);
return uploadFiles(files, scpOptions, prefix);
}
module.exports = {
export default class ScpPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'scp', options, context, queue });
}
open(context, options) {
this.scpOptions = options.scp;
this.options = options;
@ -93,8 +99,7 @@ module.exports = {
'scp'
);
this.storageManager = context.storageManager;
},
}
async processMessage(message, queue) {
if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the scp plugin is alive
@ -114,21 +119,21 @@ module.exports = {
this.storageManager.getStoragePrefix()
);
if (this.options.copyLatestFilesToBase) {
const rootPath = path.resolve(baseDir, '..');
const rootPath = resolve(baseDir, '..');
const prefix = this.storageManager.getStoragePrefix();
const firstPart = prefix.split('/')[0];
await uploadLatestFiles(rootPath, this.scpOptions, firstPart);
}
log.info('Finished upload using scp');
if (this.scpOptions.removeLocalResult) {
await fs.remove(baseDir);
await remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`);
}
} catch (e) {
queue.postMessage(make('error', e));
log.error('Could not upload using scp', e);
} catch (error) {
queue.postMessage(make('error', error));
log.error('Could not upload using scp', error);
}
queue.postMessage(make('scp.finished'));
}
}
};
}

38
lib/plugins/slack/attachements.js
View File

@ -1,17 +1,13 @@
'use strict';
const get = require('lodash.get');
const h = require('../../support/helpers');
import get from 'lodash.get';
import { time, noop, size } from '../../support/helpers/index.js';
function getMetric(metric, f) {
if (metric.median) {
return f(metric.median) + ' (' + f(metric.max) + ')';
} else {
return f(metric);
}
return metric.median
? f(metric.median) + ' (' + f(metric.max) + ')'
: f(metric);
}
module.exports = function (
export function getAttachements(
dataCollector,
resultUrls,
slackOptions,
@ -33,7 +29,7 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.timings.firstPaint'
),
f: h.time.ms
f: time.ms
},
speedIndex: {
name: 'Speed Index',
@ -41,7 +37,7 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.visualMetrics.SpeedIndex'
),
f: h.time.ms
f: time.ms
},
firstVisualChange: {
name: 'First Visual Change',
@ -49,7 +45,7 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.visualMetrics.FirstVisualChange'
),
f: h.time.ms
f: time.ms
},
visualComplete85: {
name: 'Visual Complete 85%',
@ -57,7 +53,7 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.visualMetrics.VisualComplete85'
),
f: h.time.ms
f: time.ms
},
lastVisualChange: {
name: 'Last Visual Change',
@ -65,7 +61,7 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.visualMetrics.LastVisualChange'
),
f: h.time.ms
f: time.ms
},
fullyLoaded: {
name: 'Fully Loaded',
@ -73,7 +69,7 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.timings.fullyLoaded'
),
f: h.time.ms
f: time.ms
},
domContentLoadedTime: {
name: 'domContentLoadedTime',
@ -81,22 +77,22 @@ module.exports = function (
base.browsertime,
'pageSummary.statistics.timings.pageTimings.domContentLoadedTime'
),
f: h.time.ms
f: time.ms
},
coachScore: {
name: 'Coach score',
metric: get(base.coach, 'pageSummary.advice.performance.score'),
f: h.noop
f: noop
},
transferSize: {
name: 'Page transfer size',
metric: get(base.pagexray, 'pageSummary.transferSize'),
f: h.size.format
f: size.format
},
transferRequests: {
name: 'Requests',
metric: get(base.pagexray, 'pageSummary.requests'),
f: h.noop
f: noop
}
};
@ -169,4 +165,4 @@ module.exports = function (
attachments.push(attachement);
}
return attachments;
};
}

12
lib/plugins/slack/dataCollector.js
View File

@ -1,10 +1,8 @@
'use strict';
import merge from 'lodash.merge';
import get from 'lodash.get';
import set from 'lodash.set';
const merge = require('lodash.merge'),
get = require('lodash.get'),
set = require('lodash.set');
class DataCollector {
export class DataCollector {
constructor(context) {
this.resultUrls = context.resultUrls;
this.urlRunPages = {};
@ -79,5 +77,3 @@ class DataCollector {
merge(this.summaryPage, data);
}
}
module.exports = DataCollector;

51
lib/plugins/slack/index.js
View File

@ -1,14 +1,17 @@
'use strict';
import { promisify } from 'node:util';
const throwIfMissing = require('../../support/util').throwIfMissing;
const log = require('intel').getLogger('sitespeedio.plugin.slack');
const Slack = require('node-slack');
const merge = require('lodash.merge');
const set = require('lodash.set');
const DataCollector = require('./dataCollector');
const getAttachments = require('./attachements');
const getSummary = require('./summary');
const { promisify } = require('util');
import intel from 'intel';
import Slack from 'node-slack';
import merge from 'lodash.merge';
import set from 'lodash.set';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { DataCollector } from './dataCollector.js';
import { getAttachements } from './attachements.js';
import { getSummary } from './summary.js';
import { throwIfMissing } from '../../support/util.js';
const log = intel.getLogger('sitespeedio.plugin.slack');
const defaultConfig = {
userName: 'Sitespeed.io',
@ -55,7 +58,7 @@ function send(options, dataCollector, context, screenshotType, alias) {
let attachments = [];
if (['url', 'all', 'error'].includes(type)) {
attachments = getAttachments(
attachments = getAttachements(
dataCollector,
context.resultUrls,
slackOptions,
@ -77,11 +80,11 @@ function send(options, dataCollector, context, screenshotType, alias) {
mrkdwn: true,
username: slackOptions.userName,
attachments
}).catch(e => {
if (e.errno === 'ETIMEDOUT') {
}).catch(error => {
if (error.errno === 'ETIMEDOUT') {
log.warn('Timeout sending Slack message.');
} else {
throw e;
throw error;
}
});
}
@ -101,7 +104,7 @@ function staticPagesProvider(options) {
throwIfMissing(s3Options, ['key', 'secret'], 's3');
}
return 's3';
} catch (err) {
} catch {
log.debug('s3 is not configured');
}
@ -109,14 +112,18 @@ function staticPagesProvider(options) {
try {
throwIfMissing(gcsOptions, ['projectId', 'key', 'bucketname'], 'gcs');
return 'gcs';
} catch (err) {
} catch {
log.debug('gcs is not configured');
}
return null;
return;
}
module.exports = {
export default class SlackPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'slack', options, context, queue });
}
open(context, options = {}) {
const slackOptions = options.slack || {};
throwIfMissing(slackOptions, ['hookUrl', 'userName'], 'slack');
@ -125,7 +132,7 @@ module.exports = {
this.options = options;
this.screenshotType;
this.alias = {};
},
}
processMessage(message) {
const dataCollector = this.dataCollector;
@ -219,6 +226,6 @@ module.exports = {
);
}
}
},
config: defaultConfig
};
}
}
export const config = defaultConfig;

50
lib/plugins/slack/summary.js
View File

@ -1,16 +1,22 @@
'use strict';
const util = require('util');
const get = require('lodash.get');
const h = require('../../support/helpers');
const tsdbUtil = require('../../support/tsdbUtil');
import { format } from 'node:util';
import get from 'lodash.get';
import {
time,
noop,
size,
cap,
plural,
short
} from '../../support/helpers/index.js';
import { getConnectivity } from '../../support/tsdbUtil.js';
module.exports = function (dataCollector, errors, resultUrls, name, options) {
export function getSummary(dataCollector, errors, resultUrls, name, options) {
const base = dataCollector.getSummary() || {};
const metrics = {
firstPaint: {
name: 'First paint',
metric: get(base.browsertime, 'summary.firstPaint.median'),
f: h.time.ms
f: time.ms
},
domContentLoadedTime: {
name: 'domContentLoadedTime',
@ -18,12 +24,12 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime,
'summary.pageTimings.domContentLoadedTime.median'
),
f: h.time.ms
f: time.ms
},
speedIndex: {
name: 'Speed Index',
metric: get(base.browsertime, 'summary.visualMetrics.SpeedIndex.median'),
f: h.time.ms
f: time.ms
},
firstVisualChange: {
name: 'First Visual Change',
@ -31,7 +37,7 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime,
'summary.visualMetrics.FirstVisualChange.median'
),
f: h.time.ms
f: time.ms
},
visualComplete85: {
name: 'Visual Complete 85%',
@ -39,7 +45,7 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime,
'summary.visualMetrics.VisualComplete85.median'
),
f: h.time.ms
f: time.ms
},
lastVisualChange: {
name: 'Last Visual Change',
@ -47,34 +53,34 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime,
'summary.visualMetrics.LastVisualChange.median'
),
f: h.time.ms
f: time.ms
},
fullyLoaded: {
name: 'Fully Loaded',
metric: get(base.pagexray, 'summary.fullyLoaded.median'),
f: h.time.ms
f: time.ms
},
coachScore: {
name: 'Coach score',
metric: get(base.coach, 'summary.performance.score.median'),
f: h.noop
f: noop
},
transferSize: {
name: 'Page transfer weight',
metric: get(base.pagexray, 'summary.transferSize.median'),
f: h.size.format
f: size.format
}
};
const iterations = get(options, 'browsertime.iterations', 0);
const browser = h.cap(get(options, 'browsertime.browser', 'unknown'));
const pages = h.plural(dataCollector.getURLs().length, 'page');
const testName = h.short(name || '', 30) || 'Unknown';
const browser = cap(get(options, 'browsertime.browser', 'unknown'));
const pages = plural(dataCollector.getURLs().length, 'page');
const testName = short(name || '', 30) || 'Unknown';
const device = options.mobile ? 'mobile' : 'desktop';
const runs = h.plural(iterations, 'run');
const runs = plural(iterations, 'run');
let summaryText =
`${pages} analysed for ${testName} ` +
`(${runs}, ${browser}/${device}/${tsdbUtil.getConnectivity(options)})\n`;
`(${runs}, ${browser}/${device}/${getConnectivity(options)})\n`;
let message = '';
if (resultUrls.hasBaseUrl()) {
@ -96,7 +102,7 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
let errorText = '';
if (errors.length > 0) {
errorText += util.format('%d error(s):\n', errors.length);
errorText += format('%d error(s):\n', errors.length);
logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack-hm.png';
}
@ -105,4 +111,4 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
errorText,
logo
};
};
}

Some files were not shown because too many files have changed in this diff Show More