New plugins structure and esmodule (#3769)

* New plugins structure and esmodule
This commit is contained in:
Peter Hedenskog 2023-02-25 11:16:58 +01:00 committed by GitHub
parent 1dfd8e67a0
commit 631271126f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
153 changed files with 6723 additions and 5927 deletions

View File

@ -4,7 +4,6 @@ assets/*
sitespeed-result/* sitespeed-result/*
lib/plugins/yslow/scripts/* lib/plugins/yslow/scripts/*
lib/plugins/html/assets/js/* lib/plugins/html/assets/js/*
lib/plugins/browsertime/index.js
lib/plugins/browsertime/analyzer.js
bin/browsertimeWebPageReplay.js bin/browsertimeWebPageReplay.js
test/data/* test/data/*
test/prepostscripts/*

View File

@ -5,10 +5,11 @@
"es6": true "es6": true
}, },
"parserOptions": { "parserOptions": {
"ecmaVersion": 2018 "ecmaVersion": "latest",
"sourceType": "module"
}, },
"plugins": ["prettier"], "plugins": ["prettier", "unicorn"],
"extends": "eslint:recommended", "extends": ["eslint:recommended", "plugin:unicorn/recommended"],
"rules": { "rules": {
"prettier/prettier": [ "prettier/prettier": [
"error", "error",
@ -21,6 +22,10 @@
], ],
"require-atomic-updates": 0, "require-atomic-updates": 0,
"no-extra-semi": 0, "no-extra-semi": 0,
"no-mixed-spaces-and-tabs": 0 "no-mixed-spaces-and-tabs": 0,
"unicorn/filename-case": 0,
"unicorn/prevent-abbreviations": 0,
"unicorn/no-array-reduce": 0,
"unicorn/prefer-spread":0
} }
} }

View File

@ -21,14 +21,14 @@ jobs:
- name: Start local HTTP server - name: Start local HTTP server
run: (serve test/data/html/ -l 3001&) run: (serve test/data/html/ -l 3001&)
- name: Run test on default container for Chrome - name: Run test on default container for Chrome
run: docker run --rm --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b chrome run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b chrome
- name: Run test on default container for Firefox - name: Run test on default container for Firefox
run: docker run --rm --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b firefox run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b firefox
- name: Run test on default container for Edge - name: Run test on default container for Edge
run: docker run --rm --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b edge run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001 -n 1 -b edge
- name: Run test on slim container - name: Run test on slim container
run: docker run --rm --network=host sitespeedio/sitespeed.io:slim http://127.0.0.1:3001 -n 1 --browsertime.firefox.preference "devtools.netmonitor.persistlog:true" run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io:slim http://127.0.0.1:3001 -n 1 --browsertime.firefox.preference "devtools.netmonitor.persistlog:true"
- name: Test WebPageReplay with Chrome - name: Test WebPageReplay with Chrome
run: docker run --cap-add=NET_ADMIN --rm -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b chrome run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b chrome
- name: Test WebPageReplay with Firefox - name: Test WebPageReplay with Firefox
run: docker run --cap-add=NET_ADMIN --rm --network=host -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b firefox --browsertime.firefox.acceptInsecureCerts true run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io --network=host -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b firefox --browsertime.firefox.acceptInsecureCerts true

View File

@ -1,4 +1,4 @@
FROM sitespeedio/webbrowsers:chrome-110.0-firefox-109.0-edge-109.0 FROM sitespeedio/webbrowsers:chrome-110.0-firefox-110.0-edge-110.0
ARG TARGETPLATFORM=linux/amd64 ARG TARGETPLATFORM=linux/amd64

View File

@ -1,25 +1,29 @@
#!/usr/bin/env node #!/usr/bin/env node
'use strict'; import { readFileSync } from 'node:fs';
const yargs = require('yargs'); import merge from 'lodash.merge';
const merge = require('lodash.merge'); import set from 'lodash.set';
const getURLs = require('../lib/cli/util').getURLs; import get from 'lodash.get';
const get = require('lodash.get'); import yargs from 'yargs';
const set = require('lodash.set'); import { hideBin } from 'yargs/helpers';
const findUp = require('find-up');
const fs = require('fs'); import { findUpSync } from 'find-up';
const browsertimeConfig = require('../lib/plugins/browsertime/index').config; import { BrowsertimeEngine, configureLogging } from 'browsertime';
import { getURLs } from '../lib/cli/util.js';
import {config as browsertimeConfig} from '../lib/plugins/browsertime/index.js';
const iphone6UserAgent = const iphone6UserAgent =
'Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 ' + 'Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_3 like Mac OS X) AppleWebKit/536.26 ' +
'(KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25'; '(KHTML, like Gecko) Version/6.0 Mobile/10B329 Safari/8536.25';
const configPath = findUp.sync(['.sitespeed.io.json']); const configPath = findUpSync(['.sitespeed.io.json']);
let config; let config;
try { try {
config = configPath ? JSON.parse(fs.readFileSync(configPath)) : {}; config = configPath ? JSON.parse(readFileSync(configPath)) : {};
} catch (e) { } catch (e) {
if (e instanceof SyntaxError) { if (e instanceof SyntaxError) {
/* eslint no-console: off */ /* eslint no-console: off */
@ -49,7 +53,8 @@ async function testURLs(engine, urls) {
} }
async function runBrowsertime() { async function runBrowsertime() {
let parsed = yargs let yargsInstance = yargs(hideBin(process.argv));
let parsed = yargsInstance
.env('SITESPEED_IO') .env('SITESPEED_IO')
.require(1, 'urlOrFile') .require(1, 'urlOrFile')
.option('browsertime.browser', { .option('browsertime.browser', {
@ -139,9 +144,6 @@ async function runBrowsertime() {
} }
}; };
const {BrowsertimeEngine, configureLogging} = await import ('browsertime');
const btOptions = merge({}, parsed.argv.browsertime, defaultConfig); const btOptions = merge({}, parsed.argv.browsertime, defaultConfig);
// hack to keep backward compability to --android // hack to keep backward compability to --android
if (parsed.argv.android[0] === true) { if (parsed.argv.android[0] === true) {

View File

@ -2,24 +2,30 @@
/*eslint no-console: 0*/ /*eslint no-console: 0*/
'use strict'; import { writeFileSync } from 'node:fs';
import { execSync } from 'node:child_process';
import { platform } from 'node:os';
import { parseCommandLine } from '../lib/cli/cli.js';
import { run } from '../lib/sitespeed.js';
const fs = require('fs'); async function start() {
const cli = require('../lib/cli/cli'); let parsed = await parseCommandLine();
const sitespeed = require('../lib/sitespeed'); let budgetFailing = false;
const { execSync } = require('child_process'); // hack for getting in the unchanged cli options
const os = require('os'); parsed.options.explicitOptions = parsed.explicitOptions;
parsed.options.urls = parsed.urls;
parsed.options.urlsMetaData = parsed.urlsMetaData;
async function run(options) { let options = parsed.options;
process.exitCode = 1; process.exitCode = 1;
try { try {
const result = await sitespeed.run(options); const result = await run(options);
if (options.storeResult) { if (options.storeResult) {
if (options.storeResult != 'true') { if (options.storeResult != 'true') {
fs.writeFileSync(options.storeResult, JSON.stringify(result)); writeFileSync(options.storeResult, JSON.stringify(result));
} else { } else {
fs.writeFileSync('result.json', JSON.stringify(result)); writeFileSync('result.json', JSON.stringify(result));
} }
} }
@ -27,9 +33,9 @@ async function run(options) {
throw new Error('Errors while running:\n' + result.errors.join('\n')); throw new Error('Errors while running:\n' + result.errors.join('\n'));
} }
if ((options.open || options.o) && os.platform() === 'darwin') { if ((options.open || options.o) && platform() === 'darwin') {
execSync('open ' + result.localPath + '/index.html'); execSync('open ' + result.localPath + '/index.html');
} else if ((options.open || options.o) && os.platform() === 'linux') { } else if ((options.open || options.o) && platform() === 'linux') {
execSync('xdg-open ' + result.localPath + '/index.html'); execSync('xdg-open ' + result.localPath + '/index.html');
} }
@ -47,17 +53,12 @@ async function run(options) {
) { ) {
process.exitCode = 0; process.exitCode = 0;
} }
} catch (e) { } catch (error) {
process.exitCode = 1; process.exitCode = 1;
console.log(error);
} finally { } finally {
process.exit(); process.exit();
} }
} }
let parsed = cli.parseCommandLine();
let budgetFailing = false;
// hack for getting in the unchanged cli options
parsed.options.explicitOptions = parsed.explicitOptions;
parsed.options.urls = parsed.urls;
parsed.options.urlsMetaData = parsed.urlsMetaData;
run(parsed.options); await start();

View File

@ -1,74 +0,0 @@
module.exports = {
types: [
{ value: 'feat', name: 'feat: A new feature' },
{ value: 'fix', name: 'fix: A bug fix' },
{ value: 'docs', name: 'docs: Documentation only changes' },
{
value: 'style',
name: 'style: Changes that do not affect the meaning of the code\n (white-space, formatting, missing semi-colons, etc)'
},
{
value: 'refactor',
name: 'refactor: A code change that neither fixes a bug nor adds a feature'
},
{
value: 'perf',
name: 'perf: A code change that improves performance'
},
{ value: 'test', name: 'test: Adding missing tests' },
{
value: 'chore',
name: 'chore: Changes to the build process or auxiliary tools\n and libraries such as documentation generation'
},
{ value: 'revert', name: 'revert: Revert to a commit' },
{ value: 'WIP', name: 'WIP: Work in progress' }
],
// scopes: [
// { name: 'accounts' },
// { name: 'admin' },
// { name: 'exampleScope' },
// { name: 'changeMe' }
// ],
allowTicketNumber: false,
isTicketNumberRequired: false,
// ticketNumberPrefix: 'TICKET-',
// ticketNumberRegExp: '\\d{1,5}',
// it needs to match the value for field type. Eg.: 'fix'
/*
scopeOverrides: {
fix: [
{name: 'merge'},
{name: 'style'},
{name: 'e2eTest'},
{name: 'unitTest'}
]
},
*/
// override the messages, defaults are as follows
messages: {
type: "Select the type of change that you're committing:",
// scope: '\nDenote the SCOPE of this change (optional):',
// used if allowCustomScopes is true
// customScope: 'Denote the SCOPE of this change:',
subject: 'Write a SHORT, IMPERATIVE tense description of the change:\n',
body: 'Provide a LONGER description of the change (optional). Use "|" to break new line:\n',
breaking: 'List any BREAKING CHANGES (optional):\n',
footer:
'List any ISSUES CLOSED by this change (optional). E.g.: #31, #34:\n',
confirmCommit: 'Are you sure you want to proceed with the commit above?'
},
// allowCustomScopes: true,
allowBreakingChanges: ['feat', 'fix'],
// skip any questions you want
skipQuestions: ['body'],
// limit subject length
subjectLimit: 100
// breaklineChar: '|', // It is supported for fields body and footer.
// footerPrefix : 'ISSUES CLOSED:'
// askForBreakingChangeFirst : true, // default is false
};

View File

@ -38,6 +38,18 @@ chrome
--chrome.blockDomainsExcept, --blockDomainsExcept Block all domains except this domain. Use it multiple time to keep multiple domains. You can also wildcard domains like *.sitespeed.io. Use this when you wanna block out all third parties. --chrome.blockDomainsExcept, --blockDomainsExcept Block all domains except this domain. Use it multiple time to keep multiple domains. You can also wildcard domains like *.sitespeed.io. Use this when you wanna block out all third parties.
--chrome.ignoreCertificateErrors Make Chrome ignore certificate errors. Defaults to true. [boolean] [default: true] --chrome.ignoreCertificateErrors Make Chrome ignore certificate errors. Defaults to true. [boolean] [default: true]
android
--android.powerTesting, --androidPower Enables android power testing - charging must be disabled for this.(You have to disable charging yourself for this - it depends on the phone model). [boolean]
--android.ignoreShutdownFailures, --ignoreShutdownFailures If set, shutdown failures will be ignored on Android. [boolean] [default: false]
--android.rooted, --androidRooted If your phone is rooted you can use this to set it up following Mozillas best practice for stable metrics. [boolean] [default: false]
--android.batteryTemperatureLimit, --androidBatteryTemperatureLimit Do the battery temperature need to be below a specific limit before we start the test?
--android.batteryTemperatureWaitTimeInSeconds, --androidBatteryTemperatureWaitTimeInSeconds How long time to wait (in seconds) if the androidBatteryTemperatureWaitTimeInSeconds is not met before the next try [default: 120]
--android.batteryTemperatureReboot, --androidBatteryTemperatureReboot If your phone does not get the minimum temperature aftet the wait time, reboot the phone. [boolean] [default: false]
--android.pretestPowerPress, --androidPretestPowerPress Press the power button on the phone before a test starts. [boolean] [default: false]
--android.pretestPressHomeButton, --androidPretestPressHomeButton Press the home button on the phone before a test starts. [boolean] [default: false]
--android.verifyNetwork, --androidVerifyNetwork Before a test start, verify that the device has a Internet connection by pinging 8.8.8.8 (or a configurable domain with --androidPingAddress) [boolean] [default: false]
--android.gnirehtet, --gnirehtet Start gnirehtet and reverse tethering the traffic from your Android phone. [boolean] [default: false]
firefox firefox
--firefox.binaryPath Path to custom Firefox binary (e.g. Firefox Nightly). On OS X, the path should be to the binary inside the app bundle, e.g. /Applications/Firefox.app/Contents/MacOS/firefox-bin --firefox.binaryPath Path to custom Firefox binary (e.g. Firefox Nightly). On OS X, the path should be to the binary inside the app bundle, e.g. /Applications/Firefox.app/Contents/MacOS/firefox-bin
--firefox.geckodriverPath Path to custom geckodriver binary. Make sure to use a geckodriver version that's compatible with the version of Firefox (Gecko) you're using --firefox.geckodriverPath Path to custom geckodriver binary. Make sure to use a geckodriver version that's compatible with the version of Firefox (Gecko) you're using
@ -135,25 +147,16 @@ debug
Options: Options:
--cpu Easy way to enable both chrome.timeline for Chrome and geckoProfile for Firefox [boolean] --cpu Easy way to enable both chrome.timeline for Chrome and geckoProfile for Firefox [boolean]
--androidPower Enables android power testing - charging must be disabled for this.(You have to disable charging yourself for this - it depends on the phone model). [boolean]
--video Record a video and store the video. Set it to false to remove the video that is created by turning on visualMetrics. To remove fully turn off video recordings, make sure to set video and visualMetrics to false. Requires FFMpeg to be installed. [boolean] --video Record a video and store the video. Set it to false to remove the video that is created by turning on visualMetrics. To remove fully turn off video recordings, make sure to set video and visualMetrics to false. Requires FFMpeg to be installed. [boolean]
--visualMetrics Collect Visual Metrics like First Visual Change, SpeedIndex, Perceptual Speed Index and Last Visual Change. Requires FFMpeg and Python dependencies [boolean] --visualMetrics Collect Visual Metrics like First Visual Change, SpeedIndex, Perceptual Speed Index and Last Visual Change. Requires FFMpeg and Python dependencies [boolean]
--visualElements, --visuaElements Collect Visual Metrics from elements. Works only with --visualMetrics turned on. By default you will get visual metrics from the largest image within the view port and the largest h1. You can also configure to pickup your own defined elements with --scriptInput.visualElements [boolean] --visualElements, --visuaElements Collect Visual Metrics from elements. Works only with --visualMetrics turned on. By default you will get visual metrics from the largest image within the view port and the largest h1. You can also configure to pickup your own defined elements with --scriptInput.visualElements [boolean]
--visualMetricsPerceptual Collect Perceptual Speed Index when you run --visualMetrics. [boolean] --visualMetricsPerceptual Collect Perceptual Speed Index when you run --visualMetrics. [boolean]
--visualMetricsContentful Collect Contentful Speed Index when you run --visualMetrics. [boolean] --visualMetricsContentful Collect Contentful Speed Index when you run --visualMetrics. [boolean]
--visualMetricsPortable Use the portable visual-metrics processing script (no ImageMagick dependencies). [boolean] --visualMetricsPortable Use the portable visual-metrics processing script (no ImageMagick dependencies). [boolean] [default: true]
--scriptInput.visualElements Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered. --scriptInput.visualElements Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered.
--scriptInput.longTask, --minLongTaskLength Set the minimum length of a task to be categorised as a CPU Long Task. It can never be smaller than 50. The value is in ms and only works in Chromium browsers at the moment. [number] [default: 50] --scriptInput.longTask, --minLongTaskLength Set the minimum length of a task to be categorised as a CPU Long Task. It can never be smaller than 50. The value is in ms and only works in Chromium browsers at the moment. [number] [default: 50]
-b, --browser Specify browser. Safari only works on OS X/iOS. Edge only work on OS that supports Edge. [choices: "chrome", "firefox", "edge", "safari"] [default: "chrome"] -b, --browser Specify browser. Safari only works on OS X/iOS. Edge only work on OS that supports Edge. [choices: "chrome", "firefox", "edge", "safari"] [default: "chrome"]
--ignoreShutdownFailures If set, shutdown failures will be ignored on Android. [boolean] [default: false]
--android Short key to use Android. Defaults to use com.android.chrome unless --browser is specified. [boolean] [default: false] --android Short key to use Android. Defaults to use com.android.chrome unless --browser is specified. [boolean] [default: false]
--androidRooted If your phone is rooted you can use this to set it up following Mozillas best practice for stable metrics. [boolean] [default: false]
--androidBatteryTemperatureLimit Do the battery temperature need to be below a specific limit before we start the test?
--androidBatteryTemperatureWaitTimeInSeconds How long time to wait (in seconds) if the androidBatteryTemperatureWaitTimeInSeconds is not met before the next try [default: 120]
--androidBatteryTemperatureReboot If your phone does not get the minimum temperature aftet the wait time, reboot the phone. [boolean] [default: false]
--androidPretestPowerPress Press the power button on the phone before a test starts. [boolean] [default: false]
--androidPretestPressHomeButton Press the home button on the phone before a test starts. [boolean] [default: false]
--androidVerifyNetwork Before a test start, verify that the device has a Internet connection by pinging 8.8.8.8 (or a configurable domain with --androidPingAddress) [boolean] [default: false]
--processStartTime Capture browser process start time (in milliseconds). Android only for now. [boolean] [default: false] --processStartTime Capture browser process start time (in milliseconds). Android only for now. [boolean] [default: false]
--pageCompleteCheck Supply a JavaScript (inline or JavaScript file) that decides when the browser is finished loading the page and can start to collect metrics. The JavaScript snippet is repeatedly queried to see if page has completed loading (indicated by the script returning true). Use it to fetch timings happening after the loadEventEnd. By default the tests ends 2 seconds after loadEventEnd. Also checkout --pageCompleteCheckInactivity and --pageCompleteCheckPollTimeout --pageCompleteCheck Supply a JavaScript (inline or JavaScript file) that decides when the browser is finished loading the page and can start to collect metrics. The JavaScript snippet is repeatedly queried to see if page has completed loading (indicated by the script returning true). Use it to fetch timings happening after the loadEventEnd. By default the tests ends 2 seconds after loadEventEnd. Also checkout --pageCompleteCheckInactivity and --pageCompleteCheckPollTimeout
--pageCompleteWaitTime How long time you want to wait for your pageComplteteCheck to finish, after it is signaled to closed. Extra parameter passed on to your pageCompleteCheck. [default: 8000] --pageCompleteWaitTime How long time you want to wait for your pageComplteteCheck to finish, after it is signaled to closed. Extra parameter passed on to your pageCompleteCheck. [default: 8000]
@ -198,10 +201,10 @@ Options:
--preURLDelay, --warmLoadDealy Delay between preURL and the URL you want to test (in milliseconds) [default: 1500] --preURLDelay, --warmLoadDealy Delay between preURL and the URL you want to test (in milliseconds) [default: 1500]
--userTimingWhitelist All userTimings are captured by default this option takes a regex that will whitelist which userTimings to capture in the results. --userTimingWhitelist All userTimings are captured by default this option takes a regex that will whitelist which userTimings to capture in the results.
--headless Run the browser in headless mode. Works for Firefox and Chrome. [boolean] [default: false] --headless Run the browser in headless mode. Works for Firefox and Chrome. [boolean] [default: false]
--gnirehtet Start gnirehtet and reverse tethering the traffic from your Android phone. [boolean] [default: false]
--flushDNS Flush DNS between runs, works on Mac OS and Linux. Your user needs sudo rights to be able to flush the DNS. [boolean] [default: false] --flushDNS Flush DNS between runs, works on Mac OS and Linux. Your user needs sudo rights to be able to flush the DNS. [boolean] [default: false]
--extension Path to a WebExtension to be installed in the browser. Note that --extension can be passed multiple times. --extension Path to a WebExtension to be installed in the browser. Note that --extension can be passed multiple times.
--spa Convenient parameter to use if you test a SPA application: will automatically wait for X seconds after last network activity and use hash in file names. Read more: https://www.sitespeed.io/documentation/sitespeed.io/spa/ [boolean] [default: false] --spa Convenient parameter to use if you test a SPA application: will automatically wait for X seconds after last network activity and use hash in file names. Read more: https://www.sitespeed.io/documentation/sitespeed.io/spa/ [boolean] [default: false]
--cjs Load scripting files that ends with .js as common js. Default (false) loads files as esmodules. [boolean] [default: false]
--browserRestartTries If the browser fails to start, you can retry to start it this amount of times. [number] [default: 3] --browserRestartTries If the browser fails to start, you can retry to start it this amount of times. [number] [default: 3]
--preWarmServer Do pre test requests to the URL(s) that you want to test that is not measured. Do that to make sure your web server is ready to serve. The pre test requests is done with another browser instance that is closed after pre testing is done. [boolean] [default: false] --preWarmServer Do pre test requests to the URL(s) that you want to test that is not measured. Do that to make sure your web server is ready to serve. The pre test requests is done with another browser instance that is closed after pre testing is done. [boolean] [default: false]
--preWarmServerWaitTime The wait time before you start the real testing after your pre-cache request. [number] [default: 5000] --preWarmServerWaitTime The wait time before you start the real testing after your pre-cache request. [number] [default: 5000]

View File

@ -1,29 +1,24 @@
'use strict'; import { resolve } from 'node:path';
import { platform } from 'node:os';
import { readFileSync, statSync, existsSync } from 'node:fs';
const yargs = require('yargs'); import yargs from 'yargs';
const path = require('path'); import { hideBin } from 'yargs/helpers';
const merge = require('lodash.merge'); import merge from 'lodash.merge';
const reduce = require('lodash.reduce'); import reduce from 'lodash.reduce';
const cliUtil = require('./util'); import set from 'lodash.set';
const fs = require('fs'); import get from 'lodash.get';
const set = require('lodash.set'); import { findUpSync } from 'find-up';
const get = require('lodash.get');
const findUp = require('find-up');
const os = require('os');
const toArray = require('../support/util').toArray;
const grafanaPlugin = require('../plugins/grafana/index'); import { getURLs, getAliases } from './util.js';
const graphitePlugin = require('../plugins/graphite/index'); import { toArray } from '../support/util.js';
const influxdbPlugin = require('../plugins/influxdb/index'); import friendlynames from '../support/friendlynames.js';
const cruxPlugin = require('../plugins/crux/index'); import { config as browsertimeConfig } from '../plugins/browsertime/index.js';
const matrixPlugin = require('../plugins/matrix/index'); import { config as metricsConfig } from '../plugins/metrics/index.js';
import { config as slackConfig } from '../plugins/slack/index.js';
import { config as htmlConfig } from '../plugins/html/index.js';
import { messageTypes as matrixMessageTypes } from '../plugins/matrix/index.js';
const browsertimeConfig = require('../plugins/browsertime/index').config;
const metricsConfig = require('../plugins/metrics/index').config;
const slackConfig = require('../plugins/slack/index').config;
const htmlConfig = require('../plugins/html/index').config;
const friendlynames = require('../support/friendlynames');
const metricList = Object.keys(friendlynames); const metricList = Object.keys(friendlynames);
const configFiles = ['.sitespeed.io.json']; const configFiles = ['.sitespeed.io.json'];
@ -33,20 +28,20 @@ if (process.argv.includes('--config')) {
configFiles.unshift(process.argv[index + 1]); configFiles.unshift(process.argv[index + 1]);
} }
const configPath = findUp.sync(configFiles); const configPath = findUpSync(configFiles);
let config; let config;
try { try {
config = configPath ? JSON.parse(fs.readFileSync(configPath)) : undefined; config = configPath ? JSON.parse(readFileSync(configPath)) : undefined;
} catch (e) { } catch (error) {
if (e instanceof SyntaxError) { if (error instanceof SyntaxError) {
console.error( console.error(
'Error: Could not parse the config JSON file ' + 'Error: Could not parse the config JSON file ' +
configPath + configPath +
'. Is the file really valid JSON?' '. Is the file really valid JSON?'
); );
} }
throw e; throw error;
} }
function validateInput(argv) { function validateInput(argv) {
@ -80,7 +75,7 @@ function validateInput(argv) {
} }
if (argv.slug) { if (argv.slug) {
const characters = /[^A-Za-z_\-0-9]/g; const characters = /[^\w-]/g;
if (characters.test(argv.slug)) { if (characters.test(argv.slug)) {
return 'The slug can only use characters A-Z a-z 0-9 and -_.'; return 'The slug can only use characters A-Z a-z 0-9 and -_.';
} }
@ -100,7 +95,7 @@ function validateInput(argv) {
if ( if (
argv.urlAlias && argv.urlAlias &&
argv._ && argv._ &&
cliUtil.getURLs(argv._).length !== toArray(argv.urlAlias).length getURLs(argv._).length !== toArray(argv.urlAlias).length
) { ) {
return 'Error: You have a miss match between number of alias and URLs.'; return 'Error: You have a miss match between number of alias and URLs.';
} }
@ -108,22 +103,19 @@ function validateInput(argv) {
if ( if (
argv.groupAlias && argv.groupAlias &&
argv._ && argv._ &&
cliUtil.getURLs(argv._).length !== toArray(argv.groupAlias).length getURLs(argv._).length !== toArray(argv.groupAlias).length
) { ) {
return 'Error: You have a miss match between number of alias for groups and URLs.'; return 'Error: You have a miss match between number of alias for groups and URLs.';
} }
if ( if (
argv.browsertime.connectivity && argv.browsertime.connectivity &&
argv.browsertime.connectivity.engine === 'humble' argv.browsertime.connectivity.engine === 'humble' &&
) { (!argv.browsertime.connectivity.humble ||
if ( !argv.browsertime.connectivity.humble.url)
!argv.browsertime.connectivity.humble ||
!argv.browsertime.connectivity.humble.url
) { ) {
return 'You need to specify the URL to Humble by using the --browsertime.connectivity.humble.url option.'; return 'You need to specify the URL to Humble by using the --browsertime.connectivity.humble.url option.';
} }
}
if ( if (
argv.browsertime.safari && argv.browsertime.safari &&
@ -159,8 +151,8 @@ function validateInput(argv) {
if (!urlOrFile.startsWith('http')) { if (!urlOrFile.startsWith('http')) {
// is existing file? // is existing file?
try { try {
fs.statSync(urlOrFile); statSync(urlOrFile);
} catch (e) { } catch {
return ( return (
'Error: ' + 'Error: ' +
urlOrFile + urlOrFile +
@ -182,23 +174,23 @@ function validateInput(argv) {
} }
for (let metric of toArray(argv.html.summaryBoxes)) { for (let metric of toArray(argv.html.summaryBoxes)) {
if (htmlConfig.html.summaryBoxes.indexOf(metric) === -1) { if (!htmlConfig.html.summaryBoxes.includes(metric)) {
return `Error: ${metric} is not part of summary box metric.`; return `Error: ${metric} is not part of summary box metric.`;
} }
} }
if (argv.html && argv.html.summaryBoxesThresholds) { if (argv.html && argv.html.summaryBoxesThresholds) {
try { try {
const box = fs.readFileSync( const box = readFileSync(resolve(argv.html.summaryBoxesThresholds), {
path.resolve(argv.html.summaryBoxesThresholds),
{
encoding: 'utf8' encoding: 'utf8'
} });
);
argv.html.summaryBoxesThresholds = JSON.parse(box); argv.html.summaryBoxesThresholds = JSON.parse(box);
} catch (e) { } catch (error) {
return ( return (
'Error: Could not read ' + argv.html.summaryBoxesThresholds + ' ' + e 'Error: Could not read ' +
argv.html.summaryBoxesThresholds +
' ' +
error
); );
} }
} }
@ -206,8 +198,9 @@ function validateInput(argv) {
return true; return true;
} }
module.exports.parseCommandLine = function parseCommandLine() { export async function parseCommandLine() {
let parsed = yargs let yargsInstance = yargs(hideBin(process.argv));
let parsed = yargsInstance
.parserConfiguration({ 'deep-merge-config': true }) .parserConfiguration({ 'deep-merge-config': true })
.env('SITESPEED_IO') .env('SITESPEED_IO')
.usage('$0 [options] <url>/<file>') .usage('$0 [options] <url>/<file>')
@ -344,7 +337,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
}) })
.option('browsertime.timeouts.pageCompleteCheck', { .option('browsertime.timeouts.pageCompleteCheck', {
alias: 'maxLoadTime', alias: 'maxLoadTime',
default: 120000, default: 120_000,
type: 'number', type: 'number',
describe: describe:
'The max load time to wait for a page to finish loading (in milliseconds).', 'The max load time to wait for a page to finish loading (in milliseconds).',
@ -693,7 +686,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
.option('browsertime.firefox.geckoProfilerParams.bufferSize', { .option('browsertime.firefox.geckoProfilerParams.bufferSize', {
alias: 'firefox.geckoProfilerParams.bufferSize', alias: 'firefox.geckoProfilerParams.bufferSize',
describe: 'Buffer size in elements. Default is ~90MB.', describe: 'Buffer size in elements. Default is ~90MB.',
default: 1000000, default: 1_000_000,
type: 'number', type: 'number',
group: 'Firefox' group: 'Firefox'
}) })
@ -1142,14 +1135,203 @@ module.exports.parseCommandLine = function parseCommandLine() {
describe: describe:
'Remove the files locally when the files has been copied to the other server.', 'Remove the files locally when the files has been copied to the other server.',
group: 'scp' group: 'scp'
})
.option('grafana.host', {
describe: 'The Grafana host used when sending annotations.',
group: 'Grafana'
})
.option('grafana.port', {
default: 80,
describe: 'The Grafana port used when sending annotations to Grafana.',
group: 'Grafana'
})
.option('grafana.auth', {
describe:
'The Grafana auth/bearer value used when sending annotations to Grafana. If you do not set Bearer/Auth, Bearer is automatically set. See http://docs.grafana.org/http_api/auth/#authentication-api',
group: 'Grafana'
})
.option('grafana.annotationTitle', {
describe: 'Add a title to the annotation sent for a run.',
group: 'Grafana'
})
.option('grafana.annotationMessage', {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Grafana'
})
.option('grafana.annotationTag', {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --grafana.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Grafana'
})
.option('grafana.annotationScreenshot', {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Grafana'
})
.option('graphite.host', {
describe: 'The Graphite host used to store captured metrics.',
group: 'Graphite'
})
.option('graphite.port', {
default: 2003,
describe: 'The Graphite port used to store captured metrics.',
group: 'Graphite'
})
.option('graphite.auth', {
describe:
'The Graphite user and password used for authentication. Format: user:password',
group: 'Graphite'
})
.option('graphite.httpPort', {
describe:
'The Graphite port used to access the user interface and send annotations event',
default: 8080,
group: 'Graphite'
})
.option('graphite.webHost', {
describe:
'The graphite-web host. If not specified graphite.host will be used.',
group: 'Graphite'
})
.option('graphite.namespace', {
default: 'sitespeed_io.default',
describe: 'The namespace key added to all captured metrics.',
group: 'Graphite'
})
.option('graphite.includeQueryParams', {
default: false,
describe:
'Whether to include query parameters from the URL in the Graphite keys or not',
type: 'boolean',
group: 'Graphite'
})
.option('graphite.arrayTags', {
default: true,
type: 'boolean',
describe:
'Send the tags as Array or a String. In Graphite 1.0 the tags is a array. Before a String',
group: 'Graphite'
})
.option('graphite.annotationTitle', {
describe: 'Add a title to the annotation sent for a run.',
group: 'Graphite'
})
.option('graphite.annotationMessage', {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Graphite'
})
.option('graphite.annotationScreenshot', {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Graphite'
})
.option('graphite.sendAnnotation', {
default: true,
type: 'boolean',
describe:
'Send annotations when a run is finished. You need to specify a --resultBaseURL for this to work. However if you for example use a Prometheus exporter, you may want to make sure annotations are not sent, then set it to false.',
group: 'Graphite'
})
.option('graphite.annotationRetentionMinutes', {
type: 'number',
describe:
'The retention in minutes, to make annotation match the retention in Graphite.',
group: 'Graphite'
})
.option('graphite.statsd', {
default: false,
type: 'boolean',
describe: 'Uses the StatsD interface',
group: 'Graphite'
})
.option('graphite.annotationTag', {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --graphite.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Graphite'
})
.option('graphite.addSlugToKey', {
default: true,
type: 'boolean',
describe:
'Add the slug (name of the test) as an extra key in the namespace.',
group: 'Graphite'
})
.option('graphite.bulkSize', {
default: undefined,
type: 'number',
describe: 'Break up number of metrics to send with each request.',
group: 'Graphite'
})
.option('graphite.messages', {
default: ['pageSummary', 'summary'],
options: ['pageSummary', 'summary', 'run'],
group: 'Graphite'
})
.option('influxdb.protocol', {
describe: 'The protocol used to store connect to the InfluxDB host.',
default: 'http',
group: 'InfluxDB'
})
.option('influxdb.host', {
describe: 'The InfluxDB host used to store captured metrics.',
group: 'InfluxDB'
})
.option('influxdb.port', {
default: 8086,
describe: 'The InfluxDB port used to store captured metrics.',
group: 'InfluxDB'
})
.option('influxdb.username', {
describe: 'The InfluxDB username for your InfluxDB instance.',
group: 'InfluxDB'
})
.option('influxdb.password', {
describe: 'The InfluxDB password for your InfluxDB instance.',
group: 'InfluxDB'
})
.option('influxdb.database', {
default: 'sitespeed',
describe: 'The database name used to store captured metrics.',
group: 'InfluxDB'
})
.option('influxdb.tags', {
default: 'category=default',
describe:
'A comma separated list of tags and values added to each metric',
group: 'InfluxDB'
})
.option('influxdb.includeQueryParams', {
default: false,
describe:
'Whether to include query parameters from the URL in the InfluxDB keys or not',
type: 'boolean',
group: 'InfluxDB'
})
.option('influxdb.groupSeparator', {
default: '_',
describe:
'Choose which character that will separate a group/domain. Default is underscore, set it to a dot if you wanna keep the original domain name.',
group: 'InfluxDB'
})
.option('influxdb.annotationScreenshot', {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'InfluxDB'
}); });
// Grafana CLI options
cliUtil.registerPluginOptions(parsed, grafanaPlugin);
// Graphite CLI options
cliUtil.registerPluginOptions(parsed, graphitePlugin);
parsed parsed
/** Plugins */ /** Plugins */
.option('plugins.list', { .option('plugins.list', {
@ -1244,7 +1426,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
/** /**
InfluxDB cli option InfluxDB cli option
*/ */
cliUtil.registerPluginOptions(parsed, influxdbPlugin);
parsed parsed
// Metrics // Metrics
@ -1267,6 +1448,36 @@ module.exports.parseCommandLine = function parseCommandLine() {
'Add/change/remove filters for metrics. If you want to send all metrics, use: *+ . If you want to remove all current metrics and send only the coach score: *- coach.summary.score.*', 'Add/change/remove filters for metrics. If you want to send all metrics, use: *+ . If you want to remove all current metrics and send only the coach score: *- coach.summary.score.*',
group: 'Metrics' group: 'Metrics'
}) })
.option('matrix.host', {
describe: 'The Matrix host.',
group: 'Matrix'
})
.option('matrix.accessToken', {
describe: 'The Matrix access token.',
group: 'Matrix'
})
.option('matrix.room', {
describe:
'The default Matrix room. It is alsways used. You can override the room per message type using --matrix.rooms',
group: 'Matrix'
})
.option('matrix.messages', {
describe:
'Choose what type of message to send to Matrix. There are two types of messages: Error messages and budget messages. Errors are errors that happens through the tests (failures like strarting a test) and budget is test failing against your budget.',
choices: matrixMessageTypes(),
default: matrixMessageTypes(),
group: 'Matrix'
})
.option('matrix.rooms', {
describe:
'Send messages to different rooms. Current message types are [' +
matrixMessageTypes +
']. If you want to send error messages to a specific room use --matrix.rooms.error ROOM',
group: 'Matrix'
})
/** /**
Slack options Slack options
*/ */
@ -1411,6 +1622,34 @@ module.exports.parseCommandLine = function parseCommandLine() {
type: 'boolean', type: 'boolean',
group: 'GoogleCloudStorage' group: 'GoogleCloudStorage'
}) })
.option('crux.key', {
describe:
'You need to use a key to get data from CrUx. Get the key from https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/getting-started#APIKey',
group: 'CrUx'
})
.option('crux.enable', {
default: true,
describe:
'Enable the CrUx plugin. This is on by defauly but you also need the Crux key. If you chose to disable it with this key, set this to false and you can still use the CrUx key in your configuration.',
group: 'CrUx'
})
.option('crux.formFactor', {
default: 'ALL',
type: 'string',
choices: ['ALL', 'DESKTOP', 'PHONE', 'TABLET'],
describe:
'A form factor is the type of device on which a user visits a website.',
group: 'CrUx'
})
.option('crux.collect', {
default: 'ALL',
type: 'string',
choices: ['ALL', 'URL', 'ORIGIN'],
describe:
'Choose what data to collect. URL is data for a specific URL, ORIGIN for the domain and ALL for both of them',
group: 'CrUx'
})
/** /**
Html options Html options
*/ */
@ -1517,8 +1756,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
'Instead of using the local copy of the hosting database, you can use the latest version through the Green Web Foundation API. This means sitespeed.io will make HTTP GET to the the hosting info.', 'Instead of using the local copy of the hosting database, you can use the latest version through the Green Web Foundation API. This means sitespeed.io will make HTTP GET to the the hosting info.',
group: 'Sustainable' group: 'Sustainable'
}); });
cliUtil.registerPluginOptions(parsed, cruxPlugin);
cliUtil.registerPluginOptions(parsed, matrixPlugin);
parsed parsed
.option('mobile', { .option('mobile', {
describe: describe:
@ -1600,17 +1837,19 @@ module.exports.parseCommandLine = function parseCommandLine() {
.alias('help', 'h') .alias('help', 'h')
.config(config) .config(config)
.alias('version', 'V') .alias('version', 'V')
.coerce('budget', function (arg) { .coerce('budget', function (argument) {
if (arg) { if (argument) {
if (typeof arg === 'object' && !Array.isArray(arg)) { if (typeof argument === 'object' && !Array.isArray(argument)) {
if (arg.configPath) { if (argument.configPath) {
arg.config = JSON.parse(fs.readFileSync(arg.configPath, 'utf8')); argument.config = JSON.parse(
} else if (arg.config) { readFileSync(argument.configPath, 'utf8')
arg.config = JSON.parse(arg.config); );
} else if (argument.config) {
argument.config = JSON.parse(argument.config);
} }
return arg; return argument;
} else { } else {
throw new Error( throw new TypeError(
'[ERROR] Something looks wrong with your budget configuration. Since sitespeed.io 4.4 you should pass the path to your budget file through the --budget.configPath flag instead of directly through the --budget flag.' '[ERROR] Something looks wrong with your budget configuration. Since sitespeed.io 4.4 you should pass the path to your budget file through the --budget.configPath flag instead of directly through the --budget flag.'
); );
} }
@ -1646,30 +1885,30 @@ module.exports.parseCommandLine = function parseCommandLine() {
return plugins; return plugins;
} }
}) })
.coerce('webpagetest', function (arg) { .coerce('webpagetest', function (argument) {
if (arg) { if (argument) {
// for backwards compatible reasons we check if the passed parameters is a path to a script, if so just us it (PR #1445) // for backwards compatible reasons we check if the passed parameters is a path to a script, if so just us it (PR #1445)
if (arg.script && fs.existsSync(arg.script)) { if (argument.script && existsSync(argument.script)) {
arg.script = fs.readFileSync(path.resolve(arg.script), 'utf8'); argument.script = readFileSync(resolve(argument.script), 'utf8');
/* eslint no-console: off */ /* eslint no-console: off */
console.log( console.log(
'[WARNING] Since sitespeed.io 4.4 you should pass the path to the script file through the --webpagetest.file flag (https://github.com/sitespeedio/sitespeed.io/pull/1445).' '[WARNING] Since sitespeed.io 4.4 you should pass the path to the script file through the --webpagetest.file flag (https://github.com/sitespeedio/sitespeed.io/pull/1445).'
); );
return arg; return argument;
} }
if (arg.file) { if (argument.file) {
arg.script = fs.readFileSync(path.resolve(arg.file), 'utf8'); argument.script = readFileSync(resolve(argument.file), 'utf8');
} else if (arg.script) { } else if (argument.script) {
// because the escaped characters are passed re-escaped from the console // because the escaped characters are passed re-escaped from the console
arg.script = arg.script.split('\\t').join('\t'); argument.script = argument.script.split('\\t').join('\t');
arg.script = arg.script.split('\\n').join('\n'); argument.script = argument.script.split('\\n').join('\n');
} }
return arg; return argument;
} }
}) })
// .describe('browser', 'Specify browser') // .describe('browser', 'Specify browser')
.wrap(yargs.terminalWidth()) .wrap(yargsInstance.terminalWidth())
// .check(validateInput) // .check(validateInput)
.epilog( .epilog(
'Read the docs at https://www.sitespeed.io/documentation/sitespeed.io/' 'Read the docs at https://www.sitespeed.io/documentation/sitespeed.io/'
@ -1693,8 +1932,12 @@ module.exports.parseCommandLine = function parseCommandLine() {
new Map() new Map()
); );
let explicitOptions = require('yargs/yargs')(process.argv.slice(2)).argv; let explicitOptions = yargs(hideBin(process.argv)).argv;
explicitOptions = merge(explicitOptions, yargs.getOptions().configObjects[0]);
explicitOptions = merge(
explicitOptions,
yargsInstance.getOptions().configObjects[0]
);
explicitOptions = reduce( explicitOptions = reduce(
explicitOptions, explicitOptions,
@ -1710,17 +1953,14 @@ module.exports.parseCommandLine = function parseCommandLine() {
); );
if (argv.config) { if (argv.config) {
const config = require(path.resolve(process.cwd(), argv.config)); const config = await import(resolve(process.cwd(), argv.config));
explicitOptions = merge(explicitOptions, config); explicitOptions = merge(explicitOptions, config);
} }
if (argv.webpagetest && argv.webpagetest.custom) { if (argv.webpagetest && argv.webpagetest.custom) {
argv.webpagetest.custom = fs.readFileSync( argv.webpagetest.custom = readFileSync(resolve(argv.webpagetest.custom), {
path.resolve(argv.webpagetest.custom),
{
encoding: 'utf8' encoding: 'utf8'
} });
);
} }
if (argv.summaryDetail) argv.summary = true; if (argv.summaryDetail) argv.summary = true;
@ -1750,8 +1990,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (argv.ios) { if (argv.ios) {
set(argv, 'safari.ios', true); set(argv, 'safari.ios', true);
} else if (argv.android) { } else if (argv.android && argv.browser === 'chrome') {
if (argv.browser === 'chrome') {
// Default to Chrome Android. // Default to Chrome Android.
set( set(
argv, argv,
@ -1759,7 +1998,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
get(argv, 'browsertime.chrome.android.package', 'com.android.chrome') get(argv, 'browsertime.chrome.android.package', 'com.android.chrome')
); );
} }
}
// Always use hash by default when you configure spa // Always use hash by default when you configure spa
if (argv.spa) { if (argv.spa) {
@ -1767,10 +2005,10 @@ module.exports.parseCommandLine = function parseCommandLine() {
set(argv, 'browsertime.useHash', true); set(argv, 'browsertime.useHash', true);
} }
if (argv.cpu) {
if ( if (
argv.browsertime.browser === 'chrome' || argv.cpu &&
argv.browsertime.browser === 'edge' (argv.browsertime.browser === 'chrome' ||
argv.browsertime.browser === 'edge')
) { ) {
set(argv, 'browsertime.chrome.collectLongTasks', true); set(argv, 'browsertime.chrome.collectLongTasks', true);
set(argv, 'browsertime.chrome.timeline', true); set(argv, 'browsertime.chrome.timeline', true);
@ -1780,7 +2018,6 @@ module.exports.parseCommandLine = function parseCommandLine() {
else if (argv.browsertime.browser === 'firefox') { else if (argv.browsertime.browser === 'firefox') {
set(argv, 'browsertime.firefox.geckoProfiler', true); set(argv, 'browsertime.firefox.geckoProfiler', true);
}*/ }*/
}
// we missed to populate this to Browsertime in the cli // we missed to populate this to Browsertime in the cli
// so to stay backward compatible we do it manually // so to stay backward compatible we do it manually
@ -1800,7 +2037,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (argv.browsertime.safari && argv.browsertime.safari.useSimulator) { if (argv.browsertime.safari && argv.browsertime.safari.useSimulator) {
set(argv, 'browsertime.connectivity.engine', 'throttle'); set(argv, 'browsertime.connectivity.engine', 'throttle');
} else if ( } else if (
(os.platform() === 'darwin' || os.platform() === 'linux') && (platform() === 'darwin' || platform() === 'linux') &&
!argv.browsertime.android && !argv.browsertime.android &&
!argv.browsertime.safari.ios && !argv.browsertime.safari.ios &&
!argv.browsertime.docker && !argv.browsertime.docker &&
@ -1823,7 +2060,7 @@ module.exports.parseCommandLine = function parseCommandLine() {
); );
} }
let urlsMetaData = cliUtil.getAliases(argv._, argv.urlAlias, argv.groupAlias); let urlsMetaData = getAliases(argv._, argv.urlAlias, argv.groupAlias);
// Copy the alias so it is also used by Browsertime // Copy the alias so it is also used by Browsertime
if (argv.urlAlias) { if (argv.urlAlias) {
// Browsertime has it own way of handling alias // Browsertime has it own way of handling alias
@ -1833,8 +2070,8 @@ module.exports.parseCommandLine = function parseCommandLine() {
if (!Array.isArray(argv.urlAlias)) argv.urlAlias = [argv.urlAlias]; if (!Array.isArray(argv.urlAlias)) argv.urlAlias = [argv.urlAlias];
for (let i = 0; i < urls.length; i++) { for (const [index, url] of urls.entries()) {
meta[urls[i]] = argv.urlAlias[i]; meta[url] = argv.urlAlias[index];
} }
set(argv, 'browsertime.urlMetaData', meta); set(argv, 'browsertime.urlMetaData', meta);
} else if (Object.keys(urlsMetaData).length > 0) { } else if (Object.keys(urlsMetaData).length > 0) {
@ -1847,15 +2084,15 @@ module.exports.parseCommandLine = function parseCommandLine() {
// Set the timeouts to a maximum while debugging // Set the timeouts to a maximum while debugging
if (argv.debug) { if (argv.debug) {
set(argv, 'browsertime.timeouts.pageload', 2147483647); set(argv, 'browsertime.timeouts.pageload', 2_147_483_647);
set(argv, 'browsertime.timeouts.script', 2147483647); set(argv, 'browsertime.timeouts.script', 2_147_483_647);
set(argv, 'browsertime.timeouts.pageCompleteCheck', 2147483647); set(argv, 'browsertime.timeouts.pageCompleteCheck', 2_147_483_647);
} }
return { return {
urls: argv.multi ? argv._ : cliUtil.getURLs(argv._), urls: argv.multi ? argv._ : getURLs(argv._),
urlsMetaData, urlsMetaData,
options: argv, options: argv,
explicitOptions: explicitOptions explicitOptions: explicitOptions
}; };
}; }

View File

@ -1,11 +1,10 @@
'use strict';
/*eslint no-console: 0*/ /*eslint no-console: 0*/
const fs = require('fs'); import { readFileSync } from 'node:fs';
const path = require('path'); import { resolve } from 'node:path';
const toArray = require('../support/util').toArray; import { format } from 'node:util';
const format = require('util').format;
import { toArray } from '../support/util.js';
/** /**
* *
@ -18,7 +17,7 @@ function sanitizePluginOptions(options) {
const isValidType = const isValidType =
typeof cliOptions === 'object' && typeof cliOptions === 'object' &&
cliOptions !== null && cliOptions !== undefined &&
cliOptions.constructor === Object; cliOptions.constructor === Object;
if (!isValidType) { if (!isValidType) {
@ -29,8 +28,7 @@ function sanitizePluginOptions(options) {
return cliOptions; return cliOptions;
} }
module.exports = { export function getURLs(urls) {
getURLs(urls) {
const allUrls = []; const allUrls = [];
urls = urls.map(url => url.trim()); urls = urls.map(url => url.trim());
@ -38,9 +36,9 @@ module.exports = {
if (url.startsWith('http')) { if (url.startsWith('http')) {
allUrls.push(url); allUrls.push(url);
} else { } else {
const filePath = path.resolve(url); const filePath = resolve(url);
try { try {
const lines = fs.readFileSync(filePath).toString().split('\n'); const lines = readFileSync(filePath).toString().split('\n');
for (let line of lines) { for (let line of lines) {
if (line.trim().length > 0) { if (line.trim().length > 0) {
let lineArray = line.split(' ', 2); let lineArray = line.split(' ', 2);
@ -55,22 +53,22 @@ module.exports = {
'Please use --multi if you want to run scripts. See https://www.sitespeed.io/documentation/sitespeed.io/scripting/#run' 'Please use --multi if you want to run scripts. See https://www.sitespeed.io/documentation/sitespeed.io/scripting/#run'
); );
} else { } else {
// We use skip adding it // do nada
} }
} }
} }
} }
} catch (e) { } catch (error) {
if (e.code === 'ENOENT') { if (error.code === 'ENOENT') {
throw new Error(`Couldn't find url file at ${filePath}`); throw new Error(`Couldn't find url file at ${filePath}`);
} }
throw e; throw error;
} }
} }
} }
return allUrls; return allUrls;
}, }
getAliases(urls, alias, groupAlias) { export function getAliases(urls, alias, groupAlias) {
const urlMetaData = {}; const urlMetaData = {};
urls = urls.map(url => url.trim()); urls = urls.map(url => url.trim());
let al = toArray(alias); let al = toArray(alias);
@ -88,12 +86,10 @@ module.exports = {
pos += 1; pos += 1;
} else { } else {
const filePath = url; const filePath = url;
const lines = fs.readFileSync(filePath).toString().split('\n'); const lines = readFileSync(filePath).toString().split('\n');
for (let line of lines) { for (let line of lines) {
if (line.trim().length > 0) { if (line.trim().length > 0) {
let url, let url, alias, groupAlias;
alias,
groupAlias = null;
let lineArray = line.split(' ', 3); let lineArray = line.split(' ', 3);
url = lineArray[0].trim(); url = lineArray[0].trim();
if (lineArray[1]) { if (lineArray[1]) {
@ -113,46 +109,31 @@ module.exports = {
} }
} }
return urlMetaData; return urlMetaData;
}, }
export function pluginDefaults(cliOptions) {
/**
* Retrieve a mapping of option names and their corressponding default values based on a plugin CLI configuration
*
* @param {Object<string, require('yargs').Options>} cliOptions a map of option names: yargs option
*/
pluginDefaults(cliOptions) {
let config = {}; let config = {};
try { try {
Object.entries(sanitizePluginOptions(cliOptions)).forEach(values => { for (const values of Object.entries(sanitizePluginOptions(cliOptions))) {
const [key, options] = values; const [key, options] = values;
if (typeof options.default !== 'undefined') { if (typeof options.default !== 'undefined') {
config[key] = options.default; config[key] = options.default;
} }
}); }
} catch (err) { } catch {
// In case of invalid values, just assume an empty object. // In case of invalid values, just assume an empty object.
config = {}; config = {};
} }
return config; return config;
}, }
export function registerPluginOptions(parsed, plugin) {
/** if (typeof plugin.name !== 'function' || !plugin.getName()) {
* Configure yargs options for a given plugin defining a `cliOptions` method or property. The names
* of configuration options are namespaced with the plugin name.
*
* @param {require('yargs').Argv} parsed yargs instance
* @param {object} plugin a sitespeed plugin instance
*/
registerPluginOptions(parsed, plugin) {
if (typeof plugin.name !== 'function' || !plugin.name()) {
throw new Error( throw new Error(
'Missing name() method for plugin registering CLI options' 'Missing getName() method for plugin registering CLI options'
); );
} }
const cliOptions = sanitizePluginOptions(plugin.cliOptions); const cliOptions = sanitizePluginOptions(plugin.cliOptions);
Object.entries(cliOptions).forEach(value => { for (const value of Object.entries(cliOptions)) {
const [key, yargsOptions] = value; const [key, yargsOptions] = value;
parsed.option(`${plugin.name()}.${key}`, yargsOptions); parsed.option(`${plugin.getName()}.${key}`, yargsOptions);
}); }
} }
};

View File

@ -1,51 +1,65 @@
'use strict'; import intel from 'intel';
let log = require('intel'); const {
INFO,
DEBUG,
VERBOSE,
TRACE,
NONE,
basicConfig,
addHandler,
handlers,
Formatter
} = intel;
module.exports.configure = function configure(options, logDir) { export function configure(options, logDir) {
options = options || {}; options = options || {};
let level = log.INFO; let level = INFO;
switch (options.verbose) { switch (options.verbose) {
case 1: case 1: {
level = log.DEBUG; level = DEBUG;
break; break;
case 2: }
level = log.VERBOSE; case 2: {
level = VERBOSE;
break; break;
case 3: }
level = log.TRACE; case 3: {
level = TRACE;
break; break;
default: }
default: {
break; break;
} }
}
if (options.silent) { if (options.silent) {
level = log.NONE; level = NONE;
} }
if (level === log.INFO) { if (level === INFO) {
log.basicConfig({ basicConfig({
format: '[%(date)s] %(levelname)s: %(message)s', format: '[%(date)s] %(levelname)s: %(message)s',
level: level level: level
}); });
} else { } else {
log.basicConfig({ basicConfig({
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s', format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
level: level level: level
}); });
} }
if (options.logToFile) { if (options.logToFile) {
log.addHandler( addHandler(
new log.handlers.File({ new handlers.File({
file: logDir + '/sitespeed.io.log', file: logDir + '/sitespeed.io.log',
formatter: new log.Formatter({ formatter: new Formatter({
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s', format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
level: level level: level
}) })
}) })
); );
} }
}; }

View File

@ -1,9 +1,11 @@
'use strict'; import { join, basename, resolve, dirname } from 'node:path';
import { readdir as _readdir } from 'node:fs';
import { promisify } from 'node:util';
const path = require('path'); import pkg from 'import-global';
const fs = require('fs'); const { silent } = pkg;
const { promisify } = require('util'); const readdir = promisify(_readdir);
const readdir = promisify(fs.readdir); const __dirname = dirname(import.meta.url);
const defaultPlugins = new Set([ const defaultPlugins = new Set([
'browsertime', 'browsertime',
@ -22,10 +24,9 @@ const defaultPlugins = new Set([
'remove' 'remove'
]); ]);
const pluginsDir = path.join(__dirname, '..', 'plugins'); const pluginsDir = join(__dirname, '..', 'plugins');
module.exports = { export async function parsePluginNames(options) {
async parsePluginNames(options) {
// There's a problem with Safari on iOS runninhg a big blob // There's a problem with Safari on iOS runninhg a big blob
// of JavaScript // of JavaScript
// https://github.com/sitespeedio/browsertime/issues/1275 // https://github.com/sitespeedio/browsertime/issues/1275
@ -39,6 +40,7 @@ module.exports = {
const isDefaultOrConfigured = name => const isDefaultOrConfigured = name =>
defaultPlugins.has(name) || defaultPlugins.has(name) ||
typeof possibleConfiguredPlugins[name] === 'object'; typeof possibleConfiguredPlugins[name] === 'object';
const addMessageLoggerIfDebug = pluginNames => { const addMessageLoggerIfDebug = pluginNames => {
if (options.debugMessages) { if (options.debugMessages) {
// Need to make sure logger is first, so message logs appear // Need to make sure logger is first, so message logs appear
@ -48,34 +50,46 @@ module.exports = {
return pluginNames; return pluginNames;
}; };
const files = await readdir(pluginsDir); const files = await readdir(new URL(pluginsDir));
const builtins = files.map(name => path.basename(name, '.js'));
const builtins = files.map(name => basename(name, '.js'));
// eslint-disable-next-line unicorn/no-array-callback-reference
const plugins = builtins.filter(isDefaultOrConfigured); const plugins = builtins.filter(isDefaultOrConfigured);
return addMessageLoggerIfDebug(plugins); return addMessageLoggerIfDebug(plugins);
}, }
async loadPlugins(pluginNames) { export async function loadPlugins(pluginNames, options, context, queue) {
const plugins = []; const plugins = [];
for (let name of pluginNames) { for (let name of pluginNames) {
try { try {
const plugin = require(path.join(pluginsDir, name)); let { default: plugin } = await import(
if (!plugin.name) { join(pluginsDir, name, 'index.js')
plugin.name = () => name; );
} let p = new plugin(options, context, queue);
plugins.push(plugin); plugins.push(p);
} catch (err) { } catch (error_) {
try { try {
plugins.push(require(path.resolve(process.cwd(), name))); let { default: plugin } = await import(resolve(process.cwd(), name));
} catch (error) { let p = new plugin(options, context, queue);
plugins.push(p);
} catch {
try { try {
plugins.push(require(name)); let { default: plugin } = await import(name);
let p = new plugin(options, context, queue);
plugins.push(p);
} catch (error) { } catch (error) {
console.error("Couldn't load plugin %s: %s", name, err); // eslint-disable-line no-console // try global
let plugin = silent(name);
if (plugin) {
let p = new plugin(options, context, queue);
plugins.push(p);
} else {
console.error("Couldn't load plugin %s: %s", name, error_); // eslint-disable-line no-console
// if it fails here, let it fail hard // if it fails here, let it fail hard
throw error; throw error;
} }
} }
} }
} }
}
return plugins; return plugins;
} }
};

View File

@ -1,13 +1,17 @@
'use strict';
/* eslint no-console:0 */ /* eslint no-console:0 */
const cq = require('concurrent-queue'); import cq from 'concurrent-queue';
const log = require('intel').getLogger('sitespeedio.queuehandler'); import intel from 'intel';
const messageMaker = require('../support/messageMaker');
const queueStats = require('./queueStatistics'); import { messageMaker } from '../support/messageMaker.js';
import {
registerQueueTime,
registerProcessingTime,
generateStatistics
} from './queueStatistics.js';
const make = messageMaker('queueHandler').make; const make = messageMaker('queueHandler').make;
const log = intel.getLogger('sitespeedio.queuehandler');
function shortenData(key, value) { function shortenData(key, value) {
if (key === 'data') { if (key === 'data') {
@ -16,15 +20,17 @@ function shortenData(key, value) {
return value; return value;
} }
const messageTypeDepths = {}; function validatePageSummary(message) {
const groupsPerSummaryType = {}; const type = message.type;
if (!type.endsWith('.pageSummary')) return;
if (!message.url)
throw new Error(`Page summary message (${type}) didn't specify a url`);
if (!message.group)
throw new Error(`Page summary message (${type}) didn't specify a group.`);
}
/**
* Check some message format best practices that applies to sitespeed.io.
* Throws an error if message doesn't follow the rules.
* @param message the message to check
*/
function validateMessageFormat(message) {
function validateTypeStructure(message) { function validateTypeStructure(message) {
const typeParts = message.type.split('.'), const typeParts = message.type.split('.'),
baseType = typeParts[0], baseType = typeParts[0],
@ -47,17 +53,6 @@ function validateMessageFormat(message) {
messageTypeDepths[baseType] = typeDepth; messageTypeDepths[baseType] = typeDepth;
} }
function validatePageSummary(message) {
const type = message.type;
if (!type.endsWith('.pageSummary')) return;
if (!message.url)
throw new Error(`Page summary message (${type}) didn't specify a url`);
if (!message.group)
throw new Error(`Page summary message (${type}) didn't specify a group.`);
}
function validateSummaryMessage(message) { function validateSummaryMessage(message) {
const type = message.type; const type = message.type;
if (!type.endsWith('.summary')) return; if (!type.endsWith('.summary')) return;
@ -79,16 +74,27 @@ function validateMessageFormat(message) {
groupsPerSummaryType[type] = groups.concat(message.group); groupsPerSummaryType[type] = groups.concat(message.group);
} }
const messageTypeDepths = {};
const groupsPerSummaryType = {};
/**
* Check some message format best practices that applies to sitespeed.io.
* Throws an error if message doesn't follow the rules.
* @param message the message to check
*/
function validateMessageFormat(message) {
validateTypeStructure(message); validateTypeStructure(message);
validatePageSummary(message); validatePageSummary(message);
validateSummaryMessage(message); validateSummaryMessage(message);
} }
class QueueHandler { export class QueueHandler {
constructor(plugins, options) { constructor(options) {
this.options = options; this.options = options;
this.errors = []; this.errors = [];
}
setup(plugins) {
this.createQueues(plugins); this.createQueues(plugins);
} }
@ -96,7 +102,7 @@ class QueueHandler {
this.queues = plugins this.queues = plugins
.filter(plugin => plugin.processMessage) .filter(plugin => plugin.processMessage)
.map(plugin => { .map(plugin => {
const concurrency = plugin.concurrency || Infinity; const concurrency = plugin.concurrency || Number.POSITIVE_INFINITY;
const queue = cq().limit({ concurrency }); const queue = cq().limit({ concurrency });
queue.plugin = plugin; queue.plugin = plugin;
@ -104,42 +110,42 @@ class QueueHandler {
const messageWaitingStart = {}, const messageWaitingStart = {},
messageProcessingStart = {}; messageProcessingStart = {};
queue.enqueued(obj => { queue.enqueued(object => {
const message = obj.item; const message = object.item;
messageWaitingStart[message.uuid] = process.hrtime(); messageWaitingStart[message.uuid] = process.hrtime();
}); });
queue.processingStarted(obj => { queue.processingStarted(object => {
const message = obj.item; const message = object.item;
const waitingDuration = process.hrtime( const waitingDuration = process.hrtime(
messageWaitingStart[message.uuid] messageWaitingStart[message.uuid]
), ),
waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1]; waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1];
queueStats.registerQueueTime(message, queue.plugin, waitingNanos); registerQueueTime(message, queue.plugin, waitingNanos);
messageProcessingStart[message.uuid] = process.hrtime(); messageProcessingStart[message.uuid] = process.hrtime();
}); });
// FIXME handle rejections (i.e. failures while processing messages) properly // FIXME handle rejections (i.e. failures while processing messages) properly
queue.processingEnded(obj => { queue.processingEnded(object => {
const message = obj.item; const message = object.item;
const err = obj.err; const error = object.err;
if (err) { if (error) {
let rejectionMessage = let rejectionMessage =
'Rejected ' + 'Rejected ' +
JSON.stringify(message, shortenData, 2) + JSON.stringify(message, shortenData, 2) +
' for plugin: ' + ' for plugin: ' +
plugin.name(); plugin.getName();
if (message && message.url) if (message && message.url)
rejectionMessage += ', url: ' + message.url; rejectionMessage += ', url: ' + message.url;
if (err.stack) { if (error.stack) {
log.error(err.stack); log.error(error.stack);
} }
this.errors.push(rejectionMessage + '\n' + JSON.stringify(err)); this.errors.push(rejectionMessage + '\n' + JSON.stringify(error));
} }
const processingDuration = process.hrtime( const processingDuration = process.hrtime(
@ -148,11 +154,7 @@ class QueueHandler {
const processingNanos = const processingNanos =
processingDuration[0] * 1e9 + processingDuration[1]; processingDuration[0] * 1e9 + processingDuration[1];
queueStats.registerProcessingTime( registerProcessingTime(message, queue.plugin, processingNanos);
message,
queue.plugin,
processingNanos
);
}); });
return { plugin, queue }; return { plugin, queue };
@ -172,7 +174,7 @@ class QueueHandler {
.then(() => this.drainAllQueues()) .then(() => this.drainAllQueues())
.then(async () => { .then(async () => {
for (let source of sources) { for (let source of sources) {
await source.findUrls(this); await source.findUrls(this, this.options);
} }
}) })
.then(() => this.drainAllQueues()) .then(() => this.drainAllQueues())
@ -184,7 +186,7 @@ class QueueHandler {
.then(() => this.drainAllQueues()) .then(() => this.drainAllQueues())
.then(() => { .then(() => {
if (this.options.queueStats) { if (this.options.queueStats) {
log.info(JSON.stringify(queueStats.generateStatistics(), null, 2)); log.info(JSON.stringify(generateStatistics(), undefined, 2));
} }
return this.errors; return this.errors;
}); });
@ -218,15 +220,12 @@ class QueueHandler {
async drainAllQueues() { async drainAllQueues() {
const queues = this.queues; const queues = this.queues;
return new Promise(resolve => { return new Promise(resolve => {
queues.forEach(item => for (const item of queues)
item.queue.drained(() => { item.queue.drained(() => {
if (queues.every(item => item.queue.isDrained)) { if (queues.every(item => item.queue.isDrained)) {
resolve(); resolve();
} }
}) });
);
}); });
} }
} }
module.exports = QueueHandler;

View File

@ -1,8 +1,7 @@
'use strict'; import get from 'lodash.get';
import set from 'lodash.set';
const stats = require('../support/statsHelpers'), import { pushStats, summarizeStats } from '../support/statsHelpers.js';
get = require('lodash.get'),
set = require('lodash.set');
const queueTimeByPluginName = {}, const queueTimeByPluginName = {},
queueTimeByMessageType = {}, queueTimeByMessageType = {},
@ -11,80 +10,58 @@ const queueTimeByPluginName = {},
messageTypes = new Set(), messageTypes = new Set(),
pluginNames = new Set(); pluginNames = new Set();
module.exports = { export function registerQueueTime(message, plugin, nanos) {
registerQueueTime(message, plugin, nanos) {
messageTypes.add(message.type); messageTypes.add(message.type);
pluginNames.add(plugin.name()); pluginNames.add(plugin.getName());
stats.pushStats(queueTimeByMessageType, message.type, nanos / 1000000); pushStats(queueTimeByMessageType, message.type, nanos / 1_000_000);
stats.pushStats(queueTimeByPluginName, plugin.name(), nanos / 1000000); pushStats(queueTimeByPluginName, plugin.getName(), nanos / 1_000_000);
}, }
export function registerProcessingTime(message, plugin, nanos) {
registerProcessingTime(message, plugin, nanos) {
messageTypes.add(message.type); messageTypes.add(message.type);
pluginNames.add(plugin.name()); pluginNames.add(plugin.getName());
stats.pushStats(processingTimeByMessageType, message.type, nanos / 1000000); pushStats(processingTimeByMessageType, message.type, nanos / 1_000_000);
stats.pushStats(processingTimeByPluginName, plugin.name(), nanos / 1000000); pushStats(processingTimeByPluginName, plugin.getName(), nanos / 1_000_000);
}, }
export function generateStatistics() {
generateStatistics() {
const statOptions = { const statOptions = {
percentiles: [0, 100], percentiles: [0, 100],
includeSum: true includeSum: true
}; };
const byPluginName = Array.from(pluginNames).reduce( const byPluginName = [...pluginNames].reduce((summary, pluginName) => {
(summary, pluginName) => {
set( set(
summary, summary,
['queueTime', pluginName], ['queueTime', pluginName],
stats.summarizeStats( summarizeStats(get(queueTimeByPluginName, pluginName), statOptions)
get(queueTimeByPluginName, pluginName),
statOptions
)
); );
set( set(
summary, summary,
['processingTime', pluginName], ['processingTime', pluginName],
stats.summarizeStats( summarizeStats(get(processingTimeByPluginName, pluginName), statOptions)
get(processingTimeByPluginName, pluginName),
statOptions
)
); );
return summary; return summary;
}, }, {});
{}
);
const byMessageType = Array.from(messageTypes).reduce( const byMessageType = [...messageTypes].reduce((summary, messageType) => {
(summary, messageType) => {
set( set(
summary, summary,
['queueTime', messageType], ['queueTime', messageType],
stats.summarizeStats( summarizeStats(get(queueTimeByMessageType, messageType), statOptions)
get(queueTimeByMessageType, messageType),
statOptions
)
); );
set( set(
summary, summary,
['processingTime', messageType], ['processingTime', messageType],
stats.summarizeStats( summarizeStats(get(processingTimeByMessageType, messageType), statOptions)
get(processingTimeByMessageType, messageType),
statOptions
)
); );
return summary; return summary;
}, }, {});
{}
);
return { return {
byPluginName, byPluginName,
byMessageType byMessageType
}; };
} }
};

View File

@ -1,54 +1,50 @@
'use strict'; import { parse, format } from 'node:url';
import { basename, resolve, join } from 'node:path';
const urlParser = require('url'); import { resultUrls } from './resultUrls.js';
const path = require('path'); import { storageManager } from './storageManager.js';
const resultUrls = require('./resultUrls');
const storageManager = require('./storageManager');
function getDomainOrFileName(input) { function getDomainOrFileName(input) {
let domainOrFile = input; let domainOrFile = input;
if (domainOrFile.startsWith('http')) { domainOrFile = domainOrFile.startsWith('http')
domainOrFile = urlParser.parse(domainOrFile).hostname; ? parse(domainOrFile).hostname
} else { : basename(domainOrFile).replace(/\./g, '_');
domainOrFile = path.basename(domainOrFile).replace(/\./g, '_');
}
return domainOrFile; return domainOrFile;
} }
module.exports = function (input, timestamp, options) { export function resultsStorage(input, timestamp, options) {
const outputFolder = options.outputFolder; const outputFolder = options.outputFolder;
const resultBaseURL = options.resultBaseURL; const resultBaseURL = options.resultBaseURL;
const resultsSubFolders = []; const resultsSubFolders = [];
let storageBasePath; let storageBasePath;
let storagePathPrefix; let storagePathPrefix;
let resultUrl = undefined; let resultUrl;
if (outputFolder) { if (outputFolder) {
resultsSubFolders.push(path.basename(outputFolder)); resultsSubFolders.push(basename(outputFolder));
storageBasePath = path.resolve(outputFolder); storageBasePath = resolve(outputFolder);
} else { } else {
resultsSubFolders.push( resultsSubFolders.push(
options.slug || getDomainOrFileName(input), options.slug || getDomainOrFileName(input),
timestamp.format('YYYY-MM-DD-HH-mm-ss') timestamp.format('YYYY-MM-DD-HH-mm-ss')
); );
storageBasePath = resolve('sitespeed-result', ...resultsSubFolders);
storageBasePath = path.resolve('sitespeed-result', ...resultsSubFolders);
} }
// backfill the slug // backfill the slug
options.slug = options.slug || getDomainOrFileName(input).replace(/\./g, '_'); options.slug = options.slug || getDomainOrFileName(input).replace(/\./g, '_');
storagePathPrefix = path.join(...resultsSubFolders); storagePathPrefix = join(...resultsSubFolders);
if (resultBaseURL) { if (resultBaseURL) {
const url = urlParser.parse(resultBaseURL); const url = parse(resultBaseURL);
resultsSubFolders.unshift(url.pathname.substr(1)); resultsSubFolders.unshift(url.pathname.slice(1));
url.pathname = resultsSubFolders.join('/'); url.pathname = resultsSubFolders.join('/');
resultUrl = urlParser.format(url); resultUrl = format(url);
} }
return { return {
storageManager: storageManager(storageBasePath, storagePathPrefix, options), storageManager: storageManager(storageBasePath, storagePathPrefix, options),
resultUrls: resultUrls(resultUrl, options) resultUrls: resultUrls(resultUrl, options)
}; };
}; }

View File

@ -1,23 +1,23 @@
'use strict'; import { parse } from 'node:url';
import { createHash } from 'node:crypto';
const isEmpty = require('lodash.isempty'); import isEmpty from 'lodash.isempty';
const crypto = require('crypto'); import intel from 'intel';
const log = require('intel').getLogger('sitespeedio.file');
const urlParser = require('url'); const log = intel.getLogger('sitespeedio.file');
function toSafeKey(key) { function toSafeKey(key) {
// U+2013 : EN DASH as used on https://en.wikipedia.org/wiki/201920_coronavirus_pandemic // U+2013 : EN DASH as used on https://en.wikipedia.org/wiki/201920_coronavirus_pandemic
return key.replace(/[.~ /+|,:?&%)(]|%7C/g, '-'); return key.replace(/[ %&()+,./:?|~]|%7C/g, '-');
} }
module.exports = function pathFromRootToPageDir(url, options, alias) { export function pathToFolder(url, options, alias) {
const useHash = options.useHash; const useHash = options.useHash;
const parsedUrl = urlParser.parse(decodeURIComponent(url)); const parsedUrl = parse(decodeURIComponent(url));
const pathSegments = []; const pathSegments = [];
const urlSegments = []; const urlSegments = [];
pathSegments.push('pages'); pathSegments.push('pages', parsedUrl.hostname.split('.').join('_'));
pathSegments.push(parsedUrl.hostname.split('.').join('_'));
if (options.urlMetaData && options.urlMetaData[url]) { if (options.urlMetaData && options.urlMetaData[url]) {
pathSegments.push(options.urlMetaData[url]); pathSegments.push(options.urlMetaData[url]);
@ -29,14 +29,14 @@ module.exports = function pathFromRootToPageDir(url, options, alias) {
} }
if (useHash && !isEmpty(parsedUrl.hash)) { if (useHash && !isEmpty(parsedUrl.hash)) {
const md5 = crypto.createHash('md5'), const md5 = createHash('md5'),
hash = md5.update(parsedUrl.hash).digest('hex').substring(0, 8); hash = md5.update(parsedUrl.hash).digest('hex').slice(0, 8);
urlSegments.push('hash-' + hash); urlSegments.push('hash-' + hash);
} }
if (!isEmpty(parsedUrl.search)) { if (!isEmpty(parsedUrl.search)) {
const md5 = crypto.createHash('md5'), const md5 = createHash('md5'),
hash = md5.update(parsedUrl.search).digest('hex').substring(0, 8); hash = md5.update(parsedUrl.search).digest('hex').slice(0, 8);
urlSegments.push('query-' + hash); urlSegments.push('query-' + hash);
} }
@ -49,7 +49,7 @@ module.exports = function pathFromRootToPageDir(url, options, alias) {
log.info( log.info(
`The URL ${url} hit the 255 character limit used when stored on disk, you may want to give your URL an alias to make sure it will not collide with other URLs.` `The URL ${url} hit the 255 character limit used when stored on disk, you may want to give your URL an alias to make sure it will not collide with other URLs.`
); );
pathSegments.push(folder.substr(0, 254)); pathSegments.push(folder.slice(0, 254));
} else { } else {
pathSegments.push(folder); pathSegments.push(folder);
} }
@ -58,11 +58,11 @@ module.exports = function pathFromRootToPageDir(url, options, alias) {
// pathSegments.push('data'); // pathSegments.push('data');
pathSegments.forEach(function (segment, index) { for (const [index, segment] of pathSegments.entries()) {
if (segment) { if (segment) {
pathSegments[index] = segment.replace(/[^-a-z0-9_.\u0621-\u064A]/gi, '-'); pathSegments[index] = segment.replace(/[^\w.\u0621-\u064A-]/gi, '-');
}
} }
});
return pathSegments.join('/').concat('/'); return pathSegments.join('/').concat('/');
}; }

View File

@ -1,17 +1,16 @@
'use strict'; import { parse, format } from 'node:url';
const urlParser = require('url'); import { pathToFolder } from './pathToFolder.js';
const pathToFolder = require('./pathToFolder');
function getPageUrl({ url, resultBaseUrl, options, alias }) { function getPageUrl({ url, resultBaseUrl, options, alias }) {
const pageUrl = urlParser.parse(resultBaseUrl); const pageUrl = parse(resultBaseUrl);
pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, options, alias)].join( pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, options, alias)].join(
'/' '/'
); );
return urlParser.format(pageUrl); return format(pageUrl);
} }
module.exports = function resultUrls(resultBaseUrl, options) { export function resultUrls(resultBaseUrl, options) {
return { return {
hasBaseUrl() { hasBaseUrl() {
return !!resultBaseUrl; return !!resultBaseUrl;
@ -30,4 +29,4 @@ module.exports = function resultUrls(resultBaseUrl, options) {
return pathToFolder(url, options, alias); return pathToFolder(url, options, alias);
} }
}; };
}; }

View File

@ -1,38 +1,51 @@
'use strict'; import { join } from 'node:path';
import { promisify } from 'node:util';
import {
rmdir as _rmdir,
mkdir as _mkdir,
lstat as _lstat,
readdir as _readdir,
unlink as _unlink,
writeFile as _writeFile
} from 'node:fs';
const fs = require('fs-extra'); import { copy } from 'fs-extra/esm';
const path = require('path'); import intel from 'intel';
const log = require('intel').getLogger('sitespeedio.storageManager');
const { promisify } = require('util'); import { pathToFolder } from './pathToFolder.js';
const mkdir = promisify(fs.mkdir);
const readdir = promisify(fs.readdir); const log = intel.getLogger('sitespeedio.storageManager');
const lstat = promisify(fs.lstat); const mkdir = promisify(_mkdir);
const unlink = promisify(fs.unlink); const readdir = promisify(_readdir);
const rmdir = promisify(fs.rmdir); const lstat = promisify(_lstat);
const pathToFolder = require('./pathToFolder'); const unlink = promisify(_unlink);
const rmdir = promisify(_rmdir);
const writeFile = promisify(_writeFile);
function write(dirPath, filename, data) { function write(dirPath, filename, data) {
return fs.writeFile(path.join(dirPath, filename), data); return writeFile(join(dirPath, filename), data);
} }
function isValidDirectoryName(name) { function isValidDirectoryName(name) {
return name !== undefined && name !== ''; return name !== undefined && name !== '';
} }
module.exports = function storageManager(baseDir, storagePathPrefix, options) { export function storageManager(baseDir, storagePathPrefix, options) {
return { return {
rootPathFromUrl(url, alias) { rootPathFromUrl(url, alias) {
return pathToFolder(url, options, alias) return pathToFolder(url, options, alias)
.split('/') .split('/')
.filter(isValidDirectoryName) .filter(element => isValidDirectoryName(element))
.map(() => '..') .map(() => '..')
.join('/') .join('/')
.concat('/'); .concat('/');
}, },
createDirectory(...subDirs) { createDirectory(...subDirectories) {
const pathSegments = [baseDir, ...subDirs].filter(isValidDirectoryName); const pathSegments = [baseDir, ...subDirectories].filter(element =>
isValidDirectoryName(element)
);
const dirPath = path.join.apply(null, pathSegments); const dirPath = join.apply(undefined, pathSegments);
return mkdir(dirPath, { recursive: true }).then(() => dirPath); return mkdir(dirPath, { recursive: true }).then(() => dirPath);
}, },
writeData(data, filename) { writeData(data, filename) {
@ -50,41 +63,37 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
return baseDir; return baseDir;
}, },
getFullPathToURLDir(url, alias) { getFullPathToURLDir(url, alias) {
return path.join(baseDir, pathToFolder(url, options, alias)); return join(baseDir, pathToFolder(url, options, alias));
}, },
getStoragePrefix() { getStoragePrefix() {
return storagePathPrefix; return storagePathPrefix;
}, },
copyToResultDir(filename) { copyToResultDir(filename) {
return this.createDirectory().then(dir => fs.copy(filename, dir)); return this.createDirectory().then(dir => copy(filename, dir));
}, },
copyFileToDir(filename, dir) { copyFileToDir(filename, dir) {
return fs.copy(filename, dir); return copy(filename, dir);
}, },
// TODO is missing alias // TODO is missing alias
removeDataForUrl(url) { removeDataForUrl(url) {
const dirName = path.join(baseDir, pathToFolder(url, options)); const dirName = join(baseDir, pathToFolder(url, options));
const removeDir = async dir => { const removeDir = async dir => {
try { try {
const files = await readdir(dir); const files = await readdir(dir);
await Promise.all( await Promise.all(
files.map(async file => { files.map(async file => {
try { try {
const p = path.join(dir, file); const p = join(dir, file);
const stat = await lstat(p); const stat = await lstat(p);
if (stat.isDirectory()) { await (stat.isDirectory() ? removeDir(p) : unlink(p));
await removeDir(p); } catch (error) {
} else { log.error('Could not remove file:' + file, error);
await unlink(p);
}
} catch (err) {
log.error('Could not remove file:' + file, err);
} }
}) })
); );
await rmdir(dir); await rmdir(dir);
} catch (err) { } catch (error) {
log.error('Could not remove dir:' + dir, err); log.error('Could not remove dir:' + dir, error);
} }
}; };
return removeDir(dirName); return removeDir(dirName);
@ -105,4 +114,4 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
); );
} }
}; };
}; }

View File

@ -1,15 +1,9 @@
'use strict'; import { messageMaker } from '../support/messageMaker.js';
const messageMaker = require('../support/messageMaker');
const make = messageMaker('script-reader').make; const make = messageMaker('script-reader').make;
module.exports = { export function findUrls(queue, options) {
open(context, options) {
this.options = options;
},
findUrls(queue) {
queue.postMessage( queue.postMessage(
make('browsertime.navigationScripts', {}, { url: this.options.urls }) make('browsertime.navigationScripts', {}, { url: options.urls })
); );
} }
};

View File

@ -1,15 +1,9 @@
'use strict'; import { parse } from 'node:url';
import { messageMaker } from '../support/messageMaker.js';
const urlParser = require('url');
const messageMaker = require('../support/messageMaker');
const make = messageMaker('url-reader').make; const make = messageMaker('url-reader').make;
module.exports = { export function findUrls(queue, options) {
open(context, options) { for (const url of options.urls) {
this.options = options;
},
findUrls(queue) {
for (const url of this.options.urls) {
queue.postMessage( queue.postMessage(
make( make(
'url', 'url',
@ -17,14 +11,13 @@ module.exports = {
{ {
url: url, url: url,
group: group:
this.options.urlsMetaData && options.urlsMetaData &&
this.options.urlsMetaData[url] && options.urlsMetaData[url] &&
this.options.urlsMetaData[url].groupAlias options.urlsMetaData[url].groupAlias
? this.options.urlsMetaData[url].groupAlias ? options.urlsMetaData[url].groupAlias
: urlParser.parse(url).hostname : parse(url).hostname
} }
) )
); );
} }
} }
};

View File

@ -1,8 +1,7 @@
'use strict'; import { SitespeedioPlugin } from '@sitespeed.io/plugin';
function shouldIgnoreMessage(message) { function shouldIgnoreMessage(message) {
return ( return [
[
'url', 'url',
'browsertime.navigationScripts', 'browsertime.navigationScripts',
'error', 'error',
@ -34,15 +33,18 @@ function shouldIgnoreMessage(message) {
'grafana.setup', 'grafana.setup',
'sustainable.setup', 'sustainable.setup',
'scp.setup' 'scp.setup'
].indexOf(message.type) >= 0 ].includes(message.type);
); }
export default class AnalysisstorerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'analysisstorer', options, context, queue });
} }
module.exports = {
open(context) { open(context) {
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
this.alias = {}; this.alias = {};
}, }
processMessage(message) { processMessage(message) {
if (shouldIgnoreMessage(message)) { if (shouldIgnoreMessage(message)) {
return; return;
@ -75,4 +77,4 @@ module.exports = {
return this.storageManager.writeData(jsonData, fileName); return this.storageManager.writeData(jsonData, fileName);
} }
} }
}; }

View File

@ -1,17 +1,18 @@
'use strict'; import get from 'lodash.get';
import { AssetsBySize } from './assetsBySize.js';
import { AssetsBySpeed } from './assetsBySpeed.js';
let AssetsBySize = require('./assetsBySize'), export class AssetsAggregator {
get = require('lodash.get'), constructor() {
AssetsBySpeed = require('./assetsBySpeed'); this.assets = {};
this.groups = {};
this.largestAssets = {};
this.largestAssetsByGroup = {};
this.slowestAssetsByGroup = {};
this.largestThirdPartyAssetsByGroup = {};
this.slowestThirdPartyAssetsByGroup = {};
}
module.exports = {
assets: {},
groups: {},
largestAssets: {},
largestAssetsByGroup: {},
slowestAssetsByGroup: {},
largestThirdPartyAssetsByGroup: {},
slowestThirdPartyAssetsByGroup: {},
addToAggregate(data, group, url, resultUrls, runIndex, options, alias) { addToAggregate(data, group, url, resultUrls, runIndex, options, alias) {
const maxSize = get(options, 'html.topListSize', 10); const maxSize = get(options, 'html.topListSize', 10);
let page = resultUrls.relativeSummaryPageUrl(url, alias[url]); let page = resultUrls.relativeSummaryPageUrl(url, alias[url]);
@ -53,14 +54,12 @@ module.exports = {
const url = asset.url; const url = asset.url;
if (options.firstParty) { if (options.firstParty && !options.firstParty.test(url)) {
if (!url.match(options.firstParty)) {
this.slowestAssetsThirdParty.add(asset, page, runPage); this.slowestAssetsThirdParty.add(asset, page, runPage);
this.largestAssetsThirdParty.add(asset, page, runPage); this.largestAssetsThirdParty.add(asset, page, runPage);
this.slowestThirdPartyAssetsByGroup[group].add(asset, page, runPage); this.slowestThirdPartyAssetsByGroup[group].add(asset, page, runPage);
this.largestThirdPartyAssetsByGroup[group].add(asset, page, runPage); this.largestThirdPartyAssetsByGroup[group].add(asset, page, runPage);
} }
}
const urlInfo = this.assets[url] || { const urlInfo = this.assets[url] || {
url: url, url: url,
@ -91,8 +90,7 @@ module.exports = {
urlInfoGroup.requestCount++; urlInfoGroup.requestCount++;
this.groups[group][url] = urlInfoGroup; this.groups[group][url] = urlInfoGroup;
} }
}, }
summarize() { summarize() {
const summary = { const summary = {
groups: { groups: {
@ -134,4 +132,4 @@ module.exports = {
return summary; return summary;
} }
}; }

View File

@ -1,6 +1,4 @@
'use strict'; export class AssetsBySize {
class AssetsBySize {
constructor(maxSize) { constructor(maxSize) {
this.maxSize = maxSize; this.maxSize = maxSize;
this.items = []; this.items = [];
@ -42,5 +40,3 @@ class AssetsBySize {
return this.items; return this.items;
} }
} }
module.exports = AssetsBySize;

View File

@ -1,6 +1,4 @@
'use strict'; export class AssetsBySpeed {
class AssetsBySpeed {
constructor(maxSize) { constructor(maxSize) {
this.maxSize = maxSize; this.maxSize = maxSize;
this.items = []; this.items = [];
@ -52,5 +50,3 @@ class AssetsBySpeed {
return this.items; return this.items;
} }
} }
module.exports = AssetsBySpeed;

View File

@ -1,15 +1,19 @@
'use strict'; import isEmpty from 'lodash.isempty';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const isEmpty = require('lodash.isempty'); import { AssetsAggregator } from './aggregator.js';
const aggregator = require('./aggregator');
const DEFAULT_METRICS_LARGEST_ASSETS = ['image.0.transferSize']; const DEFAULT_METRICS_LARGEST_ASSETS = ['image.0.transferSize'];
module.exports = { export default class AssetsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'assets', options, context, queue });
}
open(context, options) { open(context, options) {
this.make = context.messageMaker('assets').make; this.make = context.messageMaker('assets').make;
this.options = options; this.options = options;
this.alias = {}; this.alias = {};
this.resultUrls = context.resultUrls; this.resultUrls = context.resultUrls;
this.assetsAggregator = new AssetsAggregator();
context.filterRegistry.registerFilterForType( context.filterRegistry.registerFilterForType(
DEFAULT_METRICS_LARGEST_ASSETS, DEFAULT_METRICS_LARGEST_ASSETS,
'largestassets.summary' 'largestassets.summary'
@ -24,12 +28,12 @@ module.exports = {
[], [],
'largestthirdpartyassets.summary' 'largestthirdpartyassets.summary'
); );
}, }
processMessage(message, queue) { processMessage(message, queue) {
const make = this.make; const make = this.make;
switch (message.type) { switch (message.type) {
case 'pagexray.run': { case 'pagexray.run': {
aggregator.addToAggregate( this.assetsAggregator.addToAggregate(
message.data, message.data,
message.group, message.group,
message.url, message.url,
@ -46,7 +50,7 @@ module.exports = {
break; break;
} }
case 'sitespeedio.summarize': { case 'sitespeedio.summarize': {
const summary = aggregator.summarize(); const summary = this.assetsAggregator.summarize();
if (!isEmpty(summary)) { if (!isEmpty(summary)) {
for (let group of Object.keys(summary.groups)) { for (let group of Object.keys(summary.groups)) {
queue.postMessage( queue.postMessage(
@ -87,4 +91,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -36,10 +36,11 @@ module.exports = async function (context) {
); );
// Use the extras field in Browsertime and pass on the result // Use the extras field in Browsertime and pass on the result
context.result[context.result.length - 1].extras.axe = result; context.result[context.result.length - 1].extras.axe = result;
} catch (e) {
} catch (error) {
context.log.error( context.log.error(
'Could not run the AXE script, no AXE information collected', 'Could not run the AXE script, no AXE information collected',
e error
); );
} }
}; };

View File

@ -1,25 +1,30 @@
'use strict'; import { resolve } from 'node:path';
const path = require('path'); import { readFileSync } from 'node:fs';
const fs = require('fs'); import { fileURLToPath } from 'node:url';
const log = require('intel').getLogger('sitespeedio.plugin.axe'); import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = intel.getLogger('sitespeedio.plugin.axe');
const __dirname = fileURLToPath(new URL('.', import.meta.url));
export default class AxePlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'axe', options, context, queue });
}
module.exports = {
open(context, options) { open(context, options) {
this.options = options; this.options = options;
this.make = context.messageMaker('axe').make; this.make = context.messageMaker('axe').make;
this.pug = fs.readFileSync( this.pug = readFileSync(resolve(__dirname, 'pug', 'index.pug'), 'utf8');
path.resolve(__dirname, 'pug', 'index.pug'),
'utf8'
);
log.info('Axe plugin activated'); log.info('Axe plugin activated');
}, }
processMessage(message, queue) { processMessage(message, queue) {
const make = this.make; const make = this.make;
switch (message.type) { switch (message.type) {
case 'browsertime.setup': { case 'browsertime.setup': {
queue.postMessage( queue.postMessage(
make('browsertime.config', { make('browsertime.config', {
postURLScript: path.resolve(__dirname, 'axePostScript.cjs') postURLScript: resolve(__dirname, 'axePostScript.cjs')
}) })
); );
break; break;
@ -53,4 +58,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -1,12 +1,12 @@
'use strict'; import { resolve } from 'node:path';
import merge from 'lodash.merge';
const merge = require('lodash.merge'); import forEach from 'lodash.foreach';
const forEach = require('lodash.foreach'); import set from 'lodash.set';
const path = require('path'); import get from 'lodash.get';
const set = require('lodash.set'); import coach from 'coach-core';
const get = require('lodash.get'); const { getDomAdvice } = coach;
const coach = require('coach-core'); import intel from 'intel';
const log = require('intel').getLogger('plugin.browsertime'); const log = intel.getLogger('plugin.browsertime');
const defaultBrowsertimeOptions = { const defaultBrowsertimeOptions = {
statistics: true statistics: true
@ -53,15 +53,13 @@ async function preWarmServer(urls, options, scriptOrMultiple) {
} }
const { BrowsertimeEngine } = await import('browsertime'); const { BrowsertimeEngine } = await import('browsertime');
const engine = new BrowsertimEngine(preWarmOptions); const engine = new BrowsertimeEngine(preWarmOptions);
await engine.start(); await engine.start();
log.info('Start pre-testing/warming' + urls); log.info('Start pre-testing/warming' + urls);
if (scriptOrMultiple) { await (scriptOrMultiple
await engine.runMultiple(urls, {}); ? engine.runMultiple(urls, {})
} else { : engine.run(urls, {}));
await engine.run(urls, {});
}
await engine.stop(); await engine.stop();
log.info('Pre-testing done, closed the browser.'); log.info('Pre-testing done, closed the browser.');
return delay(options.preWarmServerWaitTime || 5000); return delay(options.preWarmServerWaitTime || 5000);
@ -73,7 +71,7 @@ async function parseUserScripts(scripts) {
const allUserScripts = {}; const allUserScripts = {};
for (let script of scripts) { for (let script of scripts) {
let myScript = await browserScripts.findAndParseScripts( let myScript = await browserScripts.findAndParseScripts(
path.resolve(script), resolve(script),
'custom' 'custom'
); );
if (!myScript['custom']) { if (!myScript['custom']) {
@ -85,7 +83,7 @@ async function parseUserScripts(scripts) {
} }
async function addCoachScripts(scripts) { async function addCoachScripts(scripts) {
const coachAdvice = await coach.getDomAdvice(); const coachAdvice = await getDomAdvice();
scripts.coach = { scripts.coach = {
coachAdvice: coachAdvice coachAdvice: coachAdvice
}; };
@ -115,8 +113,7 @@ function setupAsynScripts(asyncScripts) {
return allAsyncScripts; return allAsyncScripts;
} }
module.exports = { export async function analyzeUrl(
async analyzeUrl(
url, url,
scriptOrMultiple, scriptOrMultiple,
pluginScripts, pluginScripts,
@ -171,11 +168,7 @@ module.exports = {
try { try {
await engine.start(); await engine.start();
if (scriptOrMultiple) { if (scriptOrMultiple) {
const res = await engine.runMultiple( const res = await engine.runMultiple(url, scriptsByCategory, asyncScript);
url,
scriptsByCategory,
asyncScript
);
return res; return res;
} else { } else {
const res = await engine.run(url, scriptsByCategory, asyncScript); const res = await engine.run(url, scriptsByCategory, asyncScript);
@ -185,4 +178,3 @@ module.exports = {
await engine.stop(); await engine.stop();
} }
} }
};

View File

@ -1,7 +1,6 @@
'use strict'; import { Stats } from 'fast-stats';
const Stats = require('fast-stats').Stats; import { summarizeStats as _summarizeStats } from '../../support/statsHelpers.js';
const statsHelpers = require('../../support/statsHelpers'); export class AxeAggregator {
class AxeAggregator {
constructor(options) { constructor(options) {
this.options = options; this.options = options;
this.axeViolations = { this.axeViolations = {
@ -33,13 +32,11 @@ class AxeAggregator {
summarizeStats() { summarizeStats() {
return { return {
violations: { violations: {
critical: statsHelpers.summarizeStats(this.axeViolations.critical), critical: _summarizeStats(this.axeViolations.critical),
serious: statsHelpers.summarizeStats(this.axeViolations.serious), serious: _summarizeStats(this.axeViolations.serious),
minor: statsHelpers.summarizeStats(this.axeViolations.minor), minor: _summarizeStats(this.axeViolations.minor),
moderate: statsHelpers.summarizeStats(this.axeViolations.moderate) moderate: _summarizeStats(this.axeViolations.moderate)
} }
}; };
} }
} }
module.exports = AxeAggregator;

View File

@ -1,13 +1,13 @@
'use strict'; import forEach from 'lodash.foreach';
import { pushGroupStats, setStatsSummary } from '../../support/statsHelpers.js';
const forEach = require('lodash.foreach'),
statsHelpers = require('../../support/statsHelpers');
const timings = ['firstPaint', 'timeToDomContentFlushed']; const timings = ['firstPaint', 'timeToDomContentFlushed'];
module.exports = { export class BrowsertimeAggregator {
statsPerType: {}, constructor() {
groups: {}, this.statsPerType = {};
this.groups = {};
}
addToAggregate(browsertimeRunData, group) { addToAggregate(browsertimeRunData, group) {
if (this.groups[group] === undefined) { if (this.groups[group] === undefined) {
@ -15,7 +15,7 @@ module.exports = {
} }
if (browsertimeRunData.fullyLoaded) { if (browsertimeRunData.fullyLoaded) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['timings', 'fullyLoaded'], ['timings', 'fullyLoaded'],
@ -24,7 +24,7 @@ module.exports = {
} }
if (browsertimeRunData.memory) { if (browsertimeRunData.memory) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['memory'], ['memory'],
@ -34,7 +34,7 @@ module.exports = {
if (browsertimeRunData.googleWebVitals) { if (browsertimeRunData.googleWebVitals) {
for (let metric of Object.keys(browsertimeRunData.googleWebVitals)) { for (let metric of Object.keys(browsertimeRunData.googleWebVitals)) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['googleWebVitals', metric], ['googleWebVitals', metric],
@ -44,7 +44,7 @@ module.exports = {
} }
if (browsertimeRunData.timings.largestContentfulPaint) { if (browsertimeRunData.timings.largestContentfulPaint) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['timings', 'largestContentfulPaint'], ['timings', 'largestContentfulPaint'],
@ -53,7 +53,7 @@ module.exports = {
} }
if (browsertimeRunData.timings.interactionToNextPaint) { if (browsertimeRunData.timings.interactionToNextPaint) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['timings', 'interactionToNextPaint'], ['timings', 'interactionToNextPaint'],
@ -62,7 +62,7 @@ module.exports = {
} }
if (browsertimeRunData.pageinfo.cumulativeLayoutShift) { if (browsertimeRunData.pageinfo.cumulativeLayoutShift) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['pageinfo', 'cumulativeLayoutShift'], ['pageinfo', 'cumulativeLayoutShift'],
@ -72,7 +72,7 @@ module.exports = {
forEach(timings, timing => { forEach(timings, timing => {
if (browsertimeRunData.timings[timing]) { if (browsertimeRunData.timings[timing]) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
timing, timing,
@ -83,7 +83,7 @@ module.exports = {
forEach(browsertimeRunData.timings.navigationTiming, (value, name) => { forEach(browsertimeRunData.timings.navigationTiming, (value, name) => {
if (value) { if (value) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['navigationTiming', name], ['navigationTiming', name],
@ -94,7 +94,7 @@ module.exports = {
// pick up one level of custom metrics // pick up one level of custom metrics
forEach(browsertimeRunData.custom, (value, name) => { forEach(browsertimeRunData.custom, (value, name) => {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['custom', name], ['custom', name],
@ -103,7 +103,7 @@ module.exports = {
}); });
forEach(browsertimeRunData.timings.pageTimings, (value, name) => { forEach(browsertimeRunData.timings.pageTimings, (value, name) => {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['pageTimings', name], ['pageTimings', name],
@ -112,7 +112,7 @@ module.exports = {
}); });
forEach(browsertimeRunData.timings.paintTiming, (value, name) => { forEach(browsertimeRunData.timings.paintTiming, (value, name) => {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['paintTiming', name], ['paintTiming', name],
@ -121,7 +121,7 @@ module.exports = {
}); });
forEach(browsertimeRunData.timings.userTimings.marks, timing => { forEach(browsertimeRunData.timings.userTimings.marks, timing => {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['userTimings', 'marks', timing.name], ['userTimings', 'marks', timing.name],
@ -130,7 +130,7 @@ module.exports = {
}); });
forEach(browsertimeRunData.timings.userTimings.measures, timing => { forEach(browsertimeRunData.timings.userTimings.measures, timing => {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['userTimings', 'measures', timing.name], ['userTimings', 'measures', timing.name],
@ -141,8 +141,8 @@ module.exports = {
forEach(browsertimeRunData.visualMetrics, (value, name) => { forEach(browsertimeRunData.visualMetrics, (value, name) => {
// Sometimes visual elements fails and gives us null values // Sometimes visual elements fails and gives us null values
// And skip VisualProgress, ContentfulSpeedIndexProgress and others // And skip VisualProgress, ContentfulSpeedIndexProgress and others
if (name.indexOf('Progress') === -1 && value !== null) { if (!name.includes('Progress') && value !== null) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['visualMetrics', name], ['visualMetrics', name],
@ -153,28 +153,28 @@ module.exports = {
if (browsertimeRunData.cpu) { if (browsertimeRunData.cpu) {
if (browsertimeRunData.cpu.longTasks) { if (browsertimeRunData.cpu.longTasks) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['cpu', 'longTasks', 'tasks'], ['cpu', 'longTasks', 'tasks'],
browsertimeRunData.cpu.longTasks.tasks browsertimeRunData.cpu.longTasks.tasks
); );
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['cpu', 'longTasks', 'totalDuration'], ['cpu', 'longTasks', 'totalDuration'],
browsertimeRunData.cpu.longTasks.totalDuration browsertimeRunData.cpu.longTasks.totalDuration
); );
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['cpu', 'longTasks', 'totalBlockingTime'], ['cpu', 'longTasks', 'totalBlockingTime'],
browsertimeRunData.cpu.longTasks.totalBlockingTime browsertimeRunData.cpu.longTasks.totalBlockingTime
); );
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['cpu', 'longTasks', 'maxPotentialFid'], ['cpu', 'longTasks', 'maxPotentialFid'],
@ -185,7 +185,7 @@ module.exports = {
for (let categoryName of Object.keys( for (let categoryName of Object.keys(
browsertimeRunData.cpu.categories browsertimeRunData.cpu.categories
)) { )) {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerType, this.statsPerType,
this.groups[group], this.groups[group],
['cpu', 'categories', categoryName], ['cpu', 'categories', categoryName],
@ -194,10 +194,11 @@ module.exports = {
} }
} }
} }
}, }
summarize() { summarize() {
if (Object.keys(this.statsPerType).length === 0) { if (Object.keys(this.statsPerType).length === 0) {
return undefined; return;
} }
const summary = { const summary = {
@ -210,51 +211,51 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]); summary.groups[group] = this.summarizePerObject(this.groups[group]);
} }
return summary; return summary;
}, }
summarizePerObject(obj) { summarizePerObject(object) {
return Object.keys(obj).reduce((summary, name) => { return Object.keys(object).reduce((summary, name) => {
if (timings.indexOf(name) > -1) { if (timings.includes(name)) {
statsHelpers.setStatsSummary(summary, name, obj[name]); setStatsSummary(summary, name, object[name]);
} else if ('userTimings'.indexOf(name) > -1) { } else if ('userTimings'.includes(name)) {
summary.userTimings = {}; summary.userTimings = {};
const marksData = {}, const marksData = {},
measuresData = {}; measuresData = {};
forEach(obj.userTimings.marks, (stats, timingName) => { forEach(object.userTimings.marks, (stats, timingName) => {
statsHelpers.setStatsSummary(marksData, timingName, stats); setStatsSummary(marksData, timingName, stats);
}); });
forEach(obj.userTimings.measures, (stats, timingName) => { forEach(object.userTimings.measures, (stats, timingName) => {
statsHelpers.setStatsSummary(measuresData, timingName, stats); setStatsSummary(measuresData, timingName, stats);
}); });
summary.userTimings.marks = marksData; summary.userTimings.marks = marksData;
summary.userTimings.measures = measuresData; summary.userTimings.measures = measuresData;
} else if ('cpu'.indexOf(name) > -1) { } else if ('cpu'.includes(name)) {
const longTasks = {}; const longTasks = {};
const categories = {}; const categories = {};
summary.cpu = {}; summary.cpu = {};
forEach(obj.cpu.longTasks, (stats, name) => { forEach(object.cpu.longTasks, (stats, name) => {
statsHelpers.setStatsSummary(longTasks, name, stats); setStatsSummary(longTasks, name, stats);
}); });
forEach(obj.cpu.categories, (stats, name) => { forEach(object.cpu.categories, (stats, name) => {
statsHelpers.setStatsSummary(categories, name, stats); setStatsSummary(categories, name, stats);
}); });
summary.cpu.longTasks = longTasks; summary.cpu.longTasks = longTasks;
summary.cpu.categories = categories; summary.cpu.categories = categories;
} else if ('memory'.indexOf(name) > -1) { } else if ('memory'.includes(name)) {
const memory = {}; const memory = {};
statsHelpers.setStatsSummary(memory, 'memory', obj[name]); setStatsSummary(memory, 'memory', object[name]);
summary.memory = memory.memory; summary.memory = memory.memory;
} else { } else {
const categoryData = {}; const categoryData = {};
forEach(obj[name], (stats, timingName) => { forEach(object[name], (stats, timingName) => {
statsHelpers.setStatsSummary(categoryData, timingName, stats); setStatsSummary(categoryData, timingName, stats);
}); });
summary[name] = categoryData; summary[name] = categoryData;
} }
return summary; return summary;
}, {}); }, {});
} }
}; }

View File

@ -1,8 +1,7 @@
'use strict'; import { Stats } from 'fast-stats';
const Stats = require('fast-stats').Stats; import { summarizeStats as _summarizeStats } from '../../support/statsHelpers.js';
const statsHelpers = require('../../support/statsHelpers'); import { getGzippedFileAsJson } from './reader.js';
const { getGzippedFileAsJson } = require('./reader.js'); export class ConsoleLogAggregator {
class ConsoleLogAggregator {
constructor(options) { constructor(options) {
this.options = options; this.options = options;
this.logs = { SEVERE: new Stats(), WARNING: new Stats() }; this.logs = { SEVERE: new Stats(), WARNING: new Stats() };
@ -35,10 +34,8 @@ class ConsoleLogAggregator {
summarizeStats() { summarizeStats() {
return { return {
error: statsHelpers.summarizeStats(this.logs.SEVERE), error: _summarizeStats(this.logs.SEVERE),
warning: statsHelpers.summarizeStats(this.logs.WARNING) warning: _summarizeStats(this.logs.WARNING)
}; };
} }
} }
module.exports = ConsoleLogAggregator;

View File

@ -1,6 +1,4 @@
'use strict'; export const browsertimeDefaultSettings = {
module.exports = {
browser: 'chrome', browser: 'chrome',
iterations: 3, iterations: 3,
connectivity: { connectivity: {

View File

@ -1,6 +1,4 @@
'use strict'; export const metricsPageSummary = [
module.exports = [
'statistics.timings.pageTimings', 'statistics.timings.pageTimings',
'statistics.timings.fullyLoaded', 'statistics.timings.fullyLoaded',
'statistics.timings.firstPaint', 'statistics.timings.firstPaint',

View File

@ -1,4 +1,4 @@
module.exports = [ export const metricsRun = [
'fullyLoaded', 'fullyLoaded',
'timings.ttfb', 'timings.ttfb',
'timings.firstPaint', 'timings.firstPaint',

View File

@ -1,4 +1,4 @@
module.exports = [ export const metricsRunLimited = [
'fullyLoaded', 'fullyLoaded',
'timings.ttfb', 'timings.ttfb',
'timings.paintTiming.first-contentful-paint', 'timings.paintTiming.first-contentful-paint',

View File

@ -1,6 +1,4 @@
'use strict'; export const metricsSummary = [
module.exports = [
'firstPaint', 'firstPaint',
'timeToDomContentFlushed', 'timeToDomContentFlushed',
'fullyLoaded', 'fullyLoaded',

View File

@ -1,10 +1,9 @@
'use strict'; import { readdir as _readdir } from 'node:fs';
import { promisify } from 'node:util';
const fs = require('fs'); import { join } from 'node:path';
const { promisify } = require('util'); const readdir = promisify(_readdir);
const readdir = promisify(fs.readdir); import intel from 'intel';
const path = require('path'); const log = intel.getLogger('sitespeedio.plugin.browsertime');
const log = require('intel').getLogger('sitespeedio.plugin.browsertime');
function findFrame(videoFrames, time) { function findFrame(videoFrames, time) {
let frame = videoFrames[0]; let frame = videoFrames[0];
@ -29,7 +28,7 @@ function getMetricsFromBrowsertime(data) {
for (let mark of data.timings.userTimings.marks) { for (let mark of data.timings.userTimings.marks) {
metrics.push({ metrics.push({
metric: mark.name, metric: mark.name,
value: mark.startTime.toFixed() value: mark.startTime.toFixed(0)
}); });
userTimings++; userTimings++;
if (userTimings > maxUserTimings) { if (userTimings > maxUserTimings) {
@ -56,16 +55,18 @@ function getMetricsFromBrowsertime(data) {
}); });
if (data.timings.pageTimings) { if (data.timings.pageTimings) {
metrics.push({ metrics.push(
{
metric: 'domContentLoadedTime', metric: 'domContentLoadedTime',
name: 'DOM Content Loaded Time', name: 'DOM Content Loaded Time',
value: data.timings.pageTimings.domContentLoadedTime value: data.timings.pageTimings.domContentLoadedTime
}); },
metrics.push({ {
metric: 'pageLoadTime', metric: 'pageLoadTime',
name: 'Page Load Time', name: 'Page Load Time',
value: data.timings.pageTimings.pageLoadTime value: data.timings.pageTimings.pageLoadTime
}); }
);
} }
if ( if (
@ -84,20 +85,17 @@ function getMetricsFromBrowsertime(data) {
data.timings.largestContentfulPaint.renderTime data.timings.largestContentfulPaint.renderTime
) { ) {
let name = 'LCP'; let name = 'LCP';
if (data.timings.largestContentfulPaint.tagName === 'IMG') {
name += name +=
' <a href="' + data.timings.largestContentfulPaint.tagName === 'IMG'
? ' <a href="' +
data.timings.largestContentfulPaint.url + data.timings.largestContentfulPaint.url +
'">&#60;IMG&#62;</a>'; '">&#60;IMG&#62;</a>'
} else { : ' &#60;' +
name +=
' &#60;' +
data.timings.largestContentfulPaint.tagName + data.timings.largestContentfulPaint.tagName +
'&#62;' + '&#62;' +
(data.timings.largestContentfulPaint.id !== '' (data.timings.largestContentfulPaint.id !== ''
? ' ' + data.timings.largestContentfulPaint.id ? ' ' + data.timings.largestContentfulPaint.id
: ''); : '');
}
metrics.push({ metrics.push({
metric: 'largestContentfulPaint', metric: 'largestContentfulPaint',
name, name,
@ -106,31 +104,33 @@ function getMetricsFromBrowsertime(data) {
} }
if (data.visualMetrics) { if (data.visualMetrics) {
metrics.push({ metrics.push(
{
metric: 'FirstVisualChange', metric: 'FirstVisualChange',
name: 'First Visual Change', name: 'First Visual Change',
value: data.visualMetrics.FirstVisualChange value: data.visualMetrics.FirstVisualChange
}); },
metrics.push({ {
metric: 'LastVisualChange', metric: 'LastVisualChange',
name: 'Last Visual Change', name: 'Last Visual Change',
value: data.visualMetrics.LastVisualChange value: data.visualMetrics.LastVisualChange
}); },
metrics.push({ {
metric: 'VisualComplete85', metric: 'VisualComplete85',
name: 'Visual Complete 85%', name: 'Visual Complete 85%',
value: data.visualMetrics.VisualComplete85 value: data.visualMetrics.VisualComplete85
}); },
metrics.push({ {
metric: 'VisualComplete95', metric: 'VisualComplete95',
name: 'Visual Complete 95%', name: 'Visual Complete 95%',
value: data.visualMetrics.VisualComplete95 value: data.visualMetrics.VisualComplete95
}); },
metrics.push({ {
metric: 'VisualComplete99', metric: 'VisualComplete99',
name: 'Visual Complete 99%', name: 'Visual Complete 99%',
value: data.visualMetrics.VisualComplete99 value: data.visualMetrics.VisualComplete99
}); }
);
if (data.visualMetrics.LargestImage) { if (data.visualMetrics.LargestImage) {
metrics.push({ metrics.push({
metric: 'LargestImage', metric: 'LargestImage',
@ -169,8 +169,13 @@ function getMetricsFromBrowsertime(data) {
function findTimings(timings, start, end) { function findTimings(timings, start, end) {
return timings.filter(timing => timing.value > start && timing.value <= end); return timings.filter(timing => timing.value > start && timing.value <= end);
} }
module.exports = { export async function getFilmstrip(
async getFilmstrip(browsertimeData, run, dir, options, fullPath) { browsertimeData,
run,
dir,
options,
fullPath
) {
let doWeHaveFilmstrip = let doWeHaveFilmstrip =
options.browsertime.visualMetrics === true && options.browsertime.visualMetrics === true &&
options.browsertime.videoParams.createFilmstrip === true; options.browsertime.videoParams.createFilmstrip === true;
@ -186,9 +191,7 @@ module.exports = {
if (browsertimeData) { if (browsertimeData) {
metrics = getMetricsFromBrowsertime(browsertimeData); metrics = getMetricsFromBrowsertime(browsertimeData);
} }
const files = await readdir( const files = await readdir(join(dir, 'data', 'filmstrip', run + ''));
path.join(dir, 'data', 'filmstrip', run + '')
);
const timings = []; const timings = [];
for (let file of files) { for (let file of files) {
timings.push({ time: file.replace(/\D/g, ''), file }); timings.push({ time: file.replace(/\D/g, ''), file });
@ -199,29 +202,32 @@ module.exports = {
// We step 100 ms each step ... but if you wanna show all and the last change is late // We step 100 ms each step ... but if you wanna show all and the last change is late
// use 200 ms // use 200 ms
const step = const step =
maxTiming > 10000 && options.filmstrip && options.filmstrip.showAll maxTiming > 10_000 && options.filmstrip && options.filmstrip.showAll
? 200 ? 200
: 100; : 100;
let fileName = ''; let fileName = '';
for (let i = 0; i <= Number(maxTiming) + step; i = i + step) { for (
const entry = findFrame(timings, i); let index = 0;
const timingMetrics = findTimings(metrics, i - step, i); index <= Number(maxTiming) + step;
index = index + step
) {
const entry = findFrame(timings, index);
const timingMetrics = findTimings(metrics, index - step, index);
if ( if (
entry.file !== fileName || entry.file !== fileName ||
timingMetrics.length > 0 || timingMetrics.length > 0 ||
(options.filmstrip && options.filmstrip.showAll) (options.filmstrip && options.filmstrip.showAll)
) { ) {
toTheFront.push({ toTheFront.push({
time: i / 1000, time: index / 1000,
file: fullPath ? fullPath + entry.file : entry.file, file: fullPath ? fullPath + entry.file : entry.file,
timings: timingMetrics timings: timingMetrics
}); });
} }
fileName = entry.file; fileName = entry.file;
} }
} catch (e) { } catch (error) {
log.info('Could not read filmstrip dir', e); log.info('Could not read filmstrip dir', error);
} }
return toTheFront; return toTheFront;
} }
};

View File

@ -1,35 +1,46 @@
'use strict'; import { parse } from 'node:url';
import { default as _merge } from 'lodash.merge';
import intel from 'intel';
const log = intel.getLogger('plugin.browsertime');
import dayjs from 'dayjs';
import isEmpty from 'lodash.isempty';
import get from 'lodash.get';
import { Stats } from 'fast-stats';
import coach from 'coach-core';
const { pickAPage, analyseHar, merge } = coach;
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { summarizeStats } from '../../support/statsHelpers.js';
import { analyzeUrl } from './analyzer.js';
import { BrowsertimeAggregator } from './browsertimeAggregator.js';
import { metricsPageSummary as DEFAULT_METRICS_PAGE_SUMMARY } from './default/metricsPageSummary.js';
import { metricsSummary as DEFAULT_METRICS_SUMMARY } from './default/metricsSummary.js';
import { metricsRun as DEFAULT_METRICS_RUN } from './default/metricsRun.js';
import { metricsRunLimited as DEFAULT_METRICS_RUN_LIMITED } from './default/metricsRunLimited.js';
import { ConsoleLogAggregator } from './consoleLogAggregator.js';
import { AxeAggregator } from './axeAggregator.js';
import { getFilmstrip } from './filmstrip.js';
import { getGzippedFileAsJson } from './reader.js';
import { browsertimeDefaultSettings as defaultConfig } from './default/config.js';
const aggregator = require('./aggregator');
const api = require('coach-core');
const log = require('intel').getLogger('plugin.browsertime');
const merge = require('lodash.merge');
const analyzer = require('./analyzer');
const dayjs = require('dayjs');
const isEmpty = require('lodash.isempty');
const get = require('lodash.get');
const defaultConfig = require('./default/config');
const urlParser = require('url');
const Stats = require('fast-stats').Stats;
const statsHelpers = require('../../support/statsHelpers');
const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss'; const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
const DEFAULT_METRICS_PAGE_SUMMARY = require('./default/metricsPageSummary'); export default class BrowsertimePlugin extends SitespeedioPlugin {
const DEFAULT_METRICS_SUMMARY = require('./default/metricsSummary'); constructor(options, context, queue) {
const DEFAULT_METRICS_RUN = require('./default/metricsRun'); super({ name: 'browsertime', options, context, queue });
const DEFAULT_METRICS_RUN_LIMITED = require('./default/metricsRunLimited'); }
const ConsoleLogAggregator = require('./consoleLogAggregator');
const AxeAggregator = require('./axeAggregator'); concurrency = 1;
const filmstrip = require('./filmstrip');
const { getGzippedFileAsJson } = require('./reader.js');
module.exports = {
concurrency: 1,
open(context, options) { open(context, options) {
this.make = context.messageMaker('browsertime').make; // this.make = context.messageMaker('browsertime').make;
this.useAxe = options.axe && options.axe.enable; this.useAxe = options.axe && options.axe.enable;
this.options = merge({}, defaultConfig, options.browsertime); this.options = _merge({}, defaultConfig, options.browsertime);
this.allOptions = options; this.allOptions = options;
merge(this.options, { verbose: options.verbose, axe: options.axe }); _merge(this.options, { verbose: options.verbose, axe: options.axe });
this.firstParty = options.firstParty; this.firstParty = options.firstParty;
this.options.mobile = options.mobile; this.options.mobile = options.mobile;
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
@ -42,8 +53,10 @@ module.exports = {
'browsertime.screenshotParams.type', 'browsertime.screenshotParams.type',
defaultConfig.screenshotParams.type defaultConfig.screenshotParams.type
); );
this.scriptOrMultiple = options.multi; this.scriptOrMultiple = options.multi;
this.allAlias = {}; this.allAlias = {};
this.browsertimeAggregator = new BrowsertimeAggregator();
// hack for disabling viewport on Android that's not supported // hack for disabling viewport on Android that's not supported
if ( if (
@ -69,11 +82,10 @@ module.exports = {
'browsertime.run' 'browsertime.run'
); );
this.axeAggregatorTotal = new AxeAggregator(this.options); this.axeAggregatorTotal = new AxeAggregator(this.options);
}, }
async processMessage(message, queue) { async processMessage(message) {
const { configureLogging } = await import('browsertime'); const { configureLogging } = await import('browsertime');
configureLogging(this.options); configureLogging(this.options);
const make = this.make;
const options = this.options; const options = this.options;
switch (message.type) { switch (message.type) {
// When sistespeed.io starts, a setup messages is posted on the queue // When sistespeed.io starts, a setup messages is posted on the queue
@ -81,19 +93,17 @@ module.exports = {
// to receive configuration // to receive configuration
case 'sitespeedio.setup': { case 'sitespeedio.setup': {
// Let other plugins know that the browsertime plugin is alive // Let other plugins know that the browsertime plugin is alive
queue.postMessage(make('browsertime.setup')); super.sendMessage('browsertime.setup');
// Unfify alias setup // Unfify alias setup
if (this.options.urlMetaData) { if (this.options.urlMetaData) {
for (let url of Object.keys(this.options.urlMetaData)) { for (let url of Object.keys(this.options.urlMetaData)) {
const alias = this.options.urlMetaData[url]; const alias = this.options.urlMetaData[url];
const group = urlParser.parse(url).hostname; const group = parse(url).hostname;
this.allAlias[alias] = url; this.allAlias[alias] = url;
queue.postMessage( super.sendMessage('browsertime.alias', alias, {
make('browsertime.alias', alias, {
url, url,
group group
}) });
);
} }
} }
@ -101,18 +111,16 @@ module.exports = {
// what type of images that are used (so for exmaple the HTML pluin can create // what type of images that are used (so for exmaple the HTML pluin can create
// correct links). // correct links).
if (options.screenshot) { if (options.screenshot) {
queue.postMessage( super.sendMessage('browsertime.config', {
make('browsertime.config', {
screenshot: true, screenshot: true,
screenshotType: this.screenshotType screenshotType: this.screenshotType
}) });
);
} }
break; break;
} }
// Another plugin sent configuration options to Browsertime // Another plugin sent configuration options to Browsertime
case 'browsertime.config': { case 'browsertime.config': {
merge(options, message.data); _merge(options, message.data);
break; break;
} }
// Andother plugin got JavaScript that they want to run in Browsertime // Andother plugin got JavaScript that they want to run in Browsertime
@ -151,7 +159,7 @@ module.exports = {
// it's used in BT when we record a video // it's used in BT when we record a video
options.resultDir = await this.storageManager.getBaseDir(); options.resultDir = await this.storageManager.getBaseDir();
const consoleLogAggregator = new ConsoleLogAggregator(options); const consoleLogAggregator = new ConsoleLogAggregator(options);
const result = await analyzer.analyzeUrl( const result = await analyzeUrl(
url, url,
this.scriptOrMultiple, this.scriptOrMultiple,
this.pluginScripts, this.pluginScripts,
@ -159,28 +167,22 @@ module.exports = {
options options
); );
// We need to check for alias first, since when we send the HAR (incliude all runs) // We need to check for alias first, since when we send the HAR (include all runs)
// need to know if alias exists, else we will end up with things like // need to know if alias exists, else we will end up with things like
// https://github.com/sitespeedio/sitespeed.io/issues/2341 // https://github.com/sitespeedio/sitespeed.io/issues/2341
for ( for (const element of result) {
let resultIndex = 0;
resultIndex < result.length;
resultIndex++
) {
// Browsertime supports alias for URLS in a script // Browsertime supports alias for URLS in a script
const alias = result[resultIndex].info.alias; const alias = element.info.alias;
if (alias) { if (alias) {
if (this.scriptOrMultiple) { if (this.scriptOrMultiple) {
url = result[resultIndex].info.url; url = element.info.url;
group = urlParser.parse(url).hostname; group = parse(url).hostname;
} }
this.allAlias[url] = alias; this.allAlias[url] = alias;
queue.postMessage( super.sendMessage('browsertime.alias', alias, {
make('browsertime.alias', alias, {
url, url,
group group
}) });
);
} }
} }
@ -199,7 +201,7 @@ module.exports = {
// multiple/scripting lets do it like this for now // multiple/scripting lets do it like this for now
if (this.scriptOrMultiple) { if (this.scriptOrMultiple) {
url = result[resultIndex].info.url; url = result[resultIndex].info.url;
group = urlParser.parse(url).hostname; group = parse(url).hostname;
} }
let runIndex = 0; let runIndex = 0;
for (let run of result[resultIndex].browserScripts) { for (let run of result[resultIndex].browserScripts) {
@ -223,7 +225,7 @@ module.exports = {
if (result.har.log._android) { if (result.har.log._android) {
testInfo.android = result.har.log._android; testInfo.android = result.har.log._android;
} }
queue.postMessage(make('browsertime.browser', testInfo)); super.sendMessage('browsertime.browser', testInfo);
// Add meta data to be used when we compare multiple HARs // Add meta data to be used when we compare multiple HARs
// the meta field is added in Browsertime // the meta field is added in Browsertime
if (result.har.log.pages[harIndex]) { if (result.har.log.pages[harIndex]) {
@ -244,7 +246,7 @@ module.exports = {
_meta.result = `${base}${runIndex + 1}.html`; _meta.result = `${base}${runIndex + 1}.html`;
if (options.video) { if (options.video) {
_meta.video = `${base}data/video/${runIndex + 1}.mp4`; _meta.video = `${base}data/video/${runIndex + 1}.mp4`;
_meta.filmstrip = await filmstrip.getFilmstrip( _meta.filmstrip = await getFilmstrip(
run, run,
`${runIndex + 1}`, `${runIndex + 1}`,
`${ `${
@ -265,7 +267,7 @@ module.exports = {
url url
); );
} }
run.har = api.pickAPage(result.har, harIndex); run.har = pickAPage(result.har, harIndex);
} else { } else {
// If we do not have a HAR, use browser info from the result // If we do not have a HAR, use browser info from the result
if (result.length > 0) { if (result.length > 0) {
@ -275,13 +277,15 @@ module.exports = {
version: result[0].info.browser.version version: result[0].info.browser.version
} }
}; };
queue.postMessage(make('browsertime.browser', testInfo)); super.sendMessage('browsertime.browser', testInfo);
} }
} }
// Hack to get axe data. In the future we can make this more generic // Hack to get axe data. In the future we can make this more generic
if (result[resultIndex].extras.length > 0) { if (
if (result[resultIndex].extras[runIndex].axe) { result[resultIndex].extras.length > 0 &&
result[resultIndex].extras[runIndex].axe
) {
const order = ['critical', 'serious', 'moderate', 'minor']; const order = ['critical', 'serious', 'moderate', 'minor'];
result[resultIndex].extras[runIndex].axe.violations.sort( result[resultIndex].extras[runIndex].axe.violations.sort(
(a, b) => order.indexOf(a.impact) > order.indexOf(b.impact) (a, b) => order.indexOf(a.impact) > order.indexOf(b.impact)
@ -295,20 +299,21 @@ module.exports = {
result[resultIndex].extras[runIndex].axe result[resultIndex].extras[runIndex].axe
); );
queue.postMessage( super.sendMessage(
make('axe.run', result[resultIndex].extras[runIndex].axe, { 'axe.run',
result[resultIndex].extras[runIndex].axe,
{
url, url,
group, group,
runIndex, runIndex,
iteration: runIndex + 1 iteration: runIndex + 1
}) }
); );
// Another hack: Browsertime automatically creates statistics for alla data in extras // Another hack: Browsertime automatically creates statistics for alla data in extras
// but we don't really need that for AXE. // but we don't really need that for AXE.
delete result[resultIndex].extras[runIndex].axe; delete result[resultIndex].extras[runIndex].axe;
delete result[resultIndex].statistics.extras.axe; delete result[resultIndex].statistics.extras.axe;
} }
}
if (result[resultIndex].cpu) { if (result[resultIndex].cpu) {
run.cpu = result[resultIndex].cpu[runIndex]; run.cpu = result[resultIndex].cpu[runIndex];
} }
@ -360,7 +365,6 @@ module.exports = {
// The packaging of screenshots from browsertime // The packaging of screenshots from browsertime
// Is not optimal, the same array of screenshots hold all // Is not optimal, the same array of screenshots hold all
// screenshots from one run (the automatic ones and user generated) // screenshots from one run (the automatic ones and user generated)
// If we only test one page per run, take all screenshots (user generated etc) // If we only test one page per run, take all screenshots (user generated etc)
if (result.length === 1) { if (result.length === 1) {
run.screenshots = run.screenshots =
@ -372,12 +376,12 @@ module.exports = {
runIndex runIndex
]) { ]) {
if ( if (
screenshot.indexOf( screenshot.includes(
`${this.resultUrls.relativeSummaryPageUrl( `${this.resultUrls.relativeSummaryPageUrl(
url, url,
this.allAlias[url] this.allAlias[url]
)}data` )}data`
) > -1 )
) { ) {
run.screenshots.push(screenshot); run.screenshots.push(screenshot);
} }
@ -391,15 +395,13 @@ module.exports = {
errorStats.push(error.length); errorStats.push(error.length);
} }
queue.postMessage( super.sendMessage('browsertime.run', run, {
make('browsertime.run', run, {
url, url,
group, group,
runIndex, runIndex,
runTime: run.timestamp, runTime: run.timestamp,
iteration: runIndex + 1 iteration: runIndex + 1
}) });
);
if ( if (
options.chrome && options.chrome &&
@ -412,15 +414,13 @@ module.exports = {
result[resultIndex].files.consoleLog[runIndex] result[resultIndex].files.consoleLog[runIndex]
); );
queue.postMessage( super.sendMessage('browsertime.console', consoleData, {
make('browsertime.console', consoleData, {
url, url,
group, group,
runIndex, runIndex,
iteration: runIndex + 1 iteration: runIndex + 1
}) });
); } catch {
} catch (e) {
// This could happen if the run failed somehow // This could happen if the run failed somehow
log.error('Could not fetch the console log'); log.error('Could not fetch the console log');
} }
@ -438,14 +438,12 @@ module.exports = {
options.resultDir, options.resultDir,
`trace-${runIndex + 1}.json.gz` `trace-${runIndex + 1}.json.gz`
); );
queue.postMessage( super.sendMessage('browsertime.chrometrace', traceData, {
make('browsertime.chrometrace', traceData, {
url, url,
group, group,
name: `trace-${runIndex + 1}.json`, // backward compatible to 2.x name: `trace-${runIndex + 1}.json`,
runIndex runIndex
}) });
);
} }
// If the coach is turned on, collect the coach result // If the coach is turned on, collect the coach result
@ -458,8 +456,7 @@ module.exports = {
url, url,
coachAdvice.errors coachAdvice.errors
); );
queue.postMessage( super.sendMessage(
make(
'error', 'error',
'The coach got the following errors: ' + 'The coach got the following errors: ' +
JSON.stringify(coachAdvice.errors), JSON.stringify(coachAdvice.errors),
@ -468,7 +465,6 @@ module.exports = {
runIndex, runIndex,
iteration: runIndex + 1 iteration: runIndex + 1
} }
)
); );
} }
@ -476,26 +472,25 @@ module.exports = {
// If we run without HAR // If we run without HAR
if (result.har) { if (result.har) {
// make sure to get the right run in the HAR // make sure to get the right run in the HAR
const myHar = api.pickAPage(result.har, harIndex); const myHar = pickAPage(result.har, harIndex);
const harResult = await api.analyseHar(
const harResult = await analyseHar(
myHar, myHar,
undefined, undefined,
coachAdvice, coachAdvice,
options options
); );
advice = api.merge(coachAdvice, harResult); advice = merge(coachAdvice, harResult);
} }
queue.postMessage( super.sendMessage('coach.run', advice, {
make('coach.run', advice, {
url, url,
group, group,
runIndex, runIndex,
iteration: runIndex + 1 iteration: runIndex + 1
}) });
);
} }
aggregator.addToAggregate(run, group); this.browsertimeAggregator.addToAggregate(run, group);
runIndex++; runIndex++;
} }
@ -509,34 +504,31 @@ module.exports = {
consoleLogAggregator.summarizeStats(); consoleLogAggregator.summarizeStats();
} }
result[resultIndex].statistics.errors = result[resultIndex].statistics.errors = summarizeStats(errorStats);
statsHelpers.summarizeStats(errorStats);
// Post the result on the queue so other plugins can use it // Post the result on the queue so other plugins can use it
queue.postMessage( super.sendMessage('browsertime.pageSummary', result[resultIndex], {
make('browsertime.pageSummary', result[resultIndex], {
url, url,
group, group,
runTime: result.timestamp runTime: result.timestamp
}) });
);
// Post the HAR on the queue so other plugins can use it // Post the HAR on the queue so other plugins can use it
if (result.har) { if (result.har) {
queue.postMessage( super.sendMessage('browsertime.har', result.har, {
make('browsertime.har', result.har, {
url, url,
group group
}) });
);
} }
// Post the result on the queue so other plugins can use it // Post the result on the queue so other plugins can use it
if (this.useAxe) { if (this.useAxe) {
queue.postMessage( super.sendMessage(
make('axe.pageSummary', axeAggregatorPerURL.summarizeStats(), { 'axe.pageSummary',
axeAggregatorPerURL.summarizeStats(),
{
url, url,
group group
}) }
); );
} }
@ -544,13 +536,13 @@ module.exports = {
// [[],[],[]] where one iteration can have multiple errors // [[],[],[]] where one iteration can have multiple errors
for (let errorsForOneIteration of result[resultIndex].errors) { for (let errorsForOneIteration of result[resultIndex].errors) {
for (let error of errorsForOneIteration) { for (let error of errorsForOneIteration) {
queue.postMessage(make('error', error, merge({ url }))); super.sendMessage('error', error, _merge({ url }));
} }
} }
} }
break; break;
} catch (error) { } catch (error) {
queue.postMessage(make('error', error, merge({ url }))); super.sendMessage('error', error, _merge({ url }));
log.error('Caught error from Browsertime', error); log.error('Caught error from Browsertime', error);
break; break;
} }
@ -559,26 +551,29 @@ module.exports = {
// and post the summary on the queue // and post the summary on the queue
case 'sitespeedio.summarize': { case 'sitespeedio.summarize': {
log.debug('Generate summary metrics from Browsertime'); log.debug('Generate summary metrics from Browsertime');
const summary = aggregator.summarize(); const summary = this.browsertimeAggregator.summarize();
if (summary) { if (summary) {
for (let group of Object.keys(summary.groups)) { for (let group of Object.keys(summary.groups)) {
queue.postMessage( super.sendMessage('browsertime.summary', summary.groups[group], {
make('browsertime.summary', summary.groups[group], { group }) group
); });
} }
} }
if (this.useAxe) { if (this.useAxe) {
queue.postMessage( super.sendMessage(
make('axe.summary', this.axeAggregatorTotal.summarizeStats(), { 'axe.summary',
this.axeAggregatorTotal.summarizeStats(),
{
group: 'total' group: 'total'
}) }
); );
} }
break; break;
} }
} }
}, }
config: defaultConfig }
};
export { browsertimeDefaultSettings as config } from './default/config.js';

View File

@ -1,9 +1,8 @@
'use strict'; import { createReadStream } from 'node:fs';
const fs = require('fs'); import { join } from 'node:path';
const path = require('path'); import { gunzip as _gunzip } from 'node:zlib';
const zlib = require('zlib'); import { promisify } from 'node:util';
const { promisify } = require('util'); const gunzip = promisify(_gunzip);
const gunzip = promisify(zlib.gunzip);
async function streamToString(stream) { async function streamToString(stream) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -14,11 +13,9 @@ async function streamToString(stream) {
}); });
} }
module.exports = { export async function getGzippedFileAsJson(dir, file) {
async getGzippedFileAsJson(dir, file) { const readStream = createReadStream(join(dir, file));
const readStream = fs.createReadStream(path.join(dir, file));
const text = await streamToString(readStream); const text = await streamToString(readStream);
const unzipped = await gunzip(text); const unzipped = await gunzip(text);
return JSON.parse(unzipped.toString()); return JSON.parse(unzipped.toString());
} }
};

View File

@ -1,12 +1,10 @@
'use strict';
/** /**
* The old verificatuion and budget.json was deprecated in 8.0 * The old verificatuion and budget.json was deprecated in 8.0
*/ */
const get = require('lodash.get'); import get from 'lodash.get';
const noop = require('../../support/helpers').noop; import { noop, size } from '../../support/helpers/index.js';
const size = require('../../support/helpers').size.format; import intel from 'intel';
const log = require('intel').getLogger('sitespeedio.plugin.budget'); const log = intel.getLogger('sitespeedio.plugin.budget');
function getItem(url, type, metric, value, limit, limitType) { function getItem(url, type, metric, value, limit, limitType) {
return { return {
@ -20,20 +18,16 @@ function getItem(url, type, metric, value, limit, limitType) {
} }
function getHelperFunction(metric) { function getHelperFunction(metric) {
if ( if (metric.includes('transferSize') || metric.includes('contentSize')) {
metric.indexOf('transferSize') > -1 || return size.format;
metric.indexOf('contentSize') > -1 } else if (metric.includes('timings')) {
) {
return size;
} else if (metric.indexOf('timings') > -1) {
return function (time) { return function (time) {
return time + ' ms'; return time + ' ms';
}; };
} else return noop; } else return noop;
} }
module.exports = { export function verify(message, result, budgets) {
verify(message, result, budgets) {
const failing = []; const failing = [];
const working = []; const working = [];
// do we have an entry in the budget for this kind of message? // do we have an entry in the budget for this kind of message?
@ -77,14 +71,13 @@ module.exports = {
// group working/failing per URL // group working/failing per URL
if (failing.length > 0) { if (failing.length > 0) {
result.failing[message.url] = result.failing[message.url] result.failing[message.url] = result.failing[message.url]
? result.failing[message.url].concat(failing) ? [...result.failing[message.url], ...failing]
: failing; : failing;
} }
if (working.length > 0) { if (working.length > 0) {
result.working[message.url] = result.working[message.url] result.working[message.url] = result.working[message.url]
? result.working[message.url].concat(working) ? [...result.working[message.url], ...working]
: working; : working;
} }
} }
};

View File

@ -1,13 +1,18 @@
'use strict'; import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { verify as deprecatedVerify } from './deprecatedVerify.js';
import { verify } from './verify.js';
import { writeTap } from './tap.js';
import { writeJunit } from './junit.js';
import { writeJson } from './json.js';
const deprecatedVerify = require('./deprecatedVerify').verify; const log = intel.getLogger('sitespeedio.plugin.budget');
const verify = require('./verify').verify;
const tap = require('./tap'); export default class BudgetPlugin extends SitespeedioPlugin {
const junit = require('./junit'); constructor(options, context, queue) {
const json = require('./json'); super({ name: 'budget', options, context, queue });
const log = require('intel').getLogger('sitespeedio.plugin.budget'); }
module.exports = {
open(context, options) { open(context, options) {
this.options = options; this.options = options;
this.budgetOptions = options.budget || {}; this.budgetOptions = options.budget || {};
@ -22,7 +27,7 @@ module.exports = {
'coach.pageSummary', 'coach.pageSummary',
'axe.pageSummary' 'axe.pageSummary'
]; ];
}, }
processMessage(message, queue) { processMessage(message, queue) {
// if there's no configured budget do nothing // if there's no configured budget do nothing
if (!this.options.budget) { if (!this.options.budget) {
@ -35,7 +40,7 @@ module.exports = {
} }
const budget = this.options.budget.config; const budget = this.options.budget.config;
if (this.budgetTypes.indexOf(message.type) > -1) { if (this.budgetTypes.includes(message.type)) {
// If it doesn't have the new structure of a budget file // If it doesn't have the new structure of a budget file
// use the old verdion // use the old verdion
if (!budget.budget) { if (!budget.budget) {
@ -102,20 +107,30 @@ module.exports = {
case 'sitespeedio.render': { case 'sitespeedio.render': {
if (this.options.budget) { if (this.options.budget) {
if (this.options.budget.output === 'json') { switch (this.options.budget.output) {
json.writeJson(this.result, this.storageManager.getBaseDir()); case 'json': {
} else if (this.options.budget.output === 'tap') { writeJson(this.result, this.storageManager.getBaseDir());
tap.writeTap(this.result, this.storageManager.getBaseDir());
} else if (this.options.budget.output === 'junit') { break;
junit.writeJunit( }
case 'tap': {
writeTap(this.result, this.storageManager.getBaseDir());
break;
}
case 'junit': {
writeJunit(
this.result, this.result,
this.storageManager.getBaseDir(), this.storageManager.getBaseDir(),
this.options this.options
); );
break;
}
}
} }
} }
} }
} }
} }
} }
};

View File

@ -1,11 +1,11 @@
'use strict'; import { writeFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
const fs = require('fs'), import intel from 'intel';
log = require('intel').getLogger('sitespeedio.plugin.budget'), const log = intel.getLogger('sitespeedio.plugin.budget');
path = require('path');
exports.writeJson = function (results, dir) { export function writeJson(results, dir) {
const file = path.join(dir, 'budgetResult.json'); const file = join(dir, 'budgetResult.json');
log.info('Write budget to %s', path.resolve(file)); log.info('Write budget to %s', resolve(file));
fs.writeFileSync(file, JSON.stringify(results, null, 2)); writeFileSync(file, JSON.stringify(results, undefined, 2));
}; }

View File

@ -1,12 +1,14 @@
'use strict'; import { join, resolve } from 'node:path';
import { parse } from 'node:url';
const builder = require('junit-report-builder'), import jrp from 'junit-report-builder';
urlParser = require('url'),
log = require('intel').getLogger('sitespeedio.plugin.budget'),
path = require('path'),
merge = require('lodash.merge');
exports.writeJunit = function (results, dir, options) { import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
import merge from 'lodash.merge';
export function writeJunit(results, dir, options) {
// lets have one suite per URL // lets have one suite per URL
const urls = Object.keys(merge({}, results.failing, results.working)); const urls = Object.keys(merge({}, results.failing, results.working));
@ -14,20 +16,16 @@ exports.writeJunit = function (results, dir, options) {
// The URL can be an alias // The URL can be an alias
let name = url; let name = url;
if (url.startsWith('http')) { if (url.startsWith('http')) {
const parsedUrl = urlParser.parse(url); const parsedUrl = parse(url);
name = url.startsWith('http') ? url : url; name = url.startsWith('http') ? url : url;
parsedUrl.hostname.replace(/\./g, '_') + parsedUrl.hostname.replace(/\./g, '_') +
'.' + '.' +
parsedUrl.path.replace(/\./g, '_').replace(/\//g, '_'); parsedUrl.path.replace(/\./g, '_').replace(/\//g, '_');
} }
const suite = builder const suite = jrp
.testSuite() .testSuite()
.name( .name(options.budget.friendlyName || 'sitespeed.io' + '.' + name);
options.budget.friendlyName
? options.budget.friendlyName
: 'sitespeed.io' + '.' + name
);
if (results.failing[url]) { if (results.failing[url]) {
for (const result of results.failing[url]) { for (const result of results.failing[url]) {
@ -65,7 +63,7 @@ exports.writeJunit = function (results, dir, options) {
} }
} }
} }
const file = path.join(dir, 'junit.xml'); const file = join(dir, 'junit.xml');
log.info('Write junit budget to %s', path.resolve(file)); log.info('Write junit budget to %s', resolve(file));
builder.writeTo(file); jrp.writeTo(file);
}; }

View File

@ -1,12 +1,11 @@
'use strict'; import path from 'node:path';
import fs from 'node:fs';
import { EOL } from 'node:os';
import tap from 'tape';
import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.budget');
const tap = require('tape'), export function writeTap(results, dir) {
fs = require('fs'),
log = require('intel').getLogger('sitespeedio.plugin.budget'),
path = require('path'),
EOL = require('os').EOL;
exports.writeTap = function (results, dir) {
const file = path.join(dir, 'budget.tap'); const file = path.join(dir, 'budget.tap');
log.info('Write budget to %s', path.resolve(file)); log.info('Write budget to %s', path.resolve(file));
const tapOutput = fs.createWriteStream(file); const tapOutput = fs.createWriteStream(file);
@ -34,4 +33,4 @@ exports.writeTap = function (results, dir) {
} }
} }
} }
}; }

View File

@ -1,13 +1,12 @@
'use strict';
/** /**
* The old verificatuion and budget.json was deprecated in 8.0 * The old verificatuion and budget.json was deprecated in 8.0
*/ */
const get = require('lodash.get'); import get from 'lodash.get';
const merge = require('lodash.merge'); import merge from 'lodash.merge';
const log = require('intel').getLogger('sitespeedio.plugin.budget'); import intel from 'intel';
const friendlyNames = require('../../support/friendlynames'); const log = intel.getLogger('sitespeedio.plugin.budget');
const time = require('../../support/helpers/time'); import friendlyNames from '../../support/friendlynames.js';
import { time } from '../../support/helpers/time.js';
function getItem(friendlyName, type, metric, value, limit, limitType) { function getItem(friendlyName, type, metric, value, limit, limitType) {
return { return {
@ -22,12 +21,10 @@ function getItem(friendlyName, type, metric, value, limit, limitType) {
}; };
} }
module.exports = { export function verify(message, result, budgets, alias) {
verify(message, result, budgets, alias) {
// Let us merge the specific configuration for this URL together // Let us merge the specific configuration for this URL together
// with the generic one. In the future we can fine tune this, since // with the generic one. In the future we can fine tune this, since
// we keep all metrics within a specific URL // we keep all metrics within a specific URL
// If we have a match for the alias, use that first, if not use the URL and // If we have a match for the alias, use that first, if not use the URL and
// then the specific one // then the specific one
let budgetSetupSpecificForThisURL; let budgetSetupSpecificForThisURL;
@ -166,19 +163,14 @@ module.exports = {
} }
// group working/failing per URL // group working/failing per URL
if (failing.length > 0) { if (failing.length > 0) {
result.failing[alias || message.url] = result.failing[ result.failing[alias || message.url] = result.failing[alias || message.url]
alias || message.url ? [...result.failing[alias || message.url], ...failing]
]
? result.failing[alias || message.url].concat(failing)
: failing; : failing;
} }
if (working.length > 0) { if (working.length > 0) {
result.working[alias || message.url] = result.working[ result.working[alias || message.url] = result.working[alias || message.url]
alias || message.url ? [...result.working[alias || message.url], ...working]
]
? result.working[alias || message.url].concat(working)
: working; : working;
} }
} }
};

View File

@ -1,11 +1,11 @@
'use strict'; import forEach from 'lodash.foreach';
import { pushGroupStats, setStatsSummary } from '../../support/statsHelpers.js';
const forEach = require('lodash.foreach'), export class CoachAggregator {
statsHelpers = require('../../support/statsHelpers'); constructor() {
this.statsPerCategory = {};
module.exports = { this.groups = {};
statsPerCategory: {}, }
groups: {},
addToAggregate(coachData, group) { addToAggregate(coachData, group) {
if (this.groups[group] === undefined) { if (this.groups[group] === undefined) {
@ -13,7 +13,7 @@ module.exports = {
} }
// push the total score // push the total score
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerCategory, this.statsPerCategory,
this.groups[group], this.groups[group],
'score', 'score',
@ -25,7 +25,7 @@ module.exports = {
return; return;
} }
// Push the score per category // Push the score per category
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerCategory, this.statsPerCategory,
this.groups[group], this.groups[group],
[categoryName, 'score'], [categoryName, 'score'],
@ -33,7 +33,7 @@ module.exports = {
); );
forEach(category.adviceList, (advice, adviceName) => { forEach(category.adviceList, (advice, adviceName) => {
statsHelpers.pushGroupStats( pushGroupStats(
this.statsPerCategory, this.statsPerCategory,
this.groups[group], this.groups[group],
[categoryName, adviceName], [categoryName, adviceName],
@ -41,10 +41,10 @@ module.exports = {
); );
}); });
}); });
}, }
summarize() { summarize() {
if (Object.keys(this.statsPerCategory).length === 0) { if (Object.keys(this.statsPerCategory).length === 0) {
return undefined; return;
} }
const summary = { const summary = {
@ -57,16 +57,16 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]); summary.groups[group] = this.summarizePerObject(this.groups[group]);
} }
return summary; return summary;
}, }
summarizePerObject(type) { summarizePerObject(type) {
return Object.keys(type).reduce((summary, categoryName) => { return Object.keys(type).reduce((summary, categoryName) => {
if (categoryName === 'score') { if (categoryName === 'score') {
statsHelpers.setStatsSummary(summary, 'score', type[categoryName]); setStatsSummary(summary, 'score', type[categoryName]);
} else { } else {
const categoryData = {}; const categoryData = {};
forEach(type[categoryName], (stats, name) => { forEach(type[categoryName], (stats, name) => {
statsHelpers.setStatsSummary(categoryData, name, stats); setStatsSummary(categoryData, name, stats);
}); });
summary[categoryName] = categoryData; summary[categoryName] = categoryData;
@ -75,4 +75,4 @@ module.exports = {
return summary; return summary;
}, {}); }, {});
} }
}; }

View File

@ -1,6 +1,7 @@
'use strict'; import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const aggregator = require('./aggregator'); import { CoachAggregator } from './aggregator.js';
const log = require('intel').getLogger('plugin.coach'); import intel from 'intel';
const log = intel.getLogger('plugin.coach');
const DEFAULT_METRICS_RUN = []; const DEFAULT_METRICS_RUN = [];
@ -24,10 +25,15 @@ const DEFAULT_METRICS_PAGESUMMARY = [
'advice.info.localStorageSize' 'advice.info.localStorageSize'
]; ];
module.exports = { export default class CoachPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'coach', options, context, queue });
}
open(context, options) { open(context, options) {
this.options = options; this.options = options;
this.make = context.messageMaker('coach').make; this.make = context.messageMaker('coach').make;
this.coachAggregator = new CoachAggregator();
context.filterRegistry.registerFilterForType( context.filterRegistry.registerFilterForType(
DEFAULT_METRICS_SUMMARY, DEFAULT_METRICS_SUMMARY,
@ -41,7 +47,7 @@ module.exports = {
DEFAULT_METRICS_PAGESUMMARY, DEFAULT_METRICS_PAGESUMMARY,
'coach.pageSummary' 'coach.pageSummary'
); );
}, }
processMessage(message, queue) { processMessage(message, queue) {
const make = this.make; const make = this.make;
switch (message.type) { switch (message.type) {
@ -60,13 +66,13 @@ module.exports = {
); );
} }
aggregator.addToAggregate(message.data, message.group); this.coachAggregator.addToAggregate(message.data, message.group);
break; break;
} }
case 'sitespeedio.summarize': { case 'sitespeedio.summarize': {
log.debug('Generate summary metrics from the Coach'); log.debug('Generate summary metrics from the Coach');
let summary = aggregator.summarize(); let summary = this.coachAggregator.summarize();
if (summary) { if (summary) {
for (let group of Object.keys(summary.groups)) { for (let group of Object.keys(summary.groups)) {
queue.postMessage( queue.postMessage(
@ -78,4 +84,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -1,17 +1,22 @@
'use strict'; import { extname } from 'node:path';
import merge from 'lodash.merge';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const path = require('path'); const log = intel.getLogger('sitespeedio.plugin.crawler');
const merge = require('lodash.merge'); import Crawler from 'simplecrawler';
const log = require('intel').getLogger('sitespeedio.plugin.crawler'); import { throwIfMissing } from '../../support/util';
const Crawler = require('simplecrawler'); import { toArray } from '../../support/util';
const throwIfMissing = require('../../support/util').throwIfMissing;
const toArray = require('../../support/util').toArray;
const defaultOptions = { const defaultOptions = {
depth: 3 depth: 3
}; };
module.exports = { export default class CrawlerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'crawler', options, context, queue });
}
open(context, options) { open(context, options) {
throwIfMissing(options.crawler, ['depth'], 'crawler'); throwIfMissing(options.crawler, ['depth'], 'crawler');
this.options = merge({}, defaultOptions, options.crawler); this.options = merge({}, defaultOptions, options.crawler);
@ -22,10 +27,9 @@ module.exports = {
this.basicAuth = options.browsertime this.basicAuth = options.browsertime
? options.browsertime.basicAuth ? options.browsertime.basicAuth
: undefined; : undefined;
this.cookie = options.browsertime.cookie this.cookie = options.browsertime.cookie || undefined;
? options.browsertime.cookie }
: undefined;
},
processMessage(message, queue) { processMessage(message, queue) {
const make = this.make; const make = this.make;
if (message.type === 'url' && message.source !== 'crawler') { if (message.type === 'url' && message.source !== 'crawler') {
@ -66,9 +70,9 @@ module.exports = {
} }
crawler.addFetchCondition(queueItem => { crawler.addFetchCondition(queueItem => {
const extension = path.extname(queueItem.path); const extension = extname(queueItem.path);
// Don't try to download these, based on file name. // Don't try to download these, based on file name.
if (['png', 'jpg', 'gif', 'pdf'].indexOf(extension) !== -1) { if (['png', 'jpg', 'gif', 'pdf'].includes(extension)) {
return false; return false;
} }
@ -112,8 +116,11 @@ module.exports = {
crawler.userAgent = this.userAgent; crawler.userAgent = this.userAgent;
} }
crawler.on('fetchconditionerror', (queueItem, err) => { crawler.on('fetchconditionerror', (queueItem, error) => {
log.warn('An error occurred in the fetchCondition callback: %s', err); log.warn(
'An error occurred in the fetchCondition callback: %s',
error
);
}); });
crawler.on('fetchredirect', (queueItem, parsedURL, response) => { crawler.on('fetchredirect', (queueItem, parsedURL, response) => {
@ -157,4 +164,4 @@ module.exports = {
}); });
} }
} }
}; }

View File

@ -1,29 +0,0 @@
module.exports = {
key: {
describe:
'You need to use a key to get data from CrUx. Get the key from https://developers.google.com/web/tools/chrome-user-experience-report/api/guides/getting-started#APIKey',
group: 'CrUx'
},
enable: {
default: true,
describe:
'Enable the CrUx plugin. This is on by defauly but you also need the Crux key. If you chose to disable it with this key, set this to false and you can still use the CrUx key in your configuration.',
group: 'CrUx'
},
formFactor: {
default: 'ALL',
type: 'string',
choices: ['ALL', 'DESKTOP', 'PHONE', 'TABLET'],
describe:
'A form factor is the type of device on which a user visits a website.',
group: 'CrUx'
},
collect: {
default: 'ALL',
type: 'string',
choices: ['ALL', 'URL', 'ORIGIN'],
describe:
'Choose what data to collect. URL is data for a specific URL, ORIGIN for the domain and ALL for both of them',
group: 'CrUx'
}
};

View File

@ -1,14 +1,19 @@
'use strict'; import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import intel from 'intel';
import merge from 'lodash.merge';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { throwIfMissing } from '../../support/util.js';
import { repackage } from './repackage.js';
import { send } from './send.js';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const log = intel.getLogger('plugin.crux');
const defaultConfig = {}; const defaultConfig = {};
const log = require('intel').getLogger('plugin.crux');
const merge = require('lodash.merge');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
const send = require('./send');
const path = require('path');
const repackage = require('./repackage');
const fs = require('fs');
const DEFAULT_METRICS_PAGESUMMARY = [ const DEFAULT_METRICS_PAGESUMMARY = [
'loadingExperience.*.FIRST_CONTENTFUL_PAINT_MS.*', 'loadingExperience.*.FIRST_CONTENTFUL_PAINT_MS.*',
@ -33,10 +38,11 @@ function wait(ms) {
const CRUX_WAIT_TIME = 300; const CRUX_WAIT_TIME = 300;
module.exports = { export default class CruxPlugin extends SitespeedioPlugin {
name() { constructor(options, context, queue) {
return path.basename(__dirname); super({ name: 'crux', options, context, queue });
}, }
open(context, options) { open(context, options) {
this.make = context.messageMaker('crux').make; this.make = context.messageMaker('crux').make;
this.options = merge({}, defaultConfig, options.crux); this.options = merge({}, defaultConfig, options.crux);
@ -46,10 +52,7 @@ module.exports = {
this.formFactors = Array.isArray(this.options.formFactor) this.formFactors = Array.isArray(this.options.formFactor)
? this.options.formFactor ? this.options.formFactor
: [this.options.formFactor]; : [this.options.formFactor];
this.pug = fs.readFileSync( this.pug = readFileSync(resolve(__dirname, 'pug', 'index.pug'), 'utf8');
path.resolve(__dirname, 'pug', 'index.pug'),
'utf8'
);
if (this.options.collect === 'ALL' || this.options.collect === 'URL') { if (this.options.collect === 'ALL' || this.options.collect === 'URL') {
context.filterRegistry.registerFilterForType( context.filterRegistry.registerFilterForType(
@ -63,7 +66,7 @@ module.exports = {
'crux.summary' 'crux.summary'
); );
} }
}, }
async processMessage(message, queue) { async processMessage(message, queue) {
if (this.options.enable === true) { if (this.options.enable === true) {
const make = this.make; const make = this.make;
@ -101,7 +104,7 @@ module.exports = {
this.testedOrigins[group] = true; this.testedOrigins[group] = true;
log.info(`Get CrUx data for domain ${group}`); log.info(`Get CrUx data for domain ${group}`);
for (let formFactor of this.formFactors) { for (let formFactor of this.formFactors) {
originResult.originLoadingExperience[formFactor] = await send.get( originResult.originLoadingExperience[formFactor] = await send(
url, url,
this.options.key, this.options.key,
formFactor, formFactor,
@ -127,7 +130,7 @@ module.exports = {
originResult.originLoadingExperience[formFactor] = repackage( originResult.originLoadingExperience[formFactor] = repackage(
originResult.originLoadingExperience[formFactor] originResult.originLoadingExperience[formFactor]
); );
} catch (e) { } catch {
log.error( log.error(
'Could not repackage the JSON for origin from CrUx, is it broken? %j', 'Could not repackage the JSON for origin from CrUx, is it broken? %j',
originResult.originLoadingExperience[formFactor] originResult.originLoadingExperience[formFactor]
@ -146,7 +149,7 @@ module.exports = {
log.info(`Get CrUx data for url ${url}`); log.info(`Get CrUx data for url ${url}`);
const urlResult = { loadingExperience: {} }; const urlResult = { loadingExperience: {} };
for (let formFactor of this.formFactors) { for (let formFactor of this.formFactors) {
urlResult.loadingExperience[formFactor] = await send.get( urlResult.loadingExperience[formFactor] = await send(
url, url,
this.options.key, this.options.key,
formFactor, formFactor,
@ -172,7 +175,7 @@ module.exports = {
urlResult.loadingExperience[formFactor] = repackage( urlResult.loadingExperience[formFactor] = repackage(
urlResult.loadingExperience[formFactor] urlResult.loadingExperience[formFactor]
); );
} catch (e) { } catch {
log.error( log.error(
'Could not repackage the JSON from CrUx, is it broken? %j', 'Could not repackage the JSON from CrUx, is it broken? %j',
urlResult.loadingExperience[formFactor] urlResult.loadingExperience[formFactor]
@ -208,11 +211,5 @@ module.exports = {
} }
} }
} }
},
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
} }
}; }

View File

@ -1,6 +1,4 @@
'use strict'; export function repackage(cruxResult) {
module.exports = function (cruxResult) {
const result = {}; const result = {};
if (cruxResult.record.metrics.first_contentful_paint) { if (cruxResult.record.metrics.first_contentful_paint) {
result.FIRST_CONTENTFUL_PAINT_MS = { result.FIRST_CONTENTFUL_PAINT_MS = {
@ -77,4 +75,4 @@ module.exports = function (cruxResult) {
result.data = cruxResult; result.data = cruxResult;
return result; return result;
}; }

View File

@ -1,10 +1,8 @@
'use strict'; import { request as _request } from 'node:https';
import intel from 'intel';
const log = intel.getLogger('plugin.crux');
const https = require('https'); export async function send(url, key, formFactor, shouldWeTestTheURL) {
const log = require('intel').getLogger('plugin.crux');
module.exports = {
async get(url, key, formFactor, shouldWeTestTheURL) {
let data = shouldWeTestTheURL ? { url } : { origin: url }; let data = shouldWeTestTheURL ? { url } : { origin: url };
if (formFactor !== 'ALL') { if (formFactor !== 'ALL') {
data.formFactor = formFactor; data.formFactor = formFactor;
@ -13,7 +11,7 @@ module.exports = {
// Return new promise // Return new promise
return new Promise(function (resolve, reject) { return new Promise(function (resolve, reject) {
// Do async job // Do async job
const req = https.request( const request = _request(
{ {
host: 'chromeuxreport.googleapis.com', host: 'chromeuxreport.googleapis.com',
port: 443, port: 443,
@ -42,8 +40,7 @@ module.exports = {
); );
} }
); );
req.write(data); request.write(data);
req.end(); request.end();
}); });
} }
};

View File

@ -1,11 +1,13 @@
'use strict'; import { parse } from 'node:url';
const Stats = require('fast-stats').Stats, import { Stats } from 'fast-stats';
urlParser = require('url'), import intel from 'intel';
log = require('intel').getLogger('sitespeedio.plugin.domains'), import isEmpty from 'lodash.isempty';
statsHelpers = require('../../support/statsHelpers'), import reduce from 'lodash.reduce';
isEmpty = require('lodash.isempty'),
reduce = require('lodash.reduce'); import { summarizeStats } from '../../support/statsHelpers.js';
const log = intel.getLogger('sitespeedio.plugin.domains');
const timingNames = [ const timingNames = [
'blocked', 'blocked',
@ -18,7 +20,7 @@ const timingNames = [
]; ];
function parseDomainName(url) { function parseDomainName(url) {
return urlParser.parse(url).hostname; return parse(url).hostname;
} }
function getDomain(domainName) { function getDomain(domainName) {
@ -45,16 +47,16 @@ function calc(domains) {
domainName domainName
}; };
const stats = statsHelpers.summarizeStats(domainStats.totalTime); const stats = summarizeStats(domainStats.totalTime);
if (!isEmpty(stats)) { if (!isEmpty(stats)) {
domainSummary.totalTime = stats; domainSummary.totalTime = stats;
} }
timingNames.forEach(name => { for (const name of timingNames) {
const stats = statsHelpers.summarizeStats(domainStats[name]); const stats = summarizeStats(domainStats[name]);
if (!isEmpty(stats)) { if (!isEmpty(stats)) {
domainSummary[name] = stats; domainSummary[name] = stats;
} }
}); }
summary[domainName] = domainSummary; summary[domainName] = domainSummary;
return summary; return summary;
@ -66,12 +68,15 @@ function calc(domains) {
function isValidTiming(timing) { function isValidTiming(timing) {
// The HAR format uses -1 to indicate invalid/missing timings // The HAR format uses -1 to indicate invalid/missing timings
// isNan see https://github.com/sitespeedio/sitespeed.io/issues/2159 // isNan see https://github.com/sitespeedio/sitespeed.io/issues/2159
return typeof timing === 'number' && timing !== -1 && !isNaN(timing); return typeof timing === 'number' && timing !== -1 && !Number.isNaN(timing);
}
export class DomainsAggregator {
constructor() {
this.domains = {};
this.groups = {};
} }
module.exports = {
groups: {},
domains: {},
addToAggregate(har, url) { addToAggregate(har, url) {
const mainDomain = parseDomainName(url); const mainDomain = parseDomainName(url);
if (this.groups[mainDomain] === undefined) { if (this.groups[mainDomain] === undefined) {
@ -79,10 +84,10 @@ module.exports = {
} }
const firstPageId = har.log.pages[0].id; const firstPageId = har.log.pages[0].id;
har.log.entries.forEach(entry => { for (const entry of har.log.entries) {
if (entry.pageref !== firstPageId) { if (entry.pageref !== firstPageId) {
// Only pick the first request out of multiple runs. // Only pick the first request out of multiple runs.
return; continue;
} }
const domainName = parseDomainName(entry.request.url), const domainName = parseDomainName(entry.request.url),
@ -101,18 +106,18 @@ module.exports = {
log.debug('Missing time from har entry for url: ' + entry.request.url); log.debug('Missing time from har entry for url: ' + entry.request.url);
} }
timingNames.forEach(name => { for (const name of timingNames) {
const timing = entry.timings[name]; const timing = entry.timings[name];
if (isValidTiming(timing)) { if (isValidTiming(timing)) {
domain[name].push(timing); domain[name].push(timing);
groupDomain[name].push(timing); groupDomain[name].push(timing);
} }
}); }
this.domains[domainName] = domain; this.domains[domainName] = domain;
this.groups[mainDomain][domainName] = groupDomain; this.groups[mainDomain][domainName] = groupDomain;
}); }
}, }
summarize() { summarize() {
const summary = { const summary = {
groups: { groups: {
@ -126,4 +131,4 @@ module.exports = {
return summary; return summary;
} }
}; }

View File

@ -1,14 +1,19 @@
'use strict'; import isEmpty from 'lodash.isempty';
const isEmpty = require('lodash.isempty'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const aggregator = require('./aggregator'); import { DomainsAggregator } from './aggregator.js';
export default class DomainsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'domains', options, context, queue });
}
module.exports = {
open(context) { open(context) {
this.make = context.messageMaker('domains').make; this.make = context.messageMaker('domains').make;
// '*.requestCounts, 'domains.summary' // '*.requestCounts, 'domains.summary'
context.filterRegistry.registerFilterForType([], 'domains.summary'); context.filterRegistry.registerFilterForType([], 'domains.summary');
this.browsertime = false; this.browsertime = false;
}, this.domainsAggregator = new DomainsAggregator();
}
processMessage(message, queue) { processMessage(message, queue) {
const make = this.make; const make = this.make;
switch (message.type) { switch (message.type) {
@ -18,20 +23,20 @@ module.exports = {
} }
case 'browsertime.har': { case 'browsertime.har': {
aggregator.addToAggregate(message.data, message.url); this.domainsAggregator.addToAggregate(message.data, message.url);
break; break;
} }
case 'webpagetest.har': { case 'webpagetest.har': {
// Only collect WebPageTest data if we don't run Browsertime // Only collect WebPageTest data if we don't run Browsertime
if (this.browsertime === false) { if (this.browsertime === false) {
aggregator.addToAggregate(message.data, message.url); this.domainsAggregator.addToAggregate(message.data, message.url);
} }
break; break;
} }
case 'sitespeedio.summarize': { case 'sitespeedio.summarize': {
const summary = aggregator.summarize(); const summary = this.domainsAggregator.summarize();
if (!isEmpty(summary)) { if (!isEmpty(summary)) {
for (let group of Object.keys(summary.groups)) { for (let group of Object.keys(summary.groups)) {
queue.postMessage( queue.postMessage(
@ -43,4 +48,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -1,26 +1,26 @@
'use strict'; import { relative, join, resolve, sep } from 'node:path';
import { statSync, remove } from 'fs-extra';
const fs = require('fs-extra'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const path = require('path'); import readdir from 'recursive-readdir';
const readdir = require('recursive-readdir');
// Documentation of @google-cloud/storage: https://cloud.google.com/nodejs/docs/reference/storage/2.3.x/Bucket#upload // Documentation of @google-cloud/storage: https://cloud.google.com/nodejs/docs/reference/storage/2.3.x/Bucket#upload
const { Storage } = require('@google-cloud/storage'); import { Storage } from '@google-cloud/storage';
import intel from 'intel';
import { throwIfMissing } from '../../support/util';
const log = require('intel').getLogger('sitespeedio.plugin.gcs'); const log = intel.getLogger('sitespeedio.plugin.gcs');
const throwIfMissing = require('../../support/util').throwIfMissing;
async function uploadLatestFiles(dir, gcsOptions, prefix) { function ignoreDirectories(file, stats) {
function ignoreDirs(file, stats) {
return stats.isDirectory(); return stats.isDirectory();
} }
async function uploadLatestFiles(dir, gcsOptions, prefix) {
const storage = new Storage({ const storage = new Storage({
projectId: gcsOptions.projectId, projectId: gcsOptions.projectId,
keyFilename: gcsOptions.key keyFilename: gcsOptions.key
}); });
const bucket = storage.bucket(gcsOptions.bucketname); const bucket = storage.bucket(gcsOptions.bucketname);
const files = await readdir(dir, [ignoreDirs]); const files = await readdir(dir, [ignoreDirectories]);
const promises = []; const promises = [];
for (let file of files) { for (let file of files) {
@ -41,7 +41,7 @@ async function upload(dir, gcsOptions, prefix) {
const bucket = storage.bucket(gcsOptions.bucketname); const bucket = storage.bucket(gcsOptions.bucketname);
for (let file of files) { for (let file of files) {
const stats = fs.statSync(file); const stats = statSync(file);
if (stats.isFile()) { if (stats.isFile()) {
promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir)); promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir));
@ -60,10 +60,10 @@ async function uploadFile(
baseDir, baseDir,
noCacheTime noCacheTime
) { ) {
const subPath = path.relative(baseDir, file); const subPath = relative(baseDir, file);
const fileName = path.join(gcsOptions.path || prefix, subPath); const fileName = join(gcsOptions.path || prefix, subPath);
const params = { const parameters = {
public: !!gcsOptions.public, public: !!gcsOptions.public,
destination: fileName, destination: fileName,
resumable: false, resumable: false,
@ -71,23 +71,26 @@ async function uploadFile(
gzip: !!gcsOptions.gzip, gzip: !!gcsOptions.gzip,
metadata: { metadata: {
metadata: { metadata: {
cacheControl: 'public, max-age=' + noCacheTime ? 0 : 31536000 cacheControl: 'public, max-age=' + noCacheTime ? 0 : 31_536_000
} }
} }
}; };
return bucket.upload(file, params); return bucket.upload(file, parameters);
}
export default class GcsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'gcs', options, context, queue });
} }
module.exports = {
open(context, options) { open(context, options) {
this.gcsOptions = options.gcs; this.gcsOptions = options.gcs;
this.options = options; this.options = options;
this.make = context.messageMaker('gcs').make; this.make = context.messageMaker('gcs').make;
throwIfMissing(this.gcsOptions, ['bucketname'], 'gcs'); throwIfMissing(this.gcsOptions, ['bucketname'], 'gcs');
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
}, }
async processMessage(message, queue) { async processMessage(message, queue) {
if (message.type === 'sitespeedio.setup') { if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the GCS plugin is alive // Let other plugins know that the GCS plugin is alive
@ -108,9 +111,9 @@ module.exports = {
this.storageManager.getStoragePrefix() this.storageManager.getStoragePrefix()
); );
if (this.options.copyLatestFilesToBase) { if (this.options.copyLatestFilesToBase) {
const rootPath = path.resolve(baseDir, '..'); const rootPath = resolve(baseDir, '..');
const dirsAsArray = rootPath.split(path.sep); const directoriesAsArray = rootPath.split(sep);
const rootName = dirsAsArray.slice(-1)[0]; const rootName = directoriesAsArray.slice(-1)[0];
await uploadLatestFiles(rootPath, gcsOptions, rootName); await uploadLatestFiles(rootPath, gcsOptions, rootName);
} }
log.info('Finished upload to Google Cloud Storage'); log.info('Finished upload to Google Cloud Storage');
@ -120,18 +123,18 @@ module.exports = {
); );
} }
if (gcsOptions.removeLocalResult) { if (gcsOptions.removeLocalResult) {
await fs.remove(baseDir); await remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`); log.debug(`Removed local files and directory ${baseDir}`);
} else { } else {
log.debug( log.debug(
`Local result files and directories are stored in ${baseDir}` `Local result files and directories are stored in ${baseDir}`
); );
} }
} catch (e) { } catch (error) {
queue.postMessage(make('error', e)); queue.postMessage(make('error', error));
log.error('Could not upload to Google Cloud Storage', e); log.error('Could not upload to Google Cloud Storage', error);
} }
queue.postMessage(make('gcs.finished')); queue.postMessage(make('gcs.finished'));
} }
} }
}; }

View File

@ -1,37 +0,0 @@
module.exports = {
host: {
describe: 'The Grafana host used when sending annotations.',
group: 'Grafana'
},
port: {
default: 80,
describe: 'The Grafana port used when sending annotations to Grafana.',
group: 'Grafana'
},
auth: {
describe:
'The Grafana auth/bearer value used when sending annotations to Grafana. If you do not set Bearer/Auth, Bearer is automatically set. See http://docs.grafana.org/http_api/auth/#authentication-api',
group: 'Grafana'
},
annotationTitle: {
describe: 'Add a title to the annotation sent for a run.',
group: 'Grafana'
},
annotationMessage: {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Grafana'
},
annotationTag: {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --grafana.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Grafana'
},
annotationScreenshot: {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Grafana'
}
};

View File

@ -1,24 +1,13 @@
'use strict'; import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const path = require('path');
const sendAnnotations = require('./send-annotation');
const tsdbUtil = require('../../support/tsdbUtil');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
module.exports = { import { send } from './send-annotation.js';
name() { import { toSafeKey } from '../../support/tsdbUtil.js';
return path.basename(__dirname); import { throwIfMissing } from '../../support/util.js';
},
/** export default class GrafanaPlugin extends SitespeedioPlugin {
* Define `yargs` options with their respective default values. When displayed by the CLI help message constructor(options, context, queue) {
* all options are namespaced by its plugin name. super({ name: 'grafana', options, context, queue });
* }
* @return {Object<string, require('yargs').Options} an object mapping the name of the option and its yargs configuration
*/
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
open(context, options) { open(context, options) {
throwIfMissing(options.grafana, ['host', 'port'], 'grafana'); throwIfMissing(options.grafana, ['host', 'port'], 'grafana');
@ -31,7 +20,7 @@ module.exports = {
this.make = context.messageMaker('grafana').make; this.make = context.messageMaker('grafana').make;
this.alias = {}; this.alias = {};
this.wptExtras = {}; this.wptExtras = {};
}, }
processMessage(message, queue) { processMessage(message, queue) {
if (message.type === 'webpagetest.pageSummary') { if (message.type === 'webpagetest.pageSummary') {
@ -39,9 +28,7 @@ module.exports = {
this.wptExtras[message.url].webPageTestResultURL = this.wptExtras[message.url].webPageTestResultURL =
message.data.data.summary; message.data.data.summary;
this.wptExtras[message.url].connectivity = message.connectivity; this.wptExtras[message.url].connectivity = message.connectivity;
this.wptExtras[message.url].location = tsdbUtil.toSafeKey( this.wptExtras[message.url].location = toSafeKey(message.location);
message.location
);
} }
if (this.messageTypesToFireAnnotations.includes(message.type)) { if (this.messageTypesToFireAnnotations.includes(message.type)) {
this.receivedTypesThatFireAnnotations[message.url] this.receivedTypesThatFireAnnotations[message.url]
@ -50,28 +37,45 @@ module.exports = {
} }
// First catch if we are running Browsertime and/or WebPageTest // First catch if we are running Browsertime and/or WebPageTest
if (message.type === 'browsertime.setup') { switch (message.type) {
case 'browsertime.setup': {
this.usingBrowsertime = true; this.usingBrowsertime = true;
this.messageTypesToFireAnnotations.push('browsertime.pageSummary'); this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
} else if (message.type === 'webpagetest.setup') {
break;
}
case 'webpagetest.setup': {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary'); this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
} else if (message.type === 'sitespeedio.setup') {
break;
}
case 'sitespeedio.setup': {
// Let other plugins know that the Grafana plugin is alive // Let other plugins know that the Grafana plugin is alive
queue.postMessage(this.make('grafana.setup')); queue.postMessage(this.make('grafana.setup'));
} else if (message.type === 'influxdb.setup') {
break;
}
case 'influxdb.setup': {
// Default we use Graphite config, else use influxdb // Default we use Graphite config, else use influxdb
this.tsdbType = 'influxdb'; this.tsdbType = 'influxdb';
} else if (message.type === 'browsertime.config') {
break;
}
case 'browsertime.config': {
if (message.data.screenshot) { if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot; this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType; this.screenshotType = message.data.screenshotType;
} }
} else if (message.type === 'browsertime.browser') {
break;
}
case 'browsertime.browser': {
this.browser = message.data.browser; this.browser = message.data.browser;
} else if (
message.type === 'webpagetest.browser' && break;
!this.usingBrowsertime }
) { default: {
if (message.type === 'webpagetest.browser' && !this.usingBrowsertime) {
// We are only interested in WebPageTest browser if we run it standalone // We are only interested in WebPageTest browser if we run it standalone
this.browser = message.data.browser; this.browser = message.data.browser;
} else if (message.type === 'browsertime.alias') { } else if (message.type === 'browsertime.alias') {
@ -86,7 +90,7 @@ module.exports = {
this.alias[message.url] this.alias[message.url]
); );
this.receivedTypesThatFireAnnotations[message.url] = 0; this.receivedTypesThatFireAnnotations[message.url] = 0;
return sendAnnotations.send( return send(
message.url, message.url,
message.group, message.group,
absolutePagePath, absolutePagePath,
@ -101,15 +105,7 @@ module.exports = {
this.options this.options
); );
} }
},
/**
* At the time of this writing, this property's usage could be verified only by the CLI portion of the codebase.
* Instead of introducting a breaking change in the plugin interface, this is kept.
*
* @todo Inspect the code base and plugin dependencies to ensure this property can be removed (if necessary)
*/
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
} }
}; }
}
}

View File

@ -1,14 +1,21 @@
'use strict'; import http from 'node:http';
const http = require('http'); import https from 'node:https';
const https = require('https'); import { createRequire } from 'node:module';
const log = require('intel').getLogger('sitespeedio.plugin.grafana');
const tsdbUtil = require('../../support/tsdbUtil');
const annotationsHelper = require('../../support/annotationsHelper');
const util = require('../../support/util');
const packageInfo = require('../../../package');
module.exports = { import intel from 'intel';
send(
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
import {
getTagsAsArray,
getAnnotationMessage
} from '../../support/annotationsHelper.js';
import { toArray } from '../../support/util.js';
const require = createRequire(import.meta.url);
const version = require('../../../package.json').version;
const log = intel.getLogger('sitespeedio.plugin.grafana');
export function send(
url, url,
group, group,
absolutePagePath, absolutePagePath,
@ -26,12 +33,11 @@ module.exports = {
// templates to choose which annotations that will be showed. // templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template // That's why we need to send tags that matches the template
// variables in Grafana. // variables in Grafana.
const connectivity = tsdbUtil.getConnectivity(options); const connectivity = getConnectivity(options);
const browser = options.browser; const browser = options.browser;
// Hmm, here we have hardcoded Graphite ... // Hmm, here we have hardcoded Graphite ...
const namespace = options.graphite.namespace.split('.'); const namespace = options.graphite.namespace.split('.');
const urlAndGroup = tsdbUtil const urlAndGroup = getURLAndGroup(
.getURLAndGroup(
options, options,
group, group,
url, url,
@ -39,8 +45,7 @@ module.exports = {
? options.graphite.includeQueryParams ? options.graphite.includeQueryParams
: options.influxdb.includeQueryParams, : options.influxdb.includeQueryParams,
alias alias
) ).split('.');
.split('.');
const tags = [ const tags = [
connectivity, connectivity,
@ -54,20 +59,19 @@ module.exports = {
if (options.slug && options.slug !== urlAndGroup[0]) { if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug); tags.push(options.slug);
} }
const extraTags = util.toArray(options.grafana.annotationTag); const extraTags = toArray(options.grafana.annotationTag);
// We got some extra tag(s) from the user, let us add them to the annotation // We got some extra tag(s) from the user, let us add them to the annotation
if (extraTags.length > 0) { if (extraTags.length > 0) {
tags.push(...extraTags); tags.push(...extraTags);
} }
if (webPageTestExtraData) { if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity); tags.push(webPageTestExtraData.connectivity, webPageTestExtraData.location);
tags.push(webPageTestExtraData.location);
} }
const tagsArray = annotationsHelper.getTagsAsArray(tags); const tagsArray = getTagsAsArray(tags);
const message = annotationsHelper.getAnnotationMessage( const message = getAnnotationMessage(
absolutePagePath, absolutePagePath,
screenShotsEnabledInBrowsertime, screenShotsEnabledInBrowsertime,
screenshotType, screenshotType,
@ -78,7 +82,7 @@ module.exports = {
options options
); );
let what = let what =
packageInfo.version + version +
(browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : ''); (browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : '');
if (options.grafana.annotationTitle) { if (options.grafana.annotationTitle) {
what = options.grafana.annotationTitle; what = options.grafana.annotationTitle;
@ -98,20 +102,17 @@ module.exports = {
// If Grafana is behind auth, use it! // If Grafana is behind auth, use it!
if (options.grafana.auth) { if (options.grafana.auth) {
log.debug('Using auth for Grafana'); log.debug('Using auth for Grafana');
if ( postOptions.headers.Authorization =
options.grafana.auth.startsWith('Bearer') || options.grafana.auth.startsWith('Bearer') ||
options.grafana.auth.startsWith('Basic') options.grafana.auth.startsWith('Basic')
) { ? options.grafana.auth
postOptions.headers.Authorization = options.grafana.auth; : 'Bearer ' + options.grafana.auth;
} else {
postOptions.headers.Authorization = 'Bearer ' + options.grafana.auth;
}
} }
log.verbose('Send annotation to Grafana: %j', postData); log.verbose('Send annotation to Grafana: %j', postData);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// not perfect but maybe work for us // not perfect but maybe work for us
const lib = options.grafana.port === 443 ? https : http; const library = options.grafana.port === 443 ? https : http;
const req = lib.request(postOptions, res => { const request = library.request(postOptions, res => {
if (res.statusCode !== 200) { if (res.statusCode !== 200) {
const e = new Error( const e = new Error(
`Got ${res.statusCode} from Grafana when sending annotation` `Got ${res.statusCode} from Grafana when sending annotation`
@ -130,12 +131,11 @@ module.exports = {
resolve(); resolve();
} }
}); });
req.on('error', err => { request.on('error', error => {
log.error('Got error from Grafana when sending annotation', err); log.error('Got error from Grafana when sending annotation', error);
reject(err); reject(error);
}); });
req.write(postData); request.write(postData);
req.end(); request.end();
}); });
} }
};

View File

@ -1,104 +0,0 @@
module.exports = {
host: {
describe: 'The Graphite host used to store captured metrics.',
group: 'Graphite'
},
port: {
default: 2003,
describe: 'The Graphite port used to store captured metrics.',
group: 'Graphite'
},
auth: {
describe:
'The Graphite user and password used for authentication. Format: user:password',
group: 'Graphite'
},
httpPort: {
describe:
'The Graphite port used to access the user interface and send annotations event',
default: 8080,
group: 'Graphite'
},
webHost: {
describe:
'The graphite-web host. If not specified graphite.host will be used.',
group: 'Graphite'
},
namespace: {
default: 'sitespeed_io.default',
describe: 'The namespace key added to all captured metrics.',
group: 'Graphite'
},
includeQueryParams: {
default: false,
describe:
'Whether to include query parameters from the URL in the Graphite keys or not',
type: 'boolean',
group: 'Graphite'
},
arrayTags: {
default: true,
type: 'boolean',
describe:
'Send the tags as Array or a String. In Graphite 1.0 the tags is a array. Before a String',
group: 'Graphite'
},
annotationTitle: {
describe: 'Add a title to the annotation sent for a run.',
group: 'Graphite'
},
annotationMessage: {
describe:
'Add an extra message that will be attached to the annotation sent for a run. The message is attached after the default message and can contain HTML.',
group: 'Graphite'
},
annotationScreenshot: {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime/WebPageTest) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'Graphite'
},
sendAnnotation: {
default: true,
type: 'boolean',
describe:
'Send annotations when a run is finished. You need to specify a --resultBaseURL for this to work. However if you for example use a Prometheus exporter, you may want to make sure annotations are not sent, then set it to false.',
group: 'Graphite'
},
annotationRetentionMinutes: {
type: 'number',
describe:
'The retention in minutes, to make annotation match the retention in Graphite.',
group: 'Graphite'
},
statsd: {
default: false,
type: 'boolean',
describe: 'Uses the StatsD interface',
group: 'Graphite'
},
annotationTag: {
describe:
'Add a extra tag to the annotation sent for a run. Repeat the --graphite.annotationTag option for multiple tags. Make sure they do not collide with the other tags.',
group: 'Graphite'
},
addSlugToKey: {
default: true,
type: 'boolean',
describe:
'Add the slug (name of the test) as an extra key in the namespace.',
group: 'Graphite'
},
bulkSize: {
default: null,
type: 'number',
describe: 'Break up number of metrics to send with each request.',
group: 'Graphite'
},
messages: {
default: ['pageSummary', 'summary'],
options: ['pageSummary', 'summary', 'run'],
group: 'Graphite'
}
};

View File

@ -1,38 +1,40 @@
'use strict'; import util, { format } from 'node:util';
import reduce from 'lodash.reduce';
const flatten = require('../../support/flattenMessage'), import {
util = require('util'), getConnectivity,
graphiteUtil = require('../../support/tsdbUtil'), toSafeKey,
reduce = require('lodash.reduce'), getURLAndGroup
formatEntry = require('./helpers/format-entry'), } from '../../support/tsdbUtil.js';
isStatsd = require('./helpers/is-statsd'); import { flattenMessageData } from '../../support/flattenMessage.js';
import { formatEntry } from './helpers/format-entry.js';
import { isStatsD } from './helpers/is-statsd.js';
const STATSD = 'statsd'; const STATSD = 'statsd';
const GRAPHITE = 'graphite'; const GRAPHITE = 'graphite';
function keyPathFromMessage(message, options, includeQueryParams, alias) { function keyPathFromMessage(message, options, includeQueryParameters, alias) {
let typeParts = message.type.split('.'); let typeParts = message.type.split('.');
typeParts.push(typeParts.shift()); typeParts.push(typeParts.shift());
// always have browser and connectivity in Browsertime and related tools // always have browser and connectivity in Browsertime and related tools
if ( if (
message.type.match( /(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains|^thirdparty|^axe|^sustainable)/.test(
/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains|^thirdparty|^axe|^sustainable)/ message.type
) )
) { ) {
// if we have a friendly name for your connectivity, use that! // if we have a friendly name for your connectivity, use that!
let connectivity = graphiteUtil.getConnectivity(options); let connectivity = getConnectivity(options);
typeParts.splice(1, 0, connectivity); typeParts.splice(1, 0, connectivity);
typeParts.splice(1, 0, options.browser); typeParts.splice(1, 0, options.browser);
} else if (message.type.match(/(^webpagetest)/)) { } else if (/(^webpagetest)/.test(message.type)) {
if (message.connectivity) { if (message.connectivity) {
typeParts.splice(2, 0, message.connectivity); typeParts.splice(2, 0, message.connectivity);
} }
if (message.location) { if (message.location) {
typeParts.splice(2, 0, graphiteUtil.toSafeKey(message.location)); typeParts.splice(2, 0, toSafeKey(message.location));
} }
} else if (message.type.match(/(^gpsi)/)) { } else if (/(^gpsi)/.test(message.type)) {
typeParts.splice(2, 0, options.mobile ? 'mobile' : 'desktop'); typeParts.splice(2, 0, options.mobile ? 'mobile' : 'desktop');
} }
@ -41,17 +43,17 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
typeParts.splice( typeParts.splice(
1, 1,
0, 0,
graphiteUtil.getURLAndGroup( getURLAndGroup(
options, options,
message.group, message.group,
message.url, message.url,
includeQueryParams, includeQueryParameters,
alias alias
) )
); );
} else if (message.group) { } else if (message.group) {
// add the group of the summary message // add the group of the summary message
typeParts.splice(1, 0, graphiteUtil.toSafeKey(message.group)); typeParts.splice(1, 0, toSafeKey(message.group));
} }
if (options.graphite && options.graphite.addSlugToKey) { if (options.graphite && options.graphite.addSlugToKey) {
@ -60,13 +62,12 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
return typeParts.join('.'); return typeParts.join('.');
} }
export class GraphiteDataGenerator {
class GraphiteDataGenerator { constructor(namespace, includeQueryParameters, options) {
constructor(namespace, includeQueryParams, options) {
this.namespace = namespace; this.namespace = namespace;
this.includeQueryParams = !!includeQueryParams; this.includeQueryParams = !!includeQueryParameters;
this.options = options; this.options = options;
this.entryFormat = isStatsd(options.graphite) ? STATSD : GRAPHITE; this.entryFormat = isStatsD(options.graphite) ? STATSD : GRAPHITE;
} }
dataFromMessage(message, time, alias) { dataFromMessage(message, time, alias) {
@ -80,50 +81,46 @@ class GraphiteDataGenerator {
); );
return reduce( return reduce(
flatten.flattenMessageData(message), flattenMessageData(message),
(entries, value, key) => { (entries, value, key) => {
if (message.type === 'browsertime.run') { if (message.type === 'browsertime.run') {
if (key.includes('timings') && key.includes('marks')) { if (key.includes('timings') && key.includes('marks')) {
key = key.replace(/marks\.(\d+)/, function (match, idx) { key = key.replace(/marks\.(\d+)/, function (match, index) {
return ( return (
'marks.' + message.data.timings.userTimings.marks[idx].name 'marks.' + message.data.timings.userTimings.marks[index].name
); );
}); });
} }
if (key.includes('timings') && key.includes('measures')) { if (key.includes('timings') && key.includes('measures')) {
key = key.replace(/measures\.(\d+)/, function (match, idx) { key = key.replace(/measures\.(\d+)/, function (match, index) {
return ( return (
'measures.' + 'measures.' +
message.data.timings.userTimings.measures[idx].name message.data.timings.userTimings.measures[index].name
); );
}); });
} }
} }
if (message.type === 'pagexray.run') { if (message.type === 'pagexray.run' && key.includes('assets')) {
if (key.includes('assets')) {
key = key.replace( key = key.replace(
/assets\.(\d+)/, /assets\.(\d+)/,
function (match, idx) { function (match, index) {
let url = new URL(message.data.assets[idx].url); let url = new URL(message.data.assets[index].url);
url.search = ''; url.search = '';
return 'assets.' + graphiteUtil.toSafeKey(url.toString()); return 'assets.' + toSafeKey(url.toString());
}, },
{} {}
); );
} }
}
const fullKey = util.format('%s.%s.%s', this.namespace, keypath, key); const fullKey = format('%s.%s.%s', this.namespace, keypath, key);
const args = [formatEntry(this.entryFormat), fullKey, value]; const arguments_ = [formatEntry(this.entryFormat), fullKey, value];
this.entryFormat === GRAPHITE && args.push(timestamp); this.entryFormat === GRAPHITE && arguments_.push(timestamp);
entries.push(util.format.apply(util, args)); entries.push(format.apply(util, arguments_));
return entries; return entries;
}, },
[] []
); );
} }
} }
module.exports = GraphiteDataGenerator;

View File

@ -1,9 +1,7 @@
'use strict'; import { connect } from 'node:net';
import { Sender } from './sender.js';
const net = require('net'); export class GraphiteSender extends Sender {
const Sender = require('./sender');
class GraphiteSender extends Sender {
get facility() { get facility() {
return 'Graphite'; return 'Graphite';
} }
@ -12,7 +10,7 @@ class GraphiteSender extends Sender {
this.log(data); this.log(data);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const socket = net.connect(this.port, this.host, () => { const socket = connect(this.port, this.host, () => {
socket.write(data); socket.write(data);
socket.end(); socket.end();
resolve(); resolve();
@ -21,5 +19,3 @@ class GraphiteSender extends Sender {
}); });
} }
} }
module.exports = GraphiteSender;

View File

@ -3,12 +3,13 @@
* @param {string} [type='graphite'] ['statsd', 'graphite'] * @param {string} [type='graphite'] ['statsd', 'graphite']
* @return {string} The string template * @return {string} The string template
*/ */
module.exports = type => { export function formatEntry(type) {
switch (type) { switch (type) {
case 'statsd': case 'statsd': {
return '%s:%s|ms'; return '%s:%s|ms';
case 'graphite': }
default: default: {
return '%s %s %s'; return '%s %s %s';
} }
}; }
}

View File

@ -3,4 +3,6 @@
* @param {Object} opts graphite options * @param {Object} opts graphite options
* @return {boolean} * @return {boolean}
*/ */
module.exports = (opts = {}) => opts.statsd === true; export function isStatsD(options = {}) {
return options.statsd === true;
}

View File

@ -1,34 +1,25 @@
'use strict'; import isEmpty from 'lodash.isempty';
const path = require('path'); import merge from 'lodash.merge';
const isEmpty = require('lodash.isempty'); import get from 'lodash.get';
const GraphiteSender = require('./graphite-sender'); import dayjs from 'dayjs';
const StatsDSender = require('./statsd-sender'); import intel from 'intel';
const merge = require('lodash.merge'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const get = require('lodash.get');
const dayjs = require('dayjs');
const log = require('intel').getLogger('sitespeedio.plugin.graphite');
const sendAnnotations = require('./send-annotation');
const DataGenerator = require('./data-generator');
const graphiteUtil = require('../../support/tsdbUtil');
const isStatsd = require('./helpers/is-statsd');
const throwIfMissing = require('../../support/util').throwIfMissing;
const cliUtil = require('../../cli/util');
const toArray = require('../../support/util').toArray;
module.exports = { import { send } from './send-annotation.js';
name() { import { GraphiteDataGenerator as DataGenerator } from './data-generator.js';
return path.basename(__dirname); import { toSafeKey } from '../../support/tsdbUtil.js';
}, import { isStatsD } from './helpers/is-statsd.js';
import { throwIfMissing } from '../../support/util.js';
import { toArray } from '../../support/util.js';
import { GraphiteSender } from './graphite-sender.js';
import { StatsDSender } from './statsd-sender.js';
/** const log = intel.getLogger('sitespeedio.plugin.graphite');
* Define `yargs` options with their respective default values. When displayed by the CLI help message
* all options are namespaced by its plugin name. export default class GraphitePlugin extends SitespeedioPlugin {
* constructor(options, context, queue) {
* @return {Object<string, require('yargs').Options} an object mapping the name of the option and its yargs configuration super({ name: 'graphite', options, context, queue });
*/ }
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
open(context, options) { open(context, options) {
throwIfMissing(options.graphite, ['host'], 'graphite'); throwIfMissing(options.graphite, ['host'], 'graphite');
@ -39,62 +30,87 @@ module.exports = {
); );
} }
const opts = merge({}, this.config, options.graphite); const options_ = merge({}, this.config, options.graphite);
this.options = options; this.options = options;
this.perIteration = get(opts, 'perIteration', false); this.perIteration = get(options_, 'perIteration', false);
const SenderConstructor = isStatsd(opts) ? StatsDSender : GraphiteSender; const SenderConstructor = isStatsD(options_)
? StatsDSender
: GraphiteSender;
this.filterRegistry = context.filterRegistry; this.filterRegistry = context.filterRegistry;
this.sender = new SenderConstructor(opts.host, opts.port, opts.bulkSize); this.sender = new SenderConstructor(
options_.host,
options_.port,
options_.bulkSize
);
this.dataGenerator = new DataGenerator( this.dataGenerator = new DataGenerator(
opts.namespace, options_.namespace,
opts.includeQueryParams, options_.includeQueryParams,
options options
); );
log.debug( log.debug(
'Setting up Graphite %s:%s for namespace %s', 'Setting up Graphite %s:%s for namespace %s',
opts.host, options_.host,
opts.port, options_.port,
opts.namespace options_.namespace
); );
this.timestamp = context.timestamp; this.timestamp = context.timestamp;
this.resultUrls = context.resultUrls; this.resultUrls = context.resultUrls;
this.messageTypesToFireAnnotations = []; this.messageTypesToFireAnnotations = [];
this.receivedTypesThatFireAnnotations = {}; this.receivedTypesThatFireAnnotations = {};
this.make = context.messageMaker('graphite').make; this.make = context.messageMaker('graphite').make;
this.sendAnnotation = opts.sendAnnotation; this.sendAnnotation = options_.sendAnnotation;
this.alias = {}; this.alias = {};
this.wptExtras = {}; this.wptExtras = {};
this.usingBrowsertime = false; this.usingBrowsertime = false;
this.types = toArray(options.graphite.messages); this.types = toArray(options.graphite.messages);
}, }
processMessage(message, queue) { processMessage(message, queue) {
// First catch if we are running Browsertime and/or WebPageTest // First catch if we are running Browsertime and/or WebPageTest
if (message.type === 'browsertime.setup') { switch (message.type) {
case 'browsertime.setup': {
this.messageTypesToFireAnnotations.push('browsertime.pageSummary'); this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
this.usingBrowsertime = true; this.usingBrowsertime = true;
} else if (message.type === 'webpagetest.setup') {
break;
}
case 'webpagetest.setup': {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary'); this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
} else if (message.type === 'browsertime.config') {
break;
}
case 'browsertime.config': {
if (message.data.screenshot) { if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot; this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType; this.screenshotType = message.data.screenshotType;
} }
} else if (message.type === 'sitespeedio.setup') {
break;
}
case 'sitespeedio.setup': {
// Let other plugins know that the Graphite plugin is alive // Let other plugins know that the Graphite plugin is alive
queue.postMessage(this.make('graphite.setup')); queue.postMessage(this.make('graphite.setup'));
} else if (message.type === 'grafana.setup') {
break;
}
case 'grafana.setup': {
this.sendAnnotation = false; this.sendAnnotation = false;
} else if (message.type === 'browsertime.browser') {
break;
}
case 'browsertime.browser': {
this.browser = message.data.browser; this.browser = message.data.browser;
} else if (
message.type === 'webpagetest.browser' && break;
!this.usingBrowsertime }
) { default: {
if (message.type === 'webpagetest.browser' && !this.usingBrowsertime) {
// We are only interested in WebPageTest browser if we run it standalone // We are only interested in WebPageTest browser if we run it standalone
this.browser = message.data.browser; this.browser = message.data.browser;
} }
}
}
if (message.type === 'browsertime.alias') { if (message.type === 'browsertime.alias') {
this.alias[message.url] = message.data; this.alias[message.url] = message.data;
@ -118,9 +134,7 @@ module.exports = {
this.wptExtras[message.url].webPageTestResultURL = this.wptExtras[message.url].webPageTestResultURL =
message.data.data.summary; message.data.data.summary;
this.wptExtras[message.url].connectivity = message.connectivity; this.wptExtras[message.url].connectivity = message.connectivity;
this.wptExtras[message.url].location = graphiteUtil.toSafeKey( this.wptExtras[message.url].location = toSafeKey(message.location);
message.location
);
} }
// we only sends individual groups to Graphite, not the // we only sends individual groups to Graphite, not the
@ -162,7 +176,7 @@ module.exports = {
message.url, message.url,
this.alias[message.url] this.alias[message.url]
); );
return sendAnnotations.send( return send(
message.url, message.url,
message.group, message.group,
absolutePagePath, absolutePagePath,
@ -181,13 +195,9 @@ module.exports = {
return Promise.reject( return Promise.reject(
new Error( new Error(
'No data to send to graphite for message:\n' + 'No data to send to graphite for message:\n' +
JSON.stringify(message, null, 2) JSON.stringify(message, undefined, 2)
) )
); );
} }
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
} }
}; }

View File

@ -1,14 +1,21 @@
'use strict'; import http from 'node:http';
const http = require('http'); import https from 'node:https';
const https = require('https'); import { createRequire } from 'node:module';
const log = require('intel').getLogger('sitespeedio.plugin.graphite'); import intel from 'intel';
const graphiteUtil = require('../../support/tsdbUtil');
const annotationsHelper = require('../../support/annotationsHelper');
const util = require('../../support/util');
const packageInfo = require('../../../package');
module.exports = { import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
send( import {
getTagsAsArray,
getTagsAsString,
getAnnotationMessage
} from '../../support/annotationsHelper.js';
import { toArray } from '../../support/util.js';
const require = createRequire(import.meta.url);
const version = require('../../../package.json').version;
const log = intel.getLogger('sitespeedio.plugin.graphite');
export function send(
url, url,
group, group,
absolutePagePath, absolutePagePath,
@ -25,18 +32,16 @@ module.exports = {
// templates to choose which annotations that will be showed. // templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template // That's why we need to send tags that matches the template
// variables in Grafana. // variables in Grafana.
const connectivity = graphiteUtil.getConnectivity(options); const connectivity = getConnectivity(options);
const browser = options.browser; const browser = options.browser;
const namespace = options.graphite.namespace.split('.'); const namespace = options.graphite.namespace.split('.');
const urlAndGroup = graphiteUtil const urlAndGroup = getURLAndGroup(
.getURLAndGroup(
options, options,
group, group,
url, url,
options.graphite.includeQueryParams, options.graphite.includeQueryParams,
alias alias
) ).split('.');
.split('.');
const tags = [ const tags = [
connectivity, connectivity,
browser, browser,
@ -49,20 +54,19 @@ module.exports = {
if (options.slug && options.slug !== urlAndGroup[0]) { if (options.slug && options.slug !== urlAndGroup[0]) {
tags.push(options.slug); tags.push(options.slug);
} }
const extraTags = util.toArray(options.graphite.annotationTag); const extraTags = toArray(options.graphite.annotationTag);
// We got some extra tag(s) from the user, let us add them to the annotation // We got some extra tag(s) from the user, let us add them to the annotation
if (extraTags.length > 0) { if (extraTags.length > 0) {
tags.push(...extraTags); tags.push(...extraTags);
} }
if (webPageTestExtraData) { if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity); tags.push(webPageTestExtraData.connectivity, webPageTestExtraData.location);
tags.push(webPageTestExtraData.location);
} }
const theTags = options.graphite.arrayTags const theTags = options.graphite.arrayTags
? annotationsHelper.getTagsAsArray(tags) ? getTagsAsArray(tags)
: annotationsHelper.getTagsAsString(tags); : getTagsAsString(tags);
const message = annotationsHelper.getAnnotationMessage( const message = getAnnotationMessage(
absolutePagePath, absolutePagePath,
screenShotsEnabledInBrowsertime, screenShotsEnabledInBrowsertime,
screenshotType, screenshotType,
@ -82,7 +86,7 @@ module.exports = {
: Math.round(time.valueOf() / 1000); : Math.round(time.valueOf() / 1000);
let what = let what =
packageInfo.version + version +
(browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : ''); (browserNameAndVersion ? ` - ${browserNameAndVersion.version}` : '');
if (options.graphite.annotationTitle) { if (options.graphite.annotationTitle) {
what = options.graphite.annotationTitle; what = options.graphite.annotationTitle;
@ -108,8 +112,8 @@ module.exports = {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
log.verbose('Send annotation to Graphite: %j', postData); log.verbose('Send annotation to Graphite: %j', postData);
// not perfect but maybe work for us // not perfect but maybe work for us
const lib = options.graphite.httpPort === 443 ? https : http; const library = options.graphite.httpPort === 443 ? https : http;
const req = lib.request(postOptions, res => { const request = library.request(postOptions, res => {
if (res.statusCode !== 200) { if (res.statusCode !== 200) {
const e = new Error( const e = new Error(
`Got ${res.statusCode} from Graphite when sending annotation` `Got ${res.statusCode} from Graphite when sending annotation`
@ -122,12 +126,11 @@ module.exports = {
resolve(); resolve();
} }
}); });
req.on('error', err => { request.on('error', error => {
log.error('Got error from Graphite when sending annotation', err); log.error('Got error from Graphite when sending annotation', error);
reject(err); reject(error);
}); });
req.write(postData); request.write(postData);
req.end(); request.end();
}); });
} }
};

View File

@ -1,8 +1,7 @@
'use strict'; import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.graphite');
const log = require('intel').getLogger('sitespeedio.plugin.graphite'); export class Sender {
class Sender {
constructor(host, port, bulkSize) { constructor(host, port, bulkSize) {
this.host = host; this.host = host;
this.port = port; this.port = port;
@ -29,14 +28,12 @@ class Sender {
bulks(data) { bulks(data) {
const lines = data.split('\n'); const lines = data.split('\n');
const promises = []; const promises = [];
const bulkSize = this.bulkSize || lines.length; const bulkSize = this.bulkSize || lines.length > 0;
while (lines.length) { while (lines.length > 0) {
promises.push(this.bulk(lines.splice(-bulkSize).join('\n'))); promises.push(this.bulk(lines.splice(-bulkSize).join('\n')));
} }
return Promise.all(promises); return Promise.all(promises);
} }
} }
module.exports = Sender;

View File

@ -1,9 +1,7 @@
'use strict'; import { createSocket } from 'node:dgram';
import { Sender } from './sender.js';
const dgram = require('dgram'); export class StatsDSender extends Sender {
const Sender = require('./sender');
class StatsDSender extends Sender {
get facility() { get facility() {
return 'StatsD'; return 'StatsD';
} }
@ -12,7 +10,7 @@ class StatsDSender extends Sender {
this.log(data); this.log(data);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const client = dgram.createSocket('udp4'); const client = createSocket('udp4');
client.send(data, 0, data.length, this.port, this.host, error => client.send(data, 0, data.length, this.port, this.host, error =>
client.close() && error ? reject(error) : resolve() client.close() && error ? reject(error) : resolve()
@ -20,5 +18,3 @@ class StatsDSender extends Sender {
}); });
} }
} }
module.exports = StatsDSender;

View File

@ -1,15 +1,19 @@
'use strict'; import { gzip as _gzip } from 'node:zlib';
import { promisify } from 'node:util';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const gzip = promisify(_gzip);
const zlib = require('zlib'); export default class HarstorerPlugin extends SitespeedioPlugin {
const { promisify } = require('util'); constructor(options, context, queue) {
const gzip = promisify(zlib.gzip); super({ name: 'harstorer', options, context, queue });
}
module.exports = {
open(context, options) { open(context, options) {
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
this.gzipHAR = options.gzipHAR; this.gzipHAR = options.gzipHAR;
this.alias = {}; this.alias = {};
}, }
processMessage(message) { processMessage(message) {
switch (message.type) { switch (message.type) {
case 'browsertime.alias': { case 'browsertime.alias': {
@ -20,8 +24,8 @@ module.exports = {
case 'webpagetest.har': { case 'webpagetest.har': {
const json = JSON.stringify(message.data); const json = JSON.stringify(message.data);
if (this.gzipHAR) { return this.gzipHAR
return gzip(Buffer.from(json), { ? gzip(Buffer.from(json), {
level: 1 level: 1
}).then(gziped => }).then(gziped =>
this.storageManager.writeDataForUrl( this.storageManager.writeDataForUrl(
@ -32,9 +36,8 @@ module.exports = {
this.alias[message.url] this.alias[message.url]
) )
); )
} else { : this.storageManager.writeDataForUrl(
return this.storageManager.writeDataForUrl(
json, json,
message.type, message.type,
message.url, message.url,
@ -45,4 +48,3 @@ module.exports = {
} }
} }
} }
};

View File

@ -1,11 +1,9 @@
'use strict'; import merge from 'lodash.merge';
import get from 'lodash.get';
import reduce from 'lodash.reduce';
import set from 'lodash.set';
const merge = require('lodash.merge'), export class DataCollector {
get = require('lodash.get'),
reduce = require('lodash.reduce'),
set = require('lodash.set');
class DataCollector {
constructor(resultUrls) { constructor(resultUrls) {
this.resultUrls = resultUrls; this.resultUrls = resultUrls;
this.urlRunPages = {}; this.urlRunPages = {};
@ -138,5 +136,3 @@ class DataCollector {
return this.budget; return this.budget;
} }
} }
module.exports = DataCollector;

View File

@ -1,6 +1,4 @@
'use strict'; export default {
module.exports = {
html: { html: {
showAllWaterfallSummary: false, showAllWaterfallSummary: false,
pageSummaryMetrics: [ pageSummaryMetrics: [

View File

@ -1,9 +1,8 @@
'use strict'; import { readFile as _readFile } from 'node:fs';
const fs = require('fs'); import { promisify } from 'node:util';
const { promisify } = require('util'); const readFile = promisify(_readFile);
const readFile = promisify(fs.readFile);
module.exports = async options => { export default async options => {
const scripts = []; const scripts = [];
for (let file of options._) { for (let file of options._) {
// We could promise all these in the future // We could promise all these in the future
@ -11,7 +10,7 @@ module.exports = async options => {
try { try {
const code = await readFile(file); const code = await readFile(file);
scripts.push({ name: file, code: code }); scripts.push({ name: file, code: code });
} catch (e) { } catch {
// do nada // do nada
} }
} }

View File

@ -1,34 +1,43 @@
'use strict'; import { join } from 'node:path';
import osName from 'os-name';
import { promisify } from 'node:util';
import { platform } from 'node:os';
import { createRequire } from 'node:module';
import { fileURLToPath } from 'node:url';
import dayjs from 'dayjs';
import getos from 'getos';
import intel from 'intel';
import { markdown } from 'markdown';
import merge from 'lodash.merge';
import get from 'lodash.get';
import isEmpty from 'lodash.isempty';
import chunk from 'lodash.chunk';
const helpers = require('../../support/helpers');
const path = require('path');
const osName = require('os-name');
const getos = require('getos');
const { promisify } = require('util');
const getOS = promisify(getos); const getOS = promisify(getos);
const os = require('os'); const log = intel.getLogger('sitespeedio.plugin.html');
const merge = require('lodash.merge'); const require = createRequire(import.meta.url);
const get = require('lodash.get'); const { dependencies, version } = require('../../../package.json');
const log = require('intel').getLogger('sitespeedio.plugin.html'); import { renderTemplate } from './renderer.js';
const chunk = require('lodash.chunk'); import {
const packageInfo = require('../../../package'); pickMedianRun,
const renderer = require('./renderer'); getMetricsFromPageSummary,
const metricHelper = require('./metricHelper'); getMetricsFromRun
const markdown = require('markdown').markdown; } from './metricHelper.js';
const isEmpty = require('lodash.isempty');
const dayjs = require('dayjs');
const defaultConfigHTML = require('./defaultConfig');
const summaryBoxesSetup = require('./setup/summaryBoxes');
const detailedSetup = require('./setup/detailed');
const filmstrip = require('../browsertime/filmstrip'); import * as helpers from '../../support/helpers/index.js';
const getScripts = require('./getScripts'); import * as _html from './defaultConfig.js';
const friendlyNames = require('../../support/friendlynames'); import summaryBoxesSetup from './setup/summaryBoxes.js';
const toArray = require('../../support/util').toArray; import detailedSetup from './setup/detailed.js';
import { getFilmstrip } from '../browsertime/filmstrip.js';
import getScripts from './getScripts.js';
import friendlyNames from '../../support/friendlynames.js';
import { toArray } from '../../support/util.js';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss'; const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
class HTMLBuilder { export class HTMLBuilder {
constructor(context, options) { constructor(context, options) {
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
this.timestamp = context.timestamp.format(TIME_FORMAT); this.timestamp = context.timestamp.format(TIME_FORMAT);
@ -55,10 +64,11 @@ class HTMLBuilder {
this.summaries.push({ id, name }); this.summaries.push({ id, name });
break; break;
} }
default: default: {
log.info('Got a undefined page type ' + type); log.info('Got a undefined page type ' + type);
} }
} }
}
addInlineCSS(css) { addInlineCSS(css) {
this.inlineCSS.push(css); this.inlineCSS.push(css);
@ -197,7 +207,7 @@ class HTMLBuilder {
for (let url of Object.keys(validPages)) { for (let url of Object.keys(validPages)) {
const pageInfo = validPages[url]; const pageInfo = validPages[url];
const runPages = dataCollector.getURLRuns(url); const runPages = dataCollector.getURLRuns(url);
const medianRun = metricHelper.pickMedianRun(runPages, pageInfo); const medianRun = pickMedianRun(runPages, pageInfo);
// If we have multiple URLs in the same HAR the median run must be converted // If we have multiple URLs in the same HAR the median run must be converted
// to the right run in the HAR // to the right run in the HAR
const harIndex = pageNumber + (medianRun.runIndex - 1) * testedPages; const harIndex = pageNumber + (medianRun.runIndex - 1) * testedPages;
@ -272,7 +282,7 @@ class HTMLBuilder {
const medianPageInfo = runPages[medianRun.runIndex - 1]; const medianPageInfo = runPages[medianRun.runIndex - 1];
let filmstripData = let filmstripData =
medianPageInfo && medianPageInfo.data && medianPageInfo.data.browsertime medianPageInfo && medianPageInfo.data && medianPageInfo.data.browsertime
? await filmstrip.getFilmstrip( ? await getFilmstrip(
medianPageInfo.data.browsertime.run, medianPageInfo.data.browsertime.run,
medianRun.runIndex, medianRun.runIndex,
this.storageManager.getFullPathToURLDir(url, daurlAlias), this.storageManager.getFullPathToURLDir(url, daurlAlias),
@ -310,18 +320,18 @@ class HTMLBuilder {
this.options.browsertime.iterations, this.options.browsertime.iterations,
'run' 'run'
)} ${url} at ${summaryTimestamp}`, )} ${url} at ${summaryTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromPageSummary( pageDescription: `${getMetricsFromPageSummary(
pageInfo pageInfo
)} collected by sitespeed.io ${packageInfo.version}`, )} collected by sitespeed.io ${version}`,
headers: this.summary, headers: this.summary,
version: packageInfo.version, version: version,
timestamp: summaryTimestamp, timestamp: summaryTimestamp,
context: this.context, context: this.context,
pageSummaries pageSummaries
}; };
for (const summary of pageSummaries) { for (const summary of pageSummaries) {
pugs[summary.id] = renderer.renderTemplate(summary.id, data); pugs[summary.id] = renderTemplate(summary.id, data);
} }
data.pugs = pugs; data.pugs = pugs;
@ -339,7 +349,7 @@ class HTMLBuilder {
); );
const filmstripData = pageInfo.data.browsertime const filmstripData = pageInfo.data.browsertime
? await filmstrip.getFilmstrip( ? await getFilmstrip(
pageInfo.data.browsertime.run, pageInfo.data.browsertime.run,
iteration, iteration,
this.storageManager.getFullPathToURLDir(url, daurlAlias), this.storageManager.getFullPathToURLDir(url, daurlAlias),
@ -379,13 +389,13 @@ class HTMLBuilder {
assetsPath: assetsBaseURL || rootPath, assetsPath: assetsBaseURL || rootPath,
menu: 'pages', menu: 'pages',
pageTitle: `Run ${ pageTitle: `Run ${
parseInt(runIndex) + 1 Number.parseInt(runIndex) + 1
} for ${url} at ${runTimestamp}`, } for ${url} at ${runTimestamp}`,
pageDescription: `${metricHelper.getMetricsFromRun( pageDescription: `${getMetricsFromRun(
pageInfo pageInfo
)} collected by sitespeed.io ${packageInfo.version}`, )} collected by sitespeed.io ${version}`,
headers: this.summary, headers: this.summary,
version: packageInfo.version, version: version,
timestamp: runTimestamp, timestamp: runTimestamp,
friendlyNames, friendlyNames,
context: this.context, context: this.context,
@ -393,16 +403,21 @@ class HTMLBuilder {
}; };
// Add pugs for extra plugins // Add pugs for extra plugins
for (const run of pageRuns) { for (const run of pageRuns) {
pugs[run.id] = renderer.renderTemplate(run.id, data); pugs[run.id] = renderTemplate(run.id, data);
} }
data.pugs = pugs; data.pugs = pugs;
urlPageRenders.push( urlPageRenders.push(
this._renderUrlRunPage(url, parseInt(runIndex) + 1, data, daurlAlias) this._renderUrlRunPage(
url,
Number.parseInt(runIndex) + 1,
data,
daurlAlias
)
); );
// Do only once per URL // Do only once per URL
if (parseInt(runIndex) === 0) { if (Number.parseInt(runIndex) === 0) {
data.mySummary = mySummary; data.mySummary = mySummary;
urlPageRenders.push( urlPageRenders.push(
this._renderMetricSummaryPage(url, 'metrics', data, daurlAlias) this._renderMetricSummaryPage(url, 'metrics', data, daurlAlias)
@ -414,11 +429,10 @@ class HTMLBuilder {
// Kind of clumsy way to decide if the user changed HTML summaries, // Kind of clumsy way to decide if the user changed HTML summaries,
// so we in the pug can automatically add visual metrics // so we in the pug can automatically add visual metrics
const hasPageSummaryMetricInput = const hasPageSummaryMetricInput =
options.html.pageSummaryMetrics !== options.html.pageSummaryMetrics !== _html.pageSummaryMetrics;
defaultConfigHTML.html.pageSummaryMetrics;
let osInfo = osName(); let osInfo = osName();
if (os.platform() === 'linux') { if (platform() === 'linux') {
const linux = await getOS(); const linux = await getOS();
osInfo = `${linux.dist} ${linux.release}`; osInfo = `${linux.dist} ${linux.release}`;
} }
@ -451,8 +465,8 @@ class HTMLBuilder {
usingBrowsertime, usingBrowsertime,
usingWebPageTest, usingWebPageTest,
headers: this.summary, headers: this.summary,
version: packageInfo.version, version: version,
browsertimeVersion: packageInfo.dependencies.browsertime, browsertimeVersion: dependencies.browsertime,
timestamp: this.timestamp, timestamp: this.timestamp,
context: this.context, context: this.context,
get, get,
@ -469,11 +483,9 @@ class HTMLBuilder {
); );
let res; let res;
if (this.options.html.assetsBaseURL) { res = this.options.html.assetsBaseURL
res = Promise.resolve(); ? Promise.resolve()
} else { : this.storageManager.copyToResultDir(join(__dirname, 'assets'));
res = this.storageManager.copyToResultDir(path.join(__dirname, 'assets'));
}
return res.then(() => return res.then(() =>
Promise.allSettled(summaryRenders) Promise.allSettled(summaryRenders)
@ -487,7 +499,7 @@ class HTMLBuilder {
async _renderUrlPage(url, name, locals, alias) { async _renderUrlPage(url, name, locals, alias) {
log.debug('Render URL page %s', name); log.debug('Render URL page %s', name);
return this.storageManager.writeHtmlForUrl( return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/summary/' + name, locals), renderTemplate('url/summary/' + name, locals),
name + '.html', name + '.html',
url, url,
alias alias
@ -497,7 +509,7 @@ class HTMLBuilder {
async _renderUrlRunPage(url, name, locals, alias) { async _renderUrlRunPage(url, name, locals, alias) {
log.debug('Render URL run page %s', name); log.debug('Render URL run page %s', name);
return this.storageManager.writeHtmlForUrl( return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/iteration/index', locals), renderTemplate('url/iteration/index', locals),
name + '.html', name + '.html',
url, url,
alias alias
@ -507,7 +519,7 @@ class HTMLBuilder {
async _renderMetricSummaryPage(url, name, locals, alias) { async _renderMetricSummaryPage(url, name, locals, alias) {
log.debug('Render URL metric page %s', name); log.debug('Render URL metric page %s', name);
return this.storageManager.writeHtmlForUrl( return this.storageManager.writeHtmlForUrl(
renderer.renderTemplate('url/summary/metrics/index', locals), renderTemplate('url/summary/metrics/index', locals),
name + '.html', name + '.html',
url, url,
alias alias
@ -518,10 +530,8 @@ class HTMLBuilder {
log.debug('Render summary page %s', name); log.debug('Render summary page %s', name);
return this.storageManager.writeHtml( return this.storageManager.writeHtml(
renderer.renderTemplate(name, locals), renderTemplate(name, locals),
name + '.html' name + '.html'
); );
} }
} }
module.exports = HTMLBuilder;

View File

@ -1,17 +1,20 @@
'use strict'; import get from 'lodash.get';
import set from 'lodash.set';
const HTMLBuilder = require('./htmlBuilder'); import reduce from 'lodash.reduce';
const get = require('lodash.get'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const set = require('lodash.set'); import { DataCollector } from './dataCollector.js';
const reduce = require('lodash.reduce'); import { HTMLBuilder } from './htmlBuilder.js';
const DataCollector = require('./dataCollector'); import { addTemplate } from './renderer.js';
const renderer = require('./renderer');
// lets keep this in the HTML context, since we need things from the // lets keep this in the HTML context, since we need things from the
// regular options object in the output // regular options object in the output
const defaultConfig = require('./defaultConfig'); import defaultConfig from './defaultConfig.js';
export default class HTMLPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'html', options, context, queue });
}
module.exports = {
open(context, options) { open(context, options) {
this.make = context.messageMaker('html').make; this.make = context.messageMaker('html').make;
// we have to overwrite the default summary metrics, if given // we have to overwrite the default summary metrics, if given
@ -37,13 +40,13 @@ module.exports = {
'thirdparty.pageSummary', 'thirdparty.pageSummary',
'crux.pageSummary' 'crux.pageSummary'
]; ];
}, }
processMessage(message, queue) { processMessage(message, queue) {
const dataCollector = this.dataCollector; const dataCollector = this.dataCollector;
const make = this.make; const make = this.make;
// If this type is registered // If this type is registered
if (this.collectDataFrom.indexOf(message.type) > -1) { if (this.collectDataFrom.includes(message.type)) {
dataCollector.addDataForUrl( dataCollector.addDataForUrl(
message.url, message.url,
message.type, message.type,
@ -81,7 +84,7 @@ module.exports = {
case 'html.pug': { case 'html.pug': {
// we got a pug from plugins, let compile and cache them // we got a pug from plugins, let compile and cache them
renderer.addTemplate(message.data.id, message.data.pug); addTemplate(message.data.id, message.data.pug);
// and also keep the types so we can render them // and also keep the types so we can render them
this.HTMLBuilder.addType( this.HTMLBuilder.addType(
message.data.id, message.data.id,
@ -230,6 +233,7 @@ module.exports = {
} }
} }
} }
}, }
config: defaultConfig }
};
export { default as config } from './defaultConfig.js';

View File

@ -1,11 +1,10 @@
const get = require('lodash.get'); /* eslint-disable unicorn/no-nested-ternary */
import get from 'lodash.get';
module.exports = { export function pickMedianRun(runs, pageInfo) {
pickMedianRun(runs, pageInfo) {
// Choose the median run. Early first version, in the future we can make // Choose the median run. Early first version, in the future we can make
// this configurable through the CLI // this configurable through the CLI
// If we have SpeedIndex use that else backup with loadEventEnd // If we have SpeedIndex use that else backup with loadEventEnd
const speedIndexMedian = get( const speedIndexMedian = get(
pageInfo, pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics.SpeedIndex.median' 'data.browsertime.pageSummary.statistics.visualMetrics.SpeedIndex.median'
@ -32,8 +31,7 @@ module.exports = {
// make sure we run Browsertime for that run = 3 runs WPT and 2 runs BT // make sure we run Browsertime for that run = 3 runs WPT and 2 runs BT
if ( if (
rumRuns.data.browsertime && rumRuns.data.browsertime &&
loadEventEndMedian === loadEventEndMedian === rumRuns.data.browsertime.run.timings.loadEventEnd
rumRuns.data.browsertime.run.timings.loadEventEnd
) { ) {
return { return {
name: 'LoadEventEnd', name: 'LoadEventEnd',
@ -47,9 +45,8 @@ module.exports = {
runIndex: 1, runIndex: 1,
default: true default: true
}; };
}, }
// Get metrics from a run as a String to use in description export function getMetricsFromRun(pageInfo) {
getMetricsFromRun(pageInfo) {
const visualMetrics = get(pageInfo, 'data.browsertime.run.visualMetrics'); const visualMetrics = get(pageInfo, 'data.browsertime.run.visualMetrics');
const timings = get(pageInfo, 'data.browsertime.run.timings'); const timings = get(pageInfo, 'data.browsertime.run.timings');
if (visualMetrics) { if (visualMetrics) {
@ -62,8 +59,8 @@ module.exports = {
} else { } else {
return ''; return '';
} }
}, }
getMetricsFromPageSummary(pageInfo) { export function getMetricsFromPageSummary(pageInfo) {
const visualMetrics = get( const visualMetrics = get(
pageInfo, pageInfo,
'data.browsertime.pageSummary.statistics.visualMetrics' 'data.browsertime.pageSummary.statistics.visualMetrics'
@ -87,4 +84,3 @@ module.exports = {
return ''; return '';
} }
} }
};

View File

@ -1,9 +1,12 @@
'use strict'; import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const pug = require('pug'); import { compileFile, compile } from 'pug';
const path = require('path'); import intel from 'intel';
const log = require('intel').getLogger('sitespeedio.plugin.html');
const basePath = path.resolve(__dirname, 'templates'); const log = intel.getLogger('sitespeedio.plugin.html');
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const basePath = resolve(__dirname, 'templates');
const templateCache = {}; const templateCache = {};
@ -15,23 +18,21 @@ function getTemplate(templateName) {
return template; return template;
} }
const filename = path.resolve(basePath, templateName); const filename = resolve(basePath, templateName);
const renderedTemplate = pug.compileFile(filename); const renderedTemplate = compileFile(filename);
templateCache[templateName] = renderedTemplate; templateCache[templateName] = renderedTemplate;
return renderedTemplate; return renderedTemplate;
} }
module.exports = { export function renderTemplate(templateName, locals) {
renderTemplate(templateName, locals) {
try { try {
return getTemplate(templateName)(locals); return getTemplate(templateName)(locals);
} catch (e) { } catch (error) {
log.error('Could not generate %s, %s', templateName, e.message); log.error('Could not generate %s, %s', templateName, error.message);
} }
}, }
addTemplate(templateName, templateString) { export function addTemplate(templateName, templateString) {
const compiledTemplate = pug.compile(templateString); const compiledTemplate = compile(templateString);
templateCache[templateName + '.pug'] = compiledTemplate; templateCache[templateName + '.pug'] = compiledTemplate;
} }
};

View File

@ -1,22 +1,20 @@
'use strict'; import { noop, size, time } from '../../../support/helpers/index.js';
import get from 'lodash.get';
const h = require('../../../support/helpers');
const get = require('lodash.get');
function row(stat, name, metricName, formatter) { function row(stat, name, metricName, formatter) {
if (typeof stat === 'undefined') { if (typeof stat === 'undefined') {
return undefined; return;
} }
return { return {
name, name,
metricName, metricName,
node: stat, node: stat,
h: formatter ? formatter : h.noop h: formatter ?? noop
}; };
} }
module.exports = function (data) { export default function (data) {
if (!data) { if (!data) {
return []; return [];
} }
@ -65,41 +63,38 @@ module.exports = function (data) {
'jsRequestsPerPage' 'jsRequestsPerPage'
), ),
row(contentTypes.font.requests, 'Font requests', 'fontRequestsPerPage'), row(contentTypes.font.requests, 'Font requests', 'fontRequestsPerPage'),
row(summary.requests, 'Total requests', 'totalRequestsPerPage') row(summary.requests, 'Total requests', 'totalRequestsPerPage'),
);
rows.push(
row( row(
contentTypes.image.transferSize, contentTypes.image.transferSize,
'Image size', 'Image size',
'imageSizePerPage', 'imageSizePerPage',
h.size.format size.format
), ),
row( row(
contentTypes.html.transferSize, contentTypes.html.transferSize,
'HTML size', 'HTML size',
'htmlSizePerPage', 'htmlSizePerPage',
h.size.format size.format
), ),
row( row(
contentTypes.css.transferSize, contentTypes.css.transferSize,
'CSS size', 'CSS size',
'cssSizePerPage', 'cssSizePerPage',
h.size.format size.format
), ),
row( row(
contentTypes.javascript.transferSize, contentTypes.javascript.transferSize,
'Javascript size', 'Javascript size',
'jsSizePerPage', 'jsSizePerPage',
h.size.format size.format
), ),
row( row(
contentTypes.font.transferSize, contentTypes.font.transferSize,
'Font size', 'Font size',
'fontSizePerPage', 'fontSizePerPage',
h.size.format size.format
), ),
row(summary.transferSize, 'Total size', 'totalSizePerPage', h.size.format) row(summary.transferSize, 'Total size', 'totalSizePerPage', size.format)
); );
const responseCodes = Object.keys(summary.responseCodes); const responseCodes = Object.keys(summary.responseCodes);
@ -110,16 +105,11 @@ module.exports = function (data) {
if (browsertime) { if (browsertime) {
const summary = browsertime.summary; const summary = browsertime.summary;
rows.push(row(summary.firstPaint, 'First Paint', 'firstPaint', h.time.ms)); rows.push(row(summary.firstPaint, 'First Paint', 'firstPaint', time.ms));
if (summary.timings) { if (summary.timings) {
rows.push( rows.push(
row( row(summary.timings.fullyLoaded, 'Fully Loaded', 'fullyLoaded', time.ms)
summary.timings.fullyLoaded,
'Fully Loaded',
'fullyLoaded',
h.time.ms
)
); );
} }
@ -129,7 +119,7 @@ module.exports = function (data) {
summary.timeToDomContentFlushed, summary.timeToDomContentFlushed,
'DOMContentFlushed', 'DOMContentFlushed',
'timeToDomContentFlushed', 'timeToDomContentFlushed',
h.time.ms time.ms
) )
); );
} }
@ -140,13 +130,13 @@ module.exports = function (data) {
summary.timings.largestContentfulPaint, summary.timings.largestContentfulPaint,
'Largest Contentful Paint', 'Largest Contentful Paint',
'largestContentfulPaint', 'largestContentfulPaint',
h.time.ms time.ms
) )
); );
} }
if (summary.memory) { if (summary.memory) {
rows.push(row(summary.memory, 'Memory usage', 'memory', h.size.format)); rows.push(row(summary.memory, 'Memory usage', 'memory', size.format));
} }
if (summary.paintTiming) { if (summary.paintTiming) {
@ -156,13 +146,13 @@ module.exports = function (data) {
'first-contentful-paint': 'First Contentful Paint' 'first-contentful-paint': 'First Contentful Paint'
}; };
for (let pt of paintTimings) { for (let pt of paintTimings) {
rows.push(row(summary.paintTiming[pt], lookup[pt], pt, h.time.ms)); rows.push(row(summary.paintTiming[pt], lookup[pt], pt, time.ms));
} }
} }
const timings = Object.keys(summary.pageTimings); const timings = Object.keys(summary.pageTimings);
for (let timing of timings) { for (let timing of timings) {
rows.push(row(summary.pageTimings[timing], timing, timing, h.time.ms)); rows.push(row(summary.pageTimings[timing], timing, timing, time.ms));
} }
if (summary.custom) { if (summary.custom) {
@ -187,49 +177,49 @@ module.exports = function (data) {
summary.visualMetrics.FirstVisualChange, summary.visualMetrics.FirstVisualChange,
'First Visual Change', 'First Visual Change',
'FirstVisualChange', 'FirstVisualChange',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.SpeedIndex, summary.visualMetrics.SpeedIndex,
'Speed Index', 'Speed Index',
'SpeedIndex', 'SpeedIndex',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.PerceptualSpeedIndex, summary.visualMetrics.PerceptualSpeedIndex,
'Perceptual Speed Index', 'Perceptual Speed Index',
'PerceptualSpeedIndex', 'PerceptualSpeedIndex',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.ContentfulSpeedIndex, summary.visualMetrics.ContentfulSpeedIndex,
'Contentful Speed Index', 'Contentful Speed Index',
'ContentfulSpeedIndex', 'ContentfulSpeedIndex',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.VisualComplete85, summary.visualMetrics.VisualComplete85,
'Visual Complete 85%', 'Visual Complete 85%',
'VisualComplete85', 'VisualComplete85',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.VisualComplete95, summary.visualMetrics.VisualComplete95,
'Visual Complete 95%', 'Visual Complete 95%',
'VisualComplete95', 'VisualComplete95',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.VisualComplete99, summary.visualMetrics.VisualComplete99,
'Visual Complete 99%', 'Visual Complete 99%',
'VisualComplete99', 'VisualComplete99',
h.time.ms time.ms
), ),
row( row(
summary.visualMetrics.LastVisualChange, summary.visualMetrics.LastVisualChange,
'Last Visual Change', 'Last Visual Change',
'LastVisualChange', 'LastVisualChange',
h.time.ms time.ms
) )
); );
@ -239,17 +229,17 @@ module.exports = function (data) {
summary.visualMetrics.LargestImage, summary.visualMetrics.LargestImage,
'Largest Image', 'Largest Image',
'LargestImage', 'LargestImage',
h.time.ms time.ms
) )
); );
} }
if (summary.visualMetrics.Heading) { if (summary.visualMetrics.Heading) {
rows.push( rows.push(
row(summary.visualMetrics.Heading, 'Heading', 'Heading', h.time.ms) row(summary.visualMetrics.Heading, 'Heading', 'Heading', time.ms)
); );
} }
if (summary.visualMetrics.Logo) { if (summary.visualMetrics.Logo) {
rows.push(row(summary.visualMetrics.Logo, 'Logo', 'Logo', h.time.ms)); rows.push(row(summary.visualMetrics.Logo, 'Logo', 'Logo', time.ms));
} }
} }
@ -265,19 +255,19 @@ module.exports = function (data) {
summary.cpu.longTasks.totalDuration, summary.cpu.longTasks.totalDuration,
'CPU Long Tasks total duration', 'CPU Long Tasks total duration',
'cpuLongTasksTotalDurationPerPage', 'cpuLongTasksTotalDurationPerPage',
h.time.ms time.ms
), ),
row( row(
summary.cpu.longTasks.totalBlockingTime, summary.cpu.longTasks.totalBlockingTime,
'Total Blocking Time', 'Total Blocking Time',
'totalBlockingTime', 'totalBlockingTime',
h.time.ms time.ms
), ),
row( row(
summary.cpu.longTasks.maxPotentialFid, summary.cpu.longTasks.maxPotentialFid,
'Max Potential First Input Delay', 'Max Potential First Input Delay',
'maxPotentialFirstInputDelay', 'maxPotentialFirstInputDelay',
h.time.ms time.ms
) )
); );
} }
@ -288,31 +278,31 @@ module.exports = function (data) {
summary.cpu.categories.parseHTML, summary.cpu.categories.parseHTML,
'CPU Parse HTML', 'CPU Parse HTML',
'parseHTMLPerPage', 'parseHTMLPerPage',
h.time.ms time.ms
), ),
row( row(
summary.cpu.categories.styleLayout, summary.cpu.categories.styleLayout,
'CPU Style Layout', 'CPU Style Layout',
'styleLayoutPerPage', 'styleLayoutPerPage',
h.time.ms time.ms
), ),
row( row(
summary.cpu.categories.paintCompositeRender, summary.cpu.categories.paintCompositeRender,
'CPU Paint Composite Render', 'CPU Paint Composite Render',
'paintCompositeRenderPerPage', 'paintCompositeRenderPerPage',
h.time.ms time.ms
), ),
row( row(
summary.cpu.categories.scriptParseCompile, summary.cpu.categories.scriptParseCompile,
'CPU Script Parse Compile', 'CPU Script Parse Compile',
'scriptParseCompilePerPage', 'scriptParseCompilePerPage',
h.time.ms time.ms
), ),
row( row(
summary.cpu.categories.scriptEvaluation, summary.cpu.categories.scriptEvaluation,
'CPU Script Evaluation', 'CPU Script Evaluation',
'scriptEvaluationPerPage', 'scriptEvaluationPerPage',
h.time.ms time.ms
) )
); );
} }
@ -323,18 +313,18 @@ module.exports = function (data) {
const firstView = get(webpagetest, 'summary.timing.firstView'); const firstView = get(webpagetest, 'summary.timing.firstView');
if (firstView) { if (firstView) {
rows.push( rows.push(
row(firstView.render, 'WPT render (firstView)', 'render', h.time.ms), row(firstView.render, 'WPT render (firstView)', 'render', time.ms),
row( row(
firstView.SpeedIndex, firstView.SpeedIndex,
'WPT SpeedIndex (firstView)', 'WPT SpeedIndex (firstView)',
'SpeedIndex', 'SpeedIndex',
h.time.ms time.ms
), ),
row( row(
firstView.fullyLoaded, firstView.fullyLoaded,
'WPT Fully loaded (firstView)', 'WPT Fully loaded (firstView)',
'fullyLoaded', 'fullyLoaded',
h.time.ms time.ms
) )
); );
} }
@ -383,4 +373,4 @@ module.exports = function (data) {
} }
return rows.filter(Boolean); return rows.filter(Boolean);
}; }

View File

@ -1,15 +1,14 @@
'use strict'; import intel from 'intel';
const log = intel.getLogger('sitespeedio.plugin.html');
const log = require('intel').getLogger('sitespeedio.plugin.html'); import { toArray } from '../../../support/util.js';
const toArray = require('../../../support/util').toArray; import friendlyNames from '../../../support/friendlynames.js';
const friendlyNames = require('../../../support/friendlynames'); import get from 'lodash.get';
const get = require('lodash.get'); import defaultLimits from './summaryBoxesDefaultLimits.js';
const defaultLimits = require('./summaryBoxesDefaultLimits'); import merge from 'lodash.merge';
const merge = require('lodash.merge');
function infoBox(stat, name, formatter) { function infoBox(stat, name, formatter) {
if (typeof stat === 'undefined') { if (typeof stat === 'undefined') {
return undefined; return;
} }
return _box(stat, name, 'info', formatter, name.replace(/\s/g, '')); return _box(stat, name, 'info', formatter, name.replace(/\s/g, ''));
@ -17,7 +16,7 @@ function infoBox(stat, name, formatter) {
function scoreBox(stat, name, formatter, box, limits) { function scoreBox(stat, name, formatter, box, limits) {
if (typeof stat === 'undefined') { if (typeof stat === 'undefined') {
return undefined; return;
} }
let label = 'info'; let label = 'info';
@ -36,7 +35,7 @@ function scoreBox(stat, name, formatter, box, limits) {
function timingBox(stat, name, formatter, box, limits) { function timingBox(stat, name, formatter, box, limits) {
if (typeof stat === 'undefined') { if (typeof stat === 'undefined') {
return undefined; return;
} }
let label = 'info'; let label = 'info';
@ -56,7 +55,7 @@ function timingBox(stat, name, formatter, box, limits) {
function pagexrayBox(stat, name, formatter, box, limits) { function pagexrayBox(stat, name, formatter, box, limits) {
if (typeof stat === 'undefined') { if (typeof stat === 'undefined') {
return undefined; return;
} }
let label = 'info'; let label = 'info';
@ -75,7 +74,7 @@ function pagexrayBox(stat, name, formatter, box, limits) {
function axeBox(stat, name, formatter, url, limits) { function axeBox(stat, name, formatter, url, limits) {
if (typeof stat === 'undefined') { if (typeof stat === 'undefined') {
return undefined; return;
} }
let label = 'info'; let label = 'info';
@ -106,7 +105,7 @@ function _box(stat, name, label, formatter, url) {
}; };
} }
module.exports = function (data, html) { export default function (data, html) {
if (!data) { if (!data) {
return []; return [];
} }
@ -122,21 +121,26 @@ module.exports = function (data, html) {
if (friendly) { if (friendly) {
let boxType; let boxType;
switch (tool) { switch (tool) {
case 'coach': case 'coach': {
boxType = scoreBox; boxType = scoreBox;
break; break;
case 'axe': }
case 'axe': {
boxType = axeBox; boxType = axeBox;
break; break;
case 'pagexray': }
case 'pagexray': {
boxType = pagexrayBox; boxType = pagexrayBox;
break; break;
case 'browsertime': }
case 'browsertime': {
boxType = timingBox; boxType = timingBox;
break; break;
default: }
default: {
boxType = infoBox; boxType = infoBox;
} }
}
const stats = get(data, tool + '.summary.' + friendly.summaryPath); const stats = get(data, tool + '.summary.' + friendly.summaryPath);
const boxLimits = get(limits, box); const boxLimits = get(limits, box);
@ -158,4 +162,4 @@ module.exports = function (data, html) {
} }
} }
return boxes; return boxes;
}; }

View File

@ -1,5 +1,4 @@
'use strict'; export default {
module.exports = {
score: { score: {
score: { score: {
green: 90, green: 90,
@ -22,7 +21,6 @@ module.exports = {
yellow: 80 yellow: 80
} }
}, },
// All timings are in ms
timings: { timings: {
firstPaint: { green: 1000, yellow: 2000 }, firstPaint: { green: 1000, yellow: 2000 },
firstContentfulPaint: { green: 2000, yellow: 4000 }, firstContentfulPaint: { green: 2000, yellow: 4000 },
@ -45,16 +43,15 @@ module.exports = {
css: {}, css: {},
image: {} image: {}
}, },
// Size in byte
transferSize: { transferSize: {
total: { green: 1000000, yellow: 1500000 }, total: { green: 1_000_000, yellow: 1_500_000 },
html: {}, html: {},
css: {}, css: {},
image: {}, image: {},
javascript: {} javascript: {}
}, },
contentSize: { contentSize: {
javascript: { green: 100000, yellow: 150000 } javascript: { green: 100_000, yellow: 150_000 }
}, },
thirdParty: { thirdParty: {
requests: {}, requests: {},

View File

@ -2,8 +2,8 @@ window.addEventListener('DOMContentLoaded', function () {
let tabsRoot = document.querySelector('#tabs'); let tabsRoot = document.querySelector('#tabs');
let navigationLinks = document.querySelectorAll('#pageNavigation a'); let navigationLinks = document.querySelectorAll('#pageNavigation a');
for (let i = 0; i < navigationLinks.length; ++i) { for (const navigationLink of navigationLinks) {
navigationLinks[i].addEventListener('click', event => { navigationLink.addEventListener('click', event => {
if (!location.hash) return; if (!location.hash) return;
event.preventDefault(); event.preventDefault();
location.href = `${event.target.href}${location.hash}_ref`; location.href = `${event.target.href}${location.hash}_ref`;
@ -22,8 +22,8 @@ window.addEventListener('DOMContentLoaded', function () {
if (!currentTab) currentTab = tabsRoot.querySelector('a'); if (!currentTab) currentTab = tabsRoot.querySelector('a');
let sections = document.querySelectorAll('#tabSections section'); let sections = document.querySelectorAll('#tabSections section');
for (let i = 0; i < sections.length; i++) { for (const section of sections) {
sections[i].style.display = 'none'; section.style.display = 'none';
} }
selectTab(currentTab, false); selectTab(currentTab, false);
}); });
@ -47,14 +47,14 @@ function selectTab(newSelection, updateUrlFragment) {
section.style.display = 'block'; section.style.display = 'block';
let charts = section.querySelectorAll('.ct-chart'); let charts = section.querySelectorAll('.ct-chart');
for (let i = 0; i < charts.length; i++) { for (const chart of charts) {
if (charts[i].__chartist__) { if (chart.__chartist__) {
charts[i].__chartist__.update(); chart.__chartist__.update();
} }
} }
if (updateUrlFragment && history.replaceState) { if (updateUrlFragment && history.replaceState) {
history.replaceState(null, null, '#' + newSelection.id); history.replaceState(undefined, undefined, '#' + newSelection.id);
} }
return false; return false;

View File

@ -1,54 +0,0 @@
module.exports = {
protocol: {
describe: 'The protocol used to store connect to the InfluxDB host.',
default: 'http',
group: 'InfluxDB'
},
host: {
describe: 'The InfluxDB host used to store captured metrics.',
group: 'InfluxDB'
},
port: {
default: 8086,
describe: 'The InfluxDB port used to store captured metrics.',
group: 'InfluxDB'
},
username: {
describe: 'The InfluxDB username for your InfluxDB instance.',
group: 'InfluxDB'
},
password: {
describe: 'The InfluxDB password for your InfluxDB instance.',
group: 'InfluxDB'
},
database: {
default: 'sitespeed',
describe: 'The database name used to store captured metrics.',
group: 'InfluxDB'
},
tags: {
default: 'category=default',
describe: 'A comma separated list of tags and values added to each metric',
group: 'InfluxDB'
},
includeQueryParams: {
default: false,
describe:
'Whether to include query parameters from the URL in the InfluxDB keys or not',
type: 'boolean',
group: 'InfluxDB'
},
groupSeparator: {
default: '_',
describe:
'Choose which character that will separate a group/domain. Default is underscore, set it to a dot if you wanna keep the original domain name.',
group: 'InfluxDB'
},
annotationScreenshot: {
default: false,
type: 'boolean',
describe:
'Include screenshot (from Browsertime) in the annotation. You need to specify a --resultBaseURL for this to work.',
group: 'InfluxDB'
}
};

View File

@ -1,78 +1,174 @@
'use strict'; import merge from 'lodash.merge';
import reduce from 'lodash.reduce';
const flatten = require('../../support/flattenMessage'), import { flattenMessageData } from '../../support/flattenMessage.js';
merge = require('lodash.merge'), import {
util = require('../../support/tsdbUtil'), getConnectivity,
reduce = require('lodash.reduce'); getURLAndGroup,
toSafeKey
} from '../../support/tsdbUtil.js';
class InfluxDBDataGenerator { function getAdditionalTags(key, type) {
constructor(includeQueryParams, options) { let tags = {};
this.includeQueryParams = !!includeQueryParams; const keyArray = key.split('.');
this.options = options; if (/(^contentTypes)/.test(key)) {
this.defaultTags = {}; // contentTypes.favicon.requests.mean
for (let row of options.influxdb.tags.split(',')) { // contentTypes.favicon.requests
const keyAndValue = row.split('='); // contentTypes.css.transferSize
this.defaultTags[keyAndValue[0]] = keyAndValue[1]; tags.contentType = keyArray[1];
} else if (/(^pageTimings|^visualMetrics)/.test(key)) {
// pageTimings.serverResponseTime.max
// visualMetrics.SpeedIndex.median
tags.timings = keyArray[0];
} else
switch (type) {
case 'browsertime.pageSummary': {
// statistics.timings.pageTimings.backEndTime.median
// statistics.timings.userTimings.marks.logoTime.median
// statistics.visualMetrics.SpeedIndex.median
tags[keyArray[0]] = keyArray[1];
if (keyArray.length >= 5) {
tags[keyArray[2]] = keyArray[3];
} }
if (key.includes('cpu.categories')) {
tags.cpu = 'category';
} else if (key.includes('cpu.events')) {
tags.cpu = 'event';
} else if (key.includes('cpu.longTasks')) {
tags.cpu = 'longTask';
} }
dataFromMessage(message, time, alias) { break;
function getTagsFromMessage(
message,
includeQueryParams,
options,
defaultTags
) {
const tags = merge({}, defaultTags);
let typeParts = message.type.split('.');
tags.origin = typeParts[0];
typeParts.push(typeParts.shift());
tags.summaryType = typeParts[0];
// always have browser and connectivity in Browsertime and related tools
if (
message.type.match(
/(^pagexray|^coach|^browsertime|^thirdparty|^axe|^sustainable)/
)
) {
// if we have a friendly name for your connectivity, use that!
let connectivity = util.getConnectivity(options);
tags.connectivity = connectivity;
tags.browser = options.browser;
} else if (message.type.match(/(^webpagetest)/)) {
if (message.connectivity) {
tags.connectivity = message.connectivity;
} }
if (message.location) { case 'browsertime.summary': {
tags.location = message.location; // firstPaint.median
} // userTimings.marks.logoTime.median
} else if (message.type.match(/(^gpsi)/)) { if (key.includes('userTimings')) {
tags.strategy = options.mobile ? 'mobile' : 'desktop'; tags[keyArray[0]] = keyArray[1];
} }
// if we get a URL type, add the URL break;
if (message.url) { }
const urlAndGroup = util case 'coach.pageSummary': {
.getURLAndGroup( // advice.score
options, // advice.performance.score
message.group, if (keyArray.length > 2) {
message.url, tags.advice = keyArray[1];
includeQueryParams,
alias
)
.split('.');
tags.page = urlAndGroup[1];
tags.group = urlAndGroup[0];
} else if (message.group) {
// add the group of the summary message
tags.group = util.toSafeKey(
message.group,
options.influxdb.groupSeparator
);
} }
tags.testName = options.slug; // set the actual advice name
// advice.performance.adviceList.cacheHeaders.score
if (keyArray.length > 4) {
tags.adviceName = keyArray[3];
}
break;
}
case 'coach.summary': {
// score.max
// performance.score.median
if (keyArray.length === 3) {
tags.advice = keyArray[0];
}
break;
}
case 'webpagetest.pageSummary': {
// data.median.firstView.SpeedIndex webpagetest.pageSummary
tags.view = keyArray[2];
// data.median.firstView.breakdown.html.requests
// data.median.firstView.breakdown.html.bytes
if (key.includes('breakdown')) {
tags.contentType = keyArray[4];
}
break;
}
case 'webpagetest.summary': {
// timing.firstView.SpeedIndex.median
tags.view = keyArray[1];
// asset.firstView.breakdown.html.requests.median
if (key.includes('breakdown')) {
tags.contentType = keyArray[4];
}
break;
}
case 'pagexray.summary': {
// firstParty.requests.min pagexray.summary
// requests.median
// responseCodes.307.max pagexray.summary
// requests.min pagexray.summary
if (key.includes('responseCodes')) {
tags.responseCodes = 'response';
}
if (key.includes('firstParty') || key.includes('thirdParty')) {
tags.party = keyArray[0];
}
break;
}
case 'pagexray.pageSummary': {
// thirdParty.contentTypes.json.requests pagexray.pageSummary
// thirdParty.requests pagexray.pageSummary
// firstParty.cookieStats.max pagexray.pageSummary
// responseCodes.200 pagexray.pageSummary
// expireStats.max pagexray.pageSummary
// totalDomains pagexray.pageSummary
if (key.includes('firstParty') || key.includes('thirdParty')) {
tags.party = keyArray[0];
}
if (key.includes('responseCodes')) {
tags.responseCodes = 'response';
}
if (key.includes('contentTypes')) {
tags.contentType = keyArray[2];
}
break;
}
case 'thirdparty.pageSummary': {
tags.thirdPartyCategory = keyArray[1];
tags.thirdPartyType = keyArray[2];
break;
}
case 'lighthouse.pageSummary': {
// categories.seo.score
// categories.performance.score
if (key.includes('score')) {
tags.audit = keyArray[1];
}
if (key.includes('audits')) {
tags.audit = keyArray[1];
}
break;
}
case 'crux.pageSummary': {
tags.experience = keyArray[0];
tags.formFactor = keyArray[1];
tags.metric = keyArray[2];
break;
}
case 'gpsi.pageSummary': {
if (key.includes('googleWebVitals')) {
tags.testType = 'googleWebVitals';
} else if (key.includes('score')) {
tags.testType = 'score';
} else if (key.includes('loadingExperience')) {
tags.experience = keyArray[0];
tags.metric = keyArray[1];
tags.testType = 'crux';
}
break;
}
default:
// console.log('Missed added tags to ' + key + ' ' + type);
}
return tags; return tags;
} }
@ -91,140 +187,79 @@ class InfluxDBDataGenerator {
]; ];
const keyArray = key.split('.'); const keyArray = key.split('.');
const end = keyArray.pop(); const end = keyArray.pop();
if (functions.indexOf(end) > -1) { if (functions.includes(end)) {
return { field: end, seriesName: keyArray.pop() }; return { field: end, seriesName: keyArray.pop() };
} }
return { field: 'value', seriesName: end }; return { field: 'value', seriesName: end };
} }
export class InfluxDBDataGenerator {
function getAdditionalTags(key, type) { constructor(includeQueryParameters, options) {
let tags = {}; this.includeQueryParams = !!includeQueryParameters;
const keyArray = key.split('.'); this.options = options;
if (key.match(/(^contentTypes)/)) { this.defaultTags = {};
// contentTypes.favicon.requests.mean for (let row of options.influxdb.tags.split(',')) {
// contentTypes.favicon.requests const keyAndValue = row.split('=');
// contentTypes.css.transferSize this.defaultTags[keyAndValue[0]] = keyAndValue[1];
tags.contentType = keyArray[1];
} else if (key.match(/(^pageTimings|^visualMetrics)/)) {
// pageTimings.serverResponseTime.max
// visualMetrics.SpeedIndex.median
tags.timings = keyArray[0];
} else if (type === 'browsertime.pageSummary') {
// statistics.timings.pageTimings.backEndTime.median
// statistics.timings.userTimings.marks.logoTime.median
// statistics.visualMetrics.SpeedIndex.median
tags[keyArray[0]] = keyArray[1];
if (keyArray.length >= 5) {
tags[keyArray[2]] = keyArray[3];
} }
if (key.indexOf('cpu.categories') > -1) {
tags.cpu = 'category';
} else if (key.indexOf('cpu.events') > -1) {
tags.cpu = 'event';
} else if (key.indexOf('cpu.longTasks') > -1) {
tags.cpu = 'longTask';
}
} else if (type === 'browsertime.summary') {
// firstPaint.median
// userTimings.marks.logoTime.median
if (key.indexOf('userTimings') > -1) {
tags[keyArray[0]] = keyArray[1];
}
} else if (type === 'coach.pageSummary') {
// advice.score
// advice.performance.score
if (keyArray.length > 2) {
tags.advice = keyArray[1];
} }
// set the actual advice name dataFromMessage(message, time, alias) {
// advice.performance.adviceList.cacheHeaders.score console.log('GET DATA');
if (keyArray.length > 4) { function getTagsFromMessage(
tags.adviceName = keyArray[3]; message,
includeQueryParameters,
options,
defaultTags
) {
const tags = merge({}, defaultTags);
let typeParts = message.type.split('.');
tags.origin = typeParts[0];
typeParts.push(typeParts.shift());
tags.summaryType = typeParts[0];
// always have browser and connectivity in Browsertime and related tools
if (
/(^pagexray|^coach|^browsertime|^thirdparty|^axe|^sustainable)/.test(
message.type
)
) {
// if we have a friendly name for your connectivity, use that!
let connectivity = getConnectivity(options);
tags.connectivity = connectivity;
tags.browser = options.browser;
} else if (/(^webpagetest)/.test(message.type)) {
if (message.connectivity) {
tags.connectivity = message.connectivity;
} }
} else if (type === 'coach.summary') { if (message.location) {
// score.max tags.location = message.location;
// performance.score.median
if (keyArray.length === 3) {
tags.advice = keyArray[0];
} }
} else if (type === 'webpagetest.pageSummary') { } else if (/(^gpsi)/.test(message.type)) {
// data.median.firstView.SpeedIndex webpagetest.pageSummary tags.strategy = options.mobile ? 'mobile' : 'desktop';
tags.view = keyArray[2];
// data.median.firstView.breakdown.html.requests
// data.median.firstView.breakdown.html.bytes
if (key.indexOf('breakdown') > -1) {
tags.contentType = keyArray[4];
}
} else if (type === 'webpagetest.summary') {
// timing.firstView.SpeedIndex.median
tags.view = keyArray[1];
// asset.firstView.breakdown.html.requests.median
if (key.indexOf('breakdown') > -1) {
tags.contentType = keyArray[4];
}
} else if (type === 'pagexray.summary') {
// firstParty.requests.min pagexray.summary
// requests.median
// responseCodes.307.max pagexray.summary
// requests.min pagexray.summary
if (key.indexOf('responseCodes') > -1) {
tags.responseCodes = 'response';
} }
if (key.indexOf('firstParty') > -1 || key.indexOf('thirdParty') > -1) { // if we get a URL type, add the URL
tags.party = keyArray[0]; if (message.url) {
} const urlAndGroup = getURLAndGroup(
} else if (type === 'pagexray.pageSummary') { options,
// thirdParty.contentTypes.json.requests pagexray.pageSummary message.group,
// thirdParty.requests pagexray.pageSummary message.url,
// firstParty.cookieStats.max pagexray.pageSummary includeQueryParameters,
// responseCodes.200 pagexray.pageSummary alias
// expireStats.max pagexray.pageSummary ).split('.');
// totalDomains pagexray.pageSummary tags.page = urlAndGroup[1];
if (key.indexOf('firstParty') > -1 || key.indexOf('thirdParty') > -1) { tags.group = urlAndGroup[0];
tags.party = keyArray[0]; } else if (message.group) {
} // add the group of the summary message
if (key.indexOf('responseCodes') > -1) { tags.group = toSafeKey(message.group, options.influxdb.groupSeparator);
tags.responseCodes = 'response';
}
if (key.indexOf('contentTypes') > -1) {
tags.contentType = keyArray[2];
}
} else if (type === 'thirdparty.pageSummary') {
tags.thirdPartyCategory = keyArray[1];
tags.thirdPartyType = keyArray[2];
} else if (type === 'lighthouse.pageSummary') {
// categories.seo.score
// categories.performance.score
if (key.indexOf('score') > -1) {
tags.audit = keyArray[1];
}
if (key.indexOf('audits') > -1) {
tags.audit = keyArray[1];
}
} else if (type === 'crux.pageSummary') {
tags.experience = keyArray[0];
tags.formFactor = keyArray[1];
tags.metric = keyArray[2];
} else if (type === 'gpsi.pageSummary') {
if (key.indexOf('googleWebVitals') > -1) {
tags.testType = 'googleWebVitals';
} else if (key.indexOf('score') > -1) {
tags.testType = 'score';
} else if (key.indexOf('loadingExperience') > -1) {
tags.experience = keyArray[0];
tags.metric = keyArray[1];
tags.testType = 'crux';
}
} else {
// console.log('Missed added tags to ' + key + ' ' + type);
} }
tags.testName = options.slug;
return tags; return tags;
} }
return reduce( return reduce(
flatten.flattenMessageData(message), flattenMessageData(message),
(entries, value, key) => { (entries, value, key) => {
const fieldAndSeriesName = getFieldAndSeriesName(key); const fieldAndSeriesName = getFieldAndSeriesName(key);
let tags = getTagsFromMessage( let tags = getTagsFromMessage(
@ -249,5 +284,3 @@ class InfluxDBDataGenerator {
); );
} }
} }
module.exports = InfluxDBDataGenerator;

View File

@ -1,29 +1,17 @@
'use strict'; import isEmpty from 'lodash.isempty';
import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import { InfluxDBSender as Sender } from './sender.js';
import { toSafeKey } from '../../support/tsdbUtil.js';
import { send } from './send-annotation.js';
import { InfluxDBDataGenerator as DataGenerator } from './data-generator.js';
import { throwIfMissing } from '../../support/util.js';
const throwIfMissing = require('../../support/util').throwIfMissing; const log = intel.getLogger('sitespeedio.plugin.influxdb');
const isEmpty = require('lodash.isempty'); export default class InfluxDBPlugin extends SitespeedioPlugin {
const log = require('intel').getLogger('sitespeedio.plugin.influxdb'); constructor(options, context, queue) {
const Sender = require('./sender'); super({ name: 'influxdb', options, context, queue });
const tsdbUtil = require('../../support/tsdbUtil'); }
const sendAnnotations = require('./send-annotation');
const DataGenerator = require('./data-generator');
const path = require('path');
const cliUtil = require('../../cli/util');
module.exports = {
name() {
return path.basename(__dirname);
},
/**
* Define `yargs` options with their respective default values. When displayed by the CLI help message
* all options are namespaced by its plugin name.
*
* @return {Object<string, require('yargs').Options} an object mapping the name of the option and its yargs configuration
*/
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
open(context, options) { open(context, options) {
throwIfMissing(options.influxdb, ['host', 'database'], 'influxdb'); throwIfMissing(options.influxdb, ['host', 'database'], 'influxdb');
@ -34,38 +22,59 @@ module.exports = {
options.influxdb.database options.influxdb.database
); );
const opts = options.influxdb; const options_ = options.influxdb;
this.options = options; this.options = options;
this.sender = new Sender(opts); this.sender = new Sender(options_);
this.timestamp = context.timestamp; this.timestamp = context.timestamp;
this.resultUrls = context.resultUrls; this.resultUrls = context.resultUrls;
this.dataGenerator = new DataGenerator(opts.includeQueryParams, options); this.dataGenerator = new DataGenerator(
options_.includeQueryParams,
options
);
this.messageTypesToFireAnnotations = []; this.messageTypesToFireAnnotations = [];
this.receivedTypesThatFireAnnotations = {}; this.receivedTypesThatFireAnnotations = {};
this.make = context.messageMaker('influxdb').make; this.make = context.messageMaker('influxdb').make;
this.sendAnnotation = true; this.sendAnnotation = true;
this.alias = {}; this.alias = {};
this.wptExtras = {}; this.wptExtras = {};
}, }
processMessage(message, queue) { processMessage(message, queue) {
const filterRegistry = this.filterRegistry; const filterRegistry = this.filterRegistry;
// First catch if we are running Browsertime and/or WebPageTest // First catch if we are running Browsertime and/or WebPageTest
if (message.type === 'browsertime.setup') { switch (message.type) {
case 'browsertime.setup': {
this.messageTypesToFireAnnotations.push('browsertime.pageSummary'); this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
this.usingBrowsertime = true; this.usingBrowsertime = true;
} else if (message.type === 'webpagetest.setup') {
break;
}
case 'webpagetest.setup': {
this.messageTypesToFireAnnotations.push('webpagetest.pageSummary'); this.messageTypesToFireAnnotations.push('webpagetest.pageSummary');
} else if (message.type === 'browsertime.config') {
break;
}
case 'browsertime.config': {
if (message.data.screenshot) { if (message.data.screenshot) {
this.useScreenshots = message.data.screenshot; this.useScreenshots = message.data.screenshot;
this.screenshotType = message.data.screenshotType; this.screenshotType = message.data.screenshotType;
} }
} else if (message.type === 'sitespeedio.setup') {
break;
}
case 'sitespeedio.setup': {
// Let other plugins know that the InfluxDB plugin is alive // Let other plugins know that the InfluxDB plugin is alive
queue.postMessage(this.make('influxdb.setup')); queue.postMessage(this.make('influxdb.setup'));
} else if (message.type === 'grafana.setup') {
break;
}
case 'grafana.setup': {
this.sendAnnotation = false; this.sendAnnotation = false;
break;
}
// No default
} }
if (message.type === 'browsertime.alias') { if (message.type === 'browsertime.alias') {
@ -92,15 +101,13 @@ module.exports = {
this.wptExtras[message.url].webPageTestResultURL = this.wptExtras[message.url].webPageTestResultURL =
message.data.data.summary; message.data.data.summary;
this.wptExtras[message.url].connectivity = message.connectivity; this.wptExtras[message.url].connectivity = message.connectivity;
this.wptExtras[message.url].location = tsdbUtil.toSafeKey( this.wptExtras[message.url].location = toSafeKey(message.location);
message.location
);
} }
// Let us skip this for a while and concentrate on the real deal // Let us skip this for a while and concentrate on the real deal
if ( if (
message.type.match( /(^largestassets|^slowestassets|^aggregateassets|^domains)/.test(
/(^largestassets|^slowestassets|^aggregateassets|^domains)/ message.type
) )
) )
return; return;
@ -139,7 +146,7 @@ module.exports = {
); );
this.receivedTypesThatFireAnnotations[message.url] = 0; this.receivedTypesThatFireAnnotations[message.url] = 0;
return sendAnnotations.send( return send(
message.url, message.url,
message.group, message.group,
absolutePagePath, absolutePagePath,
@ -158,12 +165,9 @@ module.exports = {
return Promise.reject( return Promise.reject(
new Error( new Error(
'No data to send to influxdb for message:\n' + 'No data to send to influxdb for message:\n' +
JSON.stringify(message, null, 2) JSON.stringify(message, undefined, 2)
) )
); );
} }
},
get config() {
return cliUtil.pluginDefaults(this.cliOptions);
} }
}; }

View File

@ -1,14 +1,19 @@
'use strict'; import http from 'node:http';
const http = require('http'); import https from 'node:https';
const https = require('https'); import { stringify } from 'node:querystring';
const log = require('intel').getLogger('sitespeedio.plugin.influxdb');
const querystring = require('querystring');
const tsdbUtil = require('../../support/tsdbUtil');
const annotationsHelper = require('../../support/annotationsHelper');
const dayjs = require('dayjs');
module.exports = { import intel from 'intel';
send( import dayjs from 'dayjs';
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
import {
getAnnotationMessage,
getTagsAsString
} from '../../support/annotationsHelper.js';
const log = intel.getLogger('sitespeedio.plugin.influxdb');
export function send(
url, url,
group, group,
absolutePagePath, absolutePagePath,
@ -24,17 +29,15 @@ module.exports = {
// templates to choose which annotations that will be showed. // templates to choose which annotations that will be showed.
// That's why we need to send tags that matches the template // That's why we need to send tags that matches the template
// variables in Grafana. // variables in Grafana.
const connectivity = tsdbUtil.getConnectivity(options); const connectivity = getConnectivity(options);
const browser = options.browser; const browser = options.browser;
const urlAndGroup = tsdbUtil const urlAndGroup = getURLAndGroup(
.getURLAndGroup(
options, options,
group, group,
url, url,
options.influxdb.includeQueryParams, options.influxdb.includeQueryParams,
alias alias
) ).split('.');
.split('.');
let tags = [connectivity, browser, urlAndGroup[0], urlAndGroup[1]]; let tags = [connectivity, browser, urlAndGroup[0], urlAndGroup[1]];
// See https://github.com/sitespeedio/sitespeed.io/issues/3277 // See https://github.com/sitespeedio/sitespeed.io/issues/3277
@ -43,11 +46,10 @@ module.exports = {
} }
if (webPageTestExtraData) { if (webPageTestExtraData) {
tags.push(webPageTestExtraData.connectivity); tags.push(webPageTestExtraData.connectivity, webPageTestExtraData.location);
tags.push(webPageTestExtraData.location);
} }
const message = annotationsHelper.getAnnotationMessage( const message = getAnnotationMessage(
absolutePagePath, absolutePagePath,
screenShotsEnabledInBrowsertime, screenShotsEnabledInBrowsertime,
screenshotType, screenshotType,
@ -67,7 +69,7 @@ module.exports = {
tags.push(keyAndValue[1]); tags.push(keyAndValue[1]);
} }
} }
const influxDBTags = annotationsHelper.getTagsAsString(tags); const influxDBTags = getTagsAsString(tags);
const postData = `events title="Sitespeed.io",text="${message}",tags=${influxDBTags} ${timestamp}`; const postData = `events title="Sitespeed.io",text="${message}",tags=${influxDBTags} ${timestamp}`;
const postOptions = { const postOptions = {
hostname: options.influxdb.host, hostname: options.influxdb.host,
@ -84,7 +86,7 @@ module.exports = {
postOptions.path = postOptions.path =
postOptions.path + postOptions.path +
'&' + '&' +
querystring.stringify({ stringify({
u: options.influxdb.username, u: options.influxdb.username,
p: options.influxdb.password p: options.influxdb.password
}); });
@ -93,8 +95,8 @@ module.exports = {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
log.debug('Send annotation to Influx: %j', postData); log.debug('Send annotation to Influx: %j', postData);
// not perfect but maybe work for us // not perfect but maybe work for us
const lib = options.influxdb.protocol === 'https' ? https : http; const library = options.influxdb.protocol === 'https' ? https : http;
const req = lib.request(postOptions, res => { const request = library.request(postOptions, res => {
if (res.statusCode !== 204) { if (res.statusCode !== 204) {
const e = new Error( const e = new Error(
`Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}` `Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
@ -102,17 +104,16 @@ module.exports = {
log.warn(e.message); log.warn(e.message);
reject(e); reject(e);
} else { } else {
res.setEncoding('utf-8'); res.setEncoding('utf8');
log.debug('Sent annotation to InfluxDB'); log.debug('Sent annotation to InfluxDB');
resolve(); resolve();
} }
}); });
req.on('error', err => { request.on('error', error => {
log.error('Got error from InfluxDB when sending annotation', err); log.error('Got error from InfluxDB when sending annotation', error);
reject(err); reject(error);
}); });
req.write(postData); request.write(postData);
req.end(); request.end();
}); });
} }
};

View File

@ -1,10 +1,8 @@
'use strict'; import { InfluxDB } from 'influx';
const Influx = require('influx'); export class InfluxDBSender {
class InfluxDBSender {
constructor({ protocol, host, port, database, username, password }) { constructor({ protocol, host, port, database, username, password }) {
this.client = new Influx.InfluxDB({ this.client = new InfluxDB({
protocol, protocol,
host, host,
port, port,
@ -26,5 +24,3 @@ class InfluxDBSender {
return this.client.writePoints(points); return this.client.writePoints(points);
} }
} }
module.exports = InfluxDBSender;

View File

@ -1,22 +1,27 @@
'use strict'; import { resolve, join } from 'node:path';
import { platform } from 'node:os';
import { promisify } from 'node:util';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
import osName from 'os-name';
import getos from 'getos';
import get from 'lodash.get';
import { getURLAndGroup, getConnectivity } from '../../support/tsdbUtil.js';
import { cap, plural } from '../../support/helpers/index.js';
const path = require('path');
const osName = require('os-name');
const getos = require('getos');
const { promisify } = require('util');
const getOS = promisify(getos); const getOS = promisify(getos);
const os = require('os'); export default class LatestStorerPlugin extends SitespeedioPlugin {
const get = require('lodash.get'); constructor(options, context, queue) {
const graphiteUtil = require('../../support/tsdbUtil'); super({ name: 'lateststorer', options, context, queue });
const helpers = require('../../support/helpers'); }
module.exports = {
open(context, options) { open(context, options) {
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
this.alias = {}; this.alias = {};
this.options = options; this.options = options;
this.context = context; this.context = context;
}, }
async processMessage(message) { async processMessage(message) {
switch (message.type) { switch (message.type) {
// Collect alias so we can use it // Collect alias so we can use it
@ -47,7 +52,7 @@ module.exports = {
const browserData = this.browserData; const browserData = this.browserData;
const baseDir = this.storageManager.getBaseDir(); const baseDir = this.storageManager.getBaseDir();
// Hack to get out of the date dir // Hack to get out of the date dir
const newPath = path.resolve(baseDir, '..'); const newPath = resolve(baseDir, '..');
// This is a hack to get the same name as in Grafana, meaning we can // This is a hack to get the same name as in Grafana, meaning we can
// generate the path to the URL there // generate the path to the URL there
@ -55,14 +60,14 @@ module.exports = {
(options.copyLatestFilesToBaseGraphiteNamespace (options.copyLatestFilesToBaseGraphiteNamespace
? `${options.graphite.namespace}.` ? `${options.graphite.namespace}.`
: '') + : '') +
graphiteUtil.getURLAndGroup( getURLAndGroup(
options, options,
message.group, message.group,
message.url, message.url,
this.options.graphite.includeQueryParams, this.options.graphite.includeQueryParams,
this.alias this.alias
); );
const connectivity = graphiteUtil.getConnectivity(options); const connectivity = getConnectivity(options);
if (this.useScreenshots) { if (this.useScreenshots) {
let imagePath = ''; let imagePath = '';
@ -74,13 +79,13 @@ module.exports = {
screenshot screenshot
) )
) { ) {
const type = screenshot.substring( const type = screenshot.slice(
screenshot.lastIndexOf('/') + 1, screenshot.lastIndexOf('/') + 1,
screenshot.lastIndexOf('.') screenshot.lastIndexOf('.')
); );
imagePath = screenshot; imagePath = screenshot;
const imageFullPath = path.join(baseDir, imagePath); const imageFullPath = join(baseDir, imagePath);
await this.storageManager.copyFileToDir( await this.storageManager.copyFileToDir(
imageFullPath, imageFullPath,
newPath + newPath +
@ -95,12 +100,12 @@ module.exports = {
this.screenshotType this.screenshotType
); );
} else { } else {
// This is a user generated screenshot, we do not copy that // do nada
} }
} }
} }
if (options.browsertime && options.browsertime.video) { if (options.browsertime && options.browsertime.video) {
const videoFullPath = path.join(baseDir, message.data.video); const videoFullPath = join(baseDir, message.data.video);
await this.storageManager.copyFileToDir( await this.storageManager.copyFileToDir(
videoFullPath, videoFullPath,
@ -147,14 +152,12 @@ module.exports = {
} }
json.browser = {}; json.browser = {};
json.browser.name = helpers.cap( json.browser.name = cap(get(browserData, 'browser.name', 'unknown'));
get(browserData, 'browser.name', 'unknown')
);
json.browser.version = get(browserData, 'browser.version', 'unknown'); json.browser.version = get(browserData, 'browser.version', 'unknown');
json.friendlyHTML = `<b><a href="${message.url}">${ json.friendlyHTML = `<b><a href="${message.url}">${
json.alias ? json.alias : message.url json.alias ?? message.url
}</a></b> ${helpers.plural( }</a></b> ${plural(
options.browsertime.iterations, options.browsertime.iterations,
'iteration' 'iteration'
)} at <i>${json.timestamp}</i> using ${json.browser.name} ${ )} at <i>${json.timestamp}</i> using ${json.browser.name} ${
@ -189,7 +192,7 @@ module.exports = {
} else { } else {
// We are testing on desktop // We are testing on desktop
let osInfo = osName(); let osInfo = osName();
if (os.platform() === 'linux') { if (platform() === 'linux') {
const linux = await getOS(); const linux = await getOS();
osInfo = `${linux.dist} ${linux.release}`; osInfo = `${linux.dist} ${linux.release}`;
} }
@ -211,7 +214,7 @@ module.exports = {
json.result = resultURL; json.result = resultURL;
} }
const data = JSON.stringify(json, null, 0); const data = JSON.stringify(json, undefined, 0);
return this.storageManager.writeDataToDir( return this.storageManager.writeDataToDir(
data, data,
name + '.' + options.browser + '.' + connectivity + '.json', name + '.' + options.browser + '.' + connectivity + '.json',
@ -221,4 +224,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -1,31 +0,0 @@
const { messageTypes } = require('.');
module.exports = {
host: {
describe: 'The Matrix host.',
group: 'Matrix'
},
accessToken: {
describe: 'The Matrix access token.',
group: 'Matrix'
},
room: {
describe:
'The default Matrix room. It is alsways used. You can override the room per message type using --matrix.rooms',
group: 'Matrix'
},
messages: {
describe:
'Choose what type of message to send to Matrix. There are two types of messages: Error messages and budget messages. Errors are errors that happens through the tests (failures like strarting a test) and budget is test failing against your budget.',
choices: messageTypes,
default: messageTypes,
group: 'Matrix'
},
rooms: {
describe:
'Send messages to different rooms. Current message types are [' +
messageTypes +
']. If you want to send error messages to a specific room use --matrix.rooms.error ROOM',
group: 'Matrix'
}
};

View File

@ -1,15 +1,15 @@
'use strict'; import intel from 'intel';
import get from 'lodash.get';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const throwIfMissing = require('../../support/util').throwIfMissing; import send from './send.js';
const log = require('intel').getLogger('sitespeedio.plugin.matrix'); import { throwIfMissing } from '../../support/util.js';
const path = require('path');
const get = require('lodash.get'); const log = intel.getLogger('sitespeedio.plugin.matrix');
const cliUtil = require('../../cli/util');
const send = require('./send');
function getBrowserData(data) { function getBrowserData(data) {
if (data && data.browser) { return data && data.browser
return `${data.browser.name} ${data.browser.version} ${get( ? `${data.browser.name} ${data.browser.version} ${get(
data, data,
'android.model', 'android.model',
'' ''
@ -17,17 +17,15 @@ function getBrowserData(data) {
data, data,
'android.id', 'android.id',
'' ''
)} `; )} `
} else return ''; : '';
}
export default class MatrixPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'matrix', options, context, queue });
} }
module.exports = {
name() {
return path.basename(__dirname);
},
get cliOptions() {
return require(path.resolve(__dirname, 'cli.js'));
},
open(context, options = {}) { open(context, options = {}) {
this.matrixOptions = options.matrix || {}; this.matrixOptions = options.matrix || {};
this.options = options; this.options = options;
@ -37,7 +35,8 @@ module.exports = {
this.errorTexts = ''; this.errorTexts = '';
this.waitForUpload = false; this.waitForUpload = false;
this.alias = {}; this.alias = {};
}, }
async processMessage(message) { async processMessage(message) {
const options = this.matrixOptions; const options = this.matrixOptions;
switch (message.type) { switch (message.type) {
@ -67,7 +66,7 @@ module.exports = {
this.errorTexts this.errorTexts
); );
log.debug('Got %j from the matrix server', answer); log.debug('Got %j from the matrix server', answer);
} catch (e) { } catch {
// TODO what todo? // TODO what todo?
} }
} }
@ -82,17 +81,15 @@ module.exports = {
case 'error': { case 'error': {
// We can send too many messages to Matrix and get 429 so instead // We can send too many messages to Matrix and get 429 so instead
// we bulk send them all one time // we bulk send them all one time
if (options.messages.indexOf('error') > -1) { if (options.messages.includes('error')) {
this.errorTexts += `&#9888;&#65039; Error from <b>${ this.errorTexts += `&#9888;&#65039; Error from <b>${
message.source message.source
}</b> testing ${message.url ? message.url : ''} <pre>${ }</b> testing ${message.url || ''} <pre>${message.data}</pre>`;
message.data
}</pre>`;
} }
break; break;
} }
case 'budget.result': { case 'budget.result': {
if (options.messages.indexOf('budget') > -1) { if (options.messages.includes('budget')) {
let text = ''; let text = '';
// We have failing URLs in the budget // We have failing URLs in the budget
if (Object.keys(message.data.failing).length > 0) { if (Object.keys(message.data.failing).length > 0) {
@ -103,8 +100,8 @@ module.exports = {
}${getBrowserData(this.browserData)}</p>`; }${getBrowserData(this.browserData)}</p>`;
for (let url of failingURLs) { for (let url of failingURLs) {
text += `<h5>&#10060; ${url}`; text += `<h5>&#10060; ${url}`;
if (this.resultUrls.hasBaseUrl()) { text += this.resultUrls.hasBaseUrl()
text += ` (<a href="${this.resultUrls.absoluteSummaryPagePath( ? ` (<a href="${this.resultUrls.absoluteSummaryPagePath(
url, url,
this.alias[url] this.alias[url]
)}index.html">result</a> - <a href="${this.resultUrls.absoluteSummaryPagePath( )}index.html">result</a> - <a href="${this.resultUrls.absoluteSummaryPagePath(
@ -112,10 +109,8 @@ module.exports = {
this.alias[url] this.alias[url]
)}data/screenshots/1/afterPageCompleteCheck.${ )}data/screenshots/1/afterPageCompleteCheck.${
this.screenshotType this.screenshotType
}">screenshot</a>)</h5>`; }">screenshot</a>)</h5>`
} else { : '</h5>';
text += '</h5>';
}
text += '<ul>'; text += '<ul>';
for (let failing of message.data.failing[url]) { for (let failing of message.data.failing[url]) {
text += `<li>${failing.metric} : ${failing.friendlyValue} (${failing.friendlyLimit})</li>`; text += `<li>${failing.metric} : ${failing.friendlyValue} (${failing.friendlyLimit})</li>`;
@ -155,18 +150,15 @@ module.exports = {
case 'scp.finished': case 'scp.finished':
case 'ftp.finished': case 'ftp.finished':
case 's3.finished': { case 's3.finished': {
if (this.waitForUpload && options.messages.indexOf('budget') > -1) { if (this.waitForUpload && options.messages.includes('budget')) {
const room = get(options, 'rooms.budget', options.room); const room = get(options, 'rooms.budget', options.room);
await send(options.host, room, options.accessToken, this.budgetText); await send(options.host, room, options.accessToken, this.budgetText);
} }
break; break;
} }
} }
}, }
get config() { }
return cliUtil.pluginDefaults(this.cliOptions); export function messageTypes() {
},
get messageTypes() {
return ['error', 'budget']; return ['error', 'budget'];
} }
};

View File

@ -1,7 +1,6 @@
'use strict'; import { request as _request } from 'node:https';
import intel from 'intel';
const https = require('https'); const log = intel.getLogger('sitespeedio.plugin.matrix');
const log = require('intel').getLogger('sitespeedio.plugin.matrix');
function send( function send(
host, host,
@ -12,9 +11,9 @@ function send(
retries = 3, retries = 3,
backoff = 5000 backoff = 5000
) { ) {
const retryCodes = [408, 429, 500, 503]; const retryCodes = new Set([408, 429, 500, 503]);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const req = https.request( const request = _request(
{ {
host, host,
port: 443, port: 443,
@ -26,9 +25,9 @@ function send(
method: 'POST' method: 'POST'
}, },
res => { res => {
const { statusCode } = res; const { statusCode, statusMessage } = res;
if (statusCode < 200 || statusCode > 299) { if (statusCode < 200 || statusCode > 299) {
if (retries > 0 && retryCodes.includes(statusCode)) { if (retries > 0 && retryCodes.has(statusCode)) {
setTimeout(() => { setTimeout(() => {
return send( return send(
host, host,
@ -42,9 +41,9 @@ function send(
}, backoff); }, backoff);
} else { } else {
log.error( log.error(
`Got error from Matrix. Error Code: ${res.statusCode} Message: ${res.statusMessage}` `Got error from Matrix. Error Code: ${statusCode} Message: ${statusMessage}`
); );
reject(new Error(`Status Code: ${res.statusCode}`)); reject(new Error(`Status Code: ${statusCode}`));
} }
} else { } else {
const data = []; const data = [];
@ -57,12 +56,12 @@ function send(
} }
} }
); );
req.write(JSON.stringify(data)); request.write(JSON.stringify(data));
req.end(); request.end();
}); });
} }
module.exports = async (host, room, accessToken, message) => { export default async (host, room, accessToken, message) => {
const data = { const data = {
msgtype: 'm.notice', msgtype: 'm.notice',
body: '', body: '',

View File

@ -1,18 +1,20 @@
'use strict';
/* eslint no-console:0 */ /* eslint no-console:0 */
const log = require('intel').getLogger('sitespeedio.plugin.messagelogger'); import intel from 'intel';
const isEmpty = require('lodash.isempty'); import isEmpty from 'lodash.isempty';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = intel.getLogger('sitespeedio.plugin.messagelogger');
function shortenData(key, value) { function shortenData(key, value) {
if (key === 'data' && !isEmpty(value)) { if (key === 'data' && !isEmpty(value)) {
switch (typeof value) { switch (typeof value) {
case 'object': case 'object': {
return Array.isArray(value) ? '[...]' : '{...}'; return Array.isArray(value) ? '[...]' : '{...}';
}
case 'string': { case 'string': {
if (value.length > 100) { if (value.length > 100) {
return value.substring(0, 97) + '...'; return value.slice(0, 97) + '...';
} }
} }
} }
@ -20,26 +22,32 @@ function shortenData(key, value) {
return value; return value;
} }
module.exports = { export default class MessageLoggerPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'messagelogger', options, context, queue });
}
open(context, options) { open(context, options) {
this.verbose = Number(options.verbose || 0); this.verbose = Number(options.verbose || 0);
}, }
processMessage(message) { processMessage(message) {
let replacerFunc; let replacerFunction;
switch (message.type) { switch (message.type) {
case 'browsertime.har': case 'browsertime.har':
case 'browsertime.run': case 'browsertime.run':
case 'domains.summary': case 'domains.summary':
case 'webpagetest.pageSummary': case 'webpagetest.pageSummary':
case 'browsertime.screenshot': case 'browsertime.screenshot': {
replacerFunc = this.verbose > 1 ? null : shortenData; replacerFunction = this.verbose > 1 ? undefined : shortenData;
break; break;
default:
replacerFunc = this.verbose > 0 ? null : shortenData;
} }
log.info(JSON.stringify(message, replacerFunc, 2)); default: {
replacerFunction = this.verbose > 0 ? undefined : shortenData;
}
}
log.info(JSON.stringify(message, replacerFunction, 2));
}
} }
};

View File

@ -1,20 +1,24 @@
'use strict'; import merge from 'lodash.merge';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const flatten = require('../../support/flattenMessage'); import { flattenMessageData } from '../../support/flattenMessage.js';
const merge = require('lodash.merge');
const defaultConfig = { const defaultConfig = {
list: false, list: false,
filterList: false filterList: false
}; };
module.exports = { export default class MetricsPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'metrics', options, context, queue });
}
open(context, options) { open(context, options) {
this.options = merge({}, defaultConfig, options.metrics); this.options = merge({}, defaultConfig, options.metrics);
this.metrics = {}; this.metrics = {};
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
this.filterRegistry = context.filterRegistry; this.filterRegistry = context.filterRegistry;
}, }
processMessage(message) { processMessage(message) {
const filterRegistry = this.filterRegistry; const filterRegistry = this.filterRegistry;
@ -38,8 +42,7 @@ module.exports = {
} }
// only dance if we all wants to // only dance if we all wants to
if (this.options.filter) { if (this.options.filter && message.type === 'sitespeedio.setup') {
if (message.type === 'sitespeedio.setup') {
const filters = Array.isArray(this.options.filter) const filters = Array.isArray(this.options.filter)
? this.options.filter ? this.options.filter
: [this.options.filter]; : [this.options.filter];
@ -74,7 +77,6 @@ module.exports = {
} }
} }
} }
}
if (this.options.list) { if (this.options.list) {
if ( if (
@ -86,11 +88,11 @@ module.exports = {
) { ) {
return; return;
} }
let flattenMess = flatten.flattenMessageData(message); let flattenMess = flattenMessageData(message);
for (let key of Object.keys(flattenMess)) { for (let key of Object.keys(flattenMess)) {
this.metrics[message.type + '.' + key] = 1; this.metrics[message.type + '.' + key] = 1;
} }
} }
}, }
config: defaultConfig }
}; export const config = defaultConfig;

View File

@ -1,9 +1,15 @@
'use strict'; import { parse } from 'node:url';
const pagexrayAggregator = require('./pagexrayAggregator'); import intel from 'intel';
const pagexray = require('coach-core').getPageXray(); import coach from 'coach-core';
const urlParser = require('url'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = require('intel').getLogger('plugin.pagexray');
const h = require('../../support/helpers'); import { PageXrayAggregator } from './pagexrayAggregator.js';
import { short } from '../../support/helpers/index.js';
const { getPageXray } = coach;
const pagexray = getPageXray();
const log = intel.getLogger('plugin.pagexray');
const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = [ const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = [
'contentTypes', 'contentTypes',
'transferSize', 'transferSize',
@ -33,11 +39,17 @@ const DEFAULT_PAGEXRAY_SUMMARY_METRICS = [
const DEFAULT_PAGEXRAY_RUN_METRICS = []; const DEFAULT_PAGEXRAY_RUN_METRICS = [];
module.exports = { export default class PageXrayPlugin extends SitespeedioPlugin {
constructor(options, context) {
super({ name: 'pagexray', options, context });
}
open(context, options) { open(context, options) {
this.options = options; this.options = options;
this.make = context.messageMaker('pagexray').make; this.make = context.messageMaker('pagexray').make;
this.pageXrayAggregator = new PageXrayAggregator();
context.filterRegistry.registerFilterForType( context.filterRegistry.registerFilterForType(
DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS, DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS,
'pagexray.pageSummary' 'pagexray.pageSummary'
@ -54,7 +66,7 @@ module.exports = {
this.usingWebpagetest = false; this.usingWebpagetest = false;
this.usingBrowsertime = false; this.usingBrowsertime = false;
this.multi = options.multi; this.multi = options.multi;
}, }
processMessage(message, queue) { processMessage(message, queue) {
const make = this.make; const make = this.make;
switch (message.type) { switch (message.type) {
@ -75,9 +87,7 @@ module.exports = {
const group = message.group; const group = message.group;
let config = { let config = {
includeAssets: true, includeAssets: true,
firstParty: this.options.firstParty firstParty: this.options.firstParty ?? undefined
? this.options.firstParty
: undefined
}; };
const pageSummary = pagexray.convert(message.data, config); const pageSummary = pagexray.convert(message.data, config);
//check and print any http server error > 399 //check and print any http server error > 399
@ -88,20 +98,20 @@ module.exports = {
log.info( log.info(
`The server responded with a ${ `The server responded with a ${
asset.status asset.status
} status code for ${h.short(asset.url, 60)}` } status code for ${short(asset.url, 60)}`
); );
} }
} }
} }
} }
pagexrayAggregator.addToAggregate(pageSummary, group); this.pageXrayAggregator.addToAggregate(pageSummary, group);
if (this.multi) { if (this.multi) {
// The HAR file can have multiple URLs // The HAR file can have multiple URLs
const sentURL = {}; const sentURL = {};
for (let summary of pageSummary) { for (let summary of pageSummary) {
// The group can be different so take it per url // The group can be different so take it per url
const myGroup = urlParser.parse(summary.url).hostname; const myGroup = parse(summary.url).hostname;
if (!sentURL[summary.url]) { if (!sentURL[summary.url]) {
sentURL[summary.url] = 1; sentURL[summary.url] = 1;
queue.postMessage( queue.postMessage(
@ -149,7 +159,7 @@ module.exports = {
case 'sitespeedio.summarize': { case 'sitespeedio.summarize': {
log.debug('Generate summary metrics from PageXray'); log.debug('Generate summary metrics from PageXray');
let pagexraySummary = pagexrayAggregator.summarize(); let pagexraySummary = this.pageXrayAggregator.summarize();
if (pagexraySummary) { if (pagexraySummary) {
for (let group of Object.keys(pagexraySummary.groups)) { for (let group of Object.keys(pagexraySummary.groups)) {
queue.postMessage( queue.postMessage(
@ -161,4 +171,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -1,13 +1,15 @@
'use strict'; import forEach from 'lodash.foreach';
const statsHelpers = require('../../support/statsHelpers'), import { pushGroupStats, setStatsSummary } from '../../support/statsHelpers.js';
forEach = require('lodash.foreach');
const METRIC_NAMES = ['transferSize', 'contentSize', 'requests']; const METRIC_NAMES = ['transferSize', 'contentSize', 'requests'];
module.exports = { export class PageXrayAggregator {
stats: {}, constructor() {
groups: {}, this.stats = {};
this.groups = {};
}
addToAggregate(pageSummary, group) { addToAggregate(pageSummary, group) {
if (this.groups[group] === undefined) { if (this.groups[group] === undefined) {
this.groups[group] = {}; this.groups[group] = {};
@ -16,52 +18,56 @@ module.exports = {
let stats = this.stats; let stats = this.stats;
let groups = this.groups; let groups = this.groups;
pageSummary.forEach(function (summary) { for (const summary of pageSummary) {
// stats for the whole page // stats for the whole page
METRIC_NAMES.forEach(function (metric) { for (const metric of METRIC_NAMES) {
// There's a bug in Firefox/https://github.com/devtools-html/har-export-trigger // There's a bug in Firefox/https://github.com/devtools-html/har-export-trigger
// that sometimes generate content size that is null, see // that sometimes generate content size that is null, see
// https://github.com/sitespeedio/sitespeed.io/issues/2090 // https://github.com/sitespeedio/sitespeed.io/issues/2090
if (!isNaN(summary[metric])) { if (!Number.isNaN(summary[metric])) {
statsHelpers.pushGroupStats( pushGroupStats(stats, groups[group], metric, summary[metric]);
stats, }
groups[group],
metric,
summary[metric]
);
} }
});
Object.keys(summary.contentTypes).forEach(function (contentType) { for (const contentType of Object.keys(summary.contentTypes)) {
METRIC_NAMES.forEach(function (metric) { for (const metric of METRIC_NAMES) {
// There's a bug in Firefox/https://github.com/devtools-html/har-export-trigger // There's a bug in Firefox/https://github.com/devtools-html/har-export-trigger
// that sometimes generate content size that is null, see // that sometimes generate content size that is null, see
// https://github.com/sitespeedio/sitespeed.io/issues/2090 // https://github.com/sitespeedio/sitespeed.io/issues/2090
if (!isNaN(summary.contentTypes[contentType][metric])) { if (!Number.isNaN(summary.contentTypes[contentType][metric])) {
statsHelpers.pushGroupStats( pushGroupStats(
stats, stats,
groups[group], groups[group],
'contentTypes.' + contentType + '.' + metric, 'contentTypes.' + contentType + '.' + metric,
summary.contentTypes[contentType][metric] summary.contentTypes[contentType][metric]
); );
} }
}); }
}); }
Object.keys(summary.responseCodes).forEach(function (responseCode) { for (const responseCode of Object.keys(summary.responseCodes)) {
statsHelpers.pushGroupStats( pushGroupStats(
stats, stats,
groups[group], groups[group],
'responseCodes.' + responseCode, 'responseCodes.' + responseCode,
summary.responseCodes[responseCode] summary.responseCodes[responseCode]
); );
}); }
/*
for (const responseCode of Object.keys(summary.responseCodes)) {
pushGroupStats(
stats,
groups[group],
'responseCodes.' + responseCode,
summary.responseCodes[responseCode]
);
}*/
// extras for firstParty vs third // extras for firstParty vs third
if (summary.firstParty.requests) { if (summary.firstParty.requests) {
METRIC_NAMES.forEach(function (metric) { for (const metric of METRIC_NAMES) {
if (summary.firstParty[metric] !== undefined) { if (summary.firstParty[metric] !== undefined) {
statsHelpers.pushGroupStats( pushGroupStats(
stats, stats,
groups[group], groups[group],
'firstParty' + '.' + metric, 'firstParty' + '.' + metric,
@ -69,18 +75,18 @@ module.exports = {
); );
} }
if (summary.thirdParty[metric] !== undefined) { if (summary.thirdParty[metric] !== undefined) {
statsHelpers.pushGroupStats( pushGroupStats(
stats, stats,
groups[group], groups[group],
'thirdParty' + '.' + metric, 'thirdParty' + '.' + metric,
summary.thirdParty[metric] summary.thirdParty[metric]
); );
} }
}); }
} }
// Add the total amount of domains on this page // Add the total amount of domains on this page
statsHelpers.pushGroupStats( pushGroupStats(
stats, stats,
groups[group], groups[group],
'domains', 'domains',
@ -88,32 +94,22 @@ module.exports = {
); );
// And the total amounts of cookies // And the total amounts of cookies
statsHelpers.pushGroupStats( pushGroupStats(stats, groups[group], 'cookies', summary.cookies);
stats,
groups[group],
'cookies',
summary.cookies
);
forEach(summary.assets, asset => { forEach(summary.assets, asset => {
statsHelpers.pushGroupStats( pushGroupStats(stats, groups[group], 'expireStats', asset.expires);
stats, pushGroupStats(
groups[group],
'expireStats',
asset.expires
);
statsHelpers.pushGroupStats(
stats, stats,
groups[group], groups[group],
'lastModifiedStats', 'lastModifiedStats',
asset.timeSinceLastModified asset.timeSinceLastModified
); );
}); });
}); }
}, }
summarize() { summarize() {
if (Object.keys(this.stats).length === 0) { if (Object.keys(this.stats).length === 0) {
return undefined; return;
} }
const total = this.summarizePerObject(this.stats); const total = this.summarizePerObject(this.stats);
@ -128,37 +124,34 @@ module.exports = {
summary.groups[group] = this.summarizePerObject(this.groups[group]); summary.groups[group] = this.summarizePerObject(this.groups[group]);
} }
return summary; return summary;
}, }
summarizePerObject(type) { summarizePerObject(type) {
return Object.keys(type).reduce((summary, name) => { return Object.keys(type).reduce((summary, name) => {
if ( if (
METRIC_NAMES.indexOf(name) > -1 || METRIC_NAMES.includes(name) ||
name.match(/(^domains|^expireStats|^lastModifiedStats|^cookies)/) /(^domains|^expireStats|^lastModifiedStats|^cookies)/.test(name)
) { ) {
statsHelpers.setStatsSummary(summary, name, type[name]); setStatsSummary(summary, name, type[name]);
} else { } else {
if (name === 'contentTypes') { if (name === 'contentTypes') {
const contentTypeData = {}; const contentTypeData = {};
forEach(Object.keys(type[name]), contentType => { forEach(Object.keys(type[name]), contentType => {
forEach(type[name][contentType], (stats, metric) => { forEach(type[name][contentType], (stats, metric) => {
statsHelpers.setStatsSummary( setStatsSummary(contentTypeData, [contentType, metric], stats);
contentTypeData,
[contentType, metric],
stats
);
}); });
}); });
summary[name] = contentTypeData; summary[name] = contentTypeData;
} else if (name === 'responseCodes') { } else if (name === 'responseCodes') {
const responseCodeData = {}; const responseCodeData = {};
type.responseCodes.forEach((stats, metric) => { for (const [metric, stats] of type.responseCodes.entries()) {
statsHelpers.setStatsSummary(responseCodeData, metric, stats); if (stats != undefined)
}); setStatsSummary(responseCodeData, metric, stats);
}
summary[name] = responseCodeData; summary[name] = responseCodeData;
} else { } else {
const data = {}; const data = {};
forEach(type[name], (stats, metric) => { forEach(type[name], (stats, metric) => {
statsHelpers.setStatsSummary(data, metric, stats); setStatsSummary(data, metric, stats);
}); });
summary[name] = data; summary[name] = data;
} }
@ -166,4 +159,4 @@ module.exports = {
return summary; return summary;
}, {}); }, {});
} }
}; }

View File

@ -1,12 +1,16 @@
'use strict'; import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = require('intel').getLogger('sitespeedio.plugin.remove'); const log = intel.getLogger('sitespeedio.plugin.remove');
module.exports = { export default class RemovePlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'remove', options, context, queue });
}
open(context, options) { open(context, options) {
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
this.options = options; this.options = options;
}, }
async processMessage(message) { async processMessage(message) {
switch (message.type) { switch (message.type) {
case 'remove.url': { case 'remove.url': {
@ -16,4 +20,4 @@ module.exports = {
} }
} }
} }
}; }

View File

@ -1,5 +1,3 @@
'use strict';
const DEFAULT_CONTENT_TYPE = 'application/octet-stream'; const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
const types = { const types = {
@ -19,12 +17,10 @@ const types = {
log: 'text/plain' log: 'text/plain'
}; };
function getExt(filename) { function getExtension(filename) {
return filename.split('.').pop(); return filename.split('.').pop();
} }
module.exports = { export function getContentType(file) {
getContentType(file) { return types[getExtension(file)] || DEFAULT_CONTENT_TYPE;
return types[getExt(file)] || DEFAULT_CONTENT_TYPE;
} }
};

View File

@ -1,19 +1,25 @@
'use strict'; import { relative, join, resolve as _resolve, sep } from 'node:path';
const fs = require('fs-extra'); import { createReadStream, remove } from 'fs-extra';
const path = require('path'); import { Endpoint, S3 } from 'aws-sdk';
const AWS = require('aws-sdk'); import readdir from 'recursive-readdir';
const readdir = require('recursive-readdir'); import pLimit from 'p-limit';
const pLimit = require('p-limit'); import intel from 'intel';
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const log = require('intel').getLogger('sitespeedio.plugin.s3'); import { throwIfMissing } from '../../support/util';
const throwIfMissing = require('../../support/util').throwIfMissing; import { getContentType } from './contentType';
const { getContentType } = require('./contentType');
const log = intel.getLogger('sitespeedio.plugin.s3');
function ignoreDirectories(file, stats) {
return stats.isDirectory();
}
function createS3(s3Options) { function createS3(s3Options) {
let endpoint = s3Options.endpoint || 's3.amazonaws.com'; let endpoint = s3Options.endpoint || 's3.amazonaws.com';
const options = { const options = {
endpoint: new AWS.Endpoint(endpoint), endpoint: new Endpoint(endpoint),
accessKeyId: s3Options.key, accessKeyId: s3Options.key,
secretAccessKey: s3Options.secret, secretAccessKey: s3Options.secret,
signatureVersion: 'v4' signatureVersion: 'v4'
@ -21,7 +27,7 @@ function createS3(s3Options) {
// You can also set some extra options see // You can also set some extra options see
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
Object.assign(options, s3Options.options); Object.assign(options, s3Options.options);
return new AWS.S3(options); return new S3(options);
} }
async function upload(dir, s3Options, prefix) { async function upload(dir, s3Options, prefix) {
@ -38,12 +44,8 @@ async function upload(dir, s3Options, prefix) {
} }
async function uploadLatestFiles(dir, s3Options, prefix) { async function uploadLatestFiles(dir, s3Options, prefix) {
function ignoreDirs(file, stats) {
return stats.isDirectory();
}
const s3 = createS3(s3Options); const s3 = createS3(s3Options);
const files = await readdir(dir, [ignoreDirs]); const files = await readdir(dir, [ignoreDirectories]);
// Backward compability naming for old S3 plugin // Backward compability naming for old S3 plugin
const limit = pLimit(s3Options.maxAsyncS3 || 20); const limit = pLimit(s3Options.maxAsyncS3 || 20);
const promises = []; const promises = [];
@ -55,39 +57,42 @@ async function uploadLatestFiles(dir, s3Options, prefix) {
} }
async function uploadFile(file, s3, s3Options, prefix, baseDir) { async function uploadFile(file, s3, s3Options, prefix, baseDir) {
const stream = fs.createReadStream(file); const stream = createReadStream(file);
const contentType = getContentType(file); const contentType = getContentType(file);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const onUpload = err => { const onUpload = error => {
if (err) { if (error) {
reject(err); reject(error);
} else { } else {
resolve(); resolve();
} }
}; };
const options = { partSize: 10 * 1024 * 1024, queueSize: 1 }; const options = { partSize: 10 * 1024 * 1024, queueSize: 1 };
// See https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property // See https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
const subPath = path.relative(baseDir, file); const subPath = relative(baseDir, file);
const params = { const parameters = {
Body: stream, Body: stream,
Bucket: s3Options.bucketname, Bucket: s3Options.bucketname,
ContentType: contentType, ContentType: contentType,
Key: path.join(s3Options.path || prefix, subPath), Key: join(s3Options.path || prefix, subPath),
StorageClass: s3Options.storageClass || 'STANDARD' StorageClass: s3Options.storageClass || 'STANDARD'
}; };
if (s3Options.acl) { if (s3Options.acl) {
params.ACL = s3Options.acl; parameters.ACL = s3Options.acl;
} }
// Override/set all the extra options you need // Override/set all the extra options you need
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
Object.assign(params, s3Options.params); Object.assign(parameters, s3Options.params);
s3.upload(params, options, onUpload); s3.upload(parameters, options, onUpload);
}); });
} }
module.exports = { export default class S3Plugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 's3', options, context, queue });
}
open(context, options) { open(context, options) {
this.s3Options = options.s3; this.s3Options = options.s3;
this.options = options; this.options = options;
@ -97,8 +102,7 @@ module.exports = {
throwIfMissing(this.s3Options, ['key', 'secret'], 's3'); throwIfMissing(this.s3Options, ['key', 'secret'], 's3');
} }
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
}, }
async processMessage(message, queue) { async processMessage(message, queue) {
if (message.type === 'sitespeedio.setup') { if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the s3 plugin is alive // Let other plugins know that the s3 plugin is alive
@ -119,21 +123,21 @@ module.exports = {
this.storageManager.getStoragePrefix() this.storageManager.getStoragePrefix()
); );
if (this.options.copyLatestFilesToBase) { if (this.options.copyLatestFilesToBase) {
const rootPath = path.resolve(baseDir, '..'); const rootPath = _resolve(baseDir, '..');
const dirsAsArray = rootPath.split(path.sep); const directoriesAsArray = rootPath.split(sep);
const rootName = dirsAsArray.slice(-1)[0]; const rootName = directoriesAsArray.slice(-1)[0];
await uploadLatestFiles(rootPath, s3Options, rootName); await uploadLatestFiles(rootPath, s3Options, rootName);
} }
log.info('Finished upload to s3'); log.info('Finished upload to s3');
if (s3Options.removeLocalResult) { if (s3Options.removeLocalResult) {
await fs.remove(baseDir); await remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`); log.debug(`Removed local files and directory ${baseDir}`);
} }
} catch (e) { } catch (error) {
queue.postMessage(make('error', e)); queue.postMessage(make('error', error));
log.error('Could not upload to S3', e); log.error('Could not upload to S3', error);
} }
queue.postMessage(make('s3.finished')); queue.postMessage(make('s3.finished'));
} }
} }
}; }

View File

@ -1,11 +1,13 @@
'use strict'; import { join, basename, resolve } from 'node:path';
const fs = require('fs-extra'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const path = require('path'); import { readFileSync, remove } from 'fs-extra';
const { Client } = require('node-scp'); import { Client } from 'node-scp';
const readdir = require('recursive-readdir'); import readdir from 'recursive-readdir';
const log = require('intel').getLogger('sitespeedio.plugin.scp'); import intel from 'intel';
const throwIfMissing = require('../../support/util').throwIfMissing; import { throwIfMissing } from '../../support/util';
const log = intel.getLogger('sitespeedio.plugin.scp');
async function getClient(scpOptions) { async function getClient(scpOptions) {
const options = { const options = {
@ -19,7 +21,7 @@ async function getClient(scpOptions) {
options.password = scpOptions.password; options.password = scpOptions.password;
} }
if (scpOptions.privateKey) { if (scpOptions.privateKey) {
options.privateKey = fs.readFileSync(scpOptions.privateKey); options.privateKey = readFileSync(scpOptions.privateKey);
} }
if (scpOptions.passphrase) { if (scpOptions.passphrase) {
options.passphrase = scpOptions.passphrase; options.passphrase = scpOptions.passphrase;
@ -31,21 +33,21 @@ async function upload(dir, scpOptions, prefix) {
let client; let client;
try { try {
client = await getClient(scpOptions); client = await getClient(scpOptions);
const dirs = prefix.split('/'); const directories = prefix.split('/');
let fullPath = ''; let fullPath = '';
for (let dir of dirs) { for (let dir of directories) {
fullPath += dir + '/'; fullPath += dir + '/';
const doThePathExist = await client.exists( const doThePathExist = await client.exists(
path.join(scpOptions.destinationPath, fullPath) join(scpOptions.destinationPath, fullPath)
); );
if (!doThePathExist) { if (!doThePathExist) {
await client.mkdir(path.join(scpOptions.destinationPath, fullPath)); await client.mkdir(join(scpOptions.destinationPath, fullPath));
} }
} }
await client.uploadDir(dir, path.join(scpOptions.destinationPath, prefix)); await client.uploadDir(dir, join(scpOptions.destinationPath, prefix));
} catch (e) { } catch (error) {
log.error(e); log.error(error);
throw e; throw error;
} finally { } finally {
if (client) { if (client) {
client.close(); client.close();
@ -60,12 +62,12 @@ async function uploadFiles(files, scpOptions, prefix) {
for (let file of files) { for (let file of files) {
await client.uploadFile( await client.uploadFile(
file, file,
path.join(scpOptions.destinationPath, prefix, path.basename(file)) join(scpOptions.destinationPath, prefix, basename(file))
); );
} }
} catch (e) { } catch (error) {
log.error(e); log.error(error);
throw e; throw error;
} finally { } finally {
if (client) { if (client) {
client.close(); client.close();
@ -73,16 +75,20 @@ async function uploadFiles(files, scpOptions, prefix) {
} }
} }
async function uploadLatestFiles(dir, scpOptions, prefix) { function ignoreDirectories(file, stats) {
function ignoreDirs(file, stats) {
return stats.isDirectory(); return stats.isDirectory();
} }
const files = await readdir(dir, [ignoreDirs]);
async function uploadLatestFiles(dir, scpOptions, prefix) {
const files = await readdir(dir, [ignoreDirectories]);
return uploadFiles(files, scpOptions, prefix); return uploadFiles(files, scpOptions, prefix);
} }
module.exports = { export default class ScpPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'scp', options, context, queue });
}
open(context, options) { open(context, options) {
this.scpOptions = options.scp; this.scpOptions = options.scp;
this.options = options; this.options = options;
@ -93,8 +99,7 @@ module.exports = {
'scp' 'scp'
); );
this.storageManager = context.storageManager; this.storageManager = context.storageManager;
}, }
async processMessage(message, queue) { async processMessage(message, queue) {
if (message.type === 'sitespeedio.setup') { if (message.type === 'sitespeedio.setup') {
// Let other plugins know that the scp plugin is alive // Let other plugins know that the scp plugin is alive
@ -114,21 +119,21 @@ module.exports = {
this.storageManager.getStoragePrefix() this.storageManager.getStoragePrefix()
); );
if (this.options.copyLatestFilesToBase) { if (this.options.copyLatestFilesToBase) {
const rootPath = path.resolve(baseDir, '..'); const rootPath = resolve(baseDir, '..');
const prefix = this.storageManager.getStoragePrefix(); const prefix = this.storageManager.getStoragePrefix();
const firstPart = prefix.split('/')[0]; const firstPart = prefix.split('/')[0];
await uploadLatestFiles(rootPath, this.scpOptions, firstPart); await uploadLatestFiles(rootPath, this.scpOptions, firstPart);
} }
log.info('Finished upload using scp'); log.info('Finished upload using scp');
if (this.scpOptions.removeLocalResult) { if (this.scpOptions.removeLocalResult) {
await fs.remove(baseDir); await remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`); log.debug(`Removed local files and directory ${baseDir}`);
} }
} catch (e) { } catch (error) {
queue.postMessage(make('error', e)); queue.postMessage(make('error', error));
log.error('Could not upload using scp', e); log.error('Could not upload using scp', error);
} }
queue.postMessage(make('scp.finished')); queue.postMessage(make('scp.finished'));
} }
} }
}; }

View File

@ -1,17 +1,13 @@
'use strict'; import get from 'lodash.get';
import { time, noop, size } from '../../support/helpers/index.js';
const get = require('lodash.get');
const h = require('../../support/helpers');
function getMetric(metric, f) { function getMetric(metric, f) {
if (metric.median) { return metric.median
return f(metric.median) + ' (' + f(metric.max) + ')'; ? f(metric.median) + ' (' + f(metric.max) + ')'
} else { : f(metric);
return f(metric);
}
} }
module.exports = function ( export function getAttachements(
dataCollector, dataCollector,
resultUrls, resultUrls,
slackOptions, slackOptions,
@ -33,7 +29,7 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.timings.firstPaint' 'pageSummary.statistics.timings.firstPaint'
), ),
f: h.time.ms f: time.ms
}, },
speedIndex: { speedIndex: {
name: 'Speed Index', name: 'Speed Index',
@ -41,7 +37,7 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.visualMetrics.SpeedIndex' 'pageSummary.statistics.visualMetrics.SpeedIndex'
), ),
f: h.time.ms f: time.ms
}, },
firstVisualChange: { firstVisualChange: {
name: 'First Visual Change', name: 'First Visual Change',
@ -49,7 +45,7 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.visualMetrics.FirstVisualChange' 'pageSummary.statistics.visualMetrics.FirstVisualChange'
), ),
f: h.time.ms f: time.ms
}, },
visualComplete85: { visualComplete85: {
name: 'Visual Complete 85%', name: 'Visual Complete 85%',
@ -57,7 +53,7 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.visualMetrics.VisualComplete85' 'pageSummary.statistics.visualMetrics.VisualComplete85'
), ),
f: h.time.ms f: time.ms
}, },
lastVisualChange: { lastVisualChange: {
name: 'Last Visual Change', name: 'Last Visual Change',
@ -65,7 +61,7 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.visualMetrics.LastVisualChange' 'pageSummary.statistics.visualMetrics.LastVisualChange'
), ),
f: h.time.ms f: time.ms
}, },
fullyLoaded: { fullyLoaded: {
name: 'Fully Loaded', name: 'Fully Loaded',
@ -73,7 +69,7 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.timings.fullyLoaded' 'pageSummary.statistics.timings.fullyLoaded'
), ),
f: h.time.ms f: time.ms
}, },
domContentLoadedTime: { domContentLoadedTime: {
name: 'domContentLoadedTime', name: 'domContentLoadedTime',
@ -81,22 +77,22 @@ module.exports = function (
base.browsertime, base.browsertime,
'pageSummary.statistics.timings.pageTimings.domContentLoadedTime' 'pageSummary.statistics.timings.pageTimings.domContentLoadedTime'
), ),
f: h.time.ms f: time.ms
}, },
coachScore: { coachScore: {
name: 'Coach score', name: 'Coach score',
metric: get(base.coach, 'pageSummary.advice.performance.score'), metric: get(base.coach, 'pageSummary.advice.performance.score'),
f: h.noop f: noop
}, },
transferSize: { transferSize: {
name: 'Page transfer size', name: 'Page transfer size',
metric: get(base.pagexray, 'pageSummary.transferSize'), metric: get(base.pagexray, 'pageSummary.transferSize'),
f: h.size.format f: size.format
}, },
transferRequests: { transferRequests: {
name: 'Requests', name: 'Requests',
metric: get(base.pagexray, 'pageSummary.requests'), metric: get(base.pagexray, 'pageSummary.requests'),
f: h.noop f: noop
} }
}; };
@ -169,4 +165,4 @@ module.exports = function (
attachments.push(attachement); attachments.push(attachement);
} }
return attachments; return attachments;
}; }

View File

@ -1,10 +1,8 @@
'use strict'; import merge from 'lodash.merge';
import get from 'lodash.get';
import set from 'lodash.set';
const merge = require('lodash.merge'), export class DataCollector {
get = require('lodash.get'),
set = require('lodash.set');
class DataCollector {
constructor(context) { constructor(context) {
this.resultUrls = context.resultUrls; this.resultUrls = context.resultUrls;
this.urlRunPages = {}; this.urlRunPages = {};
@ -79,5 +77,3 @@ class DataCollector {
merge(this.summaryPage, data); merge(this.summaryPage, data);
} }
} }
module.exports = DataCollector;

View File

@ -1,14 +1,17 @@
'use strict'; import { promisify } from 'node:util';
const throwIfMissing = require('../../support/util').throwIfMissing; import intel from 'intel';
const log = require('intel').getLogger('sitespeedio.plugin.slack'); import Slack from 'node-slack';
const Slack = require('node-slack'); import merge from 'lodash.merge';
const merge = require('lodash.merge'); import set from 'lodash.set';
const set = require('lodash.set'); import { SitespeedioPlugin } from '@sitespeed.io/plugin';
const DataCollector = require('./dataCollector');
const getAttachments = require('./attachements'); import { DataCollector } from './dataCollector.js';
const getSummary = require('./summary'); import { getAttachements } from './attachements.js';
const { promisify } = require('util'); import { getSummary } from './summary.js';
import { throwIfMissing } from '../../support/util.js';
const log = intel.getLogger('sitespeedio.plugin.slack');
const defaultConfig = { const defaultConfig = {
userName: 'Sitespeed.io', userName: 'Sitespeed.io',
@ -55,7 +58,7 @@ function send(options, dataCollector, context, screenshotType, alias) {
let attachments = []; let attachments = [];
if (['url', 'all', 'error'].includes(type)) { if (['url', 'all', 'error'].includes(type)) {
attachments = getAttachments( attachments = getAttachements(
dataCollector, dataCollector,
context.resultUrls, context.resultUrls,
slackOptions, slackOptions,
@ -77,11 +80,11 @@ function send(options, dataCollector, context, screenshotType, alias) {
mrkdwn: true, mrkdwn: true,
username: slackOptions.userName, username: slackOptions.userName,
attachments attachments
}).catch(e => { }).catch(error => {
if (e.errno === 'ETIMEDOUT') { if (error.errno === 'ETIMEDOUT') {
log.warn('Timeout sending Slack message.'); log.warn('Timeout sending Slack message.');
} else { } else {
throw e; throw error;
} }
}); });
} }
@ -101,7 +104,7 @@ function staticPagesProvider(options) {
throwIfMissing(s3Options, ['key', 'secret'], 's3'); throwIfMissing(s3Options, ['key', 'secret'], 's3');
} }
return 's3'; return 's3';
} catch (err) { } catch {
log.debug('s3 is not configured'); log.debug('s3 is not configured');
} }
@ -109,14 +112,18 @@ function staticPagesProvider(options) {
try { try {
throwIfMissing(gcsOptions, ['projectId', 'key', 'bucketname'], 'gcs'); throwIfMissing(gcsOptions, ['projectId', 'key', 'bucketname'], 'gcs');
return 'gcs'; return 'gcs';
} catch (err) { } catch {
log.debug('gcs is not configured'); log.debug('gcs is not configured');
} }
return null; return;
}
export default class SlackPlugin extends SitespeedioPlugin {
constructor(options, context, queue) {
super({ name: 'slack', options, context, queue });
} }
module.exports = {
open(context, options = {}) { open(context, options = {}) {
const slackOptions = options.slack || {}; const slackOptions = options.slack || {};
throwIfMissing(slackOptions, ['hookUrl', 'userName'], 'slack'); throwIfMissing(slackOptions, ['hookUrl', 'userName'], 'slack');
@ -125,7 +132,7 @@ module.exports = {
this.options = options; this.options = options;
this.screenshotType; this.screenshotType;
this.alias = {}; this.alias = {};
}, }
processMessage(message) { processMessage(message) {
const dataCollector = this.dataCollector; const dataCollector = this.dataCollector;
@ -219,6 +226,6 @@ module.exports = {
); );
} }
} }
}, }
config: defaultConfig }
}; export const config = defaultConfig;

View File

@ -1,16 +1,22 @@
'use strict'; import { format } from 'node:util';
const util = require('util'); import get from 'lodash.get';
const get = require('lodash.get'); import {
const h = require('../../support/helpers'); time,
const tsdbUtil = require('../../support/tsdbUtil'); noop,
size,
cap,
plural,
short
} from '../../support/helpers/index.js';
import { getConnectivity } from '../../support/tsdbUtil.js';
module.exports = function (dataCollector, errors, resultUrls, name, options) { export function getSummary(dataCollector, errors, resultUrls, name, options) {
const base = dataCollector.getSummary() || {}; const base = dataCollector.getSummary() || {};
const metrics = { const metrics = {
firstPaint: { firstPaint: {
name: 'First paint', name: 'First paint',
metric: get(base.browsertime, 'summary.firstPaint.median'), metric: get(base.browsertime, 'summary.firstPaint.median'),
f: h.time.ms f: time.ms
}, },
domContentLoadedTime: { domContentLoadedTime: {
name: 'domContentLoadedTime', name: 'domContentLoadedTime',
@ -18,12 +24,12 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime, base.browsertime,
'summary.pageTimings.domContentLoadedTime.median' 'summary.pageTimings.domContentLoadedTime.median'
), ),
f: h.time.ms f: time.ms
}, },
speedIndex: { speedIndex: {
name: 'Speed Index', name: 'Speed Index',
metric: get(base.browsertime, 'summary.visualMetrics.SpeedIndex.median'), metric: get(base.browsertime, 'summary.visualMetrics.SpeedIndex.median'),
f: h.time.ms f: time.ms
}, },
firstVisualChange: { firstVisualChange: {
name: 'First Visual Change', name: 'First Visual Change',
@ -31,7 +37,7 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime, base.browsertime,
'summary.visualMetrics.FirstVisualChange.median' 'summary.visualMetrics.FirstVisualChange.median'
), ),
f: h.time.ms f: time.ms
}, },
visualComplete85: { visualComplete85: {
name: 'Visual Complete 85%', name: 'Visual Complete 85%',
@ -39,7 +45,7 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime, base.browsertime,
'summary.visualMetrics.VisualComplete85.median' 'summary.visualMetrics.VisualComplete85.median'
), ),
f: h.time.ms f: time.ms
}, },
lastVisualChange: { lastVisualChange: {
name: 'Last Visual Change', name: 'Last Visual Change',
@ -47,34 +53,34 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
base.browsertime, base.browsertime,
'summary.visualMetrics.LastVisualChange.median' 'summary.visualMetrics.LastVisualChange.median'
), ),
f: h.time.ms f: time.ms
}, },
fullyLoaded: { fullyLoaded: {
name: 'Fully Loaded', name: 'Fully Loaded',
metric: get(base.pagexray, 'summary.fullyLoaded.median'), metric: get(base.pagexray, 'summary.fullyLoaded.median'),
f: h.time.ms f: time.ms
}, },
coachScore: { coachScore: {
name: 'Coach score', name: 'Coach score',
metric: get(base.coach, 'summary.performance.score.median'), metric: get(base.coach, 'summary.performance.score.median'),
f: h.noop f: noop
}, },
transferSize: { transferSize: {
name: 'Page transfer weight', name: 'Page transfer weight',
metric: get(base.pagexray, 'summary.transferSize.median'), metric: get(base.pagexray, 'summary.transferSize.median'),
f: h.size.format f: size.format
} }
}; };
const iterations = get(options, 'browsertime.iterations', 0); const iterations = get(options, 'browsertime.iterations', 0);
const browser = h.cap(get(options, 'browsertime.browser', 'unknown')); const browser = cap(get(options, 'browsertime.browser', 'unknown'));
const pages = h.plural(dataCollector.getURLs().length, 'page'); const pages = plural(dataCollector.getURLs().length, 'page');
const testName = h.short(name || '', 30) || 'Unknown'; const testName = short(name || '', 30) || 'Unknown';
const device = options.mobile ? 'mobile' : 'desktop'; const device = options.mobile ? 'mobile' : 'desktop';
const runs = h.plural(iterations, 'run'); const runs = plural(iterations, 'run');
let summaryText = let summaryText =
`${pages} analysed for ${testName} ` + `${pages} analysed for ${testName} ` +
`(${runs}, ${browser}/${device}/${tsdbUtil.getConnectivity(options)})\n`; `(${runs}, ${browser}/${device}/${getConnectivity(options)})\n`;
let message = ''; let message = '';
if (resultUrls.hasBaseUrl()) { if (resultUrls.hasBaseUrl()) {
@ -96,7 +102,7 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
let errorText = ''; let errorText = '';
if (errors.length > 0) { if (errors.length > 0) {
errorText += util.format('%d error(s):\n', errors.length); errorText += format('%d error(s):\n', errors.length);
logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack-hm.png'; logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack-hm.png';
} }
@ -105,4 +111,4 @@ module.exports = function (dataCollector, errors, resultUrls, name, options) {
errorText, errorText,
logo logo
}; };
}; }

Some files were not shown because too many files have changed in this diff Show More