mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

First step to a better compare (#4064)

This commit is contained in:
Peter Hedenskog 2024-01-19 06:53:34 +01:00 committed by GitHub
parent cd54359389
commit 72b32324f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 264 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/plugins/compare/helper.js
View File

@ -226,6 +226,7 @@ function getVisualMetrics(data) {
return results; return results;
} }
/*
function getCDPPerformance(data) { function getCDPPerformance(data) {
const metricsToKeep = new Set([ const metricsToKeep = new Set([
'JSEventListeners', 'JSEventListeners',
@ -261,6 +262,7 @@ function getCDPPerformance(data) {
} }
return results; return results;
} }
*/
function getCPU(data) { function getCPU(data) {
const cpuMetrics = { const cpuMetrics = {
@ -351,8 +353,8 @@ export function getMetrics(data) {
...getElementTimings(data), ...getElementTimings(data),
...getUserTimings(data), ...getUserTimings(data),
...getCPU(data), ...getCPU(data),
...getBrowserMetrics(data), ...getBrowserMetrics(data)
...getCDPPerformance(data) // ...getCDPPerformance(data)
}; };
} }

78
lib/plugins/compare/index.js
View File

@ -58,6 +58,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
} }
async open(context, options) { async open(context, options) {
this.pageXrays = {};
this.browsertimes = {};
this.page = 0; this.page = 0;
this.make = context.messageMaker('compare').make; this.make = context.messageMaker('compare').make;
this.compareOptions = merge({}, defaultConfig, options.compare); this.compareOptions = merge({}, defaultConfig, options.compare);
@ -93,8 +95,13 @@ export default class ComparePlugin extends SitespeedioPlugin {
break; break;
} }
case 'browsertime.pageSummary': { case 'browsertime.pageSummary': {
this.browsertimes[message.url] = message;
break;
}
case 'sitespeedio.summarize': {
for (let url of Object.keys(this.browsertimes)) {
this.page++; this.page++;
const id = this.options.compare.id || urlToId(message.data.info.url); const id = this.options.compare.id || urlToId(url);
const baseline = await getBaseline( const baseline = await getBaseline(
id + '-' + this.page, id + '-' + this.page,
this.compareOptions this.compareOptions
@ -107,8 +114,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
if (baseline) { if (baseline) {
if ( if (
baseline && this.options.browsertime.iterations !==
this.options.browsertime.iterations !== baseline.timestamps.length baseline.browsertime.timestamps.length
) )
log.warning( log.warning(
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs', 'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
@ -116,8 +123,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
this.options.browsertime.iterations this.options.browsertime.iterations
); );
log.info('Got a baseline:' + id + '-' + this.page); log.info('Got a baseline:' + id + '-' + this.page);
const newMetrics = getMetrics(message.data); const newMetrics = getMetrics(this.browsertimes[url].data);
const baselineMetrics = getMetrics(baseline); const baselineMetrics = getMetrics(baseline.browsertime);
const metricsInputData = { const metricsInputData = {
options: { options: {
test_type: this.compareOptions.testType, test_type: this.compareOptions.testType,
@ -153,7 +160,9 @@ export default class ComparePlugin extends SitespeedioPlugin {
const baselineMetric = baselineMetrics[group][metricName]; const baselineMetric = baselineMetrics[group][metricName];
// Ensure these are indeed Metric instances // Ensure these are indeed Metric instances
const currentStats = getStatistics(currentMetric.getValues()); const currentStats = getStatistics(
currentMetric.getValues()
);
const baselineStats = getStatistics( const baselineStats = getStatistics(
baselineMetric.getValues() baselineMetric.getValues()
); );
@ -184,7 +193,10 @@ export default class ComparePlugin extends SitespeedioPlugin {
baselineMetrics[group][metricName].getValues() baselineMetrics[group][metricName].getValues()
); );
const cliffs = cliffsDelta(currentStats.data, baselineStats.data); const cliffs = cliffsDelta(
currentStats.data,
baselineStats.data
);
finalResult[group][metricName] = { finalResult[group][metricName] = {
current: { current: {
stdev: currentStats.stddev(), stdev: currentStats.stddev(),
@ -204,24 +216,43 @@ export default class ComparePlugin extends SitespeedioPlugin {
}; };
} }
} }
const meta = { const meta = {
baseline: { baseline: {
timestamp: dayjs(baseline.info.timestamp).format(TIME_FORMAT), timestamp: dayjs(baseline.browsertime.info.timestamp).format(
url: baseline.info.url, TIME_FORMAT
alias: baseline.info.alias ),
url: baseline.browsertime.info.url,
alias: baseline.browsertime.info.alias
}, },
current: { current: {
timestamp: dayjs(message.data.info.timestamp).format(TIME_FORMAT), timestamp: dayjs(
url: message.data.info.url, this.browsertimes[url].data.info.timestamp
alias: message.data.info.alias ).format(TIME_FORMAT),
url: url,
alias: this.browsertimes[url].data.info.alias
}, },
testOptions: this.compareOptions, testOptions: this.compareOptions,
iterations: this.options.browsertime.iterations iterations: this.options.browsertime.iterations
}; };
const raw = {
baseline: {
pagexray: baseline.pagexray,
browsertime: baseline.browsertime
},
current: {
pagexray: this.pageXrays[url].data,
browsertime: this.browsertimes[url].data
}
};
if (this.compareOptions.saveBaseline) { if (this.compareOptions.saveBaseline) {
await saveBaseline( await saveBaseline(
message.data, {
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join( join(
this.compareOptions.baselinePath || process.cwd(), this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json` `${id}-${this.page}.json`
@ -231,17 +262,20 @@ export default class ComparePlugin extends SitespeedioPlugin {
super.sendMessage( super.sendMessage(
'compare.pageSummary', 'compare.pageSummary',
{ metrics: finalResult, meta }, { metrics: finalResult, meta, raw },
{ {
url: message.url, url: url,
group: message.group, group: this.browsertimes[url].group,
runTime: message.runTime runTime: this.browsertimes[url].runTime
} }
); );
} else { } else {
if (this.compareOptions.saveBaseline) { if (this.compareOptions.saveBaseline) {
await saveBaseline( await saveBaseline(
message.data, {
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join( join(
this.compareOptions.baselinePath || process.cwd(), this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json` `${id}-${this.page}.json`
@ -249,7 +283,11 @@ export default class ComparePlugin extends SitespeedioPlugin {
); );
} }
} }
}
break;
}
case 'pagexray.pageSummary': {
this.pageXrays[message.url] = message;
break; break;
} }
} }

76
lib/plugins/compare/pug/index.pug
View File

@ -38,19 +38,85 @@ p
| . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes. | . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes.
p h2 Setup
| The baseline test table
tr
th
b Metric
th
b baseline
th
b current
tr
td Test
td
if compare.meta.baseline.alias if compare.meta.baseline.alias
a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias} a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
else else
a(href=compare.meta.baseline.url) #{compare.meta.baseline.url} a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
| was conducted at #{compare.meta.baseline.timestamp} and the current test td
if compare.meta.current.alias if compare.meta.current.alias
a(href=compare.meta.current.url) #{compare.meta.current.alias} a(href=compare.meta.current.url) #{compare.meta.current.alias}
else else
a(href=compare.meta.current.url) #{compare.meta.current.url} a(href=compare.meta.current.url) #{compare.meta.current.url}
| was done at #{compare.meta.current.timestamp}. tr
td Run time
td #{compare.meta.baseline.timestamp}
td #{compare.meta.current.timestamp}
tr
td Total
td #{compare.raw.baseline.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentSize.median)})
tr
td HTML
td #{compare.raw.baseline.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.contentSize.median)})
tr
td JavaScript
td #{compare.raw.baseline.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.contentSize.median)})
tr
td CSS requests
td #{compare.raw.baseline.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.contentSize.median)})
tr
td Image requests
td #{compare.raw.baseline.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.image.transferSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.image.transferSize.median)})
tr
td DOM Elements
td #{compare.raw.baseline.browsertime.statistics.pageinfo.domElements.median}
td #{compare.raw.current.browsertime.statistics.pageinfo.domElements.median}
if compare.raw.current.pagexray.meta.screenshot
tr
td Screenshot
td
a(href=compare.raw.baseline.pagexray.meta.screenshot)
img.screenshot(src=compare.raw.baseline.pagexray.meta.screenshot , width=200)
td
a(href=compare.raw.current.pagexray.meta.screenshot)
img.screenshot(src=compare.raw.current.pagexray.meta.screenshot , width=200)
if compare.raw.current.pagexray.meta.video
tr
td Video
td
.videoWrapper
video(controls, preload='none', width=200)
source(src=compare.raw.baseline.pagexray.meta.video type='video/mp4')
td
.videoWrapper
video(controls, preload='none', width=200)
source(src=compare.raw.current.pagexray.meta.video type='video/mp4')
if compare.raw.baseline.pagexray.meta.result
tr
td Result
td
a(href=compare.raw.baseline.pagexray.meta.result) Result
td
a(href=compare.raw.current.pagexray.meta.result) Result
h2 Comparison Data h2 Comparison Data
@ -74,7 +140,7 @@ table
td td
a(href=createGraphLink(groupName, metricName)) a(href=createGraphLink(groupName, metricName))
b #{groupName + '.' + metricName} b #{groupName + '.' + metricName}
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" || values.statisticalTestU === "No variability"
td #{values.statisticalTestU} td #{values.statisticalTestU}
else else
td #{h.decimals(values.statisticalTestU)} td #{h.decimals(values.statisticalTestU)}

2
test/compare-1.json
View File

File diff suppressed because one or more lines are too long