mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

First step to a better compare (#4064)

This commit is contained in:
Peter Hedenskog 2024-01-19 06:53:34 +01:00 committed by GitHub
parent cd54359389
commit 72b32324f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 264 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/plugins/compare/helper.js
View File

@ -226,6 +226,7 @@ function getVisualMetrics(data) {
return results;
}
/*
function getCDPPerformance(data) {
const metricsToKeep = new Set([
'JSEventListeners',
@ -261,6 +262,7 @@ function getCDPPerformance(data) {
}
return results;
}
*/
function getCPU(data) {
const cpuMetrics = {
@ -351,8 +353,8 @@ export function getMetrics(data) {
...getElementTimings(data),
...getUserTimings(data),
...getCPU(data),
...getBrowserMetrics(data),
...getCDPPerformance(data)
...getBrowserMetrics(data)
// ...getCDPPerformance(data)
};
}

78
lib/plugins/compare/index.js
View File

@ -58,6 +58,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
}
async open(context, options) {
this.pageXrays = {};
this.browsertimes = {};
this.page = 0;
this.make = context.messageMaker('compare').make;
this.compareOptions = merge({}, defaultConfig, options.compare);
@ -93,8 +95,13 @@ export default class ComparePlugin extends SitespeedioPlugin {
break;
}
case 'browsertime.pageSummary': {
this.browsertimes[message.url] = message;
break;
}
case 'sitespeedio.summarize': {
for (let url of Object.keys(this.browsertimes)) {
this.page++;
const id = this.options.compare.id || urlToId(message.data.info.url);
const id = this.options.compare.id || urlToId(url);
const baseline = await getBaseline(
id + '-' + this.page,
this.compareOptions
@ -107,8 +114,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
if (baseline) {
if (
baseline &&
this.options.browsertime.iterations !== baseline.timestamps.length
this.options.browsertime.iterations !==
baseline.browsertime.timestamps.length
)
log.warning(
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
@ -116,8 +123,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
this.options.browsertime.iterations
);
log.info('Got a baseline:' + id + '-' + this.page);
const newMetrics = getMetrics(message.data);
const baselineMetrics = getMetrics(baseline);
const newMetrics = getMetrics(this.browsertimes[url].data);
const baselineMetrics = getMetrics(baseline.browsertime);
const metricsInputData = {
options: {
test_type: this.compareOptions.testType,
@ -153,7 +160,9 @@ export default class ComparePlugin extends SitespeedioPlugin {
const baselineMetric = baselineMetrics[group][metricName];
// Ensure these are indeed Metric instances
const currentStats = getStatistics(currentMetric.getValues());
const currentStats = getStatistics(
currentMetric.getValues()
);
const baselineStats = getStatistics(
baselineMetric.getValues()
);
@ -184,7 +193,10 @@ export default class ComparePlugin extends SitespeedioPlugin {
baselineMetrics[group][metricName].getValues()
);
const cliffs = cliffsDelta(currentStats.data, baselineStats.data);
const cliffs = cliffsDelta(
currentStats.data,
baselineStats.data
);
finalResult[group][metricName] = {
current: {
stdev: currentStats.stddev(),
@ -204,24 +216,43 @@ export default class ComparePlugin extends SitespeedioPlugin {
};
}
}
const meta = {
baseline: {
timestamp: dayjs(baseline.info.timestamp).format(TIME_FORMAT),
url: baseline.info.url,
alias: baseline.info.alias
timestamp: dayjs(baseline.browsertime.info.timestamp).format(
TIME_FORMAT
),
url: baseline.browsertime.info.url,
alias: baseline.browsertime.info.alias
},
current: {
timestamp: dayjs(message.data.info.timestamp).format(TIME_FORMAT),
url: message.data.info.url,
alias: message.data.info.alias
timestamp: dayjs(
this.browsertimes[url].data.info.timestamp
).format(TIME_FORMAT),
url: url,
alias: this.browsertimes[url].data.info.alias
},
testOptions: this.compareOptions,
iterations: this.options.browsertime.iterations
};
const raw = {
baseline: {
pagexray: baseline.pagexray,
browsertime: baseline.browsertime
},
current: {
pagexray: this.pageXrays[url].data,
browsertime: this.browsertimes[url].data
}
};
if (this.compareOptions.saveBaseline) {
await saveBaseline(
message.data,
{
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
@ -231,17 +262,20 @@ export default class ComparePlugin extends SitespeedioPlugin {
super.sendMessage(
'compare.pageSummary',
{ metrics: finalResult, meta },
{ metrics: finalResult, meta, raw },
{
url: message.url,
group: message.group,
runTime: message.runTime
url: url,
group: this.browsertimes[url].group,
runTime: this.browsertimes[url].runTime
}
);
} else {
if (this.compareOptions.saveBaseline) {
await saveBaseline(
message.data,
{
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
@ -249,7 +283,11 @@ export default class ComparePlugin extends SitespeedioPlugin {
);
}
}
}
break;
}
case 'pagexray.pageSummary': {
this.pageXrays[message.url] = message;
break;
}
}

76
lib/plugins/compare/pug/index.pug
View File

@ -38,19 +38,85 @@ p
| . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes.
p
| The baseline test
h2 Setup
table
tr
th
b Metric
th
b baseline
th
b current
tr
td Test
td
if compare.meta.baseline.alias
a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
else
a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
| was conducted at #{compare.meta.baseline.timestamp} and the current test
td
if compare.meta.current.alias
a(href=compare.meta.current.url) #{compare.meta.current.alias}
else
a(href=compare.meta.current.url) #{compare.meta.current.url}
| was done at #{compare.meta.current.timestamp}.
tr
td Run time
td #{compare.meta.baseline.timestamp}
td #{compare.meta.current.timestamp}
tr
td Total
td #{compare.raw.baseline.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentSize.median)})
tr
td HTML
td #{compare.raw.baseline.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.contentSize.median)})
tr
td JavaScript
td #{compare.raw.baseline.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.contentSize.median)})
tr
td CSS requests
td #{compare.raw.baseline.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.contentSize.median)})
tr
td Image requests
td #{compare.raw.baseline.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.image.transferSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.image.transferSize.median)})
tr
td DOM Elements
td #{compare.raw.baseline.browsertime.statistics.pageinfo.domElements.median}
td #{compare.raw.current.browsertime.statistics.pageinfo.domElements.median}
if compare.raw.current.pagexray.meta.screenshot
tr
td Screenshot
td
a(href=compare.raw.baseline.pagexray.meta.screenshot)
img.screenshot(src=compare.raw.baseline.pagexray.meta.screenshot , width=200)
td
a(href=compare.raw.current.pagexray.meta.screenshot)
img.screenshot(src=compare.raw.current.pagexray.meta.screenshot , width=200)
if compare.raw.current.pagexray.meta.video
tr
td Video
td
.videoWrapper
video(controls, preload='none', width=200)
source(src=compare.raw.baseline.pagexray.meta.video type='video/mp4')
td
.videoWrapper
video(controls, preload='none', width=200)
source(src=compare.raw.current.pagexray.meta.video type='video/mp4')
if compare.raw.baseline.pagexray.meta.result
tr
td Result
td
a(href=compare.raw.baseline.pagexray.meta.result) Result
td
a(href=compare.raw.current.pagexray.meta.result) Result
h2 Comparison Data
@ -74,7 +140,7 @@ table
td
a(href=createGraphLink(groupName, metricName))
b #{groupName + '.' + metricName}
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical"
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" || values.statisticalTestU === "No variability"
td #{values.statisticalTestU}
else
td #{h.decimals(values.statisticalTestU)}

2
test/compare-1.json
View File

File diff suppressed because one or more lines are too long