mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

First step to a better compare (#4064)

This commit is contained in:
Peter Hedenskog 2024-01-19 06:53:34 +01:00 committed by GitHub
parent cd54359389
commit 72b32324f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 264 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/plugins/compare/helper.js
View File

@ -226,6 +226,7 @@ function getVisualMetrics(data) {
return results; return results;
} }
/*
function getCDPPerformance(data) { function getCDPPerformance(data) {
const metricsToKeep = new Set([ const metricsToKeep = new Set([
'JSEventListeners', 'JSEventListeners',
@ -261,6 +262,7 @@ function getCDPPerformance(data) {
} }
return results; return results;
} }
*/
function getCPU(data) { function getCPU(data) {
const cpuMetrics = { const cpuMetrics = {
@ -351,8 +353,8 @@ export function getMetrics(data) {
...getElementTimings(data), ...getElementTimings(data),
...getUserTimings(data), ...getUserTimings(data),
...getCPU(data), ...getCPU(data),
...getBrowserMetrics(data), ...getBrowserMetrics(data)
...getCDPPerformance(data) // ...getCDPPerformance(data)
}; };
} }

322
lib/plugins/compare/index.js
View File

@ -58,6 +58,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
} }
async open(context, options) { async open(context, options) {
this.pageXrays = {};
this.browsertimes = {};
this.page = 0; this.page = 0;
this.make = context.messageMaker('compare').make; this.make = context.messageMaker('compare').make;
this.compareOptions = merge({}, defaultConfig, options.compare); this.compareOptions = merge({}, defaultConfig, options.compare);
@ -93,163 +95,199 @@ export default class ComparePlugin extends SitespeedioPlugin {
break; break;
} }
case 'browsertime.pageSummary': { case 'browsertime.pageSummary': {
this.page++; this.browsertimes[message.url] = message;
const id = this.options.compare.id || urlToId(message.data.info.url); break;
const baseline = await getBaseline( }
id + '-' + this.page, case 'sitespeedio.summarize': {
this.compareOptions for (let url of Object.keys(this.browsertimes)) {
); this.page++;
if (this.options.compare.id) { const id = this.options.compare.id || urlToId(url);
log.info('Using id %s for page baseline', id); const baseline = await getBaseline(
} else { id + '-' + this.page,
log.info('Using auto generated id for the baseline: %s ', id); this.compareOptions
} );
if (this.options.compare.id) {
if (baseline) { log.info('Using id %s for page baseline', id);
if ( } else {
baseline && log.info('Using auto generated id for the baseline: %s ', id);
this.options.browsertime.iterations !== baseline.timestamps.length
)
log.warning(
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
baseline.timestamps.length,
this.options.browsertime.iterations
);
log.info('Got a baseline:' + id + '-' + this.page);
const newMetrics = getMetrics(message.data);
const baselineMetrics = getMetrics(baseline);
const metricsInputData = {
options: {
test_type: this.compareOptions.testType,
alternative: this.compareOptions.alternative
},
metrics: {}
};
if (this.compareOptions.testType === 'mannwhitneyu') {
metricsInputData.options.use_continuity =
this.compareOptions.mannwhitneyu.useContinuity;
metricsInputData.options.method =
this.compareOptions.mannwhitneyu.method;
metricsInputData.options.nan_policy = 'omit';
} else if (this.compareOptions.testType === 'wilcoxon') {
metricsInputData.options.correction =
this.compareOptions.wilcoxon.correction;
metricsInputData.options.zero_method =
this.compareOptions.wilcoxon.zeroMethod;
} }
for (let group in newMetrics) { if (baseline) {
if (baselineMetrics[group]) { if (
metricsInputData.metrics[group] = {}; this.options.browsertime.iterations !==
for (let metricName in newMetrics[group]) { baseline.browsertime.timestamps.length
// Ensure both current and baseline metrics are available )
if ( log.warning(
baselineMetrics[group][metricName] && 'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
newMetrics[group][metricName] baseline.timestamps.length,
) { this.options.browsertime.iterations
// Directly access the Metric instance );
const currentMetric = newMetrics[group][metricName]; log.info('Got a baseline:' + id + '-' + this.page);
const baselineMetric = baselineMetrics[group][metricName]; const newMetrics = getMetrics(this.browsertimes[url].data);
const baselineMetrics = getMetrics(baseline.browsertime);
const metricsInputData = {
options: {
test_type: this.compareOptions.testType,
alternative: this.compareOptions.alternative
},
metrics: {}
};
// Ensure these are indeed Metric instances if (this.compareOptions.testType === 'mannwhitneyu') {
const currentStats = getStatistics(currentMetric.getValues()); metricsInputData.options.use_continuity =
const baselineStats = getStatistics( this.compareOptions.mannwhitneyu.useContinuity;
baselineMetric.getValues() metricsInputData.options.method =
); this.compareOptions.mannwhitneyu.method;
metricsInputData.metrics[group][metricName] = { metricsInputData.options.nan_policy = 'omit';
baseline: baselineStats.data, } else if (this.compareOptions.testType === 'wilcoxon') {
current: currentStats.data metricsInputData.options.correction =
}; this.compareOptions.wilcoxon.correction;
} else { metricsInputData.options.zero_method =
log.info( this.compareOptions.wilcoxon.zeroMethod;
`Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.` }
);
for (let group in newMetrics) {
if (baselineMetrics[group]) {
metricsInputData.metrics[group] = {};
for (let metricName in newMetrics[group]) {
// Ensure both current and baseline metrics are available
if (
baselineMetrics[group][metricName] &&
newMetrics[group][metricName]
) {
// Directly access the Metric instance
const currentMetric = newMetrics[group][metricName];
const baselineMetric = baselineMetrics[group][metricName];
// Ensure these are indeed Metric instances
const currentStats = getStatistics(
currentMetric.getValues()
);
const baselineStats = getStatistics(
baselineMetric.getValues()
);
metricsInputData.metrics[group][metricName] = {
baseline: baselineStats.data,
current: currentStats.data
};
} else {
log.info(
`Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
);
}
} }
} }
} }
}
const results = await runStatisticalTests(metricsInputData); const results = await runStatisticalTests(metricsInputData);
const finalResult = {}; const finalResult = {};
for (let group in results) { for (let group in results) {
finalResult[group] = {}; finalResult[group] = {};
for (let metricName in results[group]) { for (let metricName in results[group]) {
const result = results[group][metricName]; const result = results[group][metricName];
// Again, accessing the metricName within the group // Again, accessing the metricName within the group
const currentStats = getStatistics( const currentStats = getStatistics(
newMetrics[group][metricName].getValues() newMetrics[group][metricName].getValues()
); );
const baselineStats = getStatistics( const baselineStats = getStatistics(
baselineMetrics[group][metricName].getValues() baselineMetrics[group][metricName].getValues()
); );
const cliffs = cliffsDelta(currentStats.data, baselineStats.data); const cliffs = cliffsDelta(
finalResult[group][metricName] = { currentStats.data,
current: { baselineStats.data
stdev: currentStats.stddev(), );
mean: currentStats.amean(), finalResult[group][metricName] = {
median: currentStats.median(), current: {
values: currentStats.data stdev: currentStats.stddev(),
}, mean: currentStats.amean(),
baseline: { median: currentStats.median(),
stdev: baselineStats.stddev(), values: currentStats.data
mean: baselineStats.amean(), },
median: baselineStats.median(), baseline: {
values: baselineStats.data stdev: baselineStats.stddev(),
}, mean: baselineStats.amean(),
statisticalTestU: result['p-value'], median: baselineStats.median(),
cliffsDelta: cliffs, values: baselineStats.data
isSignificant: getIsSignificant(result['p-value'], cliffs) },
}; statisticalTestU: result['p-value'],
cliffsDelta: cliffs,
isSignificant: getIsSignificant(result['p-value'], cliffs)
};
}
} }
}
const meta = {
baseline: {
timestamp: dayjs(baseline.info.timestamp).format(TIME_FORMAT),
url: baseline.info.url,
alias: baseline.info.alias
},
current: {
timestamp: dayjs(message.data.info.timestamp).format(TIME_FORMAT),
url: message.data.info.url,
alias: message.data.info.alias
},
testOptions: this.compareOptions,
iterations: this.options.browsertime.iterations
};
if (this.compareOptions.saveBaseline) { const meta = {
await saveBaseline( baseline: {
message.data, timestamp: dayjs(baseline.browsertime.info.timestamp).format(
join( TIME_FORMAT
this.compareOptions.baselinePath || process.cwd(), ),
`${id}-${this.page}.json` url: baseline.browsertime.info.url,
) alias: baseline.browsertime.info.alias
); },
} current: {
timestamp: dayjs(
this.browsertimes[url].data.info.timestamp
).format(TIME_FORMAT),
url: url,
alias: this.browsertimes[url].data.info.alias
},
testOptions: this.compareOptions,
iterations: this.options.browsertime.iterations
};
super.sendMessage( const raw = {
'compare.pageSummary', baseline: {
{ metrics: finalResult, meta }, pagexray: baseline.pagexray,
{ browsertime: baseline.browsertime
url: message.url, },
group: message.group, current: {
runTime: message.runTime pagexray: this.pageXrays[url].data,
browsertime: this.browsertimes[url].data
}
};
if (this.compareOptions.saveBaseline) {
await saveBaseline(
{
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
)
);
} }
);
} else { super.sendMessage(
if (this.compareOptions.saveBaseline) { 'compare.pageSummary',
await saveBaseline( { metrics: finalResult, meta, raw },
message.data, {
join( url: url,
this.compareOptions.baselinePath || process.cwd(), group: this.browsertimes[url].group,
`${id}-${this.page}.json` runTime: this.browsertimes[url].runTime
) }
); );
} else {
if (this.compareOptions.saveBaseline) {
await saveBaseline(
{
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
)
);
}
} }
} }
break;
}
case 'pagexray.pageSummary': {
this.pageXrays[message.url] = message;
break; break;
} }
} }

92
lib/plugins/compare/pug/index.pug
View File

@ -38,19 +38,85 @@ p
| . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes. | . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes.
p h2 Setup
| The baseline test table
if compare.meta.baseline.alias tr
a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias} th
else b Metric
a(href=compare.meta.baseline.url) #{compare.meta.baseline.url} th
| was conducted at #{compare.meta.baseline.timestamp} and the current test b baseline
if compare.meta.current.alias th
a(href=compare.meta.current.url) #{compare.meta.current.alias} b current
else tr
a(href=compare.meta.current.url) #{compare.meta.current.url} td Test
| was done at #{compare.meta.current.timestamp}. td
if compare.meta.baseline.alias
a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
else
a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
td
if compare.meta.current.alias
a(href=compare.meta.current.url) #{compare.meta.current.alias}
else
a(href=compare.meta.current.url) #{compare.meta.current.url}
tr
td Run time
td #{compare.meta.baseline.timestamp}
td #{compare.meta.current.timestamp}
tr
td Total
td #{compare.raw.baseline.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentSize.median)})
tr
td HTML
td #{compare.raw.baseline.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.contentSize.median)})
tr
td JavaScript
td #{compare.raw.baseline.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.contentSize.median)})
tr
td CSS requests
td #{compare.raw.baseline.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.contentSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.contentSize.median)})
tr
td Image requests
td #{compare.raw.baseline.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.image.transferSize.median)})
td #{compare.raw.current.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.image.transferSize.median)})
tr
td DOM Elements
td #{compare.raw.baseline.browsertime.statistics.pageinfo.domElements.median}
td #{compare.raw.current.browsertime.statistics.pageinfo.domElements.median}
if compare.raw.current.pagexray.meta.screenshot
tr
td Screenshot
td
a(href=compare.raw.baseline.pagexray.meta.screenshot)
img.screenshot(src=compare.raw.baseline.pagexray.meta.screenshot , width=200)
td
a(href=compare.raw.current.pagexray.meta.screenshot)
img.screenshot(src=compare.raw.current.pagexray.meta.screenshot , width=200)
if compare.raw.current.pagexray.meta.video
tr
td Video
td
.videoWrapper
video(controls, preload='none', width=200)
source(src=compare.raw.baseline.pagexray.meta.video type='video/mp4')
td
.videoWrapper
video(controls, preload='none', width=200)
source(src=compare.raw.current.pagexray.meta.video type='video/mp4')
if compare.raw.baseline.pagexray.meta.result
tr
td Result
td
a(href=compare.raw.baseline.pagexray.meta.result) Result
td
a(href=compare.raw.current.pagexray.meta.result) Result
h2 Comparison Data h2 Comparison Data
@ -74,7 +140,7 @@ table
td td
a(href=createGraphLink(groupName, metricName)) a(href=createGraphLink(groupName, metricName))
b #{groupName + '.' + metricName} b #{groupName + '.' + metricName}
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" || values.statisticalTestU === "No variability"
td #{values.statisticalTestU} td #{values.statisticalTestU}
else else
td #{h.decimals(values.statisticalTestU)} td #{h.decimals(values.statisticalTestU)}

2
test/compare-1.json
View File

File diff suppressed because one or more lines are too long