First step to a better compare (#4064)
This commit is contained in:
Peter Hedenskog
GitHub
parent
cd54359389
commit
72b32324f6
|
|
@ -226,6 +226,7 @@ function getVisualMetrics(data) {
|
|||
return results;
|
||||
}
|
||||
|
||||
/*
|
||||
function getCDPPerformance(data) {
|
||||
const metricsToKeep = new Set([
|
||||
'JSEventListeners',
|
||||
|
|
@ -261,6 +262,7 @@ function getCDPPerformance(data) {
|
|||
}
|
||||
return results;
|
||||
}
|
||||
*/
|
||||
|
||||
function getCPU(data) {
|
||||
const cpuMetrics = {
|
||||
|
|
@ -351,8 +353,8 @@ export function getMetrics(data) {
|
|||
...getElementTimings(data),
|
||||
...getUserTimings(data),
|
||||
...getCPU(data),
|
||||
...getBrowserMetrics(data),
|
||||
...getCDPPerformance(data)
|
||||
...getBrowserMetrics(data)
|
||||
// ...getCDPPerformance(data)
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
|
|||
}
|
||||
|
||||
async open(context, options) {
|
||||
this.pageXrays = {};
|
||||
this.browsertimes = {};
|
||||
this.page = 0;
|
||||
this.make = context.messageMaker('compare').make;
|
||||
this.compareOptions = merge({}, defaultConfig, options.compare);
|
||||
|
|
@ -93,163 +95,199 @@ export default class ComparePlugin extends SitespeedioPlugin {
|
|||
break;
|
||||
}
|
||||
case 'browsertime.pageSummary': {
|
||||
this.page++;
|
||||
const id = this.options.compare.id || urlToId(message.data.info.url);
|
||||
const baseline = await getBaseline(
|
||||
id + '-' + this.page,
|
||||
this.compareOptions
|
||||
);
|
||||
if (this.options.compare.id) {
|
||||
log.info('Using id %s for page baseline', id);
|
||||
} else {
|
||||
log.info('Using auto generated id for the baseline: %s ', id);
|
||||
}
|
||||
|
||||
if (baseline) {
|
||||
if (
|
||||
baseline &&
|
||||
this.options.browsertime.iterations !== baseline.timestamps.length
|
||||
)
|
||||
log.warning(
|
||||
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
|
||||
baseline.timestamps.length,
|
||||
this.options.browsertime.iterations
|
||||
);
|
||||
log.info('Got a baseline:' + id + '-' + this.page);
|
||||
const newMetrics = getMetrics(message.data);
|
||||
const baselineMetrics = getMetrics(baseline);
|
||||
const metricsInputData = {
|
||||
options: {
|
||||
test_type: this.compareOptions.testType,
|
||||
alternative: this.compareOptions.alternative
|
||||
},
|
||||
metrics: {}
|
||||
};
|
||||
|
||||
if (this.compareOptions.testType === 'mannwhitneyu') {
|
||||
metricsInputData.options.use_continuity =
|
||||
this.compareOptions.mannwhitneyu.useContinuity;
|
||||
metricsInputData.options.method =
|
||||
this.compareOptions.mannwhitneyu.method;
|
||||
metricsInputData.options.nan_policy = 'omit';
|
||||
} else if (this.compareOptions.testType === 'wilcoxon') {
|
||||
metricsInputData.options.correction =
|
||||
this.compareOptions.wilcoxon.correction;
|
||||
metricsInputData.options.zero_method =
|
||||
this.compareOptions.wilcoxon.zeroMethod;
|
||||
this.browsertimes[message.url] = message;
|
||||
break;
|
||||
}
|
||||
case 'sitespeedio.summarize': {
|
||||
for (let url of Object.keys(this.browsertimes)) {
|
||||
this.page++;
|
||||
const id = this.options.compare.id || urlToId(url);
|
||||
const baseline = await getBaseline(
|
||||
id + '-' + this.page,
|
||||
this.compareOptions
|
||||
);
|
||||
if (this.options.compare.id) {
|
||||
log.info('Using id %s for page baseline', id);
|
||||
} else {
|
||||
log.info('Using auto generated id for the baseline: %s ', id);
|
||||
}
|
||||
|
||||
for (let group in newMetrics) {
|
||||
if (baselineMetrics[group]) {
|
||||
metricsInputData.metrics[group] = {};
|
||||
for (let metricName in newMetrics[group]) {
|
||||
// Ensure both current and baseline metrics are available
|
||||
if (
|
||||
baselineMetrics[group][metricName] &&
|
||||
newMetrics[group][metricName]
|
||||
) {
|
||||
// Directly access the Metric instance
|
||||
const currentMetric = newMetrics[group][metricName];
|
||||
const baselineMetric = baselineMetrics[group][metricName];
|
||||
if (baseline) {
|
||||
if (
|
||||
this.options.browsertime.iterations !==
|
||||
baseline.browsertime.timestamps.length
|
||||
)
|
||||
log.warning(
|
||||
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
|
||||
baseline.timestamps.length,
|
||||
this.options.browsertime.iterations
|
||||
);
|
||||
log.info('Got a baseline:' + id + '-' + this.page);
|
||||
const newMetrics = getMetrics(this.browsertimes[url].data);
|
||||
const baselineMetrics = getMetrics(baseline.browsertime);
|
||||
const metricsInputData = {
|
||||
options: {
|
||||
test_type: this.compareOptions.testType,
|
||||
alternative: this.compareOptions.alternative
|
||||
},
|
||||
metrics: {}
|
||||
};
|
||||
|
||||
// Ensure these are indeed Metric instances
|
||||
const currentStats = getStatistics(currentMetric.getValues());
|
||||
const baselineStats = getStatistics(
|
||||
baselineMetric.getValues()
|
||||
);
|
||||
metricsInputData.metrics[group][metricName] = {
|
||||
baseline: baselineStats.data,
|
||||
current: currentStats.data
|
||||
};
|
||||
} else {
|
||||
log.info(
|
||||
`Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
|
||||
);
|
||||
if (this.compareOptions.testType === 'mannwhitneyu') {
|
||||
metricsInputData.options.use_continuity =
|
||||
this.compareOptions.mannwhitneyu.useContinuity;
|
||||
metricsInputData.options.method =
|
||||
this.compareOptions.mannwhitneyu.method;
|
||||
metricsInputData.options.nan_policy = 'omit';
|
||||
} else if (this.compareOptions.testType === 'wilcoxon') {
|
||||
metricsInputData.options.correction =
|
||||
this.compareOptions.wilcoxon.correction;
|
||||
metricsInputData.options.zero_method =
|
||||
this.compareOptions.wilcoxon.zeroMethod;
|
||||
}
|
||||
|
||||
for (let group in newMetrics) {
|
||||
if (baselineMetrics[group]) {
|
||||
metricsInputData.metrics[group] = {};
|
||||
for (let metricName in newMetrics[group]) {
|
||||
// Ensure both current and baseline metrics are available
|
||||
if (
|
||||
baselineMetrics[group][metricName] &&
|
||||
newMetrics[group][metricName]
|
||||
) {
|
||||
// Directly access the Metric instance
|
||||
const currentMetric = newMetrics[group][metricName];
|
||||
const baselineMetric = baselineMetrics[group][metricName];
|
||||
|
||||
// Ensure these are indeed Metric instances
|
||||
const currentStats = getStatistics(
|
||||
currentMetric.getValues()
|
||||
);
|
||||
const baselineStats = getStatistics(
|
||||
baselineMetric.getValues()
|
||||
);
|
||||
metricsInputData.metrics[group][metricName] = {
|
||||
baseline: baselineStats.data,
|
||||
current: currentStats.data
|
||||
};
|
||||
} else {
|
||||
log.info(
|
||||
`Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const results = await runStatisticalTests(metricsInputData);
|
||||
const finalResult = {};
|
||||
for (let group in results) {
|
||||
finalResult[group] = {};
|
||||
for (let metricName in results[group]) {
|
||||
const result = results[group][metricName];
|
||||
// Again, accessing the metricName within the group
|
||||
const currentStats = getStatistics(
|
||||
newMetrics[group][metricName].getValues()
|
||||
);
|
||||
const baselineStats = getStatistics(
|
||||
baselineMetrics[group][metricName].getValues()
|
||||
);
|
||||
const results = await runStatisticalTests(metricsInputData);
|
||||
const finalResult = {};
|
||||
for (let group in results) {
|
||||
finalResult[group] = {};
|
||||
for (let metricName in results[group]) {
|
||||
const result = results[group][metricName];
|
||||
// Again, accessing the metricName within the group
|
||||
const currentStats = getStatistics(
|
||||
newMetrics[group][metricName].getValues()
|
||||
);
|
||||
const baselineStats = getStatistics(
|
||||
baselineMetrics[group][metricName].getValues()
|
||||
);
|
||||
|
||||
const cliffs = cliffsDelta(currentStats.data, baselineStats.data);
|
||||
finalResult[group][metricName] = {
|
||||
current: {
|
||||
stdev: currentStats.stddev(),
|
||||
mean: currentStats.amean(),
|
||||
median: currentStats.median(),
|
||||
values: currentStats.data
|
||||
},
|
||||
baseline: {
|
||||
stdev: baselineStats.stddev(),
|
||||
mean: baselineStats.amean(),
|
||||
median: baselineStats.median(),
|
||||
values: baselineStats.data
|
||||
},
|
||||
statisticalTestU: result['p-value'],
|
||||
cliffsDelta: cliffs,
|
||||
isSignificant: getIsSignificant(result['p-value'], cliffs)
|
||||
};
|
||||
const cliffs = cliffsDelta(
|
||||
currentStats.data,
|
||||
baselineStats.data
|
||||
);
|
||||
finalResult[group][metricName] = {
|
||||
current: {
|
||||
stdev: currentStats.stddev(),
|
||||
mean: currentStats.amean(),
|
||||
median: currentStats.median(),
|
||||
values: currentStats.data
|
||||
},
|
||||
baseline: {
|
||||
stdev: baselineStats.stddev(),
|
||||
mean: baselineStats.amean(),
|
||||
median: baselineStats.median(),
|
||||
values: baselineStats.data
|
||||
},
|
||||
statisticalTestU: result['p-value'],
|
||||
cliffsDelta: cliffs,
|
||||
isSignificant: getIsSignificant(result['p-value'], cliffs)
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
const meta = {
|
||||
baseline: {
|
||||
timestamp: dayjs(baseline.info.timestamp).format(TIME_FORMAT),
|
||||
url: baseline.info.url,
|
||||
alias: baseline.info.alias
|
||||
},
|
||||
current: {
|
||||
timestamp: dayjs(message.data.info.timestamp).format(TIME_FORMAT),
|
||||
url: message.data.info.url,
|
||||
alias: message.data.info.alias
|
||||
},
|
||||
testOptions: this.compareOptions,
|
||||
iterations: this.options.browsertime.iterations
|
||||
};
|
||||
|
||||
if (this.compareOptions.saveBaseline) {
|
||||
await saveBaseline(
|
||||
message.data,
|
||||
join(
|
||||
this.compareOptions.baselinePath || process.cwd(),
|
||||
`${id}-${this.page}.json`
|
||||
)
|
||||
);
|
||||
}
|
||||
const meta = {
|
||||
baseline: {
|
||||
timestamp: dayjs(baseline.browsertime.info.timestamp).format(
|
||||
TIME_FORMAT
|
||||
),
|
||||
url: baseline.browsertime.info.url,
|
||||
alias: baseline.browsertime.info.alias
|
||||
},
|
||||
current: {
|
||||
timestamp: dayjs(
|
||||
this.browsertimes[url].data.info.timestamp
|
||||
).format(TIME_FORMAT),
|
||||
url: url,
|
||||
alias: this.browsertimes[url].data.info.alias
|
||||
},
|
||||
testOptions: this.compareOptions,
|
||||
iterations: this.options.browsertime.iterations
|
||||
};
|
||||
|
||||
super.sendMessage(
|
||||
'compare.pageSummary',
|
||||
{ metrics: finalResult, meta },
|
||||
{
|
||||
url: message.url,
|
||||
group: message.group,
|
||||
runTime: message.runTime
|
||||
const raw = {
|
||||
baseline: {
|
||||
pagexray: baseline.pagexray,
|
||||
browsertime: baseline.browsertime
|
||||
},
|
||||
current: {
|
||||
pagexray: this.pageXrays[url].data,
|
||||
browsertime: this.browsertimes[url].data
|
||||
}
|
||||
};
|
||||
|
||||
if (this.compareOptions.saveBaseline) {
|
||||
await saveBaseline(
|
||||
{
|
||||
browsertime: this.browsertimes[url].data,
|
||||
pagexray: this.pageXrays[url].data
|
||||
},
|
||||
join(
|
||||
this.compareOptions.baselinePath || process.cwd(),
|
||||
`${id}-${this.page}.json`
|
||||
)
|
||||
);
|
||||
}
|
||||
);
|
||||
} else {
|
||||
if (this.compareOptions.saveBaseline) {
|
||||
await saveBaseline(
|
||||
message.data,
|
||||
join(
|
||||
this.compareOptions.baselinePath || process.cwd(),
|
||||
`${id}-${this.page}.json`
|
||||
)
|
||||
|
||||
super.sendMessage(
|
||||
'compare.pageSummary',
|
||||
{ metrics: finalResult, meta, raw },
|
||||
{
|
||||
url: url,
|
||||
group: this.browsertimes[url].group,
|
||||
runTime: this.browsertimes[url].runTime
|
||||
}
|
||||
);
|
||||
} else {
|
||||
if (this.compareOptions.saveBaseline) {
|
||||
await saveBaseline(
|
||||
{
|
||||
browsertime: this.browsertimes[url].data,
|
||||
pagexray: this.pageXrays[url].data
|
||||
},
|
||||
join(
|
||||
this.compareOptions.baselinePath || process.cwd(),
|
||||
`${id}-${this.page}.json`
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'pagexray.pageSummary': {
|
||||
this.pageXrays[message.url] = message;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,19 +38,85 @@ p
|
|||
| . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes.
|
||||
|
||||
|
||||
p
|
||||
| The baseline test
|
||||
if compare.meta.baseline.alias
|
||||
a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
|
||||
else
|
||||
a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
|
||||
| was conducted at #{compare.meta.baseline.timestamp} and the current test
|
||||
if compare.meta.current.alias
|
||||
a(href=compare.meta.current.url) #{compare.meta.current.alias}
|
||||
else
|
||||
a(href=compare.meta.current.url) #{compare.meta.current.url}
|
||||
| was done at #{compare.meta.current.timestamp}.
|
||||
h2 Setup
|
||||
table
|
||||
tr
|
||||
th
|
||||
b Metric
|
||||
th
|
||||
b baseline
|
||||
th
|
||||
b current
|
||||
tr
|
||||
td Test
|
||||
td
|
||||
if compare.meta.baseline.alias
|
||||
a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
|
||||
else
|
||||
a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
|
||||
td
|
||||
if compare.meta.current.alias
|
||||
a(href=compare.meta.current.url) #{compare.meta.current.alias}
|
||||
else
|
||||
a(href=compare.meta.current.url) #{compare.meta.current.url}
|
||||
tr
|
||||
td Run time
|
||||
td #{compare.meta.baseline.timestamp}
|
||||
td #{compare.meta.current.timestamp}
|
||||
tr
|
||||
td Total
|
||||
td #{compare.raw.baseline.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentSize.median)})
|
||||
td #{compare.raw.current.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentSize.median)})
|
||||
tr
|
||||
td HTML
|
||||
td #{compare.raw.baseline.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.contentSize.median)})
|
||||
td #{compare.raw.current.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.contentSize.median)})
|
||||
tr
|
||||
td JavaScript
|
||||
td #{compare.raw.baseline.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.contentSize.median)})
|
||||
td #{compare.raw.current.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.contentSize.median)})
|
||||
tr
|
||||
td CSS requests
|
||||
td #{compare.raw.baseline.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.contentSize.median)})
|
||||
td #{compare.raw.current.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.contentSize.median)})
|
||||
tr
|
||||
td Image requests
|
||||
td #{compare.raw.baseline.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.image.transferSize.median)})
|
||||
td #{compare.raw.current.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.image.transferSize.median)})
|
||||
tr
|
||||
td DOM Elements
|
||||
td #{compare.raw.baseline.browsertime.statistics.pageinfo.domElements.median}
|
||||
td #{compare.raw.current.browsertime.statistics.pageinfo.domElements.median}
|
||||
|
||||
if compare.raw.current.pagexray.meta.screenshot
|
||||
tr
|
||||
td Screenshot
|
||||
td
|
||||
a(href=compare.raw.baseline.pagexray.meta.screenshot)
|
||||
img.screenshot(src=compare.raw.baseline.pagexray.meta.screenshot , width=200)
|
||||
td
|
||||
a(href=compare.raw.current.pagexray.meta.screenshot)
|
||||
img.screenshot(src=compare.raw.current.pagexray.meta.screenshot , width=200)
|
||||
|
||||
if compare.raw.current.pagexray.meta.video
|
||||
tr
|
||||
td Video
|
||||
td
|
||||
.videoWrapper
|
||||
video(controls, preload='none', width=200)
|
||||
source(src=compare.raw.baseline.pagexray.meta.video type='video/mp4')
|
||||
td
|
||||
.videoWrapper
|
||||
video(controls, preload='none', width=200)
|
||||
source(src=compare.raw.current.pagexray.meta.video type='video/mp4')
|
||||
|
||||
if compare.raw.baseline.pagexray.meta.result
|
||||
tr
|
||||
td Result
|
||||
td
|
||||
a(href=compare.raw.baseline.pagexray.meta.result) Result
|
||||
td
|
||||
a(href=compare.raw.current.pagexray.meta.result) Result
|
||||
|
||||
h2 Comparison Data
|
||||
|
||||
|
|
@ -74,7 +140,7 @@ table
|
|||
td
|
||||
a(href=createGraphLink(groupName, metricName))
|
||||
b #{groupName + '.' + metricName}
|
||||
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical"
|
||||
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" || values.statisticalTestU === "No variability"
|
||||
td #{values.statisticalTestU}
|
||||
else
|
||||
td #{h.decimals(values.statisticalTestU)}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue