From 0039afb460642c197d74e8f206f00b18b459e00d Mon Sep 17 00:00:00 2001
From: Peter Hedenskog <peter@soulgalore.com>
Date: Fri, 15 Dec 2023 14:42:42 +0100
Subject: [PATCH] Add isSignificant to result. (#4030)

If the change is not significant, it will be zero. Else
Cliffs delta is used to determine how large the change is.

In Graphite we could then alert on everything > 0. And then match
Cliffs delta to tell us if the change is small, medium or large.
---
 lib/plugins/compare/helper.js |  4 ++++
 lib/plugins/compare/index.js  | 18 +++++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/lib/plugins/compare/helper.js b/lib/plugins/compare/helper.js
index 10735d7cf..77cb6d3e2 100644
--- a/lib/plugins/compare/helper.js
+++ b/lib/plugins/compare/helper.js
@@ -356,6 +356,10 @@ export function getMetrics(data) {
   };
 }
 
+export function getIsSignificant(u, cliffs) {
+  return u < 0.05 ? cliffs : 0;
+}
+
 export function cliffsDelta(x, y) {
   const n_x = x.length;
   const n_y = y.length;
diff --git a/lib/plugins/compare/index.js b/lib/plugins/compare/index.js
index 90f7b8811..4fdeb287d 100644
--- a/lib/plugins/compare/index.js
+++ b/lib/plugins/compare/index.js
@@ -11,7 +11,8 @@ import {
   getStatistics,
   runStatisticalTests,
   getMetrics,
-  cliffsDelta
+  cliffsDelta,
+  getIsSignificant
 } from './helper.js';
 import { getBaseline, saveBaseline } from './baseline.js';
 
@@ -39,7 +40,16 @@ const DEFAULT_METRICS_PAGESUMMARY = [
   'metrics.renderBlocking.*.statisticalTestU',
   'metrics.elementTimings.*.statisticalTestU',
   'metrics.userTimings.*.statisticalTestU',
-  'metrics.extras.*.statisticalTestU'
+  'metrics.extras.*.statisticalTestU',
+  'metrics.timings.*.isSignificant',
+  'metrics.cpu.*.isSignificant',
+  'metrics.cdp.*.isSignificant',
+  'metrics.visualMetrics.*.isSignificant',
+  'metrics.googleWebVitals.*.isSignificant',
+  'metrics.renderBlocking.*.isSignificant',
+  'metrics.elementTimings.*.isSignificant',
+  'metrics.userTimings.*.isSignificant',
+  'metrics.extras.*.isSignificant'
 ];
 
 export default class ComparePlugin extends SitespeedioPlugin {
@@ -174,6 +184,7 @@ export default class ComparePlugin extends SitespeedioPlugin {
                 baselineMetrics[group][metricName].getValues()
               );
 
+              const cliffs = cliffsDelta(currentStats.data, baselineStats.data);
               finalResult[group][metricName] = {
                 current: {
                   stdev: currentStats.stddev(),
@@ -188,7 +199,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
                   values: baselineStats.data
                 },
                 statisticalTestU: result['p-value'],
-                cliffsDelta: cliffsDelta(currentStats.data, baselineStats.data)
+                cliffsDelta: cliffs,
+                isSignificant: getIsSignificant(result['p-value'], cliffs)
               };
             }
           }