diff --git a/lib/plugins/compare/pug/index.pug b/lib/plugins/compare/pug/index.pug index 36244a611..69c1c7ffb 100644 --- a/lib/plugins/compare/pug/index.pug +++ b/lib/plugins/compare/pug/index.pug @@ -74,8 +74,8 @@ table td a(href=createGraphLink(groupName, metricName)) b #{groupName + '.' + metricName} - if values.statisticalTestU === "N/A" - td N/A + if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" + td #{values.statisticalTestU} else td #{h.decimals(values.statisticalTestU)} td #{h.decimals(values.baseline.mean)} @@ -84,7 +84,7 @@ table td #{h.decimals(values.current.median)} td #{h.decimals(values.baseline.stdev)} td #{h.decimals(values.current.stdev)} - if values.statisticalTestU === "N/A" + if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" td No Test Conducted else td #{values.statisticalTestU < 0.05 ? 'Yes - ' + cliffDeltaHelper(values.cliffsDelta) : 'No'} @@ -96,7 +96,7 @@ each metricGroup, groupName in compare.metrics each values, metricName in metricGroup - var fullMetricName = groupName + '.' + metricName - var metricId = fullMetricName.replace(/\./g, '_') - h3 #{fullMetricName} #{values.statisticalTestU === "N/A"? '' : values.statisticalTestU < 0.05 ? '(significant change)' : ''} + h3 #{fullMetricName} #{values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" ? '' : values.statisticalTestU < 0.05 ? '(significant change)' : ''} .ct-chart(id=`chart-${metricId}`) .ct-legend span.ct-legend-item diff --git a/lib/plugins/compare/statistical.py b/lib/plugins/compare/statistical.py index 8bd423b93..74a8a3773 100644 --- a/lib/plugins/compare/statistical.py +++ b/lib/plugins/compare/statistical.py @@ -2,16 +2,17 @@ import sys import json from scipy.stats import wilcoxon, mannwhitneyu - def has_variability(sample): """Check if the sample has more than one unique value.""" return len(set(sample)) > 1 - def perform_test(test_type, baseline, current, **kwargs): """Perform the statistical test based on the test type.""" if not has_variability(baseline) or not has_variability(current): - return None, "No variability" + if baseline == current: + return None, "Datasets are identical" + else: + return None, "No variability" if test_type == 'wilcoxon': return wilcoxon(current, baseline, **kwargs) @@ -20,7 +21,6 @@ def perform_test(test_type, baseline, current, **kwargs): else: raise ValueError("Invalid test type. Choose 'wilcoxon' or 'mannwhitneyu'.") - input_data = json.loads(sys.stdin.read()) options = input_data['options'] test_type = options.pop('test_type') @@ -31,8 +31,8 @@ for group_name, metrics in input_data['metrics'].items(): group_results = {} for metric_name, metric_data in metrics.items(): stat, p = perform_test(test_type, metric_data['baseline'], metric_data['current'], **options) - if p == "No variability": - group_results[metric_name] = {'statistic': "N/A", 'p-value': "N/A"} + if p == "No variability" or p == "Datasets are identical": + group_results[metric_name] = {'statistic': "N/A", 'p-value': p} else: group_results[metric_name] = {'statistic': stat, 'p-value': p} final_results[group_name] = group_results