mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed junit test creation and better handling of tap #462 #461

This commit is contained in:
soulgalore 2014-09-19 21:20:13 +02:00
parent 05871dc10e
commit 40c9627ff1
8 changed files with 331 additions and 414 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conf/junit-rule.json
View File

@ -1,4 +0,0 @@
{
"overall": 99,
"thirdpartyversions":95
}

16
conf/junit-timings.json
View File

@ -1,16 +0,0 @@
{
"type": "median",
"default": {
"serverResponseTime":100,
"domContentLoadedTime":800,
"redirectionTime":0
},
"pages":
{
"http://www.sitespeed.io/a":
{
"serverResponseTime":250,
"domContentLoadedTime":500
}
}
}

227
lib/junitRenderer.js
View File

@ -1,227 +0,0 @@
/**
* Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
* Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
* and other contributors
* Released under the Apache 2.0 License
*/
var builder = require('xmlbuilder'),
fs = require('fs-extra'),
path = require('path'),
async = require('async'),
log = require('winston'),
util = require('./util');
function JUnitRenderer(collector, config) {
this.collector = collector;
this.config = config;
this.ruleTestsuites = builder.create('testsuites', {
version: '1.0',
encoding: 'UTF-8'
});
this.timingTestsuites = builder.create('testsuites', {
version: '1.0',
encoding: 'UTF-8'
});
}
JUnitRenderer.prototype.renderForEachPage = function(url, pageData) {
var yslowData = pageData.yslow,
ruleDictionary = yslowData.dictionary.rules,
rules = pageData.yslow.g,
score = pageData.yslow.o,
browserTimeData = pageData.browsertime;
generateRuleTestSuitePerPage(url, score, rules, ruleDictionary, this.ruleTestsuites, this.config);
if (browserTimeData) {
generateBrowserTimeTestSuitePerPage(browserTimeData, this.timingTestsuites, this.config);
}
// TODO add phantomjs timings
};
function generateBrowserTimeTestSuitePerPage(browserTimeData, timingTestsuites, config) {
browserTimeData.forEach(function(run) {
var testsuite = timingTestsuites.ele('testsuite', {
'name': 'sitespeed.io.timings.' + run.pageData.url.replace(/\./g, '_')
});
var url = run.pageData.url;
// First check if we have specific values configured for that URL else use the default ones
if (config.timingThresholds.pages) {
if (config.timingThresholds.pages.hasOwnProperty(url)) {
Object.keys(config.timingThresholds.pages[url]).forEach(function(
timing) {
run.statistics.forEach(function(stats) {
if (stats.name === timing) {
generateTimingTestCase(stats, timing, run, testsuite,
config.timingThresholds
.pages[url][timing], config);
}
});
});
}
// Use default values
else {
Object.keys(config.timingThresholds.
default).forEach(function(timing) {
run.statistics.forEach(function(stats) {
if (stats.name === timing) {
generateTimingTestCase(stats, timing, run, testsuite,
config.timingThresholds.
default [timing], config);
}
});
});
}
}
});
}
function generateTimingTestCase(stats, timing, run, testsuite, limit, config) {
var browser = run.pageData.browserName;
var version = run.pageData.browserVersion;
// The time in Jenkins needs to be in seconds
var testCase = testsuite.ele('testcase', {
'name': timing,
'time': stats[config.timingThresholds.type] / 1000
});
// is it a failure
if (stats[config.timingThresholds.type] > limit) {
testCase.ele('failure', {
'type': 'failedTiming',
'message': 'The time for ' + timing + ' is ' + stats[
config.timingThresholds.type] +
' ms, that is higher than your limit of ' + limit + ' ms. Using ' +
browser + ' ' +
version + ' ' + config.timingThresholds.type + ' value'
});
}
}
function generateRuleTestSuitePerPage(url, score, rules, ruleDictionary,
testsuites, config) {
var rule = Object.keys(rules);
var failures = 0,
skipped = 0;
var testsuite = testsuites.ele('testsuite', {
'name': 'sitespeed.io.rules.' + url.replace(/\./g, '_'),
'tests': (rule.length + 1)
});
var overallPageTestCase = testsuite.ele('testcase');
overallPageTestCase.att({
'name': 'Overall page score',
'status': score
});
if (isFailure("overall", score, config)) {
overallPageTestCase.ele('failure', {
'type': 'failedRule',
'message': 'The average overall page score ' + score +
' is below your limit'
});
failures++;
}
for (var i = 0; i < rule.length; i++) {
// is this skippable?
if (config.skipTest) {
if (config.skipTest.indexOf(rule[i]) > -1) {
skipped++;
continue;
}
}
var testCase = testsuite.ele('testcase', {
'name': '(' + rule[i] + ') ' + ruleDictionary[rule[i]].name,
'status': rules[rule[i]].score
});
if (isFailure(rule[i], rules[rule[i]].score, config)) {
failures++;
var failure = testCase.ele('failure', {
'type': 'failedRule',
'message': 'Score ' + score + ' - ' + rules[rule[i]].message
});
var comps = '';
rules[rule[i]].components.forEach(function(comp) {
comps += util.decodeURIComponent(comp) + '\n';
});
failure.txt(comps);
}
}
testsuite.att('failures', failures);
testsuite.att('skipped', skipped);
}
function isFailure(ruleid, value, config) {
if (config.thresholds) {
if (config.thresholds.hasOwnProperty(ruleid)) {
return (value < config.thresholds[ruleid]);
}
else {
return (value < config.threshold);
}
} else {
return (value < config.threshold);
}
}
function renderXMLFile(xml, fileName, cb) {
fs.writeFile(fileName, xml, function(err) {
if (err) {
log.log('error', 'Couldn\'t write JUnit xml file' + fileName);
throw err;
}
cb();
});
}
JUnitRenderer.prototype.renderAfterFullAnalyse = function(cb) {
// create testsuites and write to disk
var self = this;
var rulesXML = this.ruleTestsuites.end({
pretty: true,
indent: ' ',
newline: '\n'
});
var timingXML = this.timingTestsuites.end({
pretty: true,
indent: ' ',
newline: '\n'
});
async.parallel({
renderRules: function(callback) {
renderXMLFile(rulesXML,path.join(config.run.absResultDir,'junit.xml'),callback);
},
renderTimings: function(callback) {
renderXMLFile(timingXML,path.join(config.run.absResultDir, 'junit-timings.xml'),callback);
}
},
function(err, results) {
if (!err) {
log.log('info', 'Wrote JUnit result to ' + self.config.run.absResultDir);
}
cb();
});
};
module.exports = JUnitRenderer;

24
lib/sitespeed.js
View File

@ -8,8 +8,7 @@ var crawler = require('./crawler'),
Analyzer = require('./analyze/analyzer'),
HTMLRenderer = require('./htmlRenderer'),
Collector = require('./collector'),
JUnitRenderer = require('./junitRenderer'),
TapRenderer = require('./tapRenderer'),
TestRenderer = require('./tests/testRenderer'),
Graphite = require('./graphite'),
path = require('path'),
dateFormat = require('dateformat'),
@ -30,8 +29,7 @@ Sitespeed.prototype.run = function(config, finshedCb) {
this.analyzer = new Analyzer();
this.collector = new Collector(config);
this.htmlRenderer = new HTMLRenderer(config);
this.junitRenderer = new JUnitRenderer(this.collector);
this.tapRenderer = new TapRenderer(config);
this.testRenderer = new TestRenderer(config);
this.graphite = new Graphite(config.graphiteHost, config.graphitePort, config
.graphiteNamespace, this.collector, config);
@ -245,13 +243,10 @@ Sitespeed.prototype._analyze = function(urls, downloadErrors, callback) {
return;
}
if (self.config.tap) {
self.tapRenderer.forEachPage(url, pageData);
if (self.config.tap || self.config.junit) {
self.testRenderer.forEachPage(url, pageData);
}
if (self.config.junit) {
self.junitRenderer.renderForEachPage(url, pageData);
}
self.htmlRenderer.renderPage(url, pageData, function() {});
}, callback);
};
@ -360,9 +355,9 @@ Sitespeed.prototype._createOutput = function(downloadErrors, analysisErrors, cal
cb();
}
},
renderJUnit: function(cb) {
if (self.config.junit) {
self.junitRenderer.renderAfterFullAnalyse(cb);
renderTests: function(cb) {
if (self.config.tap ||  self.config.junit) {
self.testRenderer.render(cb);
} else {
cb();
}
@ -456,11 +451,6 @@ function setupConfiguration(config) {
config.thresholds = require(config.thresholdFile);
}
if (config.timingsThresholdFile) {
config.timingThresholds = require(config.timingsThresholdFile);
} else {
config.timingThresholds = require('../conf/junit-timings.json');
}
// decide which rules to use ...
if (config.profile === 'mobile') {
config.rules = require('../conf/mobile-rules.json');

150
lib/tapRenderer.js
View File

@ -1,150 +0,0 @@
/**
* Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
* Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
* and other contributors
* Released under the Apache 2.0 License
*/
var tap = require('tape'),
fs = require('fs-extra'),
path = require('path');
function TapRenderer(config) {
this.result = {};
this.config = config;
this.hasStream = false;
if (config.thresholdFile) {
this.thresholds = require(config.thresholdFile);
} else {
this.thresholds = require('../conf/thresholds.json');
}
}
TapRenderer.prototype.forEachPage = function(url, pageData) {
if (!this.hasStream) {
var outStream = fs.createWriteStream(path.join(this.config.run.absResultDir, 'sitespeed.tap'));
tap.createStream().pipe(outStream);
this.hasStream = true;
}
this._yslow(url, pageData);
this._wpt(url, pageData);
this._phantomJS(url, pageData);
this._gpsi(url, pageData);
this._browserTimings(url, pageData);
};
TapRenderer.prototype._gpsi = function(url, pageData) {
if (pageData.gpsi && this.thresholds.gpsi) {
var defaultLimit = this.thresholds.gpsi.
default ? this.thresholds.gpsi.
score : 90;
tap('gpsi score ' + url, function(t) {
t.ok(pageData.gpsi.score > defaultLimit, 'the gpsi score is ' + pageData.gpsi.score +
' threshold:' + defaultLimit);
t.end();
});
}
};
TapRenderer.prototype._browserTimings = function(url, pageData) {
if (pageData.browsertime && this.thresholds.timings) {
var self = this;
Object.keys(this.thresholds.timings).forEach(function(timing) {
pageData.browsertime.forEach(function(runPerBrowser) {
var browser = runPerBrowser.pageData.browserName;
runPerBrowser.statistics.forEach(function(stats) {
if (stats.name === timing) {
tap('timings ' + browser + ' ' + timing + ' ' + url, function(t) {
t.ok(stats.median < self.thresholds.timings[timing], 'the' + timing + ' is ' +
stats.median + ' threshold:' + self.thresholds.timings[timing]);
t.end();
});
}
});
});
});
}
};
TapRenderer.prototype._phantomJS = function(url, pageData) {
if (pageData.phantomjs && this.thresholds.timings) {
var self = this;
Object.keys(this.thresholds.timings).forEach(function(timing) {
var stats = pageData.phantomjs.getStats();
stats.forEach(function(stat) {
if (stat.id === timing) {
tap('phantomJS ' + timing + ' ' + url, function(t) {
t.ok(stat.stats.median < self.thresholds.timings[timing], 'the ' + timing + ' is ' +
stat.stats.median + ' threshold:' + self.thresholds.timings[timing]);
t.end();
});
}
});
});
}
};
TapRenderer.prototype._wpt = function(url, pageData) {
if (pageData.webpagetest && this.thresholds.wpt) {
var self = this;
// TODO depending on how many runs we do
var median = pageData.webpagetest.response.data.median.firstView;
Object.keys(this.thresholds.wpt).forEach(function(key) {
tap('wpt ' + key + ' ' + url, function(t) {
t.ok(median[key] < self.thresholds.wpt[key], 'the median ' + key + ' is ' + median[key] +
' threshold ' + self.thresholds.wpt[key]);
t.end();
});
});
}
};
TapRenderer.prototype._yslow = function(url, pageData) {
if (pageData.yslow && this.thresholds.yslow) {
var rules = pageData.yslow.g;
var rule = Object.keys(rules);
var self = this;
var defaultLimit = this.thresholds.yslow.
default ? this.thresholds.yslow.
default : 90;
tap('yslow ' + url, function(t) {
for (var i = 0; i < rule.length; i++) {
var score = rules[rule[i]].score;
// is this skippable?
if (self.config.skipTest) {
if (self.config.skipTest.indexOf(rule[i]) > -1) {
t.skip('Skipping ' + rule[i] + ' score ' + score);
continue;
}
}
t.ok(score > defaultLimit, 'the ' + rule[i] + ' score is:' + score);
}
t.end();
});
}
}
TapRenderer.prototype.render = function(cb) {
// what should we do when we are finished
cb();
};
module.exports = TapRenderer;

74
lib/tests/jUnitTestSuites.js Normal file
View File

@ -0,0 +1,74 @@
var builder = require('xmlbuilder'),
fs = require('fs-extra'),
path = require('path'),
async = require('async'),
log = require('winston'),
util = require('../util');
function JUnitTestSuites(filename, config) {
this.config = config;
this.filename = filename;
this.testSuites = builder.create('testsuites', {
version: '1.0',
encoding: 'UTF-8'
});
}
JUnitTestSuites.prototype.addSuite = function(name, results) {
var testsuite = this.testSuites.ele('testsuite', {
'name': 'sitespeed.io.' + name + '.' + results[0].url.replace(/\./g, '_'),
'tests': (results.length)
});
var failures = 0;
var skipped = 0;
results.forEach(function(result) {
if (result.skipped) {
skipped++;
} else {
var testCase = testsuite.ele('testcase');
testCase.att({
'name': result.title,
'status': result.value ? result.value : 'unknown'
});
if (!result.isOk) {
var failure = testCase.ele('failure', {
'type': 'failed',
'message': result.description ? result.description : 'unknown'
});
// failure.txt(comps);
failures++;
}
}
});
testsuite.att('failures', failures);
testsuite.att('skipped', skipped);
};
JUnitTestSuites.prototype.render = function(cb) {
var xml = this.testSuites.end({
pretty: true,
indent: ' ',
newline: '\n'
});
fs.writeFile(this.filename, xml, function(err) {
if (err) {
log.log('error', 'Couldn\'t write JUnit xml file' + this.filename);
throw err;
}
cb();
});
};
module.exports = JUnitTestSuites;

34
lib/tests/tap.js Normal file
View File

@ -0,0 +1,34 @@
/**
* Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
* Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
* and other contributors
* Released under the Apache 2.0 License
*/
var fs = require('fs-extra'),
tap = require('tape');
exports.writeTap = function(fileName, results, cb) {
var outStream = fs.createWriteStream(fileName);
tap.createStream().pipe(outStream);
var i = 0;
results.forEach(function(result) {
tap(result.type + ' ' + result.title + ' ' + result.url, function(t) {
i++;
if (result.skipped) {
t.skip(result.description);
} else {
t.ok(result.isOk, result.description);
}
t.end();
// check if last and fire callback
if (i === results.length) {
console.log("i:" + i + 'le:' + results.length);
cb();
}
});
});
}

216
lib/tests/testRenderer.js Normal file
View File

@ -0,0 +1,216 @@
/**
* Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
* Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
* and other contributors
* Released under the Apache 2.0 License
*/
var JUnitTestSuites = require('./jUnitTestSuites'),
tap = require('./tap'),
async = require('async'),
path = require('path');
function TestRenderer(config) {
this.result = {};
this.config = config;
this.results = [];
if (config.thresholdFile) {
this.thresholds = require(config.thresholdFile);
} else {
this.thresholds = require('../../conf/thresholds.json');
}
this.suites = new JUnitTestSuites(path.join(this.config.run.absResultDir, 'sitespeed.io.junit.xml'), this.config);
}
TestRenderer.prototype.forEachPage = function(url, pageData) {
var self = this;
var result = [this._yslow(url, pageData), , this._wpt(url, pageData),
this._phantomJS(url, pageData),
this._gpsi(url, pageData),
this._browserTimings(url, pageData)
];
result.forEach(function(r) {
self._add(r);
});
};
TestRenderer.prototype._add = function(result) {
if (result.length > 0) {
this.suites.addSuite(result[0].type, result);
}
this.results.push.apply(this.results, result);
}
TestRenderer.prototype._gpsi = function(url, pageData) {
var results = [];
if (pageData.gpsi && this.thresholds.gpsi) {
var defaultLimit = this.thresholds.gpsi.
default ? this.thresholds.gpsi.
score : 90;
var result = {};
result.title = 'score';
result.url = url;
result.isOK = pageData.gpsi.score > defaultLimit;
result.description = 'the gpsi score is ' + pageData.gpsi.score +
' threshold:' + defaultLimit;
result.value = pageData.gpsi.score;
result.type = 'gpsi';
results.push(result);
}
return results;
};
TestRenderer.prototype._browserTimings = function(url, pageData) {
var results = [];
if (pageData.browsertime && this.thresholds.timings) {
var self = this;
Object.keys(this.thresholds.timings).forEach(function(timing) {
pageData.browsertime.forEach(function(runPerBrowser) {
var browser = runPerBrowser.pageData.browserName;
runPerBrowser.statistics.forEach(function(stats) {
if (stats.name === timing) {
var result = {};
result.title = browser + ' ' + timing;
result.url = url;
result.isOk = stats.median < self.thresholds.timings[timing];
result.description = 'the' + timing + ' is ' +
stats.median + ' threshold:' + self.thresholds.timings[timing];
result.value = stats.median;
result.type = 'timings';
results.push(result);
}
});
});
});
}
return results;
};
TestRenderer.prototype._phantomJS = function(url, pageData) {
var results = [];
if (pageData.phantomjs && this.thresholds.timings) {
var self = this;
Object.keys(this.thresholds.timings).forEach(function(timing) {
var stats = pageData.phantomjs.getStats();
stats.forEach(function(stat) {
if (stat.id === timing) {
var result = {};
result.title = timing;
result.url = url;
result.isOk = stat.stats.median < self.thresholds.timings[timing];
result.description = 'the ' + timing + ' is ' +
stat.stats.median + ' threshold:' + self.thresholds.timings[timing];
result.value = stat.stats.median;
result.type = 'phantomjs';
results.push(result);
}
});
});
}
return results;
};
TestRenderer.prototype._wpt = function(url, pageData) {
var results = [];
if (pageData.webpagetest && this.thresholds.wpt) {
var self = this;
// TODO depending on how many runs we do
var median = pageData.webpagetest.response.data.median.firstView;
Object.keys(this.thresholds.wpt).forEach(function(key) {
var result = {};
result.title = key + ' ' + url;
result.url = url;
result.isOk = median[key] < self.thresholds.wpt[key];
result.description = 'the median ' + key + ' is ' + median[key] +
' threshold ' + self.thresholds.wpt[key];
result.value = median[key];
result.type = 'wpt';
results.push(result);
});
}
return results;
};
TestRenderer.prototype._yslow = function(url, pageData) {
var results = [];
if (pageData.yslow && this.thresholds.yslow) {
var rules = pageData.yslow.g;
var ruleDictionary = pageData.yslow.dictionary.rules;
var rule = Object.keys(rules);
var self = this;
var defaultLimit = this.thresholds.yslow.
default ? this.thresholds.yslow.
default : 90;
for (var i = 0; i < rule.length; i++) {
var score = rules[rule[i]].score;
// is this skippable?
if (self.config.skipTest) {
if (self.config.skipTest.indexOf(rule[i]) > -1) {
var result = {};
result.title = '(' + rule[i] + ') ' + ruleDictionary[rule[i]].name;
result.url = url;
result.skipped = true;
result.description = 'Skipping ' + rule[i] + ' score ' + score;
result.value = score;
results.push(result);
result.type = 'rule';
continue;
}
}
var result = {};
result.title = '(' + rule[i] + ') ' + ruleDictionary[rule[i]].name;
result.url = url;
result.isOk = score > defaultLimit;
result.description = 'The ' + rule[i] + ' has the score ' + score;
result.value = score;
result.type = 'rule';
results.push(result);
}
}
return results;
}
TestRenderer.prototype.render = function(cb) {
var self = this;
async.parallel({
writeTap: function(callback) {
if (self.config.tap) {
tap.writeTap(path.join(self.config.run.absResultDir, 'sitespeed.tap'), self.results, callback);
} else {
callback();
}
},
writeJUnit: function(callback) {
if (self.config.junit) {
self.suites.render(callback);
} else {
callback();
}
}
},
function(err, results) {
cb();
});
};
module.exports = TestRenderer;