From 912de08c0e2c59095917cae63d66a39d99d76cda Mon Sep 17 00:00:00 2001 From: soulgalore Date: Mon, 25 Aug 2014 21:44:50 +0200 Subject: [PATCH] first version of fetching nav timings using phantom #460 --- .../phantomjs/genericTimeMetric.js | 61 ++++++++ lib/analyze/analyzer.js | 9 +- lib/analyze/phantom.js | 118 +++++++++++++++ lib/collector.js | 1 + lib/collectors/pages.js | 13 ++ lib/htmlRenderer.js | 1 + lib/phantom.js | 142 ++++++++++++++++++ 7 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 lib/aggregators/phantomjs/genericTimeMetric.js create mode 100644 lib/analyze/phantom.js create mode 100644 lib/phantom.js diff --git a/lib/aggregators/phantomjs/genericTimeMetric.js b/lib/aggregators/phantomjs/genericTimeMetric.js new file mode 100644 index 000000000..0b980811f --- /dev/null +++ b/lib/aggregators/phantomjs/genericTimeMetric.js @@ -0,0 +1,61 @@ +/** + * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io) + * Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog + * and other contributors + * Released under the Apache 2.0 License + */ +var Stats = require('fast-stats').Stats; +var util = require('../../util'); +var timeMetrics = {}; + +exports.processPage = function(pageData) { + + if (pageData.phantomjs) { + + // The Navigation timing API + Object.keys(pageData.phantomjs.timings).forEach(function(metric) { + if (timeMetrics.hasOwnProperty(metric)) { + timeMetrics[metric].push(Number(pageData.phantomjs.timings[metric])); + } else { + timeMetrics[metric] = new Stats(); + timeMetrics[metric].push(Number(pageData.phantomjs.timings[metric])); + } + }); + + // handle User Timing API + if (pageData.phantomjs.userTimings.marks) { + pageData.phantomjs.userTimings.marks.forEach(function(mark) { + if (timeMetrics.hasOwnProperty(mark.name)) { + timeMetrics[mark.name].push(Number(mark.startTime)); + } else { + timeMetrics[mark.name] = new Stats(); + timeMetrics[mark.name].push(Number(mark.startTime)); + } + + }); + } + + + } +}; + +exports.generateResults = function() { + var keys = Object.keys(timeMetrics), + result = []; + + for (var i = 0; i < keys.length; i++) { + result.push({ + id: keys[i], + title: keys[i], + desc: util.timingMetricsDefinition[keys[i]] || 'User Timing API metric', + stats: util.getStatisticsObject(timeMetrics[keys[i]], 0), + unit: 'milliseconds' + }); + } + + return result; +}; + +exports.clear = function() { + timeMetrics = {}; +}; diff --git a/lib/analyze/analyzer.js b/lib/analyze/analyzer.js index 88977948e..9b867d332 100644 --- a/lib/analyze/analyzer.js +++ b/lib/analyze/analyzer.js @@ -10,6 +10,7 @@ var config = require('./../conf'), browsertime = require('./browsertime'), webpagetest = require('./webpagetest'), screenshots = require('./screenshots'), + phantomjs = require('./phantom'), async = require('async'); function Analyzer() {} @@ -31,6 +32,13 @@ Analyzer.prototype.analyze = function(urls, collector, downloadErrors, analysisE cb(undefined, {}); } }, + function(cb) { + if (config.runYslow) { + phantomjs.analyze(urls, cb); + } else { + cb(undefined, {}); + } + }, function(cb) { if (config.gpsiKey) { gpsi.analyze(urls, cb); @@ -90,7 +98,6 @@ Analyzer.prototype.analyze = function(urls, collector, downloadErrors, analysisE } else { pageData.har = [runPerBrowser.har]; } - }); } // WPT holds both the WPT and HAR info diff --git a/lib/analyze/phantom.js b/lib/analyze/phantom.js new file mode 100644 index 000000000..028c3a505 --- /dev/null +++ b/lib/analyze/phantom.js @@ -0,0 +1,118 @@ +/** + * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io) + * Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog + * and other contributors + * Released under the Apache 2.0 License + */ +var path = require('path'), + childProcess = require('child_process'), + config = require('./../conf'), + binPath = require('phantomjs').path, + util = require('../util'), + fs = require('fs'), + log = require('winston'), + async = require('async'); + +module.exports = { + analyze: function(urls, callback) { + + var phantomDir = path.join(config.run.absResultDir, config.dataDir, 'phantomjs'); + + fs.mkdir(phantomDir, function(err) { + if (err) { + log.log('error', 'Couldnt create the phantomjs result dir:' + phantomDir + ' ' + err); + + callback(err, { + 'type': 'phantomjs', + 'data': {}, + 'errors': {} + }); + + } else { + var queue = async.queue(phantomjs, config.threads); + + var errors = {}; + var pageData = {}; + urls.forEach(function(u) { + queue.push({ + 'url': u + }, function(data,err) { + if (err) { + errors[u] = err; + } else { + pageData[u] = data; + } + }); + }); + + queue.drain = function() { + callback(undefined, { + 'type': 'phantomjs', + 'data': pageData, + 'errors': errors + }); + }; + } + + }); + + + } +}; + +function phantomjs(args, asyncDoneCallback) { + var url = args.url; + + // PhantomJS arguments + var childArgs = ['--ssl-protocol=any', '--ignore-ssl-errors=yes']; + + // + childArgs.push(path.join(__dirname, '..', 'phantom.js')); + + childArgs.push(url); + childArgs.push(path.join(config.run.absResultDir, config.dataDir, 'phantomjs', util.getFileName(url) + + '.json')); + childArgs.push(config.viewPort.split('x')[0]); + childArgs.push(config.viewPort.split('x')[1]); + childArgs.push(config.userAgent); + + if (config.basicAuth) { + childArgs.push(config.basicAuth); + } + + if (config.requestHeaders) { + childArgs.push(JSON.stringify(config.requestHeaders)); + } else { + childArgs.push(''); + } + + log.log('info', 'Fetching data using PhantomJS for ' + url); + + childProcess.execFile(binPath, childArgs, { + timeout: 60000 + }, function(err, stdout, stderr) { + + if (stderr) { + log.log('error', 'stderr: Error getting phantomjs data ' + url + ' (' + stderr + + ')'); + } + + if (err) { + log.log('error', 'Error getting phantomjs: ' + url + ' (' + stdout + stderr + + err + ')'); + asyncDoneCallback(undefined, err + stdout); + } else { + + fs.readFile(path.join(config.run.absResultDir, config.dataDir, 'phantomjs', util.getFileName(url) + + '.json'), function(err, data) { + if (err) { + log.log('error', 'Couldnt read the phantomjs file:'); + asyncDoneCallback(undefined, err); + } else { + var phantomData = JSON.parse(data); + + asyncDoneCallback(phantomData, err); + } + }); + } +});} diff --git a/lib/collector.js b/lib/collector.js index 46fd69fe2..1f943eedc 100644 --- a/lib/collector.js +++ b/lib/collector.js @@ -22,6 +22,7 @@ function registerAggregators() { if (config.runYslow) { types.push('yslow'); + types.push('phantomjs'); } if (config.browser) { types.push('browsertime','har'); diff --git a/lib/collectors/pages.js b/lib/collectors/pages.js index 5fab1d5b0..d60f358b7 100644 --- a/lib/collectors/pages.js +++ b/lib/collectors/pages.js @@ -29,6 +29,9 @@ exports.processPage = function(pageData) { if (pageData.webpagetest) { collectWPT(pageData, p); } + if (pageData.phantomjs) { + collectPhantomJS(pageData, p); + } p.url = util.getURLFromPageData(pageData); @@ -182,6 +185,16 @@ function collectGPSI(pageData, p) { }; } +function collectPhantomJS(pageData, p) { + // example of adding phantomjs data + p.phantomjs = {}; + p.phantomjs.pageLoadTime = { + 'v': pageData.phantomjs.timings.pageLoadTime, + 'unit': 'milliseconds' + }; +} + + function collectWPT(pageData, p) { p.wpt = {}; p.wpt.speedIndex = { diff --git a/lib/htmlRenderer.js b/lib/htmlRenderer.js index 03c4d12af..9c8cd89c9 100644 --- a/lib/htmlRenderer.js +++ b/lib/htmlRenderer.js @@ -61,6 +61,7 @@ HTMLRenderer.prototype.renderPage = function (url, pageData, cb) { renderData.gpsiData = pageData.gpsi; renderData.browsertimeData = pageData.browsertime; renderData.wptData = pageData.webpagetest; + renderData.phantomjsData = pageData.phantomjs; renderData.config = config; renderData.pageMeta = { 'path': '../', diff --git a/lib/phantom.js b/lib/phantom.js new file mode 100644 index 000000000..71ef99d5b --- /dev/null +++ b/lib/phantom.js @@ -0,0 +1,142 @@ +/** + * Wait until the test condition is true or a timeout occurs. Useful for waiting + * on a server response or for a ui change (fadeIn, etc.) to occur. + * + * @param testFx javascript condition that evaluates to a boolean, + * it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or + * as a callback function. + * @param onReady what to do when testFx condition is fulfilled, + * it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or + * as a callback function. + * @param timeOutMillis the max amount of time to wait. If not specified, 3 sec is used. + */ +function waitFor(testFx, onReady, timeOutMillis) { + var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 10000, //< Default Max Timout is 10s + start = new Date().getTime(), + condition = false, + interval = setInterval(function() { + if ((new Date().getTime() - start < maxtimeOutMillis) && !condition) { + // If not time-out yet and condition not yet fulfilled + condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code + } else { + if (!condition) { + // If condition still not fulfilled (timeout but condition is 'false') + console.log("'waitFor()' timeout"); + phantom.exit(1); + } else { + // Condition fulfilled (timeout and/or condition is 'true') + typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condition is fulfilled + clearInterval(interval); //< Stop this interval + } + } + }, 250); //< repeat check every 250ms +} + +var page = require('webpage').create(), + address, output, w, h, agent, basicauth, auth, headers, fs = require('fs'); + +if (phantom.args.length < 4 || phantom.args.length > 7) { + console.log('Usage: phantom.js URL filename width height user-agent headers basic:auth'); + phantom.exit(); +} else { + address = phantom.args[0]; + output = phantom.args[1]; + w = phantom.args[2]; + h = phantom.args[3]; + agent = phantom.args[4]; + headers = phantom.args[5]; + basicauth = phantom.args[6]; + + if (basicauth) { + auth = basicauth.split(':'); + page.settings.userName = auth[0]; + page.settings.password = auth[1]; + } + + if (headers) { + page.customHeaders = JSON.parse(headers); + } + + page.viewportSize = { + width: w, + height: h + }; + + if (agent) { + page.settings.userAgent = agent; + } + + page.open(address, function(status) { + if (status !== 'success') { + console.log('Unable to load the address!'); + } else { + var self = this; + waitFor(function() { + // Check in the page if a specific element is now visible + return page.evaluate(function() { + return (window.performance.timing.loadEventEnd > 0); + }); + }, function() { + var timings = page.evaluate(function() { + + var t = window.performance.timing; + var marks = ''; + try { + marks = window.performance.getEntriesByType('mark'); + } catch (Error) { + + } + + return { + navigation: { + navigationStart: t.navigationStart, + unloadEventStart: t.unloadEventStart, + unloadEventEnd: t.unloadEventEnd, + redirectStart: t.redirectStart, + redirectEnd: t.redirectEnd, + fetchStart: t.fetchStart, + domainLookupStart: t.domainLookupStart, + domainLookupEnd: t.domainLookupEnd, + connectStart: t.connectStart, + connectEnd: t.connectEnd, + secureConnectionStart: t.secureConnectionStart, + requestStart: t.requestStart, + responseStart: t.responseStart, + responseEnd: t.responseEnd, + domLoading: t.domLoading, + domInteractive: t.domInteractive, + domContentLoadedEventStart: t.domContentLoadedEventStart, + domContentLoadedEventEnd: t.domContentLoadedEventEnd, + domComplete: t.domComplete, + loadEventStart: t.loadEventStart, + loadEventEnd: t.loadEventEnd + }, + timings: { + domainLookupTime: (t.domainLookupEnd - t.domainLookupStart), + redirectionTime: (t.fetchStart - t.navigationStart), + serverConnectionTime: (t.connectEnd - t.requestStart), + serverResponseTime: (t.responseEnd - t.responseStart), + pageDownloadTime: (t.domInteractive - t.navigationStart), + domInteractiveTime: (t.domContentLoadedEventStart - t.navigationStart), + pageLoadTime: (t.loadEventStart - t.navigationStart), + frontEndTime: (t.loadEventStart - t.responseEnd), + backEndTime: (t.responseStart - t.navigationStart) + }, + userTimings: { + marks: marks + } + }; + + }); + + timings.url = page.url; + try { + fs.write(output, JSON.stringify(timings), 'w'); + } catch (e) { + console.log(e); + } + phantom.exit(); + }); + } + }); +}