first version of fetching nav timings using phantom #460

lib/aggregators/phantomjs/genericTimeMetric.js

@@ -0,0 +1,61 @@
+/**
+ * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
+ * Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
+ * and other contributors
+ * Released under the Apache 2.0 License
+ */
+var Stats = require('fast-stats').Stats;
+var util = require('../../util');
+var timeMetrics = {};
+
+exports.processPage = function(pageData) {
+
+  if (pageData.phantomjs) {
+
+    // The Navigation timing API
+    Object.keys(pageData.phantomjs.timings).forEach(function(metric) {
+      if (timeMetrics.hasOwnProperty(metric)) {
+        timeMetrics[metric].push(Number(pageData.phantomjs.timings[metric]));
+      } else {
+        timeMetrics[metric] = new Stats();
+        timeMetrics[metric].push(Number(pageData.phantomjs.timings[metric]));
+      }
+    });
+
+    // handle User Timing API
+    if (pageData.phantomjs.userTimings.marks) {
+      pageData.phantomjs.userTimings.marks.forEach(function(mark) {
+        if (timeMetrics.hasOwnProperty(mark.name)) {
+          timeMetrics[mark.name].push(Number(mark.startTime));
+        } else {
+          timeMetrics[mark.name] = new Stats();
+          timeMetrics[mark.name].push(Number(mark.startTime));
+        }
+
+      });
+    }
+
+
+  }
+};
+
+exports.generateResults = function() {
+  var keys = Object.keys(timeMetrics),
+    result = [];
+
+  for (var i = 0; i < keys.length; i++) {
+    result.push({
+      id: keys[i],
+      title: keys[i],
+      desc: util.timingMetricsDefinition[keys[i]] || 'User Timing API metric',
+      stats: util.getStatisticsObject(timeMetrics[keys[i]], 0),
+      unit: 'milliseconds'
+    });
+  }
+
+  return result;
+};
+
+exports.clear = function() {
+  timeMetrics = {};
+};

lib/analyze/analyzer.js

@@ -10,6 +10,7 @@ var config = require('./../conf'),
   browsertime = require('./browsertime'),
   webpagetest = require('./webpagetest'),
   screenshots = require('./screenshots'),
+  phantomjs = require('./phantom'),
   async = require('async');
 
 function Analyzer() {}
@@ -31,6 +32,13 @@ Analyzer.prototype.analyze = function(urls, collector, downloadErrors, analysisE
         cb(undefined, {});
       }
     },
+    function(cb) {
+      if (config.runYslow) {
+        phantomjs.analyze(urls, cb);
+      } else {
+        cb(undefined, {});
+      }
+    },
     function(cb) {
       if (config.gpsiKey) {
         gpsi.analyze(urls, cb);
@@ -90,7 +98,6 @@ Analyzer.prototype.analyze = function(urls, collector, downloadErrors, analysisE
                   } else {
                     pageData.har = [runPerBrowser.har];
                   }
-
                 });
               }
               // WPT holds both the WPT and HAR info

lib/analyze/phantom.js

@@ -0,0 +1,118 @@
+/**
+ * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
+ * Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
+ * and other contributors
+ * Released under the Apache 2.0 License
+ */
+var path = require('path'),
+  childProcess = require('child_process'),
+  config = require('./../conf'),
+  binPath = require('phantomjs').path,
+  util = require('../util'),
+  fs = require('fs'),
+  log = require('winston'),
+  async = require('async');
+
+module.exports = {
+  analyze: function(urls, callback) {
+
+    var phantomDir = path.join(config.run.absResultDir, config.dataDir, 'phantomjs');
+
+    fs.mkdir(phantomDir, function(err) {
+      if (err) {
+        log.log('error', 'Couldnt create the phantomjs result dir:' + phantomDir + ' ' + err);
+
+        callback(err, {
+          'type': 'phantomjs',
+          'data': {},
+          'errors': {}
+        });
+
+      } else {
+        var queue = async.queue(phantomjs, config.threads);
+
+        var errors = {};
+        var pageData = {};
+        urls.forEach(function(u) {
+          queue.push({
+            'url': u
+          }, function(data,err) {
+            if (err) {
+              errors[u] = err;
+            } else {
+              pageData[u] = data;
+            }
+          });
+        });
+
+        queue.drain = function() {
+          callback(undefined, {
+            'type': 'phantomjs',
+            'data': pageData,
+            'errors': errors
+          });
+        };
+      }
+
+    });
+
+
+  }
+};
+
+function phantomjs(args, asyncDoneCallback) {
+  var url = args.url;
+
+  // PhantomJS arguments
+  var childArgs = ['--ssl-protocol=any', '--ignore-ssl-errors=yes'];
+
+  //
+  childArgs.push(path.join(__dirname, '..', 'phantom.js'));
+
+  childArgs.push(url);
+  childArgs.push(path.join(config.run.absResultDir, config.dataDir, 'phantomjs', util.getFileName(url) +
+    '.json'));
+  childArgs.push(config.viewPort.split('x')[0]);
+  childArgs.push(config.viewPort.split('x')[1]);
+  childArgs.push(config.userAgent);
+
+  if (config.basicAuth) {
+    childArgs.push(config.basicAuth);
+  }
+
+  if (config.requestHeaders) {
+    childArgs.push(JSON.stringify(config.requestHeaders));
+  } else {
+    childArgs.push('');
+  }
+
+  log.log('info', 'Fetching data using PhantomJS for ' + url);
+
+  childProcess.execFile(binPath, childArgs, {
+    timeout: 60000
+  }, function(err, stdout, stderr) {
+
+    if (stderr) {
+      log.log('error', 'stderr: Error getting phantomjs data ' + url + ' (' + stderr +
+        ')');
+    }
+
+    if (err) {
+      log.log('error', 'Error getting phantomjs: ' + url + ' (' + stdout + stderr +
+        err + ')');
+      asyncDoneCallback(undefined, err + stdout);
+    } else {
+
+      fs.readFile(path.join(config.run.absResultDir, config.dataDir, 'phantomjs', util.getFileName(url) +
+        '.json'), function(err, data) {
+        if (err) {
+          log.log('error', 'Couldnt read the phantomjs file:');
+          asyncDoneCallback(undefined, err);
+        } else {
+          var phantomData = JSON.parse(data);
+
+        asyncDoneCallback(phantomData, err);
+      }
+    });
+    }
+});}

lib/collector.js

@@ -22,6 +22,7 @@ function registerAggregators() {
 
   if (config.runYslow) {
     types.push('yslow');
+    types.push('phantomjs');
   }
   if (config.browser) {
     types.push('browsertime','har');

lib/collectors/pages.js

@@ -29,6 +29,9 @@ exports.processPage = function(pageData) {
   if (pageData.webpagetest) {
     collectWPT(pageData, p);
   }
+  if (pageData.phantomjs) {
+    collectPhantomJS(pageData, p);
+  }
 
   p.url = util.getURLFromPageData(pageData);
 
@@ -182,6 +185,16 @@ function collectGPSI(pageData, p) {
   };
 }
 
+function collectPhantomJS(pageData, p) {
+  // example of adding phantomjs data
+  p.phantomjs = {};
+  p.phantomjs.pageLoadTime = {
+    'v': pageData.phantomjs.timings.pageLoadTime,
+    'unit': 'milliseconds'
+  };
+}
+
+
 function collectWPT(pageData, p) {
   p.wpt = {};
   p.wpt.speedIndex = {

lib/htmlRenderer.js

@@ -61,6 +61,7 @@ HTMLRenderer.prototype.renderPage = function (url, pageData, cb) {
     renderData.gpsiData = pageData.gpsi;
     renderData.browsertimeData = pageData.browsertime;
     renderData.wptData = pageData.webpagetest;
+    renderData.phantomjsData = pageData.phantomjs;
     renderData.config = config;
     renderData.pageMeta = {
       'path': '../',

lib/phantom.js

@@ -0,0 +1,142 @@
+/**
+ * Wait until the test condition is true or a timeout occurs. Useful for waiting
+ * on a server response or for a ui change (fadeIn, etc.) to occur.
+ *
+ * @param testFx javascript condition that evaluates to a boolean,
+ * it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or
+ * as a callback function.
+ * @param onReady what to do when testFx condition is fulfilled,
+ * it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or
+ * as a callback function.
+ * @param timeOutMillis the max amount of time to wait. If not specified, 3 sec is used.
+ */
+function waitFor(testFx, onReady, timeOutMillis) {
+  var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 10000, //< Default Max Timout is 10s
+    start = new Date().getTime(),
+    condition = false,
+    interval = setInterval(function() {
+      if ((new Date().getTime() - start < maxtimeOutMillis) && !condition) {
+        // If not time-out yet and condition not yet fulfilled
+        condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
+      } else {
+        if (!condition) {
+          // If condition still not fulfilled (timeout but condition is 'false')
+          console.log("'waitFor()' timeout");
+          phantom.exit(1);
+        } else {
+          // Condition fulfilled (timeout and/or condition is 'true')
+          typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condition is fulfilled
+          clearInterval(interval); //< Stop this interval
+        }
+      }
+    }, 250); //< repeat check every 250ms
+}
+
+var page = require('webpage').create(),
+  address, output, w, h, agent, basicauth, auth, headers, fs = require('fs');
+
+if (phantom.args.length < 4 || phantom.args.length > 7) {
+  console.log('Usage: phantom.js URL filename width height user-agent headers basic:auth');
+  phantom.exit();
+} else {
+  address = phantom.args[0];
+  output = phantom.args[1];
+  w = phantom.args[2];
+  h = phantom.args[3];
+  agent = phantom.args[4];
+  headers = phantom.args[5];
+  basicauth = phantom.args[6];
+
+  if (basicauth) {
+    auth = basicauth.split(':');
+    page.settings.userName = auth[0];
+    page.settings.password = auth[1];
+  }
+
+  if (headers) {
+    page.customHeaders = JSON.parse(headers);
+  }
+
+  page.viewportSize = {
+    width: w,
+    height: h
+  };
+
+  if (agent) {
+    page.settings.userAgent = agent;
+  }
+
+  page.open(address, function(status) {
+    if (status !== 'success') {
+      console.log('Unable to load the address!');
+    } else {
+      var self = this;
+      waitFor(function() {
+        // Check in the page if a specific element is now visible
+        return page.evaluate(function() {
+          return (window.performance.timing.loadEventEnd > 0);
+        });
+      }, function() {
+        var timings = page.evaluate(function() {
+
+          var t = window.performance.timing;
+          var marks = '';
+          try {
+            marks = window.performance.getEntriesByType('mark');
+          } catch (Error) {
+
+          }
+
+          return {
+            navigation: {
+              navigationStart: t.navigationStart,
+              unloadEventStart: t.unloadEventStart,
+              unloadEventEnd: t.unloadEventEnd,
+              redirectStart: t.redirectStart,
+              redirectEnd: t.redirectEnd,
+              fetchStart: t.fetchStart,
+              domainLookupStart: t.domainLookupStart,
+              domainLookupEnd: t.domainLookupEnd,
+              connectStart: t.connectStart,
+              connectEnd: t.connectEnd,
+              secureConnectionStart: t.secureConnectionStart,
+              requestStart: t.requestStart,
+              responseStart: t.responseStart,
+              responseEnd: t.responseEnd,
+              domLoading: t.domLoading,
+              domInteractive: t.domInteractive,
+              domContentLoadedEventStart: t.domContentLoadedEventStart,
+              domContentLoadedEventEnd: t.domContentLoadedEventEnd,
+              domComplete: t.domComplete,
+              loadEventStart: t.loadEventStart,
+              loadEventEnd: t.loadEventEnd
+            },
+            timings: {
+              domainLookupTime: (t.domainLookupEnd - t.domainLookupStart),
+              redirectionTime: (t.fetchStart - t.navigationStart),
+              serverConnectionTime: (t.connectEnd - t.requestStart),
+              serverResponseTime: (t.responseEnd - t.responseStart),
+              pageDownloadTime: (t.domInteractive - t.navigationStart),
+              domInteractiveTime: (t.domContentLoadedEventStart - t.navigationStart),
+              pageLoadTime: (t.loadEventStart - t.navigationStart),
+              frontEndTime: (t.loadEventStart - t.responseEnd),
+              backEndTime: (t.responseStart - t.navigationStart)
+            },
+            userTimings: {
+              marks: marks
+            }
+          };
+
+        });
+
+        timings.url = page.url;
+        try {
+          fs.write(output, JSON.stringify(timings), 'w');
+        } catch (e) {
+          console.log(e);
+        }
+        phantom.exit();
+      });
+    }
+  });
+}