Merge and restructure @moos --summary option(s) to get text summary to stdout

Changed the structure so we now have a dataCollector plugin that collects the
default data and is shared through the context. Tried to make the HTML plugin
only handle HTML. And some restructuring. Lets go through the naming before
the final release.

lib/plugins/datacollector/dataCollector.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const fs = require('fs'),
+  Promise = require('bluebird'),
+  merge = require('lodash.merge'),
+  get = require('lodash.get'),
+  set = require('lodash.set');
+
+Promise.promisifyAll(fs);
+
+const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+
+class DataCollector {
+  constructor(context, options) {
+    this.storageManager = context.storageManager;
+    this.timestamp = context.timestamp.format(TIME_FORMAT);
+    this.options = options;
+    this.urlRunPages = context.dataCollection.urlRunPages;
+    this.urlPages = context.dataCollection.urlPages;
+    this.summaryPages = context.dataCollection.summaryPages;
+  }
+
+  addUrl(url) {
+    this.urlPages[url] = {
+      path: this.storageManager.pathFromRootToPageDir(url),
+      data: {}
+    };
+    this.urlRunPages[url] = [];
+  }
+
+  addErrorForUrl(url, source, data) {
+    const errors = get(this.urlPages[url], 'errors', {});
+    errors[source] = data;
+    set(this.urlPages[url], 'errors', errors);
+  }
+
+  addDataForUrl(url, typePath, data, runIndex) {
+    if (runIndex !== undefined) {
+      let runData = this.urlRunPages[url][runIndex] || {
+          runIndex,
+          data: {}
+        };
+      set(runData.data, typePath, data);
+      this.urlRunPages[url][runIndex] = runData;
+    } else {
+      set(this.urlPages[url].data, typePath, data);
+    }
+  }
+
+  addDataForSummaryPage(name, data) {
+    if (this.summaryPages[name]) {
+      merge(this.summaryPages[name], data);
+    } else {
+      set(this.summaryPages, name, merge({}, data));
+    }
+  }
+
+}
+
+module.exports = DataCollector;

lib/plugins/datacollector/index.js

@@ -0,0 +1,110 @@
+'use strict';
+
+const path = require('path'),
+  set = require('lodash.set'),
+  reduce = require('lodash.reduce'),
+  DataCollector = require('./dataCollector');
+
+module.exports = {
+  name() {
+    return path.basename(__dirname);
+  },
+
+  open(context, options) {
+    this.dataCollector = new DataCollector(context, options);
+    this.options = options;
+  },
+
+  processMessage(message) {
+    const dataCollector = this.dataCollector;
+
+    switch (message.type) {
+      case 'url':
+      {
+        return dataCollector.addUrl(message.url);
+      }
+
+      case 'error':
+      {
+        return dataCollector.addErrorForUrl(message.url, message.source, message.data);
+      }
+
+      case 'browsertime.run':
+      case 'browsertime.pageSummary':
+      case 'browsertime.har':
+      case 'webpagetest.run':
+      case 'webpagetest.pageSummary':
+      case 'gpsi.data':
+      case 'gpsi.pageSummary':
+      case 'pagexray.run':
+      case 'pagexray.pageSummary':
+      case 'coach.run':
+      case 'coach.pageSummary':
+      {
+        return dataCollector.addDataForUrl(message.url, message.type, message.data, message.runIndex);
+      }
+
+      case 'assets.aggregate':
+      {
+        const assetList = reduce(message.data, (assetList, asset) => {
+          assetList.push(asset);
+          return assetList;
+        }, []);
+
+        const count = 20,
+          fullCount = Object.keys(assetList).length,
+          topAssets = assetList
+            .sort((a, b) => b.requestCount - a.requestCount)
+            .splice(0, count);
+        return dataCollector.addDataForSummaryPage('assets', {topAssets, count, fullCount});
+      }
+
+      case 'assets.aggregateSizePerContentType':
+      {
+        if (!message.group) {
+          const assetsBySize= {};
+          assetsBySize[message.contentType] = message.data;
+          return dataCollector.addDataForSummaryPage('toplist', {assetsBySize});
+        }
+        else return;
+      }
+
+      case 'assets.slowest':
+      {
+        if (!message.group) {
+          const slowestAssets = message.data;
+          return dataCollector.addDataForSummaryPage('toplist', {slowestAssets});
+        }
+        else return;
+      }
+
+      case 'domains.summary':
+      {
+        const domainList = reduce(message.data, (domainList, domainStats) => {
+          domainList.push(domainStats);
+          return domainList;
+        }, []);
+
+        const count = 200,
+          fullCount = domainList.length,
+          topDomains = domainList
+            .sort((a, b) => b.requestCount - a.requestCount)
+            .splice(0, count);
+        return dataCollector.addDataForSummaryPage('domains', {topDomains, count, fullCount});
+      }
+
+      case 'webpagetest.summary':
+      case 'coach.summary':
+      case 'pagexray.summary':
+      case 'browsertime.summary':
+      {
+        const data = {};
+        set(data, message.type, message.data);
+        dataCollector.addDataForSummaryPage('index', data);
+        return dataCollector.addDataForSummaryPage('detailed', data);
+      }
+    }
+  },
+  close() {
+  }
+};

lib/plugins/html/htmlBuilder.js

@@ -1,16 +1,11 @@
 'use strict';
 
 const fs = require('fs'),
-  helpers = require('./helpers'),
+  helpers = require('../../support/helpers'),
   Promise = require('bluebird'),
   path = require('path'),
   merge = require('lodash.merge'),
-  reduce = require('lodash.reduce'),
-  get = require('lodash.get'),
-  set = require('lodash.set'),
   log = require('intel'),
-  summaryBoxesSetup = require('./setup/summaryBoxes'),
-  detailedSetup = require('./setup/detailed'),
   packageInfo = require('../../../package'),
   renderer = require('./renderer');
 
@@ -23,104 +18,108 @@ class HTMLBuilder {
     this.storageManager = context.storageManager;
     this.timestamp = context.timestamp.format(TIME_FORMAT);
     this.options = options;
-    this.summaryPages = {};
-    this.urlPages = {};
-    this.urlRunPages = {};
+    this.dataCollection = context.dataCollection;
+    this.summary = {};
   }
 
-  addUrl(url) {
-    this.urlPages[url] = {
-      path: this.storageManager.pathFromRootToPageDir(url),
-      data: {}
-    };
-    this.urlRunPages[url] = [];
-  }
+  render() {
+    const options = this.options;
+    const dataCollection = this.dataCollection;
 
-  addErrorForUrl(url, source, data) {
-    const errors = get(this.urlPages[url], 'errors', {});
-    errors[source] = data;
-    set(this.urlPages[url], 'errors', errors);
-  }
-
-  addDataForUrl(url, typePath, data, runIndex) {
-    if (runIndex !== undefined) {
-      let runData = this.urlRunPages[url][runIndex] || {
-          runIndex,
-          data: {}
-        };
-      set(runData.data, typePath, data);
-      this.urlRunPages[url][runIndex] = runData;
-    } else {
-      set(this.urlPages[url].data, typePath, data);
-    }
-  }
-
-  addDataForSummaryPage(name, data) {
-    if (this.summaryPages[name]) {
-      merge(this.summaryPages[name], data);
-    }
-    else {
-      set(this.summaryPages, name, merge({}, data));
-    }
-  }
-
-  renderHTML(options) {
-    log.info('Render HTML for %s page(s) ' , Object.keys(this.urlPages).length);
-    const errors = reduce(this.urlPages, (errors, urlInfo, url) => {
-      if (urlInfo.errors) {
-        errors[url] = urlInfo.errors;
-      }
-      return errors;
-    }, {});
+    log.info('Render HTML for %s page(s) ', Object.keys(dataCollection.urlPages).length);
+    const errors = dataCollection.getErrorPages();
 
     if (Object.keys(errors).length > 0) {
-      this.addDataForSummaryPage('errors', {errors, menu: 'errors'});
+      this.summary.errors = {
+        errors,
+        menu: 'errors'
+      };
     }
 
-    const validPages = reduce(this.urlPages, (validPages, urlInfo, url) => {
-      if (Object.keys(urlInfo.data).length > 0) {
-        validPages[url] = urlInfo;
-      }
-      return validPages;
-    }, {});
+    const validPages = dataCollection.getValidPages();
 
-    this.addDataForSummaryPage('pages', {pageTitle: 'Overview of all tested pages',pageDescription: 'See all the tested pages on a high level.', pages: validPages});
-    this.addDataForSummaryPage('index', {pageTitle: 'Summary of the sitespeed.io result', pageDescription: 'Executive summary of the sitespeed.io result. Act on red/yellow/green.', boxes: summaryBoxesSetup(this.summaryPages['index'])});
-    this.addDataForSummaryPage('detailed', {pageTitle: 'In details summary of the sitespeed.io result.',pageDescription: 'Get all the details you need to fast track things you need to change.', metrics: detailedSetup(this.summaryPages['detailed'])});
-    this.addDataForSummaryPage('domains', {pageTitle: 'The most used domains',pageDescription: 'A list of the most used domains and the respective timings'});
-    this.addDataForSummaryPage('assets', {pageTitle: 'Most used assets',pageDescription: 'A list of the most used assets for the analyzed pages.'});
-    this.addDataForSummaryPage('toplist', {pageTitle: 'Largest assets by type ',pageDescription: 'A list of the largest assets for the analyzed pages.'});
+    this.summary.pages = {
+      pageTitle: 'Overview of all tested pages',
+      pageDescription: 'See all the tested pages on a high level.',
+      pages: validPages
+    };
+
+    this.summary.index = {
+      pageTitle: 'Summary of the sitespeed.io result',
+      pageDescription: 'Executive summary of the sitespeed.io result. Act on red/yellow/green.',
+      boxes: dataCollection.getSummaryBoxes()
+    };
+
+    this.summary.detailed = {
+      pageTitle: 'In details summary of the sitespeed.io result.',
+      pageDescription: 'Get all the details you need to fast track things you need to change.',
+      metrics: dataCollection.getDetailedBoxes()
+    };
+
+    this.summary.domains = {
+      pageTitle: 'The most used domains',
+      pageDescription: 'A list of the most used domains and the respective timings'
+    };
+
+    this.summary.assets = {
+      pageTitle: 'Most used assets',
+      pageDescription: 'A list of the most used assets for the analyzed pages.'
+    };
+
+    this.summary.toplist = {
+      pageTitle: 'Largest assets by type ',
+      pageDescription: 'A list of the largest assets for the analyzed pages.'
+    };
 
     // TODO check that the coach is availible
-    this.addDataForSummaryPage('help', {pageTitle: 'Definitions and help in for all the used metrics',pageDescription: '', coach: validPages[Object.keys(validPages)[0]]});
+    this.summary.help = {
+      pageTitle: 'Definitions and help in for all the used metrics',
+      pageDescription: '',
+      coach: validPages[Object.keys(validPages)[0]]
+    };
 
-    const summaryRenders = Object.keys(this.summaryPages)
-      .map((name) => this._renderSummaryPage(name, merge({options, noPages: Object.keys(this.urlPages).length},this.summaryPages[name])));
+
+    const summaryRenders = Object.keys(dataCollection.summaryPages)
+      .map((name) => this._renderSummaryPage(name, merge({
+        options,
+        noPages: Object.keys(dataCollection.urlPages).length
+      }, dataCollection.summaryPages[name], this.summary[name])));
 
     const urlPageRenders = Promise.resolve(Object.keys(validPages))
       .map((url) => {
         const pageInfo = validPages[url];
 
-        const runPages = this.urlRunPages[url];
+        const runPages = dataCollection.urlRunPages[url];
 
         // only if we have some browsertime metrics, take the HAR and pass it to the summary
         let summaryPageHAR;
         if (runPages[0].data.browsertime || pageInfo.data.browsertime) {
-          summaryPageHAR = options.html.showAllWaterfallSummary ? pageInfo.data.browsertime.har: runPages[0].data.browsertime.run.har;
+          summaryPageHAR = options.html.showAllWaterfallSummary ? pageInfo.data.browsertime.har : runPages[0].data.browsertime.run.har;
         }
 
-        return this._renderUrlPage(url, 'index', {daurl: url, pageInfo, options, runPages, summaryPageHAR})
+        return this._renderUrlPage(url, 'index', {
+            daurl: url,
+            pageInfo,
+            options,
+            runPages,
+            summaryPageHAR
+          })
           .tap(() => Promise.resolve(Object.keys(runPages))
             .map((runIndex) =>
-              this._renderUrlRunPage(url, runIndex, {daurl: url, runIndex, pageInfo: runPages[runIndex], options, runPages})));
+              this._renderUrlRunPage(url, runIndex, {
+                daurl: url,
+                runIndex,
+                pageInfo: runPages[runIndex],
+                options,
+                runPages
+              })));
       });
 
     // Aggregate/summarize data and write additional files
     return Promise.all(summaryRenders)
       .then(() => Promise.all(urlPageRenders))
       .then(() => this.storageManager.copy(path.join(__dirname, 'assets')))
-      .then(() => log.info('HTML stored in %s', this.storageManager.getBaseDir()))
-      ;
+      .then(() => log.info('HTML stored in %s', this.storageManager.getBaseDir()));
   }
 
   _renderUrlPage(url, name, locals) {
@@ -131,7 +130,7 @@ class HTMLBuilder {
       menu: 'pages',
       pageTitle: 'Summary for all runs ' + url,
       pageDescription: '',
-      headers: this.summaryPages,
+      headers: this.dataCollection.summaryPages,
       version: packageInfo.version,
       timestamp: this.timestamp,
       h: helpers
@@ -146,9 +145,9 @@ class HTMLBuilder {
       JSON: JSON,
       rootPath: this.storageManager.rootPathFromUrl(url),
       menu: 'pages',
-      pageTitle: 'Run ' + (parseInt(name) + 1 )+ ' when testing ' + url,
+      pageTitle: 'Run ' + (parseInt(name) + 1) + ' when testing ' + url,
       pageDescription: '',
-      headers: this.summaryPages,
+      headers: this.dataCollection.summaryPages,
       version: packageInfo.version,
       timestamp: this.timestamp,
       h: helpers
@@ -163,7 +162,7 @@ class HTMLBuilder {
       menu: name,
       pageTitle: name,
       pageDescription: '',
-      headers: this.summaryPages,
+      headers: this.dataCollection.summaryPages,
       version: packageInfo.version,
       timestamp: this.timestamp,
       h: helpers

lib/plugins/html/index.js

@@ -1,8 +1,6 @@
 'use strict';
 
 const path = require('path'),
-  set = require('lodash.set'),
-  reduce = require('lodash.reduce'),
   HTMLBuilder = require('./htmlBuilder');
 
 module.exports = {
@@ -15,103 +13,7 @@ module.exports = {
     this.options = options;
   },
 
-  processMessage(message) {
-    switch (message.type) {
-      case 'url':
-      {
-        return this.HTMLBuilder.addUrl(message.url);
-      }
-
-      case 'error':
-      {
-        return this.HTMLBuilder.addErrorForUrl(message.url, message.source, message.data);
-      }
-
-      case 'browsertime.run':
-      case 'browsertime.pageSummary':
-      case 'browsertime.har':
-      case 'webpagetest.run':
-      case 'webpagetest.pageSummary':
-      case 'gpsi.data':
-      case 'gpsi.pageSummary':
-      case 'pagexray.run':
-      case 'pagexray.pageSummary':
-      case 'coach.run':
-      case 'coach.pageSummary':
-      {
-        return this.HTMLBuilder.addDataForUrl(message.url, message.type, message.data, message.runIndex);
-      }
-
-      case 'assets.aggregate':
-      {
-        const assetList = reduce(message.data, (assetList, asset) => {
-          assetList.push(asset);
-          return assetList;
-        }, []);
-
-        const count = 20,
-          fullCount = Object.keys(assetList).length,
-          topAssets = assetList
-            .sort((a, b) => b.requestCount - a.requestCount)
-            .splice(0, count);
-        return this.HTMLBuilder.addDataForSummaryPage('assets', {topAssets, count, fullCount});
-      }
-
-      case 'assets.aggregateSizePerContentType':
-      {
-        if (!message.group) {
-          const assetsBySize= {};
-          assetsBySize[message.contentType] = message.data;
-          return this.HTMLBuilder.addDataForSummaryPage('toplist', {assetsBySize});
-        }
-        else return;
-      }
-
-      case 'assets.slowest':
-      {
-        if (!message.group) {
-          const slowestAssets = message.data;
-          return this.HTMLBuilder.addDataForSummaryPage('toplist', {slowestAssets});
-        }
-        else return;
-      }
-
-      case 'domains.summary':
-      {
-        const domainList = reduce(message.data, (domainList, domainStats) => {
-          domainList.push(domainStats);
-          return domainList;
-        }, []);
-
-        const count = 200,
-          fullCount = domainList.length,
-          topDomains = domainList
-            .sort((a, b) => b.requestCount - a.requestCount)
-            .splice(0, count);
-        return this.HTMLBuilder.addDataForSummaryPage('domains', {topDomains, count, fullCount});
-      }
-
-      case 'webpagetest.summary':
-      case 'coach.summary':
-      case 'pagexray.summary':
-      case 'browsertime.summary':
-      {
-        const data = {};
-        set(data, message.type, message.data);
-        this.HTMLBuilder.addDataForSummaryPage('index', data);
-        return this.HTMLBuilder.addDataForSummaryPage('detailed', data);
-      }
-    }
-  },
   close() {
-    return this.HTMLBuilder.renderHTML(this.options)
-      .tap(this.buildSummary.bind(this));
-  },
-
-  buildSummary() {
-    if (!this.options.summary) return;
-    var textBuilder = require('./textBuilder'),
-      renderer = this.options.summaryDetail ? textBuilder.renderSummary : textBuilder.renderBriefSummary;
-    return renderer(this.HTMLBuilder, this.options);
+    return this.HTMLBuilder.render();
   }
 };

lib/plugins/text/index.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const path = require('path');
+const textBuilder = require('./textBuilder');
+
+module.exports = {
+  name() {
+    return path.basename(__dirname);
+  },
+
+  open(context, options) {
+    this.dataCollection = context.dataCollection;
+    this.options = options;
+  },
+
+  close(options) {
+    if (!options.summary) return;
+    const renderer = this.options.summaryDetail ? textBuilder.renderSummary : textBuilder.renderBriefSummary;
+    return renderer(this.dataCollection, options);
+  }
+};

lib/plugins/text/textBuilder.js

@@ -3,7 +3,7 @@
 const table = require('text-table'),
   flatten = require('lodash.flatten'),
   color = require('cli-color'),
-  h = require('./helpers'),
+  h = require('../../support/helpers'),
   tableOpts = {
     stringLength : color.getStrippedLength
   },
@@ -19,8 +19,8 @@ function getColor(label) {
   return {ok: 'green', warning: 'yellow', error: 'red', info: 'blackBright'}[label];
 }
 
-function getHeader(HTMLBuilder, options) {
-  const noPages = Object.keys(HTMLBuilder.urlPages).length;
+function getHeader(dataCollection, options) {
+  const noPages = Object.keys(dataCollection.urlPages).length;
   return drab([
     `${h.plural(noPages,'page')} analyzed for ${h.short(options._[0], 30)} `,
     `(${h.plural(options.browsertime.iterations, 'run')}, `,
@@ -28,8 +28,8 @@ function getHeader(HTMLBuilder, options) {
     ].join(''));
 }
 
-function getBoxes(HTMLBuilder) {
-  return flatten(HTMLBuilder.summaryPages['index'].boxes);
+function getBoxes(dataCollection) {
+  return flatten(dataCollection.getSummaryBoxes());
 }
 
 // foo bar -> fb
@@ -43,9 +43,9 @@ function noop(a) {
 }
 
 module.exports = {
-  renderSummary(HTMLBuilder, options) {
-    let out = getHeader(HTMLBuilder, options);
-    let rows = getBoxes(HTMLBuilder).map((b) => {
+  renderSummary(dataCollection, options) {
+    let out = getHeader(dataCollection, options);
+    let rows = getBoxes(dataCollection).map((b) => {
       var marker = getMarker(b.label),
         c = getColor(b.label);
       // color.xterm(ansi)(label),
@@ -55,12 +55,12 @@ module.exports = {
     options.summary = {out : `${out}\n` + table(rows, tableOpts)};
   },
 
-  renderBriefSummary(HTMLBuilder, options) {
-    let out = getHeader(HTMLBuilder, options);
+  renderBriefSummary(dataCollection, options) {
+    let out = getHeader(dataCollection, options);
     var lines = [],
       scores = [],
       size = '', reqs = '', rum = '';
-    getBoxes(HTMLBuilder).map((b) => {
+    getBoxes(dataCollection).map((b) => {
       var c =  getColor(b.label),
         val = b.median,
         name;
@@ -87,4 +87,4 @@ module.exports = {
     lines.push(rum);
     options.summary = { out: `${out}\n` + lines.join(' / ') };
   }
-};
+};

lib/sitespeed.js

@@ -9,6 +9,7 @@ const Promise = require('bluebird'),
   difference = require('lodash.difference'),
   merge = require('lodash.merge'),
   pullAll = require('lodash.pullall'),
+  DataCollection = require('./support/DataCollection'),
   packageInfo = require('../package');
 
 const QueueHandler = require('./support/queueHandler'),
@@ -54,7 +55,7 @@ module.exports = {
     }
 
     const storageManager = new StorageManager(url, timestamp, options);
-
+    const dataCollection = new DataCollection();
     return storageManager.createDataDir('logs').then((logDir) => {
         logging.configure(options, logDir);
       }).then(() => {
@@ -92,6 +93,7 @@ module.exports = {
 
             return runOptionalFunction(allPlugins, 'open', {
                 storageManager,
+                dataCollection,
                 timestamp
               }, options)
               .then(() => runOptionalFunction(allPlugins, 'postOpen', options))

lib/support/dataCollection.js

@@ -0,0 +1,51 @@
+'use strict';
+const merge = require('lodash.merge'),
+  set = require('lodash.set'),
+  reduce = require('lodash.reduce'),
+  summaryBoxesSetup = require('./setup/summaryBoxes'),
+  detailedSetup = require('./setup/detailed');
+
+class DataCollection {
+  constructor() {
+    this.summaryPages = {};
+    this.urlPages = {};
+    this.urlRunPages = {};
+  }
+
+  addDataForSummaryPage(name, data) {
+    if (this.summaryPages[name]) {
+      merge(this.summaryPages[name], data);
+    } else {
+      set(this.summaryPages, name, merge({}, data));
+    }
+  }
+
+  getSummaryBoxes() {
+    return summaryBoxesSetup(this.summaryPages['index']);
+  }
+
+  getDetailedBoxes() {
+    return detailedSetup(this.summaryPages['detailed']);
+  }
+
+  getValidPages() {
+    return reduce(this.urlPages, (validPages, urlInfo, url) => {
+      if (Object.keys(urlInfo.data).length > 0) {
+        validPages[url] = urlInfo;
+      }
+      return validPages;
+    }, {});
+  }
+
+  getErrorPages() {
+    return reduce(this.urlPages, (errors, urlInfo, url) => {
+      if (urlInfo.errors) {
+        errors[url] = urlInfo.errors;
+      }
+      return errors;
+    }, {});
+  }
+
+}
+
+module.exports = DataCollection;

lib/support/pluginLoader.js

@@ -6,7 +6,7 @@ const Promise = require('bluebird'),
 
 Promise.promisifyAll(fs);
 
-const defaultPlugins = new Set(['browsertime', 'coach', 'domains', 'assets', 'html', 'screenshot','metrics']);
+const defaultPlugins = new Set(['browsertime', 'coach', 'datacollector', 'domains', 'assets', 'html', 'screenshot','metrics','text']);
 
 const pluginsDir = path.join(__dirname, '..', 'plugins');