Configurable metrics keys (#1057)

* List metrics, filters and add/remove filters.
* List all configured filters
* List all possible metrics
* Add filters
* Remove filters

lib/plugins/coach/index.js

@@ -12,6 +12,7 @@ const make = messageMaker('coach').make;
 const DEFAULT_METRICS_SUMMARY = ['score.*','performance.score.*', 'bestpractice.score.*', 'accessibility.score.*'];
 const DEFAULT_METRICS_PAGESUMMARY = ['advice.score','advice.performance.score','advice.bestpractice.score', 'advice.accessibility.score'];
 const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes'];
+const DEFAULT_PAGEXRAY_SUMMARY_METRICS = ['*'];
 
 module.exports = {
   name() {
@@ -22,6 +23,7 @@ module.exports = {
     filterRegistry.registerFilterForType(DEFAULT_METRICS_SUMMARY, 'coach.summary');
     filterRegistry.registerFilterForType(DEFAULT_METRICS_PAGESUMMARY, 'coach.pageSummary');
     filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS, 'pagexray.pageSummary');
+    filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_SUMMARY_METRICS, 'pagexray.summary');
   },
   processMessage(message, queue) {
     switch (message.type) {

lib/plugins/graphite/data-generator.js

@@ -5,6 +5,33 @@ const flatten = require('../../support/flattenMessage'),
   supportUtil = require('../../support/util'),
   reduce = require('lodash.reduce');
 
+  function keyPathFromMessage(message, options, includeQueryParams) {
+    let typeParts = message.type.split('.');
+    typeParts.push(typeParts.shift());
+
+    // always have browser and connectivity in Browsertime and related tools
+    if (message.type.match(/(^pagexray|^coach|^browsertime|^assets|^domains)/)) {
+      typeParts.splice(1, 0, options.connectivity);
+      typeParts.splice(1, 0, options.browser);
+    } else if (message.type.match(/(^webpagetest)/)) {
+      if (message.connectivity) {
+        typeParts.splice(2, 0, message.connectivity);
+      }
+      if (message.location) {
+      typeParts.splice(2, 0, message.location);
+      }
+    }
+
+    if (message.url) {
+      typeParts.splice(1, 0, flatten.keypathFromUrl(message.url, includeQueryParams));
+    } else {
+      // it's a summary, add domain/filename
+      typeParts.splice(1, 0, supportUtil.getDomainOrFileName(options._[0]).replace(/\./g, '_'));
+    }
+
+    return typeParts.join('.');
+  }
+
 class GraphiteDataGenerator {
   constructor(namespace, includeQueryParams, options) {
     this.namespace = namespace;
@@ -14,43 +41,25 @@ class GraphiteDataGenerator {
 
   dataFromMessage(message, time) {
     const timestamp = Math.round(time.valueOf() / 1000);
-    let options = this.options;
 
-    function keyPathFromMessage(message, includeQueryParams) {
-      let typeParts = message.type.split('.');
-      typeParts.push(typeParts.shift());
-
-      // always have browser and connectivity in Browsertime and related tools
-      if (message.type.match(/(^pagexray|^coach|^browsertime|^assets|^domains)/)) {
-        typeParts.splice(1, 0, options.connectivity);
-        typeParts.splice(1, 0, options.browser);
-      } else if (message.type.match(/(^webpagetest)/)) {
-        if (message.connectivity) {
-          typeParts.splice(2, 0, message.connectivity);
-        }
-        if (message.location) {
-        typeParts.splice(2, 0, message.location);
-        }
-      }
-
-      if (message.url) {
-        typeParts.splice(1, 0, flatten.keypathFromUrl(message.url, includeQueryParams));
-      } else {
-        // it's a summary, add domain/filename
-        typeParts.splice(1, 0, supportUtil.getDomainOrFileName(options._[0]).replace(/\./g, '_'));
-      }
-
-      return typeParts.join('.');
-    }
-
-    var keypath = keyPathFromMessage(message, this.includeQueryParams);
+    var keypath = keyPathFromMessage(message,  this.options, this.includeQueryParams);
 
     return reduce(flatten.flattenMessageData(message), (entries, value, key) => {
       let fullKey = util.format('%s.%s.%s', this.namespace, keypath, key);
       entries.push(util.format('%s %s %s', fullKey, value, timestamp));
       return entries;
-    }, []).join('\n') + '\n';
+    }, []);
   }
+  /*
+  keysFromMessage(message) {
+    var keypath = keyPathFromMessage(message,  this.options, this.includeQueryParams);
+
+    return reduce(flatten.flattenMessageData(message), (entries, value, key) => {
+      let fullKey = util.format('%s.%s.%s', this.namespace, keypath, key);
+      entries.push(fullKey);
+      return entries;
+    }, []);
+  }*/
 }
 
 module.exports = GraphiteDataGenerator;

lib/plugins/graphite/index.js

@@ -30,9 +30,9 @@ module.exports = {
       return;
 
     // TODO Here we could add logic to either create a new timestamp or
-    // use the one that we haev for that run. Now just use the one for the
-    // run  
-    let data = this.dataGenerator.dataFromMessage(message, this.timestamp);
+    // use the one that we have for that run. Now just use the one for the
+    // run
+    let data = this.dataGenerator.dataFromMessage(message, this.timestamp).join('\n') + '\n';
 
     if (data.length > 0) {
       return this.sender.send(data);

lib/plugins/metrics/index.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const path = require('path'),
+  flatten = require('../../support/flattenMessage'),
+  filterRegistry = require('../../support/filterRegistry');
+
+module.exports = {
+  name() {
+    return path.basename(__dirname);
+  },
+  open(context, options) {
+    this.options = options;
+    this.metrics = {};
+    this.storageManager = context.storageManager;
+  },
+  postOpen() {
+    if (this.options.metrics && this.options.metrics.filter) {
+
+      for (let metric of this.options.metrics.filter) {
+        // for all filters
+        // cleaning all filters means (right now) that all
+        // metrics are sent
+        if (metric === '*+') {
+          filterRegistry.clearAll();
+        }
+        // all registred types will be set as unmatching,
+        // use it if you want to have a clean filter where
+        // all types are removed and then you can add your own
+        else if(metric === '*-') {
+          let types = filterRegistry.getTypes();
+          filterRegistry.clearAll();
+          for (let type of types) {
+            filterRegistry.registerFilterForType('-', type);
+          }
+        }
+        else {
+          let parts = metric.split('.');
+          // the type is "always" the first two
+          let type = parts.shift() + '.' + parts.shift();
+          let filter = parts.join('.');
+          let oldFilter = filterRegistry.getFilterForType(type);
+          if (oldFilter && typeof oldFilter === 'object') {
+            oldFilter.push(filter);
+          } else {
+            oldFilter = [filter];
+          }
+          filterRegistry.registerFilterForType(oldFilter, type);
+        }
+      }
+    }
+
+  },
+  processMessage(message) {
+    if (this.options.metrics && this.options.metrics.list) {
+      if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary')))
+        return;
+      let flattenMess = flatten.flattenMessageData(message);
+      for (let key of Object.keys(flattenMess)) {
+        this.metrics[message.type + '.' + key] = 1;
+      }
+    } else {
+      return
+    }
+  },
+  close() {
+    if (this.options.metrics && this.options.metrics.list) {
+      this.storageManager.writeData('metrics.txt', Object.keys(this.metrics).join('\n'));
+    }
+
+    if (this.options.metrics && this.options.metrics.filterList) {
+      let output = '';
+      let filtersByType = filterRegistry.getFilters();
+      for (let type of Object.keys(filtersByType)) {
+        for (let filters of filtersByType[type]) {
+            output+= type + '.' + filters + '\n';
+        }
+      }
+      return this.storageManager.writeData('configuredMetrics.txt', output);
+    }
+  }
+};

lib/support/cli.js

@@ -194,6 +194,23 @@ module.exports.parseCommandLine = function parseCommandLine() {
     })
     */
 
+    .option('metrics.list', {
+      describe: 'List all possible metrics in the data folder (metrics.txt).',
+      type: 'boolean',
+      default: false,
+      group: 'Metrics'
+    })
+    .option('metrics.filterList', {
+      describe: 'List all configured filters for metrics in the data folder (configuredMetrics.txt)',
+      type: 'boolean',
+      default: false,
+      group: 'Metrics'
+    })
+    .option('metrics.filter', {
+      type: 'array',
+      describe: 'Add/change/remove filters for metrics. If you want to send all metrics, use: *+ . If you want to remove all current metrics and send only the coach score: *- coach.summary.score.*',
+      group: 'Metrics'
+    })
 
   /*
    WebPageTest cli options

lib/support/filterRegistry.js

@@ -3,7 +3,7 @@
 const clone = require('lodash.clonedeep'),
   metricsFilter = require('./metricsFilter');
 
-const filterForType = {};
+let filterForType = {};
 
 module.exports = {
   registerFilterForType(filter, type) {
@@ -14,6 +14,22 @@ module.exports = {
     return filterForType[type];
   },
 
+  getFilters() {
+    return filterForType;
+  },
+
+  getTypes() {
+    return Object.keys(filterForType);
+  },
+
+  removeFilter(type) {
+    filterForType[type] = undefined;
+  },
+
+  clearAll() {
+    filterForType = {};
+  },
+
   filterMessage(message) {
     const filterConfig = this.getFilterForType(message.type);
 

lib/support/flattenMessage.js

@@ -41,6 +41,10 @@ module.exports = {
     }
 
     function recursiveFlatten(target, keyPrefix, value) {
+      // super simple version to avoid flatten HAR and screenshot data
+      if (keyPrefix.match(/(screenshots\.|har\.|assets\.)/)) {
+          return;
+      }
       const valueType = typeof value;
 
       switch (valueType) {
@@ -71,6 +75,11 @@ module.exports = {
           target[keyPrefix] = value ? 1 : 0;
         }
           break;
+        case 'undefined':
+        {
+          // take care of faulty values, add a log in the future
+        }
+          break;
         default:
           throw new Error('Unhandled value type ' + valueType + ' found when flattening data for prefix ' + keyPrefix);
       }

lib/support/pluginLoader.js

@@ -6,7 +6,7 @@ const Promise = require('bluebird'),
 
 Promise.promisifyAll(fs);
 
-const defaultPlugins = new Set(['browsertime', 'coach', 'domains', 'assets', 'html', 'screenshot']);
+const defaultPlugins = new Set(['browsertime', 'coach', 'domains', 'assets', 'html', 'screenshot','metrics']);
 
 const pluginsDir = path.join(__dirname, '..', 'plugins');