First go at adding configurable groups #2958 (#2964)

* First go at adding configurable groups #2958

* Fix broken group

* simplify group handling

* Adjust test to the new structure

lib/cli/cli.js

@@ -82,6 +82,14 @@ function validateInput(argv) {
     return 'You have a miss match between number of alias and URLs.';
   }
 
+  if (
+    argv.groupAlias &&
+    argv._ &&
+    cliUtil.getURLs(argv._).length !== toArray(argv.groupAlias).length
+  ) {
+    return 'You have a miss match between number of alias for groups and URLs.';
+  }
+
   // validate URLs/files
   const urlOrFiles = argv._;
   for (let urlOrFile of urlOrFiles) {
@@ -1431,6 +1439,11 @@ module.exports.parseCommandLine = function parseCommandLine() {
         'Use an alias for the URL (if you feed URLs from a file you can instead have the alias in the file). You need to pass on the same amount of alias as URLs. The alias is used as the name of the URL on the HTML report and in Graphite/InfluxDB. Pass on multiple --urlAlias for multiple alias/URLs. This will override alias in a file.',
       type: 'string'
     })
+    .option('groupAlias', {
+      describe:
+        'Use an alias for the group/domain. You need to pass on the same amount of alias as URLs. The alias is used as the name of the group in Graphite/InfluxDB. Pass on multiple --groupAlias for multiple alias/groups.',
+      type: 'string'
+    })
     .option('utc', {
       describe: 'Use Coordinated Universal Time for timestamps',
       default: false,
@@ -1627,9 +1640,11 @@ module.exports.parseCommandLine = function parseCommandLine() {
     );
   }
 
+  let urlsMetaData = cliUtil.getAliases(argv._, argv.urlAlias, argv.groupAlias);
+
   return {
     urls: argv.multi ? argv._ : cliUtil.getURLs(argv._),
-    urlsMetaData: cliUtil.getAliases(argv._, argv.urlAlias),
+    urlsMetaData,
     options: argv,
     explicitOptions: explicitOptions
   };

lib/cli/util.js

@@ -50,16 +50,20 @@ module.exports = {
     }
     return allUrls;
   },
-  getAliases(urls, alias) {
+  getAliases(urls, alias, groupAlias) {
     const urlMetaData = {};
     urls = urls.map(url => url.trim());
     let al = toArray(alias);
+    let allGroupAlias = toArray(groupAlias);
     let pos = 0;
 
     for (let url of urls) {
       if (url.startsWith('http')) {
         if (al.length > 0 && al[pos]) {
-          urlMetaData[url] = { alias: al[pos] };
+          urlMetaData[url] = { urlAlias: al[pos] };
+        }
+        if (allGroupAlias.length > 0 && allGroupAlias[pos]) {
+          urlMetaData[url] = { groupAlias: allGroupAlias[pos] };
         }
         pos += 1;
       } else {
@@ -78,7 +82,7 @@ module.exports = {
               alias = lineArray[1].trim();
             }
             if (url && alias) {
-              urlMetaData[url] = { alias: alias };
+              urlMetaData[url] = { urlAlias: alias };
             }
           }
         }

lib/core/url-source.js

@@ -11,7 +11,17 @@ module.exports = {
   findUrls(queue) {
     for (const url of this.options.urls) {
       queue.postMessage(
-        make('url', {}, { url: url, group: urlParser.parse(url).hostname })
+        make(
+          'url',
+          {},
+          {
+            url: url,
+            group:
+              this.options.urlsMetaData && this.options.urlsMetaData[url]
+                ? this.options.urlsMetaData[url].groupAlias
+                : urlParser.parse(url).hostname
+          }
+        )
       );
     }
   }

lib/plugins/thirdparty/index.js

@@ -18,6 +18,7 @@ module.exports = {
     this.make = context.messageMaker('thirdparty').make;
     this.groups = {};
     this.runsPerUrl = {};
+    this.urlAndGroup = {};
 
     if (options.thirdParty && options.thirdParty.cpu) {
       DEFAULT_THIRDPARTY_PAGESUMMARY_METRICS.push('tool.*');
@@ -154,11 +155,12 @@ module.exports = {
       queue.postMessage(
         make('thirdparty.run', runData, {
           url: message.url,
-          group: urlParser.parse(message.url).hostname,
+          group: message.group,
           runIndex: message.runIndex
         })
       );
-
+      // Store the group to use in summarize
+      this.urlAndGroup[message.url] = message.group;
       if (this.runsPerUrl[message.url]) {
         this.runsPerUrl[message.url].push(runData);
       } else {
@@ -171,7 +173,7 @@ module.exports = {
         queue.postMessage(
           make('thirdparty.pageSummary', summary[url], {
             url,
-            group: urlParser.parse(url).hostname
+            group: this.urlAndGroup[url]
           })
         );
       }

lib/plugins/webpagetest/index.js

@@ -122,6 +122,7 @@ module.exports = {
     this.filterRegistry = context.filterRegistry;
 
     this.options = merge({}, defaultConfig, options.webpagetest);
+    this.allOptions = options;
 
     if (get(this.options, 'ssio.domainsDashboard')) {
       // that adds a lot of disk space need into graphite, so we keep it hidden for now
@@ -197,7 +198,7 @@ module.exports = {
       // We got a URL that we should test
       case 'url': {
         const url = message.url;
-        const group = message.group;
+        let group = message.group;
         return analyzer
           .analyzeUrl(url, this.storageManager, this.log, wptOptions)
           .then(result => {

lib/support/flattenMessage.js

@@ -13,7 +13,7 @@ function toSafeKey(key) {
 }
 
 module.exports = {
-  keypathFromUrl(url, includeQueryParams, useHash) {
+  keypathFromUrl(url, includeQueryParams, useHash, group) {
     function flattenQueryParams(params) {
       return Object.keys(params).reduce(
         (result, key) => joinNonEmpty([result, key, params[key]], '_'),
@@ -35,7 +35,7 @@ module.exports = {
       path = joinNonEmpty([path, toSafeKey(url.hash)], '_');
     }
 
-    const keys = [toSafeKey(url.hostname), path];
+    const keys = [toSafeKey(group || url.hostname), path];
 
     return joinNonEmpty(keys, '.');
   },

lib/support/tsdbUtil.js

@@ -29,7 +29,12 @@ module.exports = {
     } else if (alias && alias[url]) {
       return this.toSafeKey(group) + '.' + this.toSafeKey(alias[url]);
     } else {
-      return flatten.keypathFromUrl(url, includeQueryParams, options.useHash);
+      return flatten.keypathFromUrl(
+        url,
+        includeQueryParams,
+        options.useHash,
+        group
+      );
     }
   }
 };

test/cliUtilTests.js

@@ -28,7 +28,9 @@ describe('cliUtil', function() {
       aliases = cliUtil.getAliases([
         'test/fixtures/sitespeed-urls-aliases.txt'
       ]);
-      expect(aliases['https://www.sitespeed.io'].alias).to.equal('Home_Page');
+      expect(aliases['https://www.sitespeed.io'].urlAlias).to.equal(
+        'Home_Page'
+      );
       expect(
         aliases[
           'https://www.sitespeed.io/documentation/sitespeed.io/webpagetest/'