Possibility to send asset toplists to Graphite (#1301)

Restructure the toplist (largest/slowest/aggregated assets) to follow the message key structure with summary/pageSummary to simplify key generation. We also now includes the URL in the slowestassets list that can be sent to Graphite. Ping #1289 and closes #1294

lib/plugins/assets/aggregator.js

@@ -47,6 +47,7 @@ module.exports = {
         cacheTime: asset.expires,
         size: asset.contentSize,
         page,
+        timing: asset.timing,
         requestCount: 0
       };
       urlInfo.requestCount++;
@@ -59,6 +60,7 @@ module.exports = {
         cacheTime: asset.expires,
         size: asset.contentSize,
         page,
+        timing: asset.timing,
         requestCount: 0
       };
 

lib/plugins/assets/assetsBySize.js

@@ -34,7 +34,7 @@ class AssetsBySize {
       }
     }
   }
-  
+
   getItems() {
     this.items.sort(function(asset, asset2) {
       return asset2.contentSize - asset.contentSize;

lib/plugins/assets/index.js

@@ -9,7 +9,7 @@ let messageMaker = require('../../support/messageMaker'),
 const make = messageMaker('assets').make;
 
 const DEFAULT_METRICS_LARGEST_ASSETS = [
-  'total.image.0.transferSize'
+  'image.0.transferSize'
 ];
 
 module.exports = {
@@ -18,7 +18,9 @@ module.exports = {
   },
   open(context) {
     this.context = context;
-    filterRegistry.registerFilterForType(DEFAULT_METRICS_LARGEST_ASSETS, 'assets.largest');
+    filterRegistry.registerFilterForType(DEFAULT_METRICS_LARGEST_ASSETS, 'largestassets.summary');
+    filterRegistry.registerFilterForType([], 'slowestassets.summary');
+    filterRegistry.registerFilterForType([], 'aggregateassets.summary');
   },
   processMessage(message, queue) {
     switch (message.type) {
@@ -33,28 +35,29 @@ module.exports = {
         const summary = aggregator.summarize();
         if (!isEmpty(summary)) {
           for (let group of Object.keys(summary.groups)) {
-            const data = {};
-            data[group] = summary.groups[group];
-            queue.postMessage(make('assets.aggregate', data));
+            queue.postMessage(make('aggregateassets.summary', summary.groups[group], {group}));
 
             for (let contentType of Object.keys(summary.size[group])) {
               const d = {};
-              d[group] = {};
-              d[group][contentType] = summary.size[group][contentType];
-              queue.postMessage(make('assets.largest', d));
+              d[contentType] = summary.size[group][contentType];
+              queue.postMessage(make('largestassets.summary', d, {group}));
             }
-            const slow = {};
-            slow[group] = summary.timing[group];
-            queue.postMessage(make('assets.slowest', slow));
+
+            // hack for possibility to add URL to metrics sent to graphite
+            const d = [];
+            for (const item of summary.timing[group]) {
+              var obj = {};
+              obj[item.url] = item;
+              d.push(obj);
+            }
+            queue.postMessage(make('slowestassets.summary', d, {group}));
           }
 
           for (let contentType of Object.keys(summary.size.total)) {
-            const data = {};
-            data[contentType] = summary.size.total[contentType];
-            queue.postMessage(make('assets.largest', contentType));
+            queue.postMessage(make('largestassets.summary', summary.size.total[contentType], {group: 'total'}));
           }
 
-          queue.postMessage(make('assets.slowest', summary.timing.total));
+          queue.postMessage(make('slowestassets.summary', summary.timing.total, {group: 'total'}));
         }
         break;
       }

lib/plugins/datacollector/index.js

@@ -44,10 +44,10 @@ module.exports = {
           return dataCollector.addDataForUrl(message.url, message.type, message.data, message.runIndex);
         }
 
-      case 'assets.aggregate':
+      case 'aggregateassets.summary':
         {
-          if (message.data.total) {
-            const assetList = reduce(message.data.total, (assetList, asset) => {
+          if (message.group === 'total') {
+            const assetList = reduce(message.data, (assetList, asset) => {
               assetList.push(asset);
               return assetList;
             }, []);
@@ -65,22 +65,22 @@ module.exports = {
           } else return;
         }
 
-      case 'assets.largest':
+      case 'largestassets.summary':
         {
-          if (message.data.total) {
+          if (message.group === 'total') {
             const assetsBySize = {};
-            const contentType = Object.keys(message.data.total)[0];
-            assetsBySize[contentType] = message.data.total[contentType];
+            const contentType = Object.keys(message.data)[0];
+            assetsBySize[contentType] = message.data[contentType];
             return dataCollector.addDataForSummaryPage('toplist', {
               assetsBySize
             });
           } else return;
         }
 
-      case 'assets.slowest':
+      case 'slowestassets.summary':
         {
-          if (message.data.total) {
-            const slowestAssets = message.data.total;
+          if (message.group === 'total') {
+            const slowestAssets = message.data;
             return dataCollector.addDataForSummaryPage('toplist', {
               slowestAssets
             });

lib/plugins/graphite/data-generator.js

@@ -9,7 +9,7 @@ function keyPathFromMessage(message, options, includeQueryParams) {
   typeParts.push(typeParts.shift());
 
   // always have browser and connectivity in Browsertime and related tools
-  if (message.type.match(/(^pagexray|^coach|^browsertime|^assets|^domains)/)) {
+  if (message.type.match(/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains)/)) {
     typeParts.splice(1, 0, options.connectivity);
     typeParts.splice(1, 0, options.browser);
   } else if (message.type.match(/(^webpagetest)/)) {
@@ -23,7 +23,7 @@ function keyPathFromMessage(message, options, includeQueryParams) {
   // if we get a URL type, add the URL
   if (message.url) {
     typeParts.splice(1, 0, flatten.keypathFromUrl(message.url, includeQueryParams));
-  } else {
+  } else if (message.group){
     // add the group of the summary message
     // the group in the current version is the domain so lets
     // change the dots to underscore

lib/plugins/html/templates/assets.pug

@@ -13,7 +13,6 @@ mixin rows(assets)
       +durationCell('cache time',asset.cacheTime)
       +kbSizeCell('size', asset.size)
       +numberCell('count', asset.requestCount)
-
 block content
   include runInfo
   if fullCount > 0

lib/plugins/metrics/index.js

@@ -60,7 +60,7 @@ module.exports = {
   },
   processMessage(message) {
     if (this.options.list) {
-      if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary') || message.type.endsWith('assets.largest')))
+      if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary')))
         return;
       let flattenMess = flatten.flattenMessageData(message);
       for (let key of Object.keys(flattenMess)) {

lib/support/flattenMessage.js

@@ -7,7 +7,7 @@ function joinNonEmpty(strings, delimeter) {
 }
 
 function toSafeKey(key) {
-  return key.replace(/[.~ /+|,]|%7C/g, '_');
+  return key.replace(/[.~ /+|,:]|%7C/g, '_');
 }
 
 module.exports = {
@@ -42,7 +42,7 @@ module.exports = {
 
     function recursiveFlatten(target, keyPrefix, value) {
       // super simple version to avoid flatten HAR and screenshot data
-      if (keyPrefix.match(/(screenshots\.|har\.|assets\.)/)) {
+      if (keyPrefix.match(/(screenshots\.|har\.)/)) {
           return;
       }
       const valueType = typeof value;