Send annotations to Graphite for each tested URL (#1434)

lib/plugins/graphite/data-generator.js

@@ -2,13 +2,9 @@
 
 const flatten = require('../../support/flattenMessage'),
   util = require('util'),
-  get = require('lodash.get'),
+  graphiteUtil = require('./util'),
   reduce = require('lodash.reduce');
 
-function toSafeKey(key) {
-  return key.replace(/[.~ /+|,:?&%]|%7C/g, '_');
-}
-
 function keyPathFromMessage(message, options, includeQueryParams) {
   let typeParts = message.type.split('.');
   typeParts.push(typeParts.shift());
@@ -17,12 +13,8 @@ function keyPathFromMessage(message, options, includeQueryParams) {
   if (message.type.match(/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains)/)) {
 
     // if we have a friendly name for your conectivity, use that!
-    let connectivity = get(options, 'browsertime.connectivity.alias');
-    if (connectivity) {
-      connectivity = toSafeKey(connectivity);
-    } else {
-      connectivity = options.connectivity;
-    }
+    let connectivity = graphiteUtil.getConnectivity(options);
+
     typeParts.splice(1, 0, connectivity);
     typeParts.splice(1, 0, options.browser);
   } else if (message.type.match(/(^webpagetest)/)) {
@@ -35,15 +27,10 @@ function keyPathFromMessage(message, options, includeQueryParams) {
   }
   // if we get a URL type, add the URL
   if (message.url) {
-    if(message.group && options.urlsMetaData[message.url]) {
-      let alias = options.urlsMetaData[message.url].alias;
-      typeParts.splice(1, 0, toSafeKey(message.group) + "." + toSafeKey(alias));
-    } else {
-      typeParts.splice(1, 0, flatten.keypathFromUrl(message.url, includeQueryParams));
-    }
+    typeParts.splice(1, 0, graphiteUtil.getURLAndGroup(options, message.group, message.url, includeQueryParams));
   } else if (message.group) {
     // add the group of the summary message
-    typeParts.splice(1, 0, toSafeKey(message.group));
+    typeParts.splice(1, 0, graphiteUtil.toSafeKey(message.group));
   }
 
   return typeParts.join('.');

lib/plugins/graphite/index.js

@@ -6,6 +6,7 @@ let path = require('path'),
   Sender = require('./sender'),
   merge = require('lodash.merge'),
   log = require('intel'),
+  sendAnnotations = require('./send-annotation'),
   DataGenerator = require('./data-generator');
 
 
@@ -21,10 +22,12 @@ module.exports = {
   },
   open(context, options) {
     const opts = merge({}, defaultConfig, options.graphite);
+    this.options = options;
     this.sender = new Sender(opts.host, opts.port);
     this.dataGenerator = new DataGenerator(opts.namespace, opts.includeQueryParams, options);
     log.debug('Setting up Graphite %s:%s for namespace %s', opts.host, opts.port, opts.namespace);
     this.timestamp = context.timestamp;
+    this.storageManager = context.storageManager;
   },
   processMessage(message) {
     if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary')))
@@ -46,7 +49,15 @@ module.exports = {
     let data = this.dataGenerator.dataFromMessage(message, this.timestamp).join('\n') + '\n';
 
     if (data.length > 0) {
-      return this.sender.send(data);
+      const storageManager = this.storageManager;
+      const resultBaseURL = this.options.resultBaseURL;
+      return this.sender.send(data).then(() => {
+        // make sure we only send once per URL (and browsertime is the most important)
+        // and you need to configure a base URL where you get the HTML result
+        if (message.type === 'browsertime.pageSummary' && resultBaseURL) {
+          return sendAnnotations.send(this.options, message.group, message.url, storageManager.getRelativeBaseDir(), storageManager.pathFromRootToPageDir(message.url));
+        } else return;
+      });
     } else {
       return Promise.reject(new Error('No data to send to graphite for message:\n' +
         JSON.stringify(message, null, 2)));

lib/plugins/graphite/send-annotation.js

@@ -0,0 +1,62 @@
+'use strict';
+const http = require('http');
+const https = require('https');
+const log = require('intel');
+const Promise = require('bluebird');
+const graphiteUtil = require('./util');
+
+module.exports = {
+  send(options, group, url, baseDir, pagePath) {
+
+    // The tags make it possible for the dashboard to use the
+    // templates to choose which annotations that will be showed.
+    // That's why we need to send tags that matches the template
+    // variables in Grafana.
+    const connectivity = graphiteUtil.getConnectivity(options);
+    const browser = options.browser;
+    const namespace = options.graphite.namespace.split('.').join(',');
+    const urlAndGroup = graphiteUtil.getURLAndGroup(options, group, url, options.graphite.includeQueryParams).split('.').join(',');
+    const tags =  `${connectivity},${browser},${namespace},${urlAndGroup}`;
+    const message = `<a href='${options.resultBaseURL}/${baseDir}/${pagePath}' target='_blank'>Result ${options.browsertime.iterations} run(s)</a>`;
+
+    const postData =
+      `{"what": "Sitespeed.io", "tags": "${tags}", "data": "${message}"}`;
+    const postOptions = {
+      hostname: options.graphite.host,
+      port: options.graphite.httpPort || 8080,
+      path: '/events/',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Content-Length': Buffer.byteLength(postData)
+      }
+    };
+
+    // If Graphite is behind auth, use it!
+    if (options.graphite.auth) {
+      postOptions.auth = options.graphite.auth;
+    }
+
+    return new Promise((resolve, reject) => {
+      log.trace('Send annotation to Graphite: %j', postData);
+      // not perfect but maybe work for us
+      const lib = options.graphite.httpPort === 443 ? https : http;
+      const req = lib.request(postOptions, (res) => {
+        if (res.statusCode !== 200) {
+          log.error('Got %s from Graphite when sending annotation', res.statusCode);
+          reject();
+        } else {
+        res.setEncoding('utf8');
+        log.info('Sent annotation to Graphite');
+        resolve();
+      }
+      });
+      req.on('error', (err) => {
+        log.error('Got error from Graphite when sending annotation', err);
+        reject(err)
+      });
+      req.write(postData);
+      req.end();
+    })
+  }
+}

lib/plugins/graphite/sender.js

@@ -1,6 +1,6 @@
 'use strict';
 
-var net = require('net'),
+const net = require('net'),
   log = require('intel'),
   Promise = require('bluebird');
 

lib/plugins/graphite/util.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const get = require('lodash.get');
+const flatten = require('../../support/flattenMessage');
+
+module.exports = {
+  toSafeKey(key) {
+    return key.replace(/[.~ /+|,:?&%]|%7C/g, '_');
+  },
+  getConnectivity(options) {
+    // if we have a friendly name for your conectivity, use that!
+    let connectivity = get(options, 'browsertime.connectivity.alias');
+    if (connectivity) {
+      return this.toSafeKey(connectivity);
+    } else {
+      return options.connectivity;
+    }
+  },
+  getURLAndGroup(options, group, url, includeQueryParams) {
+    if(group && options.urlsMetaData[url]) {
+      let alias = options.urlsMetaData[url].alias;
+       return this.toSafeKey(group) + "." + this.toSafeKey(alias);
+    } else {
+      return flatten.keypathFromUrl(url, includeQueryParams);
+    }
+  }
+}