From 5e80d8af84e54e378e6ccb18c1e5fb2b2f40aa0c Mon Sep 17 00:00:00 2001 From: Peter Hedenskog Date: Fri, 10 Feb 2017 08:21:35 +0100 Subject: [PATCH] Send annotations to Graphite for each tested URL (#1434) --- lib/plugins/graphite/data-generator.js | 23 ++------- lib/plugins/graphite/index.js | 13 +++++- lib/plugins/graphite/send-annotation.js | 62 +++++++++++++++++++++++++ lib/plugins/graphite/sender.js | 2 +- lib/plugins/graphite/util.js | 27 +++++++++++ 5 files changed, 107 insertions(+), 20 deletions(-) create mode 100644 lib/plugins/graphite/send-annotation.js create mode 100644 lib/plugins/graphite/util.js diff --git a/lib/plugins/graphite/data-generator.js b/lib/plugins/graphite/data-generator.js index 322a2c540..fa1597bed 100644 --- a/lib/plugins/graphite/data-generator.js +++ b/lib/plugins/graphite/data-generator.js @@ -2,13 +2,9 @@ const flatten = require('../../support/flattenMessage'), util = require('util'), - get = require('lodash.get'), + graphiteUtil = require('./util'), reduce = require('lodash.reduce'); -function toSafeKey(key) { - return key.replace(/[.~ /+|,:?&%]|%7C/g, '_'); -} - function keyPathFromMessage(message, options, includeQueryParams) { let typeParts = message.type.split('.'); typeParts.push(typeParts.shift()); @@ -17,12 +13,8 @@ function keyPathFromMessage(message, options, includeQueryParams) { if (message.type.match(/(^pagexray|^coach|^browsertime|^largestassets|^slowestassets|^aggregateassets|^domains)/)) { // if we have a friendly name for your conectivity, use that! - let connectivity = get(options, 'browsertime.connectivity.alias'); - if (connectivity) { - connectivity = toSafeKey(connectivity); - } else { - connectivity = options.connectivity; - } + let connectivity = graphiteUtil.getConnectivity(options); + typeParts.splice(1, 0, connectivity); typeParts.splice(1, 0, options.browser); } else if (message.type.match(/(^webpagetest)/)) { @@ -35,15 +27,10 @@ function keyPathFromMessage(message, options, includeQueryParams) { } // if we get a URL type, add the URL if (message.url) { - if(message.group && options.urlsMetaData[message.url]) { - let alias = options.urlsMetaData[message.url].alias; - typeParts.splice(1, 0, toSafeKey(message.group) + "." + toSafeKey(alias)); - } else { - typeParts.splice(1, 0, flatten.keypathFromUrl(message.url, includeQueryParams)); - } + typeParts.splice(1, 0, graphiteUtil.getURLAndGroup(options, message.group, message.url, includeQueryParams)); } else if (message.group) { // add the group of the summary message - typeParts.splice(1, 0, toSafeKey(message.group)); + typeParts.splice(1, 0, graphiteUtil.toSafeKey(message.group)); } return typeParts.join('.'); diff --git a/lib/plugins/graphite/index.js b/lib/plugins/graphite/index.js index 8f984ef38..4196aa5a9 100644 --- a/lib/plugins/graphite/index.js +++ b/lib/plugins/graphite/index.js @@ -6,6 +6,7 @@ let path = require('path'), Sender = require('./sender'), merge = require('lodash.merge'), log = require('intel'), + sendAnnotations = require('./send-annotation'), DataGenerator = require('./data-generator'); @@ -21,10 +22,12 @@ module.exports = { }, open(context, options) { const opts = merge({}, defaultConfig, options.graphite); + this.options = options; this.sender = new Sender(opts.host, opts.port); this.dataGenerator = new DataGenerator(opts.namespace, opts.includeQueryParams, options); log.debug('Setting up Graphite %s:%s for namespace %s', opts.host, opts.port, opts.namespace); this.timestamp = context.timestamp; + this.storageManager = context.storageManager; }, processMessage(message) { if (!(message.type.endsWith('.summary') || message.type.endsWith('.pageSummary'))) @@ -46,7 +49,15 @@ module.exports = { let data = this.dataGenerator.dataFromMessage(message, this.timestamp).join('\n') + '\n'; if (data.length > 0) { - return this.sender.send(data); + const storageManager = this.storageManager; + const resultBaseURL = this.options.resultBaseURL; + return this.sender.send(data).then(() => { + // make sure we only send once per URL (and browsertime is the most important) + // and you need to configure a base URL where you get the HTML result + if (message.type === 'browsertime.pageSummary' && resultBaseURL) { + return sendAnnotations.send(this.options, message.group, message.url, storageManager.getRelativeBaseDir(), storageManager.pathFromRootToPageDir(message.url)); + } else return; + }); } else { return Promise.reject(new Error('No data to send to graphite for message:\n' + JSON.stringify(message, null, 2))); diff --git a/lib/plugins/graphite/send-annotation.js b/lib/plugins/graphite/send-annotation.js new file mode 100644 index 000000000..33a536b20 --- /dev/null +++ b/lib/plugins/graphite/send-annotation.js @@ -0,0 +1,62 @@ +'use strict'; +const http = require('http'); +const https = require('https'); +const log = require('intel'); +const Promise = require('bluebird'); +const graphiteUtil = require('./util'); + +module.exports = { + send(options, group, url, baseDir, pagePath) { + + // The tags make it possible for the dashboard to use the + // templates to choose which annotations that will be showed. + // That's why we need to send tags that matches the template + // variables in Grafana. + const connectivity = graphiteUtil.getConnectivity(options); + const browser = options.browser; + const namespace = options.graphite.namespace.split('.').join(','); + const urlAndGroup = graphiteUtil.getURLAndGroup(options, group, url, options.graphite.includeQueryParams).split('.').join(','); + const tags = `${connectivity},${browser},${namespace},${urlAndGroup}`; + const message = `Result ${options.browsertime.iterations} run(s)`; + + const postData = + `{"what": "Sitespeed.io", "tags": "${tags}", "data": "${message}"}`; + const postOptions = { + hostname: options.graphite.host, + port: options.graphite.httpPort || 8080, + path: '/events/', + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': Buffer.byteLength(postData) + } + }; + + // If Graphite is behind auth, use it! + if (options.graphite.auth) { + postOptions.auth = options.graphite.auth; + } + + return new Promise((resolve, reject) => { + log.trace('Send annotation to Graphite: %j', postData); + // not perfect but maybe work for us + const lib = options.graphite.httpPort === 443 ? https : http; + const req = lib.request(postOptions, (res) => { + if (res.statusCode !== 200) { + log.error('Got %s from Graphite when sending annotation', res.statusCode); + reject(); + } else { + res.setEncoding('utf8'); + log.info('Sent annotation to Graphite'); + resolve(); + } + }); + req.on('error', (err) => { + log.error('Got error from Graphite when sending annotation', err); + reject(err) + }); + req.write(postData); + req.end(); + }) + } +} diff --git a/lib/plugins/graphite/sender.js b/lib/plugins/graphite/sender.js index bf5d0a48a..dc56d17e5 100644 --- a/lib/plugins/graphite/sender.js +++ b/lib/plugins/graphite/sender.js @@ -1,6 +1,6 @@ 'use strict'; -var net = require('net'), +const net = require('net'), log = require('intel'), Promise = require('bluebird'); diff --git a/lib/plugins/graphite/util.js b/lib/plugins/graphite/util.js new file mode 100644 index 000000000..a98f18658 --- /dev/null +++ b/lib/plugins/graphite/util.js @@ -0,0 +1,27 @@ +'use strict'; + +const get = require('lodash.get'); +const flatten = require('../../support/flattenMessage'); + +module.exports = { + toSafeKey(key) { + return key.replace(/[.~ /+|,:?&%]|%7C/g, '_'); + }, + getConnectivity(options) { + // if we have a friendly name for your conectivity, use that! + let connectivity = get(options, 'browsertime.connectivity.alias'); + if (connectivity) { + return this.toSafeKey(connectivity); + } else { + return options.connectivity; + } + }, + getURLAndGroup(options, group, url, includeQueryParams) { + if(group && options.urlsMetaData[url]) { + let alias = options.urlsMetaData[url].alias; + return this.toSafeKey(group) + "." + this.toSafeKey(alias); + } else { + return flatten.keypathFromUrl(url, includeQueryParams); + } + } +}