Move InfluxDB to it own standalone plugin (#4451)
This commit is contained in:
Peter Hedenskog
GitHub
parent
b0a98cc973
commit
e78200da3e
|
|
@ -70,10 +70,6 @@ jobs:
|
|||
run: bin/sitespeed.js http://127.0.0.1:3001/simple/ -n 1 --graphite.host 127.0.0.1 --xvfb
|
||||
- name: Run test without a CLI
|
||||
run: xvfb-run node test/runWithoutCli.js
|
||||
- name: Run test with Influx 1.8
|
||||
run: bin/sitespeed.js http://127.0.0.1:3001/simple/ -n 1 --influxdb.host 127.0.0.1 --xvfb --logToFile --resultBaseUrl https://result.sitespeed.io --influxdb.annotationScreenshot=true
|
||||
- name: Run test with Influx 2.6.1
|
||||
run: bin/sitespeed.js http://127.0.0.1:3001/simple/ -n 1 --influxdb.host 127.0.0.1 --influxdb.port 8087 --influxdb.version 2 --influxdb.organisation sitespeed --influxdb.token sitespeed --xvfb --resultBaseUrl https://result.sitespeed.io --influxdb.annotationScreenshot=true
|
||||
- name: Run Chrome test with config
|
||||
run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
|
||||
- name: Run Chrome test using compare plugin
|
||||
|
|
|
|||
|
|
@ -421,7 +421,7 @@ export async function parseCommandLine() {
|
|||
})
|
||||
.option('browsertime.script', {
|
||||
describe:
|
||||
'Add custom Javascript that collect metrics and run after the page has finished loading. Note that --script can be passed multiple times if you want to collect multiple metrics. The metrics will automatically be pushed to the summary/detailed summary and each individual page + sent to Graphite/InfluxDB.',
|
||||
'Add custom Javascript that collect metrics and run after the page has finished loading. Note that --script can be passed multiple times if you want to collect multiple metrics. The metrics will automatically be pushed to the summary/detailed summary and each individual page + sent to Graphite',
|
||||
alias: ['script'],
|
||||
group: 'Browser'
|
||||
})
|
||||
|
|
@ -1346,78 +1346,6 @@ export async function parseCommandLine() {
|
|||
describe:
|
||||
'Define which messages to send to Graphite. By default we do not send data per run, but you can change that by adding run as one of the options',
|
||||
group: 'Graphite'
|
||||
})
|
||||
|
||||
.option('influxdb.protocol', {
|
||||
describe: 'The protocol used to store connect to the InfluxDB host.',
|
||||
default: 'http',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.host', {
|
||||
describe: 'The InfluxDB host used to store captured metrics.',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.port', {
|
||||
default: 8086,
|
||||
describe: 'The InfluxDB port used to store captured metrics.',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.username', {
|
||||
describe:
|
||||
'The InfluxDB username for your InfluxDB instance (only for InfluxDB v1)',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.password', {
|
||||
describe:
|
||||
'The InfluxDB password for your InfluxDB instance (only for InfluxDB v1).',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.organisation', {
|
||||
describe:
|
||||
'The InfluxDB organisation for your InfluxDB instance (only for InfluxDB v2)',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.token', {
|
||||
describe:
|
||||
'The InfluxDB token for your InfluxDB instance (only for InfluxDB v2)',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.version', {
|
||||
default: 1,
|
||||
describe: 'The InfluxDB version of your InfluxDB instance.',
|
||||
type: 'integer',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.database', {
|
||||
default: 'sitespeed',
|
||||
describe: 'The database name used to store captured metrics.',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.tags', {
|
||||
default: 'category=default',
|
||||
describe:
|
||||
'A comma separated list of tags and values added to each metric',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.includeQueryParams', {
|
||||
default: false,
|
||||
describe:
|
||||
'Whether to include query parameters from the URL in the InfluxDB keys or not',
|
||||
type: 'boolean',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.groupSeparator', {
|
||||
default: '_',
|
||||
describe:
|
||||
'Choose which character that will separate a group/domain. Default is underscore, set it to a dot if you wanna keep the original domain name.',
|
||||
group: 'InfluxDB'
|
||||
})
|
||||
.option('influxdb.annotationScreenshot', {
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
describe:
|
||||
'Include screenshot (from Browsertime) in the annotation. You need to specify a --resultBaseURL for this to work.',
|
||||
group: 'InfluxDB'
|
||||
});
|
||||
|
||||
parsed
|
||||
|
|
@ -1527,10 +1455,6 @@ export async function parseCommandLine() {
|
|||
default: browsertimeConfig.screenshotParams.maxSize,
|
||||
group: 'Screenshot'
|
||||
});
|
||||
/**
|
||||
InfluxDB cli option
|
||||
*/
|
||||
|
||||
parsed
|
||||
// Metrics
|
||||
.option('metrics.list', {
|
||||
|
|
@ -2028,12 +1952,12 @@ export async function parseCommandLine() {
|
|||
})
|
||||
.option('urlAlias', {
|
||||
describe:
|
||||
'Use an alias for the URL (if you feed URLs from a file you can instead have the alias in the file). You need to pass on the same amount of alias as URLs. The alias is used as the name of the URL on the HTML report and in Graphite/InfluxDB. Pass on multiple --urlAlias for multiple alias/URLs. This will override alias in a file.',
|
||||
'Use an alias for the URL (if you feed URLs from a file you can instead have the alias in the file). You need to pass on the same amount of alias as URLs. The alias is used as the name of the URL on the HTML report and in Graphite. Pass on multiple --urlAlias for multiple alias/URLs. This will override alias in a file.',
|
||||
type: 'string'
|
||||
})
|
||||
.option('groupAlias', {
|
||||
describe:
|
||||
'Use an alias for the group/domain. You need to pass on the same amount of alias as URLs. The alias is used as the name of the group in Graphite/InfluxDB. Pass on multiple --groupAlias for multiple alias/groups. This do not work for scripting at the moment.',
|
||||
'Use an alias for the group/domain. You need to pass on the same amount of alias as URLs. The alias is used as the name of the group in Graphite. Pass on multiple --groupAlias for multiple alias/groups. This do not work for scripting at the moment.',
|
||||
type: 'string'
|
||||
})
|
||||
.option('utc', {
|
||||
|
|
|
|||
|
|
@ -41,9 +41,7 @@ export function send(
|
|||
options,
|
||||
group,
|
||||
url,
|
||||
tsdbType === 'graphite'
|
||||
? options.graphite.includeQueryParams
|
||||
: options.influxdb.includeQueryParams,
|
||||
options.graphite.includeQueryParams,
|
||||
alias
|
||||
).split('.');
|
||||
|
||||
|
|
|
|||
|
|
@ -1,262 +0,0 @@
|
|||
import merge from 'lodash.merge';
|
||||
|
||||
import { flattenMessageData } from '../../support/flattenMessage.js';
|
||||
import {
|
||||
getConnectivity,
|
||||
getURLAndGroup,
|
||||
toSafeKey
|
||||
} from '../../support/tsdbUtil.js';
|
||||
|
||||
function getAdditionalTags(key, type) {
|
||||
let tags = {};
|
||||
const keyArray = key.split('.');
|
||||
if (/(^contentTypes)/.test(key)) {
|
||||
// contentTypes.favicon.requests.mean
|
||||
// contentTypes.favicon.requests
|
||||
// contentTypes.css.transferSize
|
||||
tags.contentType = keyArray[1];
|
||||
} else if (/(^pageTimings|^visualMetrics)/.test(key)) {
|
||||
// pageTimings.serverResponseTime.max
|
||||
// visualMetrics.SpeedIndex.median
|
||||
tags.timings = keyArray[0];
|
||||
} else
|
||||
switch (type) {
|
||||
case 'browsertime.pageSummary': {
|
||||
// statistics.timings.pageTimings.backEndTime.median
|
||||
// statistics.timings.userTimings.marks.logoTime.median
|
||||
// statistics.visualMetrics.SpeedIndex.median
|
||||
tags[keyArray[0]] = keyArray[1];
|
||||
if (keyArray.length >= 5) {
|
||||
tags[keyArray[2]] = keyArray[3];
|
||||
}
|
||||
if (key.includes('cpu.categories')) {
|
||||
tags.cpu = 'category';
|
||||
} else if (key.includes('cpu.events')) {
|
||||
tags.cpu = 'event';
|
||||
} else if (key.includes('cpu.longTasks')) {
|
||||
tags.cpu = 'longTask';
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'browsertime.summary': {
|
||||
// firstPaint.median
|
||||
// userTimings.marks.logoTime.median
|
||||
if (key.includes('userTimings')) {
|
||||
tags[keyArray[0]] = keyArray[1];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'axe.pageSummary': {
|
||||
tags.axeType = keyArray[0];
|
||||
break;
|
||||
}
|
||||
case 'coach.pageSummary': {
|
||||
// advice.score
|
||||
// advice.performance.score
|
||||
if (keyArray.length > 2) {
|
||||
tags.advice = keyArray[1];
|
||||
}
|
||||
|
||||
// set the actual advice name
|
||||
// advice.performance.adviceList.cacheHeaders.score
|
||||
if (keyArray.length > 4) {
|
||||
tags.adviceName = keyArray[3];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'coach.summary': {
|
||||
// score.max
|
||||
// performance.score.median
|
||||
if (keyArray.length === 3) {
|
||||
tags.advice = keyArray[0];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'pagexray.summary': {
|
||||
// firstParty.requests.min pagexray.summary
|
||||
// requests.median
|
||||
// responseCodes.307.max pagexray.summary
|
||||
// requests.min pagexray.summary
|
||||
if (key.includes('responseCodes')) {
|
||||
tags.responseCodes = 'response';
|
||||
}
|
||||
|
||||
if (key.includes('firstParty') || key.includes('thirdParty')) {
|
||||
tags.party = keyArray[0];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'pagexray.pageSummary': {
|
||||
// thirdParty.contentTypes.json.requests pagexray.pageSummary
|
||||
// thirdParty.requests pagexray.pageSummary
|
||||
// firstParty.cookieStats.max pagexray.pageSummary
|
||||
// responseCodes.200 pagexray.pageSummary
|
||||
// expireStats.max pagexray.pageSummary
|
||||
// totalDomains pagexray.pageSummary
|
||||
if (key.includes('firstParty') || key.includes('thirdParty')) {
|
||||
tags.party = keyArray[0];
|
||||
}
|
||||
if (key.includes('responseCodes')) {
|
||||
tags.responseCodes = 'response';
|
||||
}
|
||||
if (key.includes('contentTypes')) {
|
||||
tags.contentType = keyArray[2];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'thirdparty.pageSummary': {
|
||||
tags.thirdPartyCategory = keyArray[1];
|
||||
tags.thirdPartyType = keyArray[2];
|
||||
|
||||
break;
|
||||
}
|
||||
case 'lighthouse.pageSummary': {
|
||||
// categories.seo.score
|
||||
// categories.performance.score
|
||||
if (key.includes('score')) {
|
||||
tags.audit = keyArray[1];
|
||||
}
|
||||
if (key.includes('audits')) {
|
||||
tags.audit = keyArray[1];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'crux.pageSummary': {
|
||||
tags.experience = keyArray[0];
|
||||
tags.formFactor = keyArray[1];
|
||||
tags.metric = keyArray[2];
|
||||
|
||||
break;
|
||||
}
|
||||
case 'gpsi.pageSummary': {
|
||||
if (key.includes('googleWebVitals')) {
|
||||
tags.testType = 'googleWebVitals';
|
||||
} else if (key.includes('score')) {
|
||||
tags.testType = 'score';
|
||||
} else if (key.includes('loadingExperience')) {
|
||||
tags.experience = keyArray[0];
|
||||
tags.metric = keyArray[1];
|
||||
tags.testType = 'crux';
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
// console.log('Missed added tags to ' + key + ' ' + type);
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
|
||||
function getFieldAndSeriesName(key) {
|
||||
const functions = [
|
||||
'min',
|
||||
'p10',
|
||||
'median',
|
||||
'mean',
|
||||
'avg',
|
||||
'max',
|
||||
'p90',
|
||||
'p99',
|
||||
'mdev',
|
||||
'stddev',
|
||||
'rsd'
|
||||
];
|
||||
const keyArray = key.split('.');
|
||||
const end = keyArray.pop();
|
||||
if (functions.includes(end)) {
|
||||
return { field: end, seriesName: keyArray.pop() };
|
||||
}
|
||||
return { field: 'value', seriesName: end };
|
||||
}
|
||||
export class InfluxDBDataGenerator {
|
||||
constructor(includeQueryParameters, options) {
|
||||
this.includeQueryParams = !!includeQueryParameters;
|
||||
this.options = options;
|
||||
this.defaultTags = {};
|
||||
for (let row of options.influxdb.tags.split(',')) {
|
||||
const keyAndValue = row.split('=');
|
||||
this.defaultTags[keyAndValue[0]] = keyAndValue[1];
|
||||
}
|
||||
}
|
||||
|
||||
dataFromMessage(message, time, alias) {
|
||||
function getTagsFromMessage(
|
||||
message,
|
||||
includeQueryParameters,
|
||||
options,
|
||||
defaultTags
|
||||
) {
|
||||
const tags = merge({}, defaultTags);
|
||||
let typeParts = message.type.split('.');
|
||||
tags.origin = typeParts[0];
|
||||
typeParts.push(typeParts.shift());
|
||||
tags.summaryType = typeParts[0];
|
||||
|
||||
// always have browser and connectivity in Browsertime and related tools
|
||||
if (
|
||||
/(^pagexray|^coach|^browsertime|^thirdparty|^axe|^sustainable)/.test(
|
||||
message.type
|
||||
)
|
||||
) {
|
||||
// if we have a friendly name for your connectivity, use that!
|
||||
let connectivity = getConnectivity(options);
|
||||
tags.connectivity = connectivity;
|
||||
tags.browser = options.browser;
|
||||
} else if (/(^gpsi)/.test(message.type)) {
|
||||
tags.strategy = options.mobile ? 'mobile' : 'desktop';
|
||||
}
|
||||
|
||||
// if we get a URL type, add the URL
|
||||
if (message.url) {
|
||||
const urlAndGroup = getURLAndGroup(
|
||||
options,
|
||||
message.group,
|
||||
message.url,
|
||||
includeQueryParameters,
|
||||
alias
|
||||
).split('.');
|
||||
tags.page = urlAndGroup[1];
|
||||
tags.group = urlAndGroup[0];
|
||||
} else if (message.group) {
|
||||
// add the group of the summary message
|
||||
tags.group = toSafeKey(message.group, options.influxdb.groupSeparator);
|
||||
}
|
||||
|
||||
if (options.slug) {
|
||||
tags.testName = options.slug;
|
||||
}
|
||||
|
||||
return tags;
|
||||
}
|
||||
return Object.entries(flattenMessageData(message)).reduce(
|
||||
(entries, [key, value]) => {
|
||||
const fieldAndSeriesName = getFieldAndSeriesName(key);
|
||||
let tags = getTagsFromMessage(
|
||||
message,
|
||||
this.includeQueryParams,
|
||||
this.options,
|
||||
this.defaultTags
|
||||
);
|
||||
tags = { ...getAdditionalTags(key, message.type), ...tags };
|
||||
const point = {
|
||||
time: time.valueOf(),
|
||||
[fieldAndSeriesName.field]: value
|
||||
};
|
||||
entries.push({
|
||||
tags,
|
||||
seriesName: fieldAndSeriesName.seriesName,
|
||||
point
|
||||
});
|
||||
return entries;
|
||||
},
|
||||
[]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
import { getLogger } from '@sitespeed.io/log';
|
||||
import dayjs from 'dayjs';
|
||||
|
||||
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
|
||||
import { InfluxDBSender as Sender } from './sender.js';
|
||||
import { InfluxDB2Sender as SenderV2 } from './senderV2.js';
|
||||
import { sendV1 } from './send-annotation.js';
|
||||
import { sendV2 } from './send-annotationV2.js';
|
||||
import { InfluxDBDataGenerator as DataGenerator } from './data-generator.js';
|
||||
import { throwIfMissing, isEmpty } from '../../support/util.js';
|
||||
|
||||
const log = getLogger('sitespeedio.plugin.influxdb');
|
||||
export default class InfluxDBPlugin extends SitespeedioPlugin {
|
||||
constructor(options, context, queue) {
|
||||
super({ name: 'influxdb', options, context, queue });
|
||||
}
|
||||
|
||||
open(context, options) {
|
||||
throwIfMissing(options.influxdb, ['host', 'database'], 'influxdb');
|
||||
this.filterRegistry = context.filterRegistry;
|
||||
log.debug(
|
||||
'Setup InfluxDB host %s and database %s',
|
||||
options.influxdb.host,
|
||||
options.influxdb.database
|
||||
);
|
||||
|
||||
const options_ = options.influxdb;
|
||||
this.options = options;
|
||||
this.sender =
|
||||
options_.version == 1 ? new Sender(options_) : new SenderV2(options_);
|
||||
this.timestamp = context.timestamp;
|
||||
this.resultUrls = context.resultUrls;
|
||||
this.dataGenerator = new DataGenerator(
|
||||
options_.includeQueryParams,
|
||||
options
|
||||
);
|
||||
this.messageTypesToFireAnnotations = [];
|
||||
this.receivedTypesThatFireAnnotations = {};
|
||||
this.make = context.messageMaker('influxdb').make;
|
||||
this.sendAnnotation = true;
|
||||
this.alias = {};
|
||||
}
|
||||
|
||||
processMessage(message, queue) {
|
||||
const filterRegistry = this.filterRegistry;
|
||||
|
||||
// First catch if we are running Browsertime and/or WebPageTest
|
||||
switch (message.type) {
|
||||
case 'browsertime.setup': {
|
||||
this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
|
||||
this.usingBrowsertime = true;
|
||||
|
||||
break;
|
||||
}
|
||||
case 'browsertime.config': {
|
||||
if (message.data.screenshot) {
|
||||
this.useScreenshots = message.data.screenshot;
|
||||
this.screenshotType = message.data.screenshotType;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'sitespeedio.setup': {
|
||||
// Let other plugins know that the InfluxDB plugin is alive
|
||||
queue.postMessage(this.make('influxdb.setup'));
|
||||
|
||||
break;
|
||||
}
|
||||
case 'grafana.setup': {
|
||||
this.sendAnnotation = false;
|
||||
|
||||
break;
|
||||
}
|
||||
// No default
|
||||
}
|
||||
|
||||
if (message.type === 'browsertime.alias') {
|
||||
this.alias[message.url] = message.data;
|
||||
}
|
||||
|
||||
if (
|
||||
!(
|
||||
message.type.endsWith('.summary') ||
|
||||
message.type.endsWith('.pageSummary')
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.messageTypesToFireAnnotations.includes(message.type)) {
|
||||
this.receivedTypesThatFireAnnotations[message.url]
|
||||
? this.receivedTypesThatFireAnnotations[message.url]++
|
||||
: (this.receivedTypesThatFireAnnotations[message.url] = 1);
|
||||
}
|
||||
|
||||
// Let us skip this for a while and concentrate on the real deal
|
||||
if (
|
||||
/(^largestassets|^slowestassets|^aggregateassets|^domains)/.test(
|
||||
message.type
|
||||
)
|
||||
)
|
||||
return;
|
||||
|
||||
// we only sends individual groups to Influx, not the
|
||||
// total of all groups (you can calculate that yourself)
|
||||
if (message.group === 'total') {
|
||||
return;
|
||||
}
|
||||
|
||||
message = filterRegistry.filterMessage(message);
|
||||
if (isEmpty(message.data)) return;
|
||||
|
||||
let data = this.dataGenerator.dataFromMessage(
|
||||
message,
|
||||
message.type === 'browsertime.pageSummary'
|
||||
? dayjs(message.runTime)
|
||||
: this.timestamp,
|
||||
this.alias
|
||||
);
|
||||
|
||||
if (data.length > 0) {
|
||||
log.debug('Send the following data to InfluxDB: %:2j', data);
|
||||
return this.sender.send(data).then(() => {
|
||||
// Make sure we only send the annotation once per URL:
|
||||
// If we run browsertime, always send on browsertime.pageSummary
|
||||
// If we run WebPageTest standalone, send on webPageTestSummary
|
||||
// when we configured a base url
|
||||
if (
|
||||
this.receivedTypesThatFireAnnotations[message.url] ===
|
||||
this.messageTypesToFireAnnotations.length &&
|
||||
this.resultUrls.hasBaseUrl() &&
|
||||
this.sendAnnotation
|
||||
) {
|
||||
const absolutePagePath = this.resultUrls.absoluteSummaryPagePath(
|
||||
message.url,
|
||||
this.alias[message.url]
|
||||
);
|
||||
this.receivedTypesThatFireAnnotations[message.url] = 0;
|
||||
|
||||
return this.options.influxdb.version == 2
|
||||
? sendV2(
|
||||
message.url,
|
||||
message.group,
|
||||
absolutePagePath,
|
||||
this.useScreenshots,
|
||||
this.screenshotType,
|
||||
// Browsertime pass on when the first run was done for that URL
|
||||
message.runTime,
|
||||
this.alias,
|
||||
this.usingBrowsertime,
|
||||
this.options
|
||||
)
|
||||
: sendV1(
|
||||
message.url,
|
||||
message.group,
|
||||
absolutePagePath,
|
||||
this.useScreenshots,
|
||||
this.screenshotType,
|
||||
// Browsertime pass on when the first run was done for that URL
|
||||
message.runTime,
|
||||
this.alias,
|
||||
this.usingBrowsertime,
|
||||
this.options
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
return Promise.reject(
|
||||
new Error(
|
||||
'No data to send to influxdb for message:\n' +
|
||||
JSON.stringify(message, undefined, 2)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import { stringify } from 'node:querystring';
|
||||
|
||||
import { getLogger } from '@sitespeed.io/log';
|
||||
import dayjs from 'dayjs';
|
||||
|
||||
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
|
||||
import {
|
||||
getAnnotationMessage,
|
||||
getTagsAsString
|
||||
} from '../../support/annotationsHelper.js';
|
||||
|
||||
const log = getLogger('sitespeedio.plugin.influxdb');
|
||||
|
||||
export function sendV1(
|
||||
url,
|
||||
group,
|
||||
absolutePagePath,
|
||||
screenShotsEnabledInBrowsertime,
|
||||
screenshotType,
|
||||
runTime,
|
||||
alias,
|
||||
usingBrowsertime,
|
||||
options
|
||||
) {
|
||||
// The tags make it possible for the dashboard to use the
|
||||
// templates to choose which annotations that will be showed.
|
||||
// That's why we need to send tags that matches the template
|
||||
// variables in Grafana.
|
||||
const connectivity = getConnectivity(options);
|
||||
const browser = options.browser;
|
||||
const urlAndGroup = getURLAndGroup(
|
||||
options,
|
||||
group,
|
||||
url,
|
||||
options.influxdb.includeQueryParams,
|
||||
alias
|
||||
).split('.');
|
||||
let tags = [connectivity, browser, urlAndGroup[0], urlAndGroup[1]];
|
||||
|
||||
if (options.slug) {
|
||||
tags.push(options.slug);
|
||||
}
|
||||
|
||||
const message = getAnnotationMessage(
|
||||
absolutePagePath,
|
||||
screenShotsEnabledInBrowsertime,
|
||||
screenshotType,
|
||||
undefined,
|
||||
usingBrowsertime,
|
||||
options
|
||||
);
|
||||
const timestamp = runTime
|
||||
? Math.round(dayjs(runTime) / 1000)
|
||||
: Math.round(dayjs() / 1000);
|
||||
// if we have a category, let us send that category too
|
||||
if (options.influxdb.tags) {
|
||||
for (let row of options.influxdb.tags.split(',')) {
|
||||
const keyAndValue = row.split('=');
|
||||
tags.push(keyAndValue[1]);
|
||||
}
|
||||
}
|
||||
const influxDBTags = getTagsAsString(tags);
|
||||
const postData = `events title="Sitespeed.io",text="${message}",tags=${influxDBTags} ${timestamp}`;
|
||||
const postOptions = {
|
||||
hostname: options.influxdb.host,
|
||||
port: options.influxdb.port,
|
||||
path: '/write?db=' + options.influxdb.database + '&precision=s',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Content-Length': Buffer.byteLength(postData)
|
||||
}
|
||||
};
|
||||
|
||||
if (options.influxdb.username) {
|
||||
postOptions.path =
|
||||
postOptions.path +
|
||||
'&' +
|
||||
stringify({
|
||||
u: options.influxdb.username,
|
||||
p: options.influxdb.password
|
||||
});
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
log.debug('Send annotation to Influx: %j', postData);
|
||||
// not perfect but maybe work for us
|
||||
const library = options.influxdb.protocol === 'https' ? https : http;
|
||||
const request = library.request(postOptions, res => {
|
||||
if (res.statusCode === 204) {
|
||||
res.setEncoding('utf8');
|
||||
log.debug('Sent annotation to InfluxDB');
|
||||
resolve();
|
||||
} else {
|
||||
const e = new Error(
|
||||
`Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
|
||||
);
|
||||
log.warn(e.message);
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
request.on('error', error => {
|
||||
log.error('Got error from InfluxDB when sending annotation', error);
|
||||
reject(error);
|
||||
});
|
||||
request.write(postData);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
|
||||
import { getLogger } from '@sitespeed.io/log';
|
||||
import dayjs from 'dayjs';
|
||||
|
||||
import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
|
||||
import {
|
||||
getAnnotationMessage,
|
||||
getTagsAsString
|
||||
} from '../../support/annotationsHelper.js';
|
||||
|
||||
const log = getLogger('sitespeedio.plugin.influxdb');
|
||||
|
||||
export function sendV2(
|
||||
url,
|
||||
group,
|
||||
absolutePagePath,
|
||||
screenShotsEnabledInBrowsertime,
|
||||
screenshotType,
|
||||
runTime,
|
||||
alias,
|
||||
usingBrowsertime,
|
||||
options
|
||||
) {
|
||||
// The tags make it possible for the dashboard to use the
|
||||
// templates to choose which annotations that will be showed.
|
||||
// That's why we need to send tags that matches the template
|
||||
// variables in Grafana.
|
||||
const connectivity = getConnectivity(options);
|
||||
const browser = options.browser;
|
||||
const urlAndGroup = getURLAndGroup(
|
||||
options,
|
||||
group,
|
||||
url,
|
||||
options.influxdb.includeQueryParams,
|
||||
alias
|
||||
).split('.');
|
||||
let tags = [
|
||||
`connectivity=${connectivity}`,
|
||||
`browser=${browser}`,
|
||||
`group=${urlAndGroup[0]}`,
|
||||
`page=${urlAndGroup[1]}`
|
||||
];
|
||||
|
||||
if (options.slug) {
|
||||
tags.push(`slug=${options.slug}`);
|
||||
}
|
||||
|
||||
const message = getAnnotationMessage(
|
||||
absolutePagePath,
|
||||
screenShotsEnabledInBrowsertime,
|
||||
screenshotType,
|
||||
undefined,
|
||||
usingBrowsertime,
|
||||
options
|
||||
);
|
||||
const timestamp = runTime
|
||||
? Math.round(dayjs(runTime) / 1000)
|
||||
: Math.round(dayjs() / 1000);
|
||||
// if we have a category, let us send that category too
|
||||
if (options.influxdb.tags) {
|
||||
for (const tag of options.influxdb.tags.split(',')) {
|
||||
tags.push(tag);
|
||||
}
|
||||
}
|
||||
const influxDBTags = tags.join(',');
|
||||
const grafanaTags = getTagsAsString(tags.map(pair => pair.split('=')[1]));
|
||||
const postData = `annotations,${influxDBTags} title="Sitespeed.io",text="${message}",tags=${grafanaTags} ${timestamp}`;
|
||||
const postOptions = {
|
||||
hostname: options.influxdb.host,
|
||||
port: options.influxdb.port,
|
||||
path: `/api/v2/write?org=${options.influxdb.organisation}&bucket=${options.influxdb.database}&precision=s`,
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Content-Length': Buffer.byteLength(postData),
|
||||
Authorization: `Token ${options.influxdb.token}`
|
||||
}
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
log.debug('Send annotation to Influx: %j', postData);
|
||||
// not perfect but maybe work for us
|
||||
const library = options.influxdb.protocol === 'https' ? https : http;
|
||||
const request = library.request(postOptions, res => {
|
||||
if (res.statusCode === 204) {
|
||||
res.setEncoding('utf8');
|
||||
log.debug('Sent annotation to InfluxDB');
|
||||
resolve();
|
||||
} else {
|
||||
const e = new Error(
|
||||
`Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
|
||||
);
|
||||
log.warn(e.message);
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
request.on('error', error => {
|
||||
log.error('Got error from InfluxDB when sending annotation', error);
|
||||
reject(error);
|
||||
});
|
||||
request.write(postData);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
import { InfluxDB } from 'influx';
|
||||
|
||||
export class InfluxDBSender {
|
||||
constructor({ protocol, host, port, database, username, password }) {
|
||||
this.client = new InfluxDB({
|
||||
protocol,
|
||||
host,
|
||||
port,
|
||||
database,
|
||||
username,
|
||||
password
|
||||
});
|
||||
}
|
||||
|
||||
send(data) {
|
||||
const points = [];
|
||||
for (let point of data) {
|
||||
points.push({
|
||||
tags: point.tags,
|
||||
measurement: point.seriesName,
|
||||
fields: point.point
|
||||
});
|
||||
}
|
||||
return this.client.writePoints(points);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
import { InfluxDB, Point, HttpError } from '@influxdata/influxdb-client';
|
||||
|
||||
export class InfluxDB2Sender {
|
||||
constructor({ protocol, host, port, database, organisation, token }) {
|
||||
this.client = new InfluxDB({
|
||||
url: `${protocol}://${host}:${port}`,
|
||||
token
|
||||
}).getWriteApi(organisation, database);
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
send(data) {
|
||||
const points = [];
|
||||
for (let point of data) {
|
||||
const influxPoint = new Point(point.seriesName);
|
||||
for (const key of Object.keys(point.tags)) {
|
||||
influxPoint.tag(key, point.tags[key]);
|
||||
}
|
||||
for (const key of Object.keys(point.point)) {
|
||||
if (key === 'time') {
|
||||
influxPoint.timestamp(new Date(point.point[key]));
|
||||
} else {
|
||||
influxPoint.floatField(key, point.point[key]);
|
||||
}
|
||||
}
|
||||
points.push(influxPoint);
|
||||
}
|
||||
this.client.writePoints(points);
|
||||
return this.client.flush().catch(error => {
|
||||
if (error instanceof HttpError && error.statusCode === 401) {
|
||||
throw new Error(
|
||||
`The InfluxDB database: ${this.database} doesn't exist.`
|
||||
);
|
||||
}
|
||||
throw new Error('Writing to influx failed');
|
||||
});
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -83,7 +83,6 @@
|
|||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.717.0",
|
||||
"@google-cloud/storage": "7.14.0",
|
||||
"@influxdata/influxdb-client": "1.33.2",
|
||||
"@sitespeed.io/log": "0.2.6",
|
||||
"@sitespeed.io/plugin": "1.0.0",
|
||||
"@tgwf/co2": "0.16.4",
|
||||
|
|
@ -95,7 +94,6 @@
|
|||
"fast-crc32c": "2.0.0",
|
||||
"fast-stats": "0.0.7",
|
||||
"import-global": "1.1.1",
|
||||
"influx": "5.9.3",
|
||||
"lodash.get": "4.4.2",
|
||||
"lodash.merge": "4.6.2",
|
||||
"lodash.set": "4.3.2",
|
||||
|
|
|
|||
|
|
@ -4,21 +4,4 @@ services:
|
|||
image: sitespeedio/graphite:1.1.5-12
|
||||
ports:
|
||||
- "2003:2003"
|
||||
- "8080:80"
|
||||
influxdb_v1.8:
|
||||
image: influxdb:1.8
|
||||
ports:
|
||||
- '8086:8086'
|
||||
environment:
|
||||
- INFLUXDB_DB=sitespeed
|
||||
influxdb_v2.6:
|
||||
image: influxdb:2.6.1
|
||||
ports:
|
||||
- '8087:8086'
|
||||
environment:
|
||||
- DOCKER_INFLUXDB_INIT_MODE=setup
|
||||
- DOCKER_INFLUXDB_INIT_USERNAME=sitespeed
|
||||
- DOCKER_INFLUXDB_INIT_PASSWORD=sitespeed
|
||||
- DOCKER_INFLUXDB_INIT_ORG=sitespeed
|
||||
- DOCKER_INFLUXDB_INIT_BUCKET=sitespeed
|
||||
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=sitespeed
|
||||
- "8080:80"
|
||||
|
|
@ -1,359 +0,0 @@
|
|||
import dayjs from 'dayjs';
|
||||
import test from 'ava';
|
||||
|
||||
import { InfluxDBDataGenerator as DataGenerator } from '../lib/plugins/influxdb/data-generator.js';
|
||||
|
||||
test(`Test influxdb dataGenerator`, t => {
|
||||
const message = {
|
||||
uuid: '33774328-e781-4152-babe-a367cee27153',
|
||||
type: 'coach.summary',
|
||||
timestamp: '2017-04-04T09:55:59+02:00',
|
||||
source: 'coach',
|
||||
data: {
|
||||
score: {
|
||||
median: '96',
|
||||
mean: '96',
|
||||
min: '96',
|
||||
p90: '96',
|
||||
max: '96'
|
||||
},
|
||||
accessibility: {
|
||||
score: {
|
||||
median: '95',
|
||||
mean: '95',
|
||||
min: '95',
|
||||
p90: '95',
|
||||
max: '95'
|
||||
},
|
||||
altImages: {
|
||||
median: '80',
|
||||
mean: '80',
|
||||
min: '80',
|
||||
p90: '80',
|
||||
max: '80'
|
||||
},
|
||||
headings: {
|
||||
median: '90',
|
||||
mean: '90',
|
||||
min: '90',
|
||||
p90: '90',
|
||||
max: '90'
|
||||
},
|
||||
labelOnInput: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
landmarks: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
neverSuppressZoom: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
sections: {
|
||||
median: '0',
|
||||
mean: '0',
|
||||
min: '0',
|
||||
p90: '0',
|
||||
max: '0'
|
||||
},
|
||||
table: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
}
|
||||
},
|
||||
bestpractice: {
|
||||
score: {
|
||||
median: '85',
|
||||
mean: '85',
|
||||
min: '85',
|
||||
p90: '85',
|
||||
max: '85'
|
||||
},
|
||||
charset: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
doctype: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
https: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
httpsH2: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
language: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
metaDescription: {
|
||||
median: '50',
|
||||
mean: '50',
|
||||
min: '50',
|
||||
p90: '50',
|
||||
max: '50'
|
||||
},
|
||||
optimizely: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
pageTitle: {
|
||||
median: '50',
|
||||
mean: '50',
|
||||
min: '50',
|
||||
p90: '50',
|
||||
max: '50'
|
||||
},
|
||||
spdy: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
url: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
}
|
||||
},
|
||||
performance: {
|
||||
score: {
|
||||
median: '98',
|
||||
mean: '98',
|
||||
min: '98',
|
||||
p90: '98',
|
||||
max: '98'
|
||||
},
|
||||
avoidScalingImages: {
|
||||
median: '50',
|
||||
mean: '50',
|
||||
min: '50',
|
||||
p90: '50',
|
||||
max: '50'
|
||||
},
|
||||
cssPrint: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
fastRender: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
inlineCss: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
jquery: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
spof: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
thirdPartyAsyncJs: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
assetsRedirects: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
cacheHeaders: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
cacheHeadersLong: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
compressAssets: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
connectionKeepAlive: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
cssSize: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
documentRedirect: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
favicon: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
fewFonts: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
fewRequestsPerDomain: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
headerSize: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
imageSize: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
javascriptSize: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
mimeTypes: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
optimalCssSize: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
pageSize: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
privateAssets: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
},
|
||||
responseOk: {
|
||||
median: '100',
|
||||
mean: '100',
|
||||
min: '100',
|
||||
p90: '100',
|
||||
max: '100'
|
||||
}
|
||||
}
|
||||
},
|
||||
group: 'www.sitespeed.io'
|
||||
};
|
||||
|
||||
let generator = new DataGenerator(false, {
|
||||
_: ['filename'],
|
||||
browser: 'chrome',
|
||||
connectivity: 'cable',
|
||||
influxdb: {
|
||||
tags: 'tool=sitespeed.io'
|
||||
}
|
||||
});
|
||||
|
||||
const data = generator.dataFromMessage(message, dayjs());
|
||||
const seriesName = data[0].seriesName;
|
||||
const numberOfTags = Object.keys(data[0].tags).length;
|
||||
t.is(seriesName, 'score');
|
||||
t.is(numberOfTags, 6);
|
||||
});
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
version: '2'
|
||||
services:
|
||||
grafana:
|
||||
image: grafana/grafana
|
||||
depends_on:
|
||||
- influxdb
|
||||
links:
|
||||
- influxdb
|
||||
ports:
|
||||
- "3000:3000"
|
||||
influxdb:
|
||||
image: tutum/influxdb
|
||||
environment:
|
||||
- PRE_CREATE_DB="sitespeed"
|
||||
- ADMIN_USER="root"
|
||||
- INFLUXDB_INIT_PWD="root"
|
||||
ports:
|
||||
- "8083:8083"
|
||||
- "8086:8086"
|
||||
- "8090:8090"
|
||||
- "8099:8099"
|
||||
Loading…
Reference in New Issue