Remove hooks and use messages instead (#1758)
* remove hooks and use messages instead * remove hooks * fix the ifs * send
This commit is contained in:
Peter Hedenskog
GitHub
parent
10def76541
commit
9d18533579
|
|
@ -23,35 +23,39 @@ module.exports = {
|
|||
verify(message, this.result, this.options.budget.config);
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
close() {
|
||||
if (this.options.budget) {
|
||||
if (this.options.budget.output === 'tap') {
|
||||
tap.writeTap(this.result, this.storageManager.getBaseDir());
|
||||
} else if (this.options.budget.output === 'junit') {
|
||||
junit.writeJunit(this.result, this.storageManager.getBaseDir());
|
||||
} else {
|
||||
let failing = 0;
|
||||
let working = 0;
|
||||
for (const url of Object.keys(this.result.failing)) {
|
||||
for (const result of this.result.failing[url]) {
|
||||
log.error(
|
||||
'Failing budget %s.%s for %s with value %s %s limit %s',
|
||||
result.type,
|
||||
result.metric,
|
||||
url,
|
||||
result.value,
|
||||
result.limitType,
|
||||
result.limit
|
||||
case 'sitespeedio.render': {
|
||||
if (this.options.budget) {
|
||||
if (this.options.budget.output === 'tap') {
|
||||
tap.writeTap(this.result, this.storageManager.getBaseDir());
|
||||
} else if (this.options.budget.output === 'junit') {
|
||||
junit.writeJunit(this.result, this.storageManager.getBaseDir());
|
||||
} else {
|
||||
let failing = 0;
|
||||
let working = 0;
|
||||
for (const url of Object.keys(this.result.failing)) {
|
||||
for (const result of this.result.failing[url]) {
|
||||
log.error(
|
||||
'Failing budget %s.%s for %s with value %s %s limit %s',
|
||||
result.type,
|
||||
result.metric,
|
||||
url,
|
||||
result.value,
|
||||
result.limitType,
|
||||
result.limit
|
||||
);
|
||||
failing = failing + 1;
|
||||
}
|
||||
}
|
||||
for (const url of Object.keys(this.result.working)) {
|
||||
working = working + this.result.working[url].length;
|
||||
}
|
||||
log.info(
|
||||
'Budget: %d working and %d failing tests',
|
||||
working,
|
||||
failing
|
||||
);
|
||||
failing = failing + 1;
|
||||
}
|
||||
}
|
||||
for (const url of Object.keys(this.result.working)) {
|
||||
working = working + this.result.working[url].length;
|
||||
}
|
||||
log.info('Budget: %d working and %d failing tests', working, failing);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,11 @@ const HTMLBuilder = require('./htmlBuilder');
|
|||
const get = require('lodash.get');
|
||||
const set = require('lodash.set');
|
||||
const reduce = require('lodash.reduce');
|
||||
const messageMaker = require('../../support/messageMaker');
|
||||
const DataCollector = require('../../support/dataCollector');
|
||||
|
||||
const make = messageMaker('html').make;
|
||||
|
||||
// lets keep this in the HTML context, since we need things from the
|
||||
// regular options object in the output
|
||||
const defaultConfig = {
|
||||
|
|
@ -22,7 +25,7 @@ module.exports = {
|
|||
this.dataCollector = new DataCollector(context);
|
||||
this.maxAssets = get(options, 'maxAssets', 20);
|
||||
},
|
||||
processMessage(message) {
|
||||
processMessage(message, queue) {
|
||||
const dataCollector = this.dataCollector;
|
||||
|
||||
if (
|
||||
|
|
@ -146,11 +149,13 @@ module.exports = {
|
|||
dataCollector.addSummary('detailed', data);
|
||||
break;
|
||||
}
|
||||
case 'sitespeedio.render': {
|
||||
return this.HTMLBuilder.render(dataCollector).then(() => {
|
||||
queue.postMessage(make('html.finished'));
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
close() {
|
||||
return this.HTMLBuilder.render(this.dataCollector);
|
||||
},
|
||||
config: defaultConfig
|
||||
};
|
||||
|
|
|
|||
|
|
@ -15,76 +15,75 @@ module.exports = {
|
|||
this.metrics = {};
|
||||
this.storageManager = context.storageManager;
|
||||
},
|
||||
postOpen() {
|
||||
processMessage(message) {
|
||||
// only dance if we all wants to
|
||||
if (this.options.filter) {
|
||||
const filters = Array.isArray(this.options.filter)
|
||||
? this.options.filter
|
||||
: [this.options.filter];
|
||||
if (message.type === 'sitespeedio.setup') {
|
||||
const filters = Array.isArray(this.options.filter)
|
||||
? this.options.filter
|
||||
: [this.options.filter];
|
||||
|
||||
for (let metric of filters) {
|
||||
// for all filters
|
||||
// cleaning all filters means (right now) that all
|
||||
// metrics are sent
|
||||
if (metric === '*+') {
|
||||
filterRegistry.clearAll();
|
||||
} else if (metric === '*-') {
|
||||
// all registered types will be set as unmatching,
|
||||
// use it if you want to have a clean filter where
|
||||
// all types are removed and then you can add your own
|
||||
let types = filterRegistry.getTypes();
|
||||
filterRegistry.clearAll();
|
||||
for (let type of types) {
|
||||
filterRegistry.registerFilterForType('-', type);
|
||||
}
|
||||
} else {
|
||||
let parts = metric.split('.');
|
||||
// the type is "always" the first two
|
||||
let type = parts.shift() + '.' + parts.shift();
|
||||
let filter = parts.join('.');
|
||||
let oldFilter = filterRegistry.getFilterForType(type);
|
||||
if (oldFilter && typeof oldFilter === 'object') {
|
||||
oldFilter.push(filter);
|
||||
for (let metric of filters) {
|
||||
// for all filters
|
||||
// cleaning all filters means (right now) that all
|
||||
// metrics are sent
|
||||
if (metric === '*+') {
|
||||
filterRegistry.clearAll();
|
||||
} else if (metric === '*-') {
|
||||
// all registered types will be set as unmatching,
|
||||
// use it if you want to have a clean filter where
|
||||
// all types are removed and then you can add your own
|
||||
let types = filterRegistry.getTypes();
|
||||
filterRegistry.clearAll();
|
||||
for (let type of types) {
|
||||
filterRegistry.registerFilterForType('-', type);
|
||||
}
|
||||
} else {
|
||||
oldFilter = [filter];
|
||||
let parts = metric.split('.');
|
||||
// the type is "always" the first two
|
||||
let type = parts.shift() + '.' + parts.shift();
|
||||
let filter = parts.join('.');
|
||||
let oldFilter = filterRegistry.getFilterForType(type);
|
||||
if (oldFilter && typeof oldFilter === 'object') {
|
||||
oldFilter.push(filter);
|
||||
} else {
|
||||
oldFilter = [filter];
|
||||
}
|
||||
filterRegistry.registerFilterForType(oldFilter, type);
|
||||
}
|
||||
filterRegistry.registerFilterForType(oldFilter, type);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
processMessage(message) {
|
||||
if (this.options.list) {
|
||||
if (
|
||||
!(
|
||||
message.type.endsWith('.summary') ||
|
||||
message.type.endsWith('.pageSummary')
|
||||
)
|
||||
if (message.type === 'sitespeedio.render') {
|
||||
if (this.options.list) {
|
||||
this.storageManager.writeData(
|
||||
'metrics.txt',
|
||||
Object.keys(this.metrics).join('\n')
|
||||
);
|
||||
}
|
||||
if (this.options.filterList) {
|
||||
let output = '';
|
||||
let filtersByType = filterRegistry.getFilters();
|
||||
for (let type of Object.keys(filtersByType)) {
|
||||
for (let filters of filtersByType[type]) {
|
||||
output += type + '.' + filters + '\n';
|
||||
}
|
||||
}
|
||||
return this.storageManager.writeData('configuredMetrics.txt', output);
|
||||
}
|
||||
} else if (
|
||||
!(
|
||||
message.type.endsWith('.summary') ||
|
||||
message.type.endsWith('.pageSummary')
|
||||
)
|
||||
return;
|
||||
) {
|
||||
return;
|
||||
} else {
|
||||
let flattenMess = flatten.flattenMessageData(message);
|
||||
for (let key of Object.keys(flattenMess)) {
|
||||
this.metrics[message.type + '.' + key] = 1;
|
||||
}
|
||||
}
|
||||
},
|
||||
close() {
|
||||
if (this.options.list) {
|
||||
this.storageManager.writeData(
|
||||
'metrics.txt',
|
||||
Object.keys(this.metrics).join('\n')
|
||||
);
|
||||
}
|
||||
|
||||
if (this.options.filterList) {
|
||||
let output = '';
|
||||
let filtersByType = filterRegistry.getFilters();
|
||||
for (let type of Object.keys(filtersByType)) {
|
||||
for (let filters of filtersByType[type]) {
|
||||
output += type + '.' + filters + '\n';
|
||||
}
|
||||
}
|
||||
return this.storageManager.writeData('configuredMetrics.txt', output);
|
||||
}
|
||||
},
|
||||
config: defaultConfig
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ const s3 = require('s3');
|
|||
const throwIfMissing = require('../../support/util').throwIfMissing;
|
||||
const fs = require('fs-extra');
|
||||
const pick = require('lodash.pick');
|
||||
const messageMaker = require('../../support/messageMaker');
|
||||
const make = messageMaker('s3').make;
|
||||
|
||||
function createS3Client(s3Options) {
|
||||
const clientOptions = pick(s3Options, [
|
||||
|
|
@ -26,60 +28,63 @@ function createS3Client(s3Options) {
|
|||
|
||||
module.exports = {
|
||||
open(context, options) {
|
||||
const s3Options = options.s3;
|
||||
this.s3Options = options.s3;
|
||||
|
||||
throwIfMissing(s3Options, ['bucketname'], 's3');
|
||||
if (s3Options.key || s3Options.secret) {
|
||||
throwIfMissing(s3Options, ['key', 'secret'], 's3');
|
||||
throwIfMissing(this.s3Options, ['bucketname'], 's3');
|
||||
if (this.s3Options.key || this.s3Options.secret) {
|
||||
throwIfMissing(this.s3Options, ['key', 'secret'], 's3');
|
||||
}
|
||||
this.storageManager = context.storageManager;
|
||||
},
|
||||
|
||||
postClose(options) {
|
||||
const s3Options = options.s3;
|
||||
const client = createS3Client(s3Options);
|
||||
processMessage(message, queue) {
|
||||
if (message.type === 'html.finished') {
|
||||
const s3Options = this.s3Options;
|
||||
const client = createS3Client(s3Options);
|
||||
|
||||
const baseDir = this.storageManager.getBaseDir();
|
||||
const baseDir = this.storageManager.getBaseDir();
|
||||
|
||||
const params = {
|
||||
localDir: baseDir,
|
||||
s3Params: {
|
||||
Bucket: s3Options.bucketname,
|
||||
Prefix: s3Options.path || this.storageManager.getStoragePrefix(),
|
||||
const params = {
|
||||
localDir: baseDir,
|
||||
s3Params: {
|
||||
Bucket: s3Options.bucketname,
|
||||
Prefix: s3Options.path || this.storageManager.getStoragePrefix(),
|
||||
|
||||
// Allow user to set default StorageClass on objects and default to STANDARD if not set
|
||||
// Possible options [STANDARD | REDUCED_REDUNDANCY | STANDARD_IA]
|
||||
StorageClass: s3Options.storageClass || 'STANDARD'
|
||||
// Allow user to set default StorageClass on objects and default to STANDARD if not set
|
||||
// Possible options [STANDARD | REDUCED_REDUNDANCY | STANDARD_IA]
|
||||
StorageClass: s3Options.storageClass || 'STANDARD'
|
||||
|
||||
// other options supported by putObject, except Body and ContentLength.
|
||||
// See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
||||
}
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
log.info(
|
||||
`Uploading ${baseDir} to S3 bucket ${s3Options.bucketname}, this can take a while ...`
|
||||
);
|
||||
|
||||
const uploader = client.uploadDir(params);
|
||||
|
||||
uploader.on('error', err => {
|
||||
log.error('Could not upload to S3', err);
|
||||
reject(err);
|
||||
});
|
||||
uploader.on('end', () => {
|
||||
if (s3Options.removeLocalResult) {
|
||||
fs
|
||||
.remove(baseDir)
|
||||
.then(() => {
|
||||
log.debug(`Removed local files and directory ${baseDir}`);
|
||||
resolve();
|
||||
})
|
||||
.catch(e => reject(e));
|
||||
} else {
|
||||
resolve();
|
||||
// other options supported by putObject, except Body and ContentLength.
|
||||
// See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
||||
}
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
log.info(
|
||||
`Uploading ${baseDir} to S3 bucket ${s3Options.bucketname}, this can take a while ...`
|
||||
);
|
||||
|
||||
const uploader = client.uploadDir(params);
|
||||
|
||||
uploader.on('error', err => {
|
||||
log.error('Could not upload to S3', err);
|
||||
reject(err);
|
||||
});
|
||||
uploader.on('end', () => {
|
||||
if (s3Options.removeLocalResult) {
|
||||
fs
|
||||
.remove(baseDir)
|
||||
.then(() => {
|
||||
log.debug(`Removed local files and directory ${baseDir}`);
|
||||
resolve();
|
||||
})
|
||||
.catch(e => reject(e));
|
||||
} else {
|
||||
queue.postMessage(make('s3.finished'));
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -20,12 +20,77 @@ const defaultConfig = {
|
|||
limitMetric: 'coachScore'
|
||||
};
|
||||
|
||||
function send(options) {
|
||||
const slackOptions = merge({}, defaultConfig, options.slack);
|
||||
const slack = new Slack(slackOptions.hookUrl);
|
||||
const type = slackOptions.type;
|
||||
const pageErrors = [];
|
||||
const dataCollector = this.dataCollector;
|
||||
let logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack.png';
|
||||
|
||||
let channel = slackOptions.channel;
|
||||
if (channel && !channel.startsWith('#')) {
|
||||
channel = `#${channel}`;
|
||||
}
|
||||
|
||||
for (const url of dataCollector.getURLs()) {
|
||||
const errors = dataCollector.getURLData(url).errors;
|
||||
if (errors) {
|
||||
pageErrors.push(errors);
|
||||
}
|
||||
}
|
||||
|
||||
let text = '';
|
||||
|
||||
if (['summary', 'all', 'error'].includes(type)) {
|
||||
const sum = getSummary(
|
||||
dataCollector,
|
||||
pageErrors,
|
||||
this.resultUrls,
|
||||
this.context,
|
||||
options
|
||||
);
|
||||
text += sum.summaryText + '\n' + sum.errorText;
|
||||
logo = sum.logo;
|
||||
}
|
||||
|
||||
let attachments = [];
|
||||
if (['url', 'all', 'error'].includes(type)) {
|
||||
attachments = getAttachments(dataCollector, this.resultUrls, slackOptions);
|
||||
}
|
||||
|
||||
if ((type === 'error' && pageErrors.length > 0) || type !== 'error') {
|
||||
log.debug(
|
||||
'Sending message to Slack channel %s and username %s',
|
||||
slackOptions.channel,
|
||||
slackOptions.userName
|
||||
);
|
||||
return slack
|
||||
.sendAsync({
|
||||
text,
|
||||
icon_url: logo,
|
||||
channel,
|
||||
mrkdwn: true,
|
||||
username: slackOptions.userName,
|
||||
attachments
|
||||
})
|
||||
.catch(e => {
|
||||
if (e.errno === 'ETIMEDOUT') {
|
||||
log.warn('Timeout sending Slack message.');
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
open(context, options) {
|
||||
throwIfMissing(options.slack, ['hookUrl', 'userName'], 'slack');
|
||||
this.dataCollector = new DataCollector(context);
|
||||
this.resultUrls = context.resultUrls;
|
||||
this.context = context;
|
||||
this.options = options;
|
||||
},
|
||||
processMessage(message) {
|
||||
const dataCollector = this.dataCollector;
|
||||
|
|
@ -59,73 +124,21 @@ module.exports = {
|
|||
dataCollector.addSummary('index', data);
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
final(options) {
|
||||
const slackOptions = merge({}, defaultConfig, options.slack);
|
||||
const slack = new Slack(slackOptions.hookUrl);
|
||||
const type = slackOptions.type;
|
||||
const pageErrors = [];
|
||||
const dataCollector = this.dataCollector;
|
||||
let logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack.png';
|
||||
|
||||
let channel = slackOptions.channel;
|
||||
if (channel && !channel.startsWith('#')) {
|
||||
channel = `#${channel}`;
|
||||
}
|
||||
|
||||
for (const url of dataCollector.getURLs()) {
|
||||
const errors = dataCollector.getURLData(url).errors;
|
||||
if (errors) {
|
||||
pageErrors.push(errors);
|
||||
case 'html.finished': {
|
||||
// if we have set a result base URL wait on
|
||||
// the content to be uploaded
|
||||
if (!this.options.resultBaseURL) {
|
||||
return send(this.options);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let text = '';
|
||||
|
||||
if (['summary', 'all', 'error'].includes(type)) {
|
||||
const sum = getSummary(
|
||||
dataCollector,
|
||||
pageErrors,
|
||||
this.resultUrls,
|
||||
this.context,
|
||||
options
|
||||
);
|
||||
text += sum.summaryText + '\n' + sum.errorText;
|
||||
logo = sum.logo;
|
||||
}
|
||||
|
||||
let attachments = [];
|
||||
if (['url', 'all', 'error'].includes(type)) {
|
||||
attachments = getAttachments(
|
||||
dataCollector,
|
||||
this.resultUrls,
|
||||
slackOptions
|
||||
);
|
||||
}
|
||||
|
||||
if ((type === 'error' && pageErrors.length > 0) || type !== 'error') {
|
||||
log.debug(
|
||||
'Sending message to Slack channel %s and username %s',
|
||||
slackOptions.channel,
|
||||
slackOptions.userName
|
||||
);
|
||||
return slack
|
||||
.sendAsync({
|
||||
text,
|
||||
icon_url: logo,
|
||||
channel,
|
||||
mrkdwn: true,
|
||||
username: slackOptions.userName,
|
||||
attachments
|
||||
})
|
||||
.catch(e => {
|
||||
if (e.errno === 'ETIMEDOUT') {
|
||||
log.warn('Timeout sending Slack message.');
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
case 'gc.finished':
|
||||
case 'ftp.finished':
|
||||
case 's3.finished': {
|
||||
return send(this.options);
|
||||
}
|
||||
}
|
||||
},
|
||||
config: defaultConfig
|
||||
|
|
|
|||
|
|
@ -148,16 +148,9 @@ module.exports = {
|
|||
log
|
||||
};
|
||||
return runOptionalFunction(allPlugins, 'open', context, options)
|
||||
.then(() => runOptionalFunction(allPlugins, 'postOpen', options))
|
||||
.then(() => queueHandler.run(urlSources))
|
||||
.tap(errors =>
|
||||
runOptionalFunction(allPlugins, 'close', options, errors)
|
||||
)
|
||||
.tap(errors =>
|
||||
runOptionalFunction(allPlugins, 'postClose', options, errors)
|
||||
)
|
||||
.tap(errors =>
|
||||
runOptionalFunction(allPlugins, 'final', options, errors)
|
||||
);
|
||||
});
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue