mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introducing slug for your test and a new experimental setup (#3203)

This commit is contained in:
Peter Hedenskog 2020-12-08 19:48:51 +01:00 committed by GitHub
parent 1c0add2643
commit 5c32deeaef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 140 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/cli/cli.js
View File

@ -75,6 +75,16 @@ function validateInput(argv) {
return 'Error: You can only run with one browser at a time.';
}
if (argv.slug) {
const characters = /[^A-Za-z_\-0-9]/g;
if (characters.test(argv.slug)) {
return 'The slug can only use characters A-Z a-z 0-9 and -_.';
}
if (argv.slug.length > 200) {
return 'The max length for the slug is 200 characters.';
}
}
if (argv.crawler && argv.crawler.depth && argv.multi) {
return 'Error: Crawl do not work running in multi mode.';
}
@ -1414,6 +1424,10 @@ module.exports.parseCommandLine = function parseCommandLine() {
.option('name', {
describe: 'Give your test a name.'
})
.option('slug', {
describe:
'Give your test a slug. The slug is used when you send the metrics to your data storage to identify the test and the folder of the tests. The max length of the slug is 200 characters and it can only contain a-z A-Z 0-9 and -_ characters.'
})
.help('h')
.alias('help', 'h')
.config(config)
@ -1603,6 +1617,11 @@ module.exports.parseCommandLine = function parseCommandLine() {
);
}
if (argv.experimentalNewSetup) {
set(argv, 'browsertime.storeURLsAsFlatPageOnDisk', true);
set(argv, 'storeURLsAsFlatPageOnDisk', true);
}
let urlsMetaData = cliUtil.getAliases(argv._, argv.urlAlias, argv.groupAlias);
return {

25
lib/core/resultsStorage/index.js
View File

@ -2,9 +2,13 @@
const urlParser = require('url');
const path = require('path');
const dayjs = require('dayjs');
const resultUrls = require('./resultUrls');
const storageManager = require('./storageManager');
const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo;
const roundDownTo10Minutes = roundDownTo(1000 * 60 * 10);
function getDomainOrFileName(input) {
let domainOrFile = input;
if (domainOrFile.startsWith('http')) {
@ -27,13 +31,24 @@ module.exports = function(input, timestamp, options) {
resultsSubFolders.push(path.basename(outputFolder));
storageBasePath = path.resolve(outputFolder);
} else {
resultsSubFolders.push(
getDomainOrFileName(input),
timestamp.format('YYYY-MM-DD-HH-mm-ss')
);
if (options.experimentalNewSetup) {
const ten = dayjs(roundDownTo10Minutes(timestamp.valueOf()));
resultsSubFolders.push(
options.slug || getDomainOrFileName(input),
ten.format('YYYY-MM-DD-HH-mm')
);
} else {
resultsSubFolders.push(
options.slug || getDomainOrFileName(input),
timestamp.format('YYYY-MM-DD-HH-mm-ss')
);
}
storageBasePath = path.resolve('sitespeed-result', ...resultsSubFolders);
}
// backfill the slug
options.slug = options.slug || getDomainOrFileName(input).replace(/\./g, '_');
storagePathPrefix = path.join(...resultsSubFolders);
if (resultBaseURL) {
@ -45,6 +60,6 @@ module.exports = function(input, timestamp, options) {
return {
storageManager: storageManager(storageBasePath, storagePathPrefix, options),
resultUrls: resultUrls(resultUrl, options.useHash)
resultUrls: resultUrls(resultUrl, options)
};
};

82
lib/core/resultsStorage/pathToFolder.js
View File

@ -1,25 +1,66 @@
'use strict';
const isEmpty = require('lodash.isempty'),
crypto = require('crypto'),
urlParser = require('url');
const isEmpty = require('lodash.isempty');
const crypto = require('crypto');
const log = require('intel').getLogger('sitespeedio.file');
const urlParser = require('url');
module.exports = function pathFromRootToPageDir(url, useHash) {
const parsedUrl = urlParser.parse(decodeURIComponent(url)),
pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
function toSafeKey(key) {
// U+2013 : EN DASH – as used on https://en.wikipedia.org/wiki/2019–20_coronavirus_pandemic
return key.replace(/[.~ /+|,:?&%–)(]|%7C/g, '-');
}
if (useHash && !isEmpty(parsedUrl.hash)) {
const md5 = crypto.createHash('md5'),
hash = md5
.update(parsedUrl.hash)
.digest('hex')
.substring(0, 8);
pathSegments.push('hash-' + hash);
module.exports = function pathFromRootToPageDir(url, options) {
const useHash = options.useHash;
const parsedUrl = urlParser.parse(decodeURIComponent(url));
const pathSegments = [];
const urlSegments = [];
pathSegments.push('pages');
pathSegments.push(parsedUrl.hostname.split('.').join('_'));
if (options.urlMetaData && options.urlMetaData[url]) {
pathSegments.push(options.urlMetaData[url]);
} else {
if (!isEmpty(parsedUrl.pathname)) {
urlSegments.push(...parsedUrl.pathname.split('/').filter(Boolean));
}
if (useHash && !isEmpty(parsedUrl.hash)) {
const md5 = crypto.createHash('md5'),
hash = md5
.update(parsedUrl.hash)
.digest('hex')
.substring(0, 8);
urlSegments.push('hash-' + hash);
}
if (!isEmpty(parsedUrl.search)) {
const md5 = crypto.createHash('md5'),
hash = md5
.update(parsedUrl.search)
.digest('hex')
.substring(0, 8);
urlSegments.push('query-' + hash);
}
// This is used from sitespeed.io to match URLs on Graphite
if (!options.storeURLsAsFlatPageOnDisk) {
pathSegments.push(...urlSegments);
} else {
const folder = toSafeKey(urlSegments.join('_').concat('_'));
if (folder.length > 255) {
log.info(
`The URL ${url} hit the 255 character limit used when stored on disk, you may want to give your URL an alias to make sure it will not collide with other URLs.`
);
pathSegments.push(folder.substr(0, 254));
} else {
pathSegments.push(folder);
}
}
}
pathSegments.unshift(parsedUrl.hostname);
pathSegments.unshift('pages');
// pathSegments.push('data');
pathSegments.forEach(function(segment, index) {
if (segment) {
@ -27,14 +68,5 @@ module.exports = function pathFromRootToPageDir(url, useHash) {
}
});
if (!isEmpty(parsedUrl.search)) {
const md5 = crypto.createHash('md5'),
hash = md5
.update(parsedUrl.search)
.digest('hex')
.substring(0, 8);
pathSegments.push('query-' + hash);
}
return pathSegments.join('/').concat('/');
};

12
lib/core/resultsStorage/resultUrls.js
View File

@ -3,13 +3,13 @@
const urlParser = require('url');
const pathToFolder = require('./pathToFolder');
function getPageUrl({ url, resultBaseUrl, useHash }) {
function getPageUrl({ url, resultBaseUrl, options }) {
const pageUrl = urlParser.parse(resultBaseUrl);
pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, useHash)].join('/');
pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, options)].join('/');
return urlParser.format(pageUrl);
}
module.exports = function resultUrls(resultBaseUrl, useHash) {
module.exports = function resultUrls(resultBaseUrl, options) {
return {
hasBaseUrl() {
return !!resultBaseUrl;
@ -19,13 +19,13 @@ module.exports = function resultUrls(resultBaseUrl, useHash) {
},
// In the future this one shoudl include the full URL including /index.html
absoluteSummaryPageUrl(url) {
return getPageUrl({ url, resultBaseUrl, useHash });
return getPageUrl({ url, resultBaseUrl, options });
},
absoluteSummaryPagePath(url) {
return getPageUrl({ url, resultBaseUrl, useHash });
return getPageUrl({ url, resultBaseUrl, options });
},
relativeSummaryPageUrl(url) {
return pathToFolder(url, useHash);
return pathToFolder(url, options);
}
};
};

11
lib/core/resultsStorage/storageManager.js
View File

@ -20,10 +20,9 @@ function isValidDirectoryName(name) {
}
module.exports = function storageManager(baseDir, storagePathPrefix, options) {
const useHash = options.useHash;
return {
rootPathFromUrl(url) {
return pathToFolder(url, useHash)
return pathToFolder(url, options)
.split('/')
.filter(isValidDirectoryName)
.map(() => '..')
@ -48,7 +47,7 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
return baseDir;
},
getFullPathToURLDir(url) {
return path.join(baseDir, pathToFolder(url, useHash));
return path.join(baseDir, pathToFolder(url, options));
},
getStoragePrefix() {
return storagePathPrefix;
@ -57,7 +56,7 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
return this.createDirectory().then(dir => fs.copy(filename, dir));
},
removeDataForUrl(url) {
const dirName = path.join(baseDir, pathToFolder(url, useHash));
const dirName = path.join(baseDir, pathToFolder(url, options));
const removeDir = async dir => {
try {
const files = await readdir(dir);
@ -84,11 +83,11 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
return removeDir(dirName);
},
createDirForUrl(url, subDir) {
return this.createDirectory(pathToFolder(url, useHash), subDir);
return this.createDirectory(pathToFolder(url, options), subDir);
},
writeDataForUrl(data, filename, url, subDir) {
return this.createDirectory(
pathToFolder(url, useHash),
pathToFolder(url, options),
'data',
subDir
).then(dir => write(dir, filename, data));

13
lib/plugins/graphite/data-generator.js
View File

@ -35,6 +35,7 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
} else if (message.type.match(/(^gpsi)/)) {
typeParts.splice(2, 0, options.mobile ? 'mobile' : 'desktop');
}
// if we get a URL type, add the URL
if (message.url) {
typeParts.splice(
@ -62,6 +63,10 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
typeParts.splice(0, 1, 'run-' + message.iteration);
}
if (options.experimentalNewSetup) {
typeParts.splice(1, 0, options.slug);
}
return typeParts.join('.');
}
@ -74,7 +79,13 @@ class GraphiteDataGenerator {
}
dataFromMessage(message, time, alias) {
const timestamp = Math.round(time.valueOf() / 1000);
const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo;
const roundDownTo10Minutes = roundDownTo(1000 * 60 * 10);
let timestamp = time;
if (this.options.experimentalNewSetup) {
timestamp = Math.round(roundDownTo10Minutes(time.valueOf()) / 1000);
}
const keypath = keyPathFromMessage(
message,

3
lib/plugins/influxdb/data-generator.js
View File

@ -70,6 +70,9 @@ class InfluxDBDataGenerator {
options.influxdb.groupSeparator
);
}
tags.testName = options.slug;
return tags;
}

2
test/influxdbTests.js
View File

@ -361,7 +361,7 @@ describe('influxdb', function() {
const seriesName = data[0].seriesName;
const numberOfTags = Object.keys(data[0].tags).length;
expect(seriesName).to.match(/score/);
expect(numberOfTags).to.equal(6);
expect(numberOfTags).to.equal(7);
});
});
});

16
test/pathToFolderTests.js
View File

@ -5,22 +5,22 @@ const expect = require('chai').expect;
describe('pathFromRootToPageDir', function() {
it('should create path from site root', function() {
const path = pathFromRootToPageDir('http://www.foo.bar');
expect(path).to.equal('pages/www.foo.bar/');
const path = pathFromRootToPageDir('http://www.foo.bar', {});
expect(path).to.equal('pages/www_foo_bar/');
});
it('should create path from url', function() {
const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z.html');
expect(path).to.equal('pages/www.foo.bar/x/y/z.html/');
const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z.html', {});
expect(path).to.equal('pages/www_foo_bar/x/y/z.html/');
});
it('should create path from url with sanitized characters', function() {
const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z:200.html');
expect(path).to.equal('pages/www.foo.bar/x/y/z-200.html/');
const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z:200.html', {});
expect(path).to.equal('pages/www_foo_bar/x/y/z-200.html/');
});
it('should create path from url with query string', function() {
const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z?foo=bar');
expect(path).to.equal('pages/www.foo.bar/x/y/z/query-115ffe20/');
const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z?foo=bar', {});
expect(path).to.equal('pages/www_foo_bar/x/y/z/query-115ffe20/');
});
});

12
test/resultUrlTests.js
View File

@ -71,7 +71,7 @@ describe('resultUrls', function() {
expect(
resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal(
`http://results.com/www.foo.bar/${timestampString}/pages/www.foo.bar/xyz/`
`http://results.com/www.foo.bar/${timestampString}/pages/www_foo_bar/xyz/`
);
});
it('should create url with absolute output folder', function() {
@ -82,7 +82,7 @@ describe('resultUrls', function() {
);
expect(
resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
).to.equal('http://results.com/leaf/pages/www_foo_bar/xyz/');
});
it('should create url with relative output folder', function() {
const resultUrls = createResultUrls(
@ -92,7 +92,7 @@ describe('resultUrls', function() {
);
expect(
resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
).to.equal('http://results.com/leaf/pages/www_foo_bar/xyz/');
});
});
describe('#relativeSummaryPageUrl', function() {
@ -104,7 +104,7 @@ describe('resultUrls', function() {
);
expect(
resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('pages/www.foo.bar/xyz/');
).to.equal('pages/www_foo_bar/xyz/');
});
it('should create url with absolute output folder', function() {
const resultUrls = createResultUrls(
@ -114,7 +114,7 @@ describe('resultUrls', function() {
);
expect(
resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('pages/www.foo.bar/xyz/');
).to.equal('pages/www_foo_bar/xyz/');
});
it('should create url with relative output folder', function() {
const resultUrls = createResultUrls(
@ -124,7 +124,7 @@ describe('resultUrls', function() {
);
expect(
resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
).to.equal('pages/www.foo.bar/xyz/');
).to.equal('pages/www_foo_bar/xyz/');
});
});
});

5
test/slackTests.js
View File

@ -21,7 +21,7 @@ const defaultContextFactory = (context = {}) => {
filterRegistry,
intel,
statsHelpers,
resultUrls: resultUrls()
resultUrls: resultUrls('', {})
},
context
);
@ -228,7 +228,8 @@ describe('slack', () => {
name: 'Simple test'
});
context.resultUrls = resultUrls(
'https://results.sitespeed.io/absolute/path'
'https://results.sitespeed.io/absolute/path',
{}
);
const plugin = pluginFactory(context);
const mock = mockSend();