diff --git a/lib/cli/cli.js b/lib/cli/cli.js index c30ec687c..f04536b97 100644 --- a/lib/cli/cli.js +++ b/lib/cli/cli.js @@ -75,6 +75,16 @@ function validateInput(argv) { return 'Error: You can only run with one browser at a time.'; } + if (argv.slug) { + const characters = /[^A-Za-z_\-0-9]/g; + if (characters.test(argv.slug)) { + return 'The slug can only use characters A-Z a-z 0-9 and -_.'; + } + if (argv.slug.length > 200) { + return 'The max length for the slug is 200 characters.'; + } + } + if (argv.crawler && argv.crawler.depth && argv.multi) { return 'Error: Crawl do not work running in multi mode.'; } @@ -1414,6 +1424,10 @@ module.exports.parseCommandLine = function parseCommandLine() { .option('name', { describe: 'Give your test a name.' }) + .option('slug', { + describe: + 'Give your test a slug. The slug is used when you send the metrics to your data storage to identify the test and the folder of the tests. The max length of the slug is 200 characters and it can only contain a-z A-Z 0-9 and -_ characters.' + }) .help('h') .alias('help', 'h') .config(config) @@ -1603,6 +1617,11 @@ module.exports.parseCommandLine = function parseCommandLine() { ); } + if (argv.experimentalNewSetup) { + set(argv, 'browsertime.storeURLsAsFlatPageOnDisk', true); + set(argv, 'storeURLsAsFlatPageOnDisk', true); + } + let urlsMetaData = cliUtil.getAliases(argv._, argv.urlAlias, argv.groupAlias); return { diff --git a/lib/core/resultsStorage/index.js b/lib/core/resultsStorage/index.js index 0337891c8..27d9db728 100644 --- a/lib/core/resultsStorage/index.js +++ b/lib/core/resultsStorage/index.js @@ -2,9 +2,13 @@ const urlParser = require('url'); const path = require('path'); +const dayjs = require('dayjs'); const resultUrls = require('./resultUrls'); const storageManager = require('./storageManager'); +const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo; +const roundDownTo10Minutes = roundDownTo(1000 * 60 * 10); + function getDomainOrFileName(input) { let domainOrFile = input; if (domainOrFile.startsWith('http')) { @@ -27,13 +31,24 @@ module.exports = function(input, timestamp, options) { resultsSubFolders.push(path.basename(outputFolder)); storageBasePath = path.resolve(outputFolder); } else { - resultsSubFolders.push( - getDomainOrFileName(input), - timestamp.format('YYYY-MM-DD-HH-mm-ss') - ); + if (options.experimentalNewSetup) { + const ten = dayjs(roundDownTo10Minutes(timestamp.valueOf())); + resultsSubFolders.push( + options.slug || getDomainOrFileName(input), + ten.format('YYYY-MM-DD-HH-mm') + ); + } else { + resultsSubFolders.push( + options.slug || getDomainOrFileName(input), + timestamp.format('YYYY-MM-DD-HH-mm-ss') + ); + } storageBasePath = path.resolve('sitespeed-result', ...resultsSubFolders); } + // backfill the slug + options.slug = options.slug || getDomainOrFileName(input).replace(/\./g, '_'); + storagePathPrefix = path.join(...resultsSubFolders); if (resultBaseURL) { @@ -45,6 +60,6 @@ module.exports = function(input, timestamp, options) { return { storageManager: storageManager(storageBasePath, storagePathPrefix, options), - resultUrls: resultUrls(resultUrl, options.useHash) + resultUrls: resultUrls(resultUrl, options) }; }; diff --git a/lib/core/resultsStorage/pathToFolder.js b/lib/core/resultsStorage/pathToFolder.js index 094cd38b4..1b26fff8f 100644 --- a/lib/core/resultsStorage/pathToFolder.js +++ b/lib/core/resultsStorage/pathToFolder.js @@ -1,25 +1,66 @@ 'use strict'; -const isEmpty = require('lodash.isempty'), - crypto = require('crypto'), - urlParser = require('url'); +const isEmpty = require('lodash.isempty'); +const crypto = require('crypto'); +const log = require('intel').getLogger('sitespeedio.file'); +const urlParser = require('url'); -module.exports = function pathFromRootToPageDir(url, useHash) { - const parsedUrl = urlParser.parse(decodeURIComponent(url)), - pathSegments = parsedUrl.pathname.split('/').filter(Boolean); +function toSafeKey(key) { + // U+2013 : EN DASH – as used on https://en.wikipedia.org/wiki/2019–20_coronavirus_pandemic + return key.replace(/[.~ /+|,:?&%–)(]|%7C/g, '-'); +} - if (useHash && !isEmpty(parsedUrl.hash)) { - const md5 = crypto.createHash('md5'), - hash = md5 - .update(parsedUrl.hash) - .digest('hex') - .substring(0, 8); - pathSegments.push('hash-' + hash); +module.exports = function pathFromRootToPageDir(url, options) { + const useHash = options.useHash; + const parsedUrl = urlParser.parse(decodeURIComponent(url)); + + const pathSegments = []; + const urlSegments = []; + pathSegments.push('pages'); + pathSegments.push(parsedUrl.hostname.split('.').join('_')); + + if (options.urlMetaData && options.urlMetaData[url]) { + pathSegments.push(options.urlMetaData[url]); + } else { + if (!isEmpty(parsedUrl.pathname)) { + urlSegments.push(...parsedUrl.pathname.split('/').filter(Boolean)); + } + + if (useHash && !isEmpty(parsedUrl.hash)) { + const md5 = crypto.createHash('md5'), + hash = md5 + .update(parsedUrl.hash) + .digest('hex') + .substring(0, 8); + urlSegments.push('hash-' + hash); + } + + if (!isEmpty(parsedUrl.search)) { + const md5 = crypto.createHash('md5'), + hash = md5 + .update(parsedUrl.search) + .digest('hex') + .substring(0, 8); + urlSegments.push('query-' + hash); + } + + // This is used from sitespeed.io to match URLs on Graphite + if (!options.storeURLsAsFlatPageOnDisk) { + pathSegments.push(...urlSegments); + } else { + const folder = toSafeKey(urlSegments.join('_').concat('_')); + if (folder.length > 255) { + log.info( + `The URL ${url} hit the 255 character limit used when stored on disk, you may want to give your URL an alias to make sure it will not collide with other URLs.` + ); + pathSegments.push(folder.substr(0, 254)); + } else { + pathSegments.push(folder); + } + } } - pathSegments.unshift(parsedUrl.hostname); - - pathSegments.unshift('pages'); + // pathSegments.push('data'); pathSegments.forEach(function(segment, index) { if (segment) { @@ -27,14 +68,5 @@ module.exports = function pathFromRootToPageDir(url, useHash) { } }); - if (!isEmpty(parsedUrl.search)) { - const md5 = crypto.createHash('md5'), - hash = md5 - .update(parsedUrl.search) - .digest('hex') - .substring(0, 8); - pathSegments.push('query-' + hash); - } - return pathSegments.join('/').concat('/'); }; diff --git a/lib/core/resultsStorage/resultUrls.js b/lib/core/resultsStorage/resultUrls.js index a6973a47d..303bf5c95 100644 --- a/lib/core/resultsStorage/resultUrls.js +++ b/lib/core/resultsStorage/resultUrls.js @@ -3,13 +3,13 @@ const urlParser = require('url'); const pathToFolder = require('./pathToFolder'); -function getPageUrl({ url, resultBaseUrl, useHash }) { +function getPageUrl({ url, resultBaseUrl, options }) { const pageUrl = urlParser.parse(resultBaseUrl); - pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, useHash)].join('/'); + pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, options)].join('/'); return urlParser.format(pageUrl); } -module.exports = function resultUrls(resultBaseUrl, useHash) { +module.exports = function resultUrls(resultBaseUrl, options) { return { hasBaseUrl() { return !!resultBaseUrl; @@ -19,13 +19,13 @@ module.exports = function resultUrls(resultBaseUrl, useHash) { }, // In the future this one shoudl include the full URL including /index.html absoluteSummaryPageUrl(url) { - return getPageUrl({ url, resultBaseUrl, useHash }); + return getPageUrl({ url, resultBaseUrl, options }); }, absoluteSummaryPagePath(url) { - return getPageUrl({ url, resultBaseUrl, useHash }); + return getPageUrl({ url, resultBaseUrl, options }); }, relativeSummaryPageUrl(url) { - return pathToFolder(url, useHash); + return pathToFolder(url, options); } }; }; diff --git a/lib/core/resultsStorage/storageManager.js b/lib/core/resultsStorage/storageManager.js index 626b03552..401cc18d1 100644 --- a/lib/core/resultsStorage/storageManager.js +++ b/lib/core/resultsStorage/storageManager.js @@ -20,10 +20,9 @@ function isValidDirectoryName(name) { } module.exports = function storageManager(baseDir, storagePathPrefix, options) { - const useHash = options.useHash; return { rootPathFromUrl(url) { - return pathToFolder(url, useHash) + return pathToFolder(url, options) .split('/') .filter(isValidDirectoryName) .map(() => '..') @@ -48,7 +47,7 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) { return baseDir; }, getFullPathToURLDir(url) { - return path.join(baseDir, pathToFolder(url, useHash)); + return path.join(baseDir, pathToFolder(url, options)); }, getStoragePrefix() { return storagePathPrefix; @@ -57,7 +56,7 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) { return this.createDirectory().then(dir => fs.copy(filename, dir)); }, removeDataForUrl(url) { - const dirName = path.join(baseDir, pathToFolder(url, useHash)); + const dirName = path.join(baseDir, pathToFolder(url, options)); const removeDir = async dir => { try { const files = await readdir(dir); @@ -84,11 +83,11 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) { return removeDir(dirName); }, createDirForUrl(url, subDir) { - return this.createDirectory(pathToFolder(url, useHash), subDir); + return this.createDirectory(pathToFolder(url, options), subDir); }, writeDataForUrl(data, filename, url, subDir) { return this.createDirectory( - pathToFolder(url, useHash), + pathToFolder(url, options), 'data', subDir ).then(dir => write(dir, filename, data)); diff --git a/lib/plugins/graphite/data-generator.js b/lib/plugins/graphite/data-generator.js index 8db553d08..aa809cad5 100644 --- a/lib/plugins/graphite/data-generator.js +++ b/lib/plugins/graphite/data-generator.js @@ -35,6 +35,7 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) { } else if (message.type.match(/(^gpsi)/)) { typeParts.splice(2, 0, options.mobile ? 'mobile' : 'desktop'); } + // if we get a URL type, add the URL if (message.url) { typeParts.splice( @@ -62,6 +63,10 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) { typeParts.splice(0, 1, 'run-' + message.iteration); } + if (options.experimentalNewSetup) { + typeParts.splice(1, 0, options.slug); + } + return typeParts.join('.'); } @@ -74,7 +79,13 @@ class GraphiteDataGenerator { } dataFromMessage(message, time, alias) { - const timestamp = Math.round(time.valueOf() / 1000); + const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo; + const roundDownTo10Minutes = roundDownTo(1000 * 60 * 10); + + let timestamp = time; + if (this.options.experimentalNewSetup) { + timestamp = Math.round(roundDownTo10Minutes(time.valueOf()) / 1000); + } const keypath = keyPathFromMessage( message, diff --git a/lib/plugins/influxdb/data-generator.js b/lib/plugins/influxdb/data-generator.js index d1d54fc6b..6cd85c271 100644 --- a/lib/plugins/influxdb/data-generator.js +++ b/lib/plugins/influxdb/data-generator.js @@ -70,6 +70,9 @@ class InfluxDBDataGenerator { options.influxdb.groupSeparator ); } + + tags.testName = options.slug; + return tags; } diff --git a/test/influxdbTests.js b/test/influxdbTests.js index 881e910f5..44b70a6a9 100644 --- a/test/influxdbTests.js +++ b/test/influxdbTests.js @@ -361,7 +361,7 @@ describe('influxdb', function() { const seriesName = data[0].seriesName; const numberOfTags = Object.keys(data[0].tags).length; expect(seriesName).to.match(/score/); - expect(numberOfTags).to.equal(6); + expect(numberOfTags).to.equal(7); }); }); }); diff --git a/test/pathToFolderTests.js b/test/pathToFolderTests.js index 7db1c7d9e..8df2b12c6 100644 --- a/test/pathToFolderTests.js +++ b/test/pathToFolderTests.js @@ -5,22 +5,22 @@ const expect = require('chai').expect; describe('pathFromRootToPageDir', function() { it('should create path from site root', function() { - const path = pathFromRootToPageDir('http://www.foo.bar'); - expect(path).to.equal('pages/www.foo.bar/'); + const path = pathFromRootToPageDir('http://www.foo.bar', {}); + expect(path).to.equal('pages/www_foo_bar/'); }); it('should create path from url', function() { - const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z.html'); - expect(path).to.equal('pages/www.foo.bar/x/y/z.html/'); + const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z.html', {}); + expect(path).to.equal('pages/www_foo_bar/x/y/z.html/'); }); it('should create path from url with sanitized characters', function() { - const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z:200.html'); - expect(path).to.equal('pages/www.foo.bar/x/y/z-200.html/'); + const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z:200.html', {}); + expect(path).to.equal('pages/www_foo_bar/x/y/z-200.html/'); }); it('should create path from url with query string', function() { - const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z?foo=bar'); - expect(path).to.equal('pages/www.foo.bar/x/y/z/query-115ffe20/'); + const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z?foo=bar', {}); + expect(path).to.equal('pages/www_foo_bar/x/y/z/query-115ffe20/'); }); }); diff --git a/test/resultUrlTests.js b/test/resultUrlTests.js index a6b08602c..79ea1dcfa 100644 --- a/test/resultUrlTests.js +++ b/test/resultUrlTests.js @@ -71,7 +71,7 @@ describe('resultUrls', function() { expect( resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz') ).to.equal( - `http://results.com/www.foo.bar/${timestampString}/pages/www.foo.bar/xyz/` + `http://results.com/www.foo.bar/${timestampString}/pages/www_foo_bar/xyz/` ); }); it('should create url with absolute output folder', function() { @@ -82,7 +82,7 @@ describe('resultUrls', function() { ); expect( resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz') - ).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/'); + ).to.equal('http://results.com/leaf/pages/www_foo_bar/xyz/'); }); it('should create url with relative output folder', function() { const resultUrls = createResultUrls( @@ -92,7 +92,7 @@ describe('resultUrls', function() { ); expect( resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz') - ).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/'); + ).to.equal('http://results.com/leaf/pages/www_foo_bar/xyz/'); }); }); describe('#relativeSummaryPageUrl', function() { @@ -104,7 +104,7 @@ describe('resultUrls', function() { ); expect( resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz') - ).to.equal('pages/www.foo.bar/xyz/'); + ).to.equal('pages/www_foo_bar/xyz/'); }); it('should create url with absolute output folder', function() { const resultUrls = createResultUrls( @@ -114,7 +114,7 @@ describe('resultUrls', function() { ); expect( resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz') - ).to.equal('pages/www.foo.bar/xyz/'); + ).to.equal('pages/www_foo_bar/xyz/'); }); it('should create url with relative output folder', function() { const resultUrls = createResultUrls( @@ -124,7 +124,7 @@ describe('resultUrls', function() { ); expect( resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz') - ).to.equal('pages/www.foo.bar/xyz/'); + ).to.equal('pages/www_foo_bar/xyz/'); }); }); }); diff --git a/test/slackTests.js b/test/slackTests.js index 32416801d..8a83b2140 100644 --- a/test/slackTests.js +++ b/test/slackTests.js @@ -21,7 +21,7 @@ const defaultContextFactory = (context = {}) => { filterRegistry, intel, statsHelpers, - resultUrls: resultUrls() + resultUrls: resultUrls('', {}) }, context ); @@ -228,7 +228,8 @@ describe('slack', () => { name: 'Simple test' }); context.resultUrls = resultUrls( - 'https://results.sitespeed.io/absolute/path' + 'https://results.sitespeed.io/absolute/path', + {} ); const plugin = pluginFactory(context); const mock = mockSend();