Introducing slug for your test and a new experimental setup (#3203)

lib/cli/cli.js

@@ -75,6 +75,16 @@ function validateInput(argv) {
     return 'Error: You can only run with one browser at a time.';
   }
 
+  if (argv.slug) {
+    const characters = /[^A-Za-z_\-0-9]/g;
+    if (characters.test(argv.slug)) {
+      return 'The slug can only use characters A-Z a-z 0-9 and -_.';
+    }
+    if (argv.slug.length > 200) {
+      return 'The max length for the slug is 200 characters.';
+    }
+  }
+
   if (argv.crawler && argv.crawler.depth && argv.multi) {
     return 'Error: Crawl do not work running in multi mode.';
   }
@@ -1414,6 +1424,10 @@ module.exports.parseCommandLine = function parseCommandLine() {
     .option('name', {
       describe: 'Give your test a name.'
     })
+    .option('slug', {
+      describe:
+        'Give your test a slug. The slug is used when you send the metrics to your data storage to identify the test and the folder of the tests. The max length of the slug is 200 characters and it can only contain a-z A-Z 0-9 and -_ characters.'
+    })
     .help('h')
     .alias('help', 'h')
     .config(config)
@@ -1603,6 +1617,11 @@ module.exports.parseCommandLine = function parseCommandLine() {
     );
   }
 
+  if (argv.experimentalNewSetup) {
+    set(argv, 'browsertime.storeURLsAsFlatPageOnDisk', true);
+    set(argv, 'storeURLsAsFlatPageOnDisk', true);
+  }
+
   let urlsMetaData = cliUtil.getAliases(argv._, argv.urlAlias, argv.groupAlias);
 
   return {

lib/core/resultsStorage/index.js

@@ -2,9 +2,13 @@
 
 const urlParser = require('url');
 const path = require('path');
+const dayjs = require('dayjs');
 const resultUrls = require('./resultUrls');
 const storageManager = require('./storageManager');
 
+const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo;
+const roundDownTo10Minutes = roundDownTo(1000 * 60 * 10);
+
 function getDomainOrFileName(input) {
   let domainOrFile = input;
   if (domainOrFile.startsWith('http')) {
@@ -27,13 +31,24 @@ module.exports = function(input, timestamp, options) {
     resultsSubFolders.push(path.basename(outputFolder));
     storageBasePath = path.resolve(outputFolder);
   } else {
-    resultsSubFolders.push(
+    if (options.experimentalNewSetup) {
-      getDomainOrFileName(input),
+      const ten = dayjs(roundDownTo10Minutes(timestamp.valueOf()));
-      timestamp.format('YYYY-MM-DD-HH-mm-ss')
+      resultsSubFolders.push(
-    );
+        options.slug || getDomainOrFileName(input),
+        ten.format('YYYY-MM-DD-HH-mm')
+      );
+    } else {
+      resultsSubFolders.push(
+        options.slug || getDomainOrFileName(input),
+        timestamp.format('YYYY-MM-DD-HH-mm-ss')
+      );
+    }
     storageBasePath = path.resolve('sitespeed-result', ...resultsSubFolders);
   }
 
+  // backfill the slug
+  options.slug = options.slug || getDomainOrFileName(input).replace(/\./g, '_');
+
   storagePathPrefix = path.join(...resultsSubFolders);
 
   if (resultBaseURL) {
@@ -45,6 +60,6 @@ module.exports = function(input, timestamp, options) {
 
   return {
     storageManager: storageManager(storageBasePath, storagePathPrefix, options),
-    resultUrls: resultUrls(resultUrl, options.useHash)
+    resultUrls: resultUrls(resultUrl, options)
   };
 };

lib/core/resultsStorage/pathToFolder.js

@@ -1,25 +1,66 @@
 'use strict';
 
-const isEmpty = require('lodash.isempty'),
+const isEmpty = require('lodash.isempty');
-  crypto = require('crypto'),
+const crypto = require('crypto');
-  urlParser = require('url');
+const log = require('intel').getLogger('sitespeedio.file');
+const urlParser = require('url');
 
-module.exports = function pathFromRootToPageDir(url, useHash) {
+function toSafeKey(key) {
-  const parsedUrl = urlParser.parse(decodeURIComponent(url)),
+  // U+2013 : EN DASH – as used on https://en.wikipedia.org/wiki/2019–20_coronavirus_pandemic
-    pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
+  return key.replace(/[.~ /+|,:?&%–)(]|%7C/g, '-');
+}
 
-  if (useHash && !isEmpty(parsedUrl.hash)) {
+module.exports = function pathFromRootToPageDir(url, options) {
-    const md5 = crypto.createHash('md5'),
+  const useHash = options.useHash;
-      hash = md5
+  const parsedUrl = urlParser.parse(decodeURIComponent(url));
-        .update(parsedUrl.hash)
+
-        .digest('hex')
+  const pathSegments = [];
-        .substring(0, 8);
+  const urlSegments = [];
-    pathSegments.push('hash-' + hash);
+  pathSegments.push('pages');
+  pathSegments.push(parsedUrl.hostname.split('.').join('_'));
+
+  if (options.urlMetaData && options.urlMetaData[url]) {
+    pathSegments.push(options.urlMetaData[url]);
+  } else {
+    if (!isEmpty(parsedUrl.pathname)) {
+      urlSegments.push(...parsedUrl.pathname.split('/').filter(Boolean));
+    }
+
+    if (useHash && !isEmpty(parsedUrl.hash)) {
+      const md5 = crypto.createHash('md5'),
+        hash = md5
+          .update(parsedUrl.hash)
+          .digest('hex')
+          .substring(0, 8);
+      urlSegments.push('hash-' + hash);
+    }
+
+    if (!isEmpty(parsedUrl.search)) {
+      const md5 = crypto.createHash('md5'),
+        hash = md5
+          .update(parsedUrl.search)
+          .digest('hex')
+          .substring(0, 8);
+      urlSegments.push('query-' + hash);
+    }
+
+    // This is used from sitespeed.io to match URLs on Graphite
+    if (!options.storeURLsAsFlatPageOnDisk) {
+      pathSegments.push(...urlSegments);
+    } else {
+      const folder = toSafeKey(urlSegments.join('_').concat('_'));
+      if (folder.length > 255) {
+        log.info(
+          `The URL ${url} hit the 255 character limit used when stored on disk, you may want to give your URL an alias to make sure it will not collide with other URLs.`
+        );
+        pathSegments.push(folder.substr(0, 254));
+      } else {
+        pathSegments.push(folder);
+      }
+    }
   }
 
-  pathSegments.unshift(parsedUrl.hostname);
+  // pathSegments.push('data');
-
-  pathSegments.unshift('pages');
 
   pathSegments.forEach(function(segment, index) {
     if (segment) {
@@ -27,14 +68,5 @@ module.exports = function pathFromRootToPageDir(url, useHash) {
     }
   });
 
-  if (!isEmpty(parsedUrl.search)) {
-    const md5 = crypto.createHash('md5'),
-      hash = md5
-        .update(parsedUrl.search)
-        .digest('hex')
-        .substring(0, 8);
-    pathSegments.push('query-' + hash);
-  }
-
   return pathSegments.join('/').concat('/');
 };

lib/core/resultsStorage/resultUrls.js

@@ -3,13 +3,13 @@
 const urlParser = require('url');
 const pathToFolder = require('./pathToFolder');
 
-function getPageUrl({ url, resultBaseUrl, useHash }) {
+function getPageUrl({ url, resultBaseUrl, options }) {
   const pageUrl = urlParser.parse(resultBaseUrl);
-  pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, useHash)].join('/');
+  pageUrl.pathname = [pageUrl.pathname, pathToFolder(url, options)].join('/');
   return urlParser.format(pageUrl);
 }
 
-module.exports = function resultUrls(resultBaseUrl, useHash) {
+module.exports = function resultUrls(resultBaseUrl, options) {
   return {
     hasBaseUrl() {
       return !!resultBaseUrl;
@@ -19,13 +19,13 @@ module.exports = function resultUrls(resultBaseUrl, useHash) {
     },
     // In the future this one shoudl include the full URL including /index.html
     absoluteSummaryPageUrl(url) {
-      return getPageUrl({ url, resultBaseUrl, useHash });
+      return getPageUrl({ url, resultBaseUrl, options });
     },
     absoluteSummaryPagePath(url) {
-      return getPageUrl({ url, resultBaseUrl, useHash });
+      return getPageUrl({ url, resultBaseUrl, options });
     },
     relativeSummaryPageUrl(url) {
-      return pathToFolder(url, useHash);
+      return pathToFolder(url, options);
     }
   };
 };

lib/core/resultsStorage/storageManager.js

@@ -20,10 +20,9 @@ function isValidDirectoryName(name) {
 }
 
 module.exports = function storageManager(baseDir, storagePathPrefix, options) {
-  const useHash = options.useHash;
   return {
     rootPathFromUrl(url) {
-      return pathToFolder(url, useHash)
+      return pathToFolder(url, options)
         .split('/')
         .filter(isValidDirectoryName)
         .map(() => '..')
@@ -48,7 +47,7 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
       return baseDir;
     },
     getFullPathToURLDir(url) {
-      return path.join(baseDir, pathToFolder(url, useHash));
+      return path.join(baseDir, pathToFolder(url, options));
     },
     getStoragePrefix() {
       return storagePathPrefix;
@@ -57,7 +56,7 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
       return this.createDirectory().then(dir => fs.copy(filename, dir));
     },
     removeDataForUrl(url) {
-      const dirName = path.join(baseDir, pathToFolder(url, useHash));
+      const dirName = path.join(baseDir, pathToFolder(url, options));
       const removeDir = async dir => {
         try {
           const files = await readdir(dir);
@@ -84,11 +83,11 @@ module.exports = function storageManager(baseDir, storagePathPrefix, options) {
       return removeDir(dirName);
     },
     createDirForUrl(url, subDir) {
-      return this.createDirectory(pathToFolder(url, useHash), subDir);
+      return this.createDirectory(pathToFolder(url, options), subDir);
     },
     writeDataForUrl(data, filename, url, subDir) {
       return this.createDirectory(
-        pathToFolder(url, useHash),
+        pathToFolder(url, options),
         'data',
         subDir
       ).then(dir => write(dir, filename, data));

lib/plugins/graphite/data-generator.js

@@ -35,6 +35,7 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
   } else if (message.type.match(/(^gpsi)/)) {
     typeParts.splice(2, 0, options.mobile ? 'mobile' : 'desktop');
   }
+
   // if we get a URL type, add the URL
   if (message.url) {
     typeParts.splice(
@@ -62,6 +63,10 @@ function keyPathFromMessage(message, options, includeQueryParams, alias) {
     typeParts.splice(0, 1, 'run-' + message.iteration);
   }
 
+  if (options.experimentalNewSetup) {
+    typeParts.splice(1, 0, options.slug);
+  }
+
   return typeParts.join('.');
 }
 
@@ -74,7 +79,13 @@ class GraphiteDataGenerator {
   }
 
   dataFromMessage(message, time, alias) {
-    const timestamp = Math.round(time.valueOf() / 1000);
+    const roundDownTo = roundTo => x => Math.floor(x / roundTo) * roundTo;
+    const roundDownTo10Minutes = roundDownTo(1000 * 60 * 10);
+
+    let timestamp = time;
+    if (this.options.experimentalNewSetup) {
+      timestamp = Math.round(roundDownTo10Minutes(time.valueOf()) / 1000);
+    }
 
     const keypath = keyPathFromMessage(
       message,

lib/plugins/influxdb/data-generator.js

@@ -70,6 +70,9 @@ class InfluxDBDataGenerator {
           options.influxdb.groupSeparator
         );
       }
+
+      tags.testName = options.slug;
+
       return tags;
     }
 

test/influxdbTests.js

@@ -361,7 +361,7 @@ describe('influxdb', function() {
       const seriesName = data[0].seriesName;
       const numberOfTags = Object.keys(data[0].tags).length;
       expect(seriesName).to.match(/score/);
-      expect(numberOfTags).to.equal(6);
+      expect(numberOfTags).to.equal(7);
     });
   });
 });

test/pathToFolderTests.js

@@ -5,22 +5,22 @@ const expect = require('chai').expect;
 
 describe('pathFromRootToPageDir', function() {
   it('should create path from site root', function() {
-    const path = pathFromRootToPageDir('http://www.foo.bar');
+    const path = pathFromRootToPageDir('http://www.foo.bar', {});
-    expect(path).to.equal('pages/www.foo.bar/');
+    expect(path).to.equal('pages/www_foo_bar/');
   });
 
   it('should create path from url', function() {
-    const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z.html');
+    const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z.html', {});
-    expect(path).to.equal('pages/www.foo.bar/x/y/z.html/');
+    expect(path).to.equal('pages/www_foo_bar/x/y/z.html/');
   });
 
   it('should create path from url with sanitized characters', function() {
-    const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z:200.html');
+    const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z:200.html', {});
-    expect(path).to.equal('pages/www.foo.bar/x/y/z-200.html/');
+    expect(path).to.equal('pages/www_foo_bar/x/y/z-200.html/');
   });
 
   it('should create path from url with query string', function() {
-    const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z?foo=bar');
+    const path = pathFromRootToPageDir('http://www.foo.bar/x/y/z?foo=bar', {});
-    expect(path).to.equal('pages/www.foo.bar/x/y/z/query-115ffe20/');
+    expect(path).to.equal('pages/www_foo_bar/x/y/z/query-115ffe20/');
   });
 });

test/resultUrlTests.js

@@ -71,7 +71,7 @@ describe('resultUrls', function() {
       expect(
         resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
       ).to.equal(
-        `http://results.com/www.foo.bar/${timestampString}/pages/www.foo.bar/xyz/`
+        `http://results.com/www.foo.bar/${timestampString}/pages/www_foo_bar/xyz/`
       );
     });
     it('should create url with absolute output folder', function() {
@@ -82,7 +82,7 @@ describe('resultUrls', function() {
       );
       expect(
         resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
-      ).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
+      ).to.equal('http://results.com/leaf/pages/www_foo_bar/xyz/');
     });
     it('should create url with relative output folder', function() {
       const resultUrls = createResultUrls(
@@ -92,7 +92,7 @@ describe('resultUrls', function() {
       );
       expect(
         resultUrls.absoluteSummaryPageUrl('http://www.foo.bar/xyz')
-      ).to.equal('http://results.com/leaf/pages/www.foo.bar/xyz/');
+      ).to.equal('http://results.com/leaf/pages/www_foo_bar/xyz/');
     });
   });
   describe('#relativeSummaryPageUrl', function() {
@@ -104,7 +104,7 @@ describe('resultUrls', function() {
       );
       expect(
         resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
-      ).to.equal('pages/www.foo.bar/xyz/');
+      ).to.equal('pages/www_foo_bar/xyz/');
     });
     it('should create url with absolute output folder', function() {
       const resultUrls = createResultUrls(
@@ -114,7 +114,7 @@ describe('resultUrls', function() {
       );
       expect(
         resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
-      ).to.equal('pages/www.foo.bar/xyz/');
+      ).to.equal('pages/www_foo_bar/xyz/');
     });
     it('should create url with relative output folder', function() {
       const resultUrls = createResultUrls(
@@ -124,7 +124,7 @@ describe('resultUrls', function() {
       );
       expect(
         resultUrls.relativeSummaryPageUrl('http://www.foo.bar/xyz')
-      ).to.equal('pages/www.foo.bar/xyz/');
+      ).to.equal('pages/www_foo_bar/xyz/');
     });
   });
 });

test/slackTests.js

@@ -21,7 +21,7 @@ const defaultContextFactory = (context = {}) => {
       filterRegistry,
       intel,
       statsHelpers,
-      resultUrls: resultUrls()
+      resultUrls: resultUrls('', {})
     },
     context
   );
@@ -228,7 +228,8 @@ describe('slack', () => {
         name: 'Simple test'
       });
       context.resultUrls = resultUrls(
-        'https://results.sitespeed.io/absolute/path'
+        'https://results.sitespeed.io/absolute/path',
+        {}
       );
       const plugin = pluginFactory(context);
       const mock = mockSend();