New scp plugin for copying result to another server (#3691)

* New scp plugin for copying result to another server * sync up * use old * new version
2022-07-10 10:39:05 +02:00 · 2022-07-10 10:39:05 +02:00 · c3f215c312
parent 5150694e6f
commit c3f215c312
7 changed files with 286 additions and 19 deletions
--- a/lib/cli/cli.js
+++ b/lib/cli/cli.js
@ -1088,6 +1088,42 @@ module.exports.parseCommandLine = function parseCommandLine() {
      default: false,
      describe: 'Ignore robots.txt rules of the crawled domain.',
      group: 'Crawler'
+    })
+
+    .option('scp.host', {
+      describe: 'The host.',
+      group: 'scp'
+    })
+    .option('scp. destinationPath', {
+      describe:
+        'The destionation path on the remote server where the files will be copied.',
+      group: 'scp'
+    })
+    .option('scp.port', {
+      default: 22,
+      describe: 'The port.',
+      group: 'scp'
+    })
+    .option('scp.username', {
+      describe: 'The username. Use username/password or privateKey/pem.',
+      group: 'scp'
+    })
+    .option('scp.password', {
+      describe: 'The password.',
+      group: 'scp'
+    })
+    .option('scp.privateKey', {
+      describe: 'Path to the pem file.',
+      group: 'scp'
+    })
+    .option('scp.passphrase', {
+      describe: 'The passphrase.',
+      group: 'scp'
+    })
+    .option('scp.removeLocalResult', {
+      default: true,
+      describe: 'Remove the files locally when the files has been copied',
+      group: 'scp'
    });

  // Grafana CLI options
--- a/lib/plugins/analysisstorer/index.js
+++ b/lib/plugins/analysisstorer/index.js
@ -25,12 +25,14 @@ function shouldIgnoreMessage(message) {
      'html.css',
      'html.pug',
      's3.finished',
+      'scp.finished',
      'gcs.finished',
      'ftp.finished',
      'graphite.setup',
      'influxdb.setup',
      'grafana.setup',
-      'sustainable.setup'
+      'sustainable.setup',
+      'scp.setup'
    ].indexOf(message.type) >= 0
  );
 }
--- a/lib/plugins/matrix/index.js
+++ b/lib/plugins/matrix/index.js
@ -152,6 +152,7 @@ module.exports = {
        break;
      }
      case 'gcs.finished':
+      case 'scp.finished':
      case 'ftp.finished':
      case 's3.finished': {
        if (this.waitForUpload && options.messages.indexOf('budget') > -1) {
--- a/lib/plugins/scp/index.js
+++ b/lib/plugins/scp/index.js
@ -0,0 +1,128 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const { Client } = require('node-scp');
+const readdir = require('recursive-readdir');
+const log = require('intel').getLogger('sitespeedio.plugin.scp');
+const throwIfMissing = require('../../support/util').throwIfMissing;
+
+async function getClient(scpOptions) {
+  const options = {
+    host: scpOptions.host,
+    port: scpOptions.port || 22
+  };
+  if (scpOptions.username) {
+    options.username = scpOptions.username;
+  }
+  if (scpOptions.password) {
+    options.password = scpOptions.password;
+  }
+  if (scpOptions.privateKey) {
+    options.privateKey = fs.readFileSync(scpOptions.privateKey);
+  }
+  if (scpOptions.passphrase) {
+    options.passphrase = scpOptions.passphrase;
+  }
+  return await Client(options);
+}
+
+async function upload(dir, scpOptions, prefix) {
+  let client;
+  try {
+    client = await getClient(scpOptions);
+    const dirs = prefix.split('/');
+    let fullPath = '';
+    for (let dir of dirs) {
+      fullPath += dir + '/';
+      const doThePathExist = await client.exists(
+        path.join(scpOptions.destinationPath, fullPath)
+      );
+      if (!doThePathExist) {
+        await client.mkdir(path.join(scpOptions.destinationPath, fullPath));
+      }
+    }
+    await client.uploadDir(dir, path.join(scpOptions.destinationPath, prefix));
+  } catch (e) {
+    log.error(e);
+    throw e;
+  } finally {
+    if (client) {
+      client.close();
+    }
+  }
+}
+
+async function uploadFiles(files, scpOptions) {
+  let client;
+  try {
+    client = await getClient(scpOptions);
+    for (let file of files) {
+      await client.uploadFile(
+        file,
+        path.join(scpOptions.destinationPath, path.basename(file))
+      );
+    }
+  } catch (e) {
+    log.error(e);
+    throw e;
+  } finally {
+    if (client) {
+      client.close();
+    }
+  }
+}
+
+async function uploadLatestFiles(dir, scpOptions) {
+  function ignoreDirs(file, stats) {
+    return stats.isDirectory();
+  }
+  const files = await readdir(dir, [ignoreDirs]);
+
+  return uploadFiles(files, scpOptions);
+}
+
+module.exports = {
+  open(context, options) {
+    this.scpOptions = options.scp;
+    this.options = options;
+    this.make = context.messageMaker('scp').make;
+    throwIfMissing(this.scpOptions, ['host', 'destinationPath'], 'scp');
+    this.storageManager = context.storageManager;
+  },
+
+  async processMessage(message, queue) {
+    if (message.type === 'sitespeedio.setup') {
+      // Let other plugins know that the scp plugin is alive
+      queue.postMessage(this.make('scp.setup'));
+    } else if (message.type === 'html.finished') {
+      const make = this.make;
+      const baseDir = this.storageManager.getBaseDir();
+
+      log.info(
+        `Uploading ${baseDir} using scp bucket, this can take a while ...`
+      );
+
+      try {
+        await upload(
+          baseDir,
+          this.scpOptions,
+          this.storageManager.getStoragePrefix()
+        );
+        if (this.options.copyLatestFilesToBase) {
+          const rootPath = path.resolve(baseDir, '..');
+          await uploadLatestFiles(rootPath, this.scpOptions);
+        }
+        log.info('Finished upload using scp');
+        if (this.scpOptions.removeLocalResult) {
+          await fs.remove(baseDir);
+          log.debug(`Removed local files and directory ${baseDir}`);
+        }
+      } catch (e) {
+        queue.postMessage(make('error', e));
+        log.error('Could not upload using scp', e);
+      }
+      queue.postMessage(make('scp.finished'));
+    }
+  }
+};
--- a/lib/plugins/slack/index.js
+++ b/lib/plugins/slack/index.js
@ -208,6 +208,7 @@ module.exports = {

      case 'gcs.finished':
      case 'ftp.finished':
+      case 'scp.finished':
      case 's3.finished': {
        return send(
          this.options,
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@ -40,6 +40,7 @@
        "lodash.set": "4.3.2",
        "lodash.union": "4.6.0",
        "markdown": "0.5.0",
+        "node-scp": "0.0.18",
        "node-slack": "0.0.7",
        "os-name": "4.0.0",
        "p-limit": "2.2.2",
@ -1237,9 +1238,12 @@
      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
    },
    "node_modules/asn1": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
-      "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
+      "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
+      "dependencies": {
+        "safer-buffer": "~2.1.0"
+      }
    },
    "node_modules/assert-never": {
      "version": "1.2.1",
@ -1613,10 +1617,9 @@
      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
    },
    "node_modules/bcrypt-pbkdf": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
-      "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
-      "optional": true,
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
      "dependencies": {
        "tweetnacl": "^0.14.3"
      }
@ -1756,6 +1759,15 @@
      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
    },
+    "node_modules/buildcheck": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.3.tgz",
+      "integrity": "sha512-pziaA+p/wdVImfcbsZLNF32EiWyujlQLwolMqUQE8xpKNOH7KmZQaY8sXN7DGOEzPAElo9QTaeNRfGnf3iOJbA==",
+      "optional": true,
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
    "node_modules/builtin-modules": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
@ -2365,6 +2377,20 @@
      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
    },
+    "node_modules/cpu-features": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.4.tgz",
+      "integrity": "sha512-fKiZ/zp1mUwQbnzb9IghXtHtDoTMtNeb8oYGx6kX2SYfhnG0HNdBEBIzB9b5KlXu5DQPhfy3mInbBxFcgwAr3A==",
+      "hasInstallScript": true,
+      "optional": true,
+      "dependencies": {
+        "buildcheck": "0.0.3",
+        "nan": "^2.15.0"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
    "node_modules/cross-spawn": {
      "version": "7.0.3",
      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@ -5764,6 +5790,12 @@
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
    },
+    "node_modules/nan": {
+      "version": "2.16.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz",
+      "integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==",
+      "optional": true
+    },
    "node_modules/natural-compare": {
      "version": "1.4.0",
      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@ -5819,6 +5851,14 @@
        "node": ">= 6.0.0"
      }
    },
+    "node_modules/node-scp": {
+      "version": "0.0.18",
+      "resolved": "https://registry.npmjs.org/node-scp/-/node-scp-0.0.18.tgz",
+      "integrity": "sha512-PHRXMAb8Wfjtrsr6zz2UbOG+XcyDZUM/r8ZW0aUVxbuXfYex+wR0OD/jqw6TpdkvV+ddNLh+UBNV8oJ3R6Q9RA==",
+      "dependencies": {
+        "ssh2": "^1.6.0"
+      }
+    },
    "node_modules/node-slack": {
      "version": "0.0.7",
      "resolved": "https://registry.npmjs.org/node-slack/-/node-slack-0.0.7.tgz",
@ -7464,6 +7504,23 @@
        "node": ">=4"
      }
    },
+    "node_modules/ssh2": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.11.0.tgz",
+      "integrity": "sha512-nfg0wZWGSsfUe/IBJkXVll3PEZ//YH2guww+mP88gTpuSU4FtZN7zu9JoeTGOyCNx2dTDtT9fOpWwlzyj4uOOw==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "asn1": "^0.2.4",
+        "bcrypt-pbkdf": "^1.0.2"
+      },
+      "engines": {
+        "node": ">=10.16.0"
+      },
+      "optionalDependencies": {
+        "cpu-features": "~0.0.4",
+        "nan": "^2.16.0"
+      }
+    },
    "node_modules/sshpk": {
      "version": "1.14.1",
      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz",
@ -7947,8 +8004,7 @@
    "node_modules/tweetnacl": {
      "version": "0.14.5",
      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
-      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
-      "optional": true
+      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
    },
    "node_modules/type": {
      "version": "1.2.0",
@ -9438,9 +9494,12 @@
      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
    },
    "asn1": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
-      "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
+      "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
+      "requires": {
+        "safer-buffer": "~2.1.0"
+      }
    },
    "assert-never": {
      "version": "1.2.1",
@ -9719,10 +9778,9 @@
      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
    },
    "bcrypt-pbkdf": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
-      "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
-      "optional": true,
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
      "requires": {
        "tweetnacl": "^0.14.3"
      }
@ -9836,6 +9894,12 @@
      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
    },
+    "buildcheck": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.3.tgz",
+      "integrity": "sha512-pziaA+p/wdVImfcbsZLNF32EiWyujlQLwolMqUQE8xpKNOH7KmZQaY8sXN7DGOEzPAElo9QTaeNRfGnf3iOJbA==",
+      "optional": true
+    },
    "builtin-modules": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
@ -10315,6 +10379,16 @@
      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
    },
+    "cpu-features": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.4.tgz",
+      "integrity": "sha512-fKiZ/zp1mUwQbnzb9IghXtHtDoTMtNeb8oYGx6kX2SYfhnG0HNdBEBIzB9b5KlXu5DQPhfy3mInbBxFcgwAr3A==",
+      "optional": true,
+      "requires": {
+        "buildcheck": "0.0.3",
+        "nan": "^2.15.0"
+      }
+    },
    "cross-spawn": {
      "version": "7.0.3",
      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@ -12938,6 +13012,12 @@
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
    },
+    "nan": {
+      "version": "2.16.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz",
+      "integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==",
+      "optional": true
+    },
    "natural-compare": {
      "version": "1.4.0",
      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@ -12973,6 +13053,14 @@
      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
      "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA=="
    },
+    "node-scp": {
+      "version": "0.0.18",
+      "resolved": "https://registry.npmjs.org/node-scp/-/node-scp-0.0.18.tgz",
+      "integrity": "sha512-PHRXMAb8Wfjtrsr6zz2UbOG+XcyDZUM/r8ZW0aUVxbuXfYex+wR0OD/jqw6TpdkvV+ddNLh+UBNV8oJ3R6Q9RA==",
+      "requires": {
+        "ssh2": "^1.6.0"
+      }
+    },
    "node-slack": {
      "version": "0.0.7",
      "resolved": "https://registry.npmjs.org/node-slack/-/node-slack-0.0.7.tgz",
@ -14244,6 +14332,17 @@
        "node-addon-api": "^1.3.0"
      }
    },
+    "ssh2": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.11.0.tgz",
+      "integrity": "sha512-nfg0wZWGSsfUe/IBJkXVll3PEZ//YH2guww+mP88gTpuSU4FtZN7zu9JoeTGOyCNx2dTDtT9fOpWwlzyj4uOOw==",
+      "requires": {
+        "asn1": "^0.2.4",
+        "bcrypt-pbkdf": "^1.0.2",
+        "cpu-features": "~0.0.4",
+        "nan": "^2.16.0"
+      }
+    },
    "sshpk": {
      "version": "1.14.1",
      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz",
@ -14621,8 +14720,7 @@
    "tweetnacl": {
      "version": "0.14.5",
      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
-      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
-      "optional": true
+      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
    },
    "type": {
      "version": "1.2.0",
--- a/package.json
+++ b/package.json
@ -107,6 +107,7 @@
    "lodash.set": "4.3.2",
    "lodash.union": "4.6.0",
    "markdown": "0.5.0",
+    "node-scp":"0.0.18",
    "node-slack": "0.0.7",
    "os-name": "4.0.0",
    "p-limit": "2.2.2",