Replace p-limit (#4394)

lib/plugins/s3/index.js

@@ -1,7 +1,6 @@
 import path from 'node:path';
 import { promises as fsPromises } from 'node:fs';
 
-import pLimit from 'p-limit';
 import { getLogger } from '@sitespeed.io/log';
 import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
 
@@ -9,6 +8,7 @@ import { SitespeedioPlugin } from '@sitespeed.io/plugin';
 import { throwIfMissing } from '../../support/util.js';
 import { recursiveReaddir } from '../../support/fileUtil.js';
 import { getContentType } from './contentType.js';
+import { runWithConcurrencyLimit } from './limit.js';
 
 const log = getLogger('sitespeedio.plugin.s3');
 
@@ -86,31 +86,36 @@ export default class S3Plugin extends SitespeedioPlugin {
 
       try {
         const files = await recursiveReaddir(baseDir, true);
-        const limit = pLimit(s3Options.maxAsyncS3 || 20);
-        const uploadPromises = files.map(file =>
-          limit(() =>
-            uploadFile(
-              file,
-              this.s3Client,
-              s3Options,
-              this.storageManager.getStoragePrefix(),
-              baseDir
-            )
-          )
-        );
-        await Promise.all(uploadPromises);
+        const tasks = files.map(file => async () => {
+          return uploadFile(
+            file,
+            this.s3Client,
+            s3Options,
+            this.storageManager.getStoragePrefix(),
+            baseDir
+          );
+        });
+
+        await runWithConcurrencyLimit(tasks, s3Options.maxAsyncS3 || 20);
 
         if (this.options.copyLatestFilesToBase) {
           const rootPath = path.join(baseDir, '..');
           const directoriesAsArray = rootPath.split(path.sep);
           const rootName = directoriesAsArray.at(-1);
           const latestFiles = await recursiveReaddir(rootPath, true);
-          const latestUploadPromises = latestFiles.map(file =>
-            limit(() =>
-              uploadFile(file, this.s3Client, s3Options, rootName, rootPath)
-            )
+          const latestTasks = latestFiles.map(file => async () => {
+            return uploadFile(
+              file,
+              this.s3Client,
+              s3Options,
+              rootName,
+              rootPath
+            );
+          });
+          await runWithConcurrencyLimit(
+            latestTasks,
+            s3Options.maxAsyncS3 || 20
           );
-          await Promise.all(latestUploadPromises);
         }
 
         log.info('Finished upload to S3');

lib/plugins/s3/limit.js

@@ -0,0 +1,34 @@
+export async function runWithConcurrencyLimit(tasks, limit) {
+  const running = new Set();
+
+  async function runNext() {
+    if (tasks.length === 0) {
+      return;
+    }
+
+    const task = tasks.shift();
+    const promise = task()
+      .catch(error => {
+        throw error;
+      })
+      .finally(() => {
+        running.delete(promise);
+        void runNext();
+      });
+
+    running.add(promise);
+    if (running.size < limit) {
+      void runNext();
+    }
+  }
+
+  const starters = [];
+  for (let index = 0; index < limit && tasks.length > 0; index++) {
+    starters.push(runNext());
+  }
+
+  await Promise.allSettled(starters);
+  if (running.size > 0) {
+    await Promise.allSettled(Array.from(running));
+  }
+}

npm-shrinkwrap.json

@@ -33,7 +33,6 @@
         "markdown": "0.5.0",
         "node-scp": "0.0.23",
         "ora": "8.0.1",
-        "p-limit": "6.1.0",
         "pug": "3.0.3",
         "simplecrawler": "1.1.9",
         "tape": "5.8.1",
@@ -7768,31 +7767,6 @@
         "os-tmpdir": "^1.0.0"
       }
     },
-    "node_modules/p-limit": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz",
-      "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==",
-      "dependencies": {
-        "yocto-queue": "^1.1.1"
-      },
-      "engines": {
-        "node": ">=18"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/p-limit/node_modules/yocto-queue": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz",
-      "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==",
-      "engines": {
-        "node": ">=12.20"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/p-map": {
       "version": "7.0.2",
       "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.2.tgz",

package.json

@@ -104,7 +104,6 @@
     "markdown": "0.5.0",
     "node-scp": "0.0.23",
     "ora": "8.0.1",
-    "p-limit": "6.1.0",
     "pug": "3.0.3",
     "simplecrawler": "1.1.9",
     "tape": "5.8.1",