feat: option for synchronous file reading

src/config/app.ts

@@ -92,4 +92,7 @@ export const storageConfig: MemoryStorageConfig = {
 
 	// Number of milliseconds to keep item in storage after last use, > minExpiration
 	cleanupAfter: 0,
+
+	// Asynchronous reading of cache from file system
+	asyncRead: true,
 };

src/data/storage/load.ts

@@ -1,4 +1,4 @@
-import { readFile } from 'node:fs';
+import { readFile, readFileSync } from 'node:fs';
 import type { MemoryStorage, MemoryStorageItem } from '../../types/storage';
 import { runStorageCallbacks } from './callbacks';
 import { addStorageToCleanup } from './cleanup';
@@ -20,22 +20,44 @@ export function loadStoredItem<T>(storage: MemoryStorage<T>, storedItem: MemoryS
 	}
 
 	// Load file
-	pendingReads.add(storedItem);
+	let failed = (error: unknown) => {
-	readFile(config.filename, 'utf8', (err, dataStr) => {
+		console.error(error);
-		pendingReads.delete(storedItem);
+		runStorageCallbacks(storedItem, true);
-
+	};
-		if (err) {
+	let loaded = (dataStr: string) => {
-			// Failed
-			console.error(err);
-			runStorageCallbacks(storedItem, true);
-			return;
-		}
-
 		// Loaded
 		storedItem.data = JSON.parse(dataStr) as T;
 		runStorageCallbacks(storedItem);
 
 		// Add to cleanup queue
 		addStorageToCleanup(storage, storedItem);
-	});
+	};
+
+	if (storage.config.asyncRead) {
+		// Load asynchronously
+		pendingReads.add(storedItem);
+		readFile(config.filename, 'utf8', (err, dataStr) => {
+			pendingReads.delete(storedItem);
+
+			if (err) {
+				// Failed
+				failed(err);
+				return;
+			}
+
+			// Loaded
+			loaded(dataStr);
+		});
+	} else {
+		// Load synchronously
+		let dataStr: string;
+		try {
+			dataStr = readFileSync(config.filename, 'utf8');
+		} catch (err) {
+			// Failed
+			failed(err);
+			return;
+		}
+		loaded(dataStr);
+	}
 }

src/types/storage.ts

@@ -52,6 +52,9 @@ export interface MemoryStorageConfig {
 	// Number of milliseconds to keep item in storage after last use, > minExpiration
 	cleanupAfter?: number;
 
+	// Asynchronous reading
+	asyncRead?: boolean;
+
 	// Timer callback, used for debugging and testing. Called before cleanup when its triggered by timer
 	timerCallback?: () => void;
 }

tests/data/storage-long-test.ts

@@ -18,6 +18,7 @@ describe('Advanced storage tests', () => {
 				const storage = createStorage<Item>({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Create items

tests/data/storage-read-test.ts

@@ -13,12 +13,14 @@ describe('Reading stored data', () => {
 				const storage = createStorage({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Config
 				expect(storage.config).toEqual({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Timer should not exist
@@ -70,12 +72,14 @@ describe('Reading stored data', () => {
 				const storage = createStorage({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Config
 				expect(storage.config).toEqual({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Timer should not exist
@@ -127,12 +131,14 @@ describe('Reading stored data', () => {
 				const storage = createStorage({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Config
 				expect(storage.config).toEqual({
 					cacheDir,
 					maxCount: 2,
+					asyncRead: true,
 				});
 
 				// Timer should not exist
@@ -206,7 +212,7 @@ describe('Reading stored data', () => {
 		});
 	});
 
-	test('Error reading cache', () => {
+	test('Error reading cache asynchronously', () => {
 		return new Promise((fulfill, reject) => {
 			try {
 				const dir = uniqueCacheDir();
@@ -216,6 +222,53 @@ describe('Reading stored data', () => {
 				const storage = createStorage({
 					cacheDir,
 					maxCount: 1,
+					asyncRead: true,
+				});
+
+				// Add one item
+				const content = {
+					test: true,
+				};
+				createStoredItem(storage, content, 'foo.json', true, (item, err) => {
+					try {
+						// Data should be written to cache
+						expect(item.data).toBeUndefined();
+
+						// Remove cache file
+						unlinkSync(actualCacheDir + '/foo.json');
+
+						// Attempt to read data
+						getStoredItem(storage, item, (data) => {
+							try {
+								expect(data).toBeFalsy();
+								fulfill(true);
+							} catch (err) {
+								reject(err);
+							}
+						});
+					} catch (err) {
+						reject(err);
+					}
+				});
+
+				// Test continues in callback in createStoredItem()...
+			} catch (err) {
+				reject(err);
+			}
+		});
+	});
+
+	test('Error reading cache synchronously', () => {
+		return new Promise((fulfill, reject) => {
+			try {
+				const dir = uniqueCacheDir();
+				const cacheDir = '{cache}/' + dir;
+				const actualCacheDir = cacheDir.replace('{cache}', appConfig.cacheRootDir);
+
+				const storage = createStorage({
+					cacheDir,
+					maxCount: 1,
+					asyncRead: false,
 				});
 
 				// Add one item

tests/icon-set/get-stored-icon-test.ts

@@ -173,7 +173,53 @@ describe('Loading icon data from storage', () => {
 				const name = 'star';
 				let isSync1 = true;
 
-				// First run should be async
+				// First run should be async if loader uses async read, synchronous if loaded uses sync read
+				getStoredIconData(storedIconSet, name, () => {
+					let isSync2 = true;
+
+					// Second run should be synchronous
+					getStoredIconData(storedIconSet, name, () => {
+						fulfill(isSync2 === true && isSync1 === true);
+					});
+					isSync2 = false;
+				});
+				isSync1 = false;
+			});
+		}
+
+		// Load icon
+		expect(await syncTest()).toBeTruthy();
+	});
+
+	test('Asynchronous loading', async () => {
+		const iconSet = JSON.parse(await loadFixture('json/mdi-light.json')) as IconifyJSON;
+
+		function store(): Promise<StoredIconSet> {
+			return new Promise((fulfill, reject) => {
+				// Create storage
+				const dir = uniqueCacheDir();
+				const cacheDir = '{cache}/' + dir;
+				const storage = createStorage<IconifyIcons>({
+					cacheDir,
+					maxCount: 2,
+					asyncRead: true,
+				});
+
+				// Split icon set
+				storeLoadedIconSet(iconSet, fulfill, storage, {
+					chunkSize: 5000,
+					minIconsPerChunk: 10,
+				});
+			});
+		}
+		const storedIconSet = await store();
+
+		function syncTest(): Promise<boolean> {
+			return new Promise((fulfill, reject) => {
+				const name = 'star';
+				let isSync1 = true;
+
+				// First run should be async if loader uses async read, synchronous if loaded uses sync read
 				getStoredIconData(storedIconSet, name, () => {
 					let isSync2 = true;
 

tests/icon-set/get-stored-icons-test.ts

@@ -188,7 +188,53 @@ describe('Loading icons from storage', () => {
 				const names: string[] = ['abacus', 'floor-1', 'star', 'wifi'];
 				let isSync1 = true;
 
-				// First run should be async
+				// First run should be async if loader uses async read, synchronous if loaded uses sync read
+				getStoredIconsData(storedIconSet, names, () => {
+					let isSync2 = true;
+
+					// Second run should be synchronous
+					getStoredIconsData(storedIconSet, names, () => {
+						fulfill(isSync2 === true && isSync1 === true);
+					});
+					isSync2 = false;
+				});
+				isSync1 = false;
+			});
+		}
+
+		// Load icons
+		expect(await syncTest()).toBeTruthy();
+	});
+
+	test('Asynchronous loading', async () => {
+		const iconSet = JSON.parse(await loadFixture('json/mdi-light.json')) as IconifyJSON;
+
+		function store(): Promise<StoredIconSet> {
+			return new Promise((fulfill, reject) => {
+				// Create storage
+				const dir = uniqueCacheDir();
+				const cacheDir = '{cache}/' + dir;
+				const storage = createStorage<IconifyIcons>({
+					cacheDir,
+					maxCount: 5,
+					asyncRead: true,
+				});
+
+				// Split icon set
+				storeLoadedIconSet(iconSet, fulfill, storage, {
+					chunkSize: 5000,
+					minIconsPerChunk: 10,
+				});
+			});
+		}
+		const storedIconSet = await store();
+
+		function syncTest(): Promise<boolean> {
+			return new Promise((fulfill, reject) => {
+				const names: string[] = ['abacus', 'floor-1', 'star', 'wifi'];
+				let isSync1 = true;
+
+				// First run should be async if loader uses async read, synchronous if loaded uses sync read
 				getStoredIconsData(storedIconSet, names, () => {
 					let isSync2 = true;