From a701024701462a7d40ec433e8a6824960b22fbce Mon Sep 17 00:00:00 2001 From: Vjacheslav Trushkin Date: Mon, 14 Nov 2022 12:30:01 +0200 Subject: [PATCH] feat: option for synchronous file reading --- src/config/app.ts | 3 ++ src/data/storage/load.ts | 48 +++++++++++++++------ src/types/storage.ts | 3 ++ tests/data/storage-long-test.ts | 1 + tests/data/storage-read-test.ts | 55 ++++++++++++++++++++++++- tests/icon-set/get-stored-icon-test.ts | 48 ++++++++++++++++++++- tests/icon-set/get-stored-icons-test.ts | 48 ++++++++++++++++++++- 7 files changed, 190 insertions(+), 16 deletions(-) diff --git a/src/config/app.ts b/src/config/app.ts index 3648bfc..6a2c0e9 100644 --- a/src/config/app.ts +++ b/src/config/app.ts @@ -92,4 +92,7 @@ export const storageConfig: MemoryStorageConfig = { // Number of milliseconds to keep item in storage after last use, > minExpiration cleanupAfter: 0, + + // Asynchronous reading of cache from file system + asyncRead: true, }; diff --git a/src/data/storage/load.ts b/src/data/storage/load.ts index 1b70bc9..c6f80f6 100644 --- a/src/data/storage/load.ts +++ b/src/data/storage/load.ts @@ -1,4 +1,4 @@ -import { readFile } from 'node:fs'; +import { readFile, readFileSync } from 'node:fs'; import type { MemoryStorage, MemoryStorageItem } from '../../types/storage'; import { runStorageCallbacks } from './callbacks'; import { addStorageToCleanup } from './cleanup'; @@ -20,22 +20,44 @@ export function loadStoredItem(storage: MemoryStorage, storedItem: MemoryS } // Load file - pendingReads.add(storedItem); - readFile(config.filename, 'utf8', (err, dataStr) => { - pendingReads.delete(storedItem); - - if (err) { - // Failed - console.error(err); - runStorageCallbacks(storedItem, true); - return; - } - + let failed = (error: unknown) => { + console.error(error); + runStorageCallbacks(storedItem, true); + }; + let loaded = (dataStr: string) => { // Loaded storedItem.data = JSON.parse(dataStr) as T; runStorageCallbacks(storedItem); // Add to cleanup queue addStorageToCleanup(storage, storedItem); - }); + }; + + if (storage.config.asyncRead) { + // Load asynchronously + pendingReads.add(storedItem); + readFile(config.filename, 'utf8', (err, dataStr) => { + pendingReads.delete(storedItem); + + if (err) { + // Failed + failed(err); + return; + } + + // Loaded + loaded(dataStr); + }); + } else { + // Load synchronously + let dataStr: string; + try { + dataStr = readFileSync(config.filename, 'utf8'); + } catch (err) { + // Failed + failed(err); + return; + } + loaded(dataStr); + } } diff --git a/src/types/storage.ts b/src/types/storage.ts index 8111d0b..f61b782 100644 --- a/src/types/storage.ts +++ b/src/types/storage.ts @@ -52,6 +52,9 @@ export interface MemoryStorageConfig { // Number of milliseconds to keep item in storage after last use, > minExpiration cleanupAfter?: number; + // Asynchronous reading + asyncRead?: boolean; + // Timer callback, used for debugging and testing. Called before cleanup when its triggered by timer timerCallback?: () => void; } diff --git a/tests/data/storage-long-test.ts b/tests/data/storage-long-test.ts index 128cd5d..64dba89 100644 --- a/tests/data/storage-long-test.ts +++ b/tests/data/storage-long-test.ts @@ -18,6 +18,7 @@ describe('Advanced storage tests', () => { const storage = createStorage({ cacheDir, maxCount: 2, + asyncRead: true, }); // Create items diff --git a/tests/data/storage-read-test.ts b/tests/data/storage-read-test.ts index 916a332..a602c4a 100644 --- a/tests/data/storage-read-test.ts +++ b/tests/data/storage-read-test.ts @@ -13,12 +13,14 @@ describe('Reading stored data', () => { const storage = createStorage({ cacheDir, maxCount: 2, + asyncRead: true, }); // Config expect(storage.config).toEqual({ cacheDir, maxCount: 2, + asyncRead: true, }); // Timer should not exist @@ -70,12 +72,14 @@ describe('Reading stored data', () => { const storage = createStorage({ cacheDir, maxCount: 2, + asyncRead: true, }); // Config expect(storage.config).toEqual({ cacheDir, maxCount: 2, + asyncRead: true, }); // Timer should not exist @@ -127,12 +131,14 @@ describe('Reading stored data', () => { const storage = createStorage({ cacheDir, maxCount: 2, + asyncRead: true, }); // Config expect(storage.config).toEqual({ cacheDir, maxCount: 2, + asyncRead: true, }); // Timer should not exist @@ -206,7 +212,7 @@ describe('Reading stored data', () => { }); }); - test('Error reading cache', () => { + test('Error reading cache asynchronously', () => { return new Promise((fulfill, reject) => { try { const dir = uniqueCacheDir(); @@ -216,6 +222,53 @@ describe('Reading stored data', () => { const storage = createStorage({ cacheDir, maxCount: 1, + asyncRead: true, + }); + + // Add one item + const content = { + test: true, + }; + createStoredItem(storage, content, 'foo.json', true, (item, err) => { + try { + // Data should be written to cache + expect(item.data).toBeUndefined(); + + // Remove cache file + unlinkSync(actualCacheDir + '/foo.json'); + + // Attempt to read data + getStoredItem(storage, item, (data) => { + try { + expect(data).toBeFalsy(); + fulfill(true); + } catch (err) { + reject(err); + } + }); + } catch (err) { + reject(err); + } + }); + + // Test continues in callback in createStoredItem()... + } catch (err) { + reject(err); + } + }); + }); + + test('Error reading cache synchronously', () => { + return new Promise((fulfill, reject) => { + try { + const dir = uniqueCacheDir(); + const cacheDir = '{cache}/' + dir; + const actualCacheDir = cacheDir.replace('{cache}', appConfig.cacheRootDir); + + const storage = createStorage({ + cacheDir, + maxCount: 1, + asyncRead: false, }); // Add one item diff --git a/tests/icon-set/get-stored-icon-test.ts b/tests/icon-set/get-stored-icon-test.ts index f8cfc03..e059083 100644 --- a/tests/icon-set/get-stored-icon-test.ts +++ b/tests/icon-set/get-stored-icon-test.ts @@ -173,7 +173,53 @@ describe('Loading icon data from storage', () => { const name = 'star'; let isSync1 = true; - // First run should be async + // First run should be async if loader uses async read, synchronous if loaded uses sync read + getStoredIconData(storedIconSet, name, () => { + let isSync2 = true; + + // Second run should be synchronous + getStoredIconData(storedIconSet, name, () => { + fulfill(isSync2 === true && isSync1 === true); + }); + isSync2 = false; + }); + isSync1 = false; + }); + } + + // Load icon + expect(await syncTest()).toBeTruthy(); + }); + + test('Asynchronous loading', async () => { + const iconSet = JSON.parse(await loadFixture('json/mdi-light.json')) as IconifyJSON; + + function store(): Promise { + return new Promise((fulfill, reject) => { + // Create storage + const dir = uniqueCacheDir(); + const cacheDir = '{cache}/' + dir; + const storage = createStorage({ + cacheDir, + maxCount: 2, + asyncRead: true, + }); + + // Split icon set + storeLoadedIconSet(iconSet, fulfill, storage, { + chunkSize: 5000, + minIconsPerChunk: 10, + }); + }); + } + const storedIconSet = await store(); + + function syncTest(): Promise { + return new Promise((fulfill, reject) => { + const name = 'star'; + let isSync1 = true; + + // First run should be async if loader uses async read, synchronous if loaded uses sync read getStoredIconData(storedIconSet, name, () => { let isSync2 = true; diff --git a/tests/icon-set/get-stored-icons-test.ts b/tests/icon-set/get-stored-icons-test.ts index 476e448..281dae7 100644 --- a/tests/icon-set/get-stored-icons-test.ts +++ b/tests/icon-set/get-stored-icons-test.ts @@ -188,7 +188,53 @@ describe('Loading icons from storage', () => { const names: string[] = ['abacus', 'floor-1', 'star', 'wifi']; let isSync1 = true; - // First run should be async + // First run should be async if loader uses async read, synchronous if loaded uses sync read + getStoredIconsData(storedIconSet, names, () => { + let isSync2 = true; + + // Second run should be synchronous + getStoredIconsData(storedIconSet, names, () => { + fulfill(isSync2 === true && isSync1 === true); + }); + isSync2 = false; + }); + isSync1 = false; + }); + } + + // Load icons + expect(await syncTest()).toBeTruthy(); + }); + + test('Asynchronous loading', async () => { + const iconSet = JSON.parse(await loadFixture('json/mdi-light.json')) as IconifyJSON; + + function store(): Promise { + return new Promise((fulfill, reject) => { + // Create storage + const dir = uniqueCacheDir(); + const cacheDir = '{cache}/' + dir; + const storage = createStorage({ + cacheDir, + maxCount: 5, + asyncRead: true, + }); + + // Split icon set + storeLoadedIconSet(iconSet, fulfill, storage, { + chunkSize: 5000, + minIconsPerChunk: 10, + }); + }); + } + const storedIconSet = await store(); + + function syncTest(): Promise { + return new Promise((fulfill, reject) => { + const names: string[] = ['abacus', 'floor-1', 'star', 'wifi']; + let isSync1 = true; + + // First run should be async if loader uses async read, synchronous if loaded uses sync read getStoredIconsData(storedIconSet, names, () => { let isSync2 = true;