From ec354b9d33a87ba00bd9cbc1e728115ab5faf6f7 Mon Sep 17 00:00:00 2001 From: Vjacheslav Trushkin Date: Sun, 16 Oct 2022 11:43:17 +0300 Subject: [PATCH] chore: change icon names list structure for better performance --- src/data/icon-set/lists/icons-v2.ts | 26 ++---- src/data/icon-set/lists/icons.ts | 2 + src/data/icon-set/lists/validate.ts | 26 ++++-- src/data/icon-set/store/storage.ts | 4 +- src/data/icon-set/utils/get-icon.ts | 2 +- src/data/icon-set/utils/get-icons.ts | 126 ++++++++------------------- src/http/responses/collection-v1.ts | 28 +++--- src/http/responses/collection-v2.ts | 6 +- src/http/responses/svg.ts | 13 ++- src/types/icon-set/extra.ts | 52 +++++------ tests/icon-set/validate-test.ts | 4 +- 11 files changed, 122 insertions(+), 167 deletions(-) diff --git a/src/data/icon-set/lists/icons-v2.ts b/src/data/icon-set/lists/icons-v2.ts index 4cee235..41b072b 100644 --- a/src/data/icon-set/lists/icons-v2.ts +++ b/src/data/icon-set/lists/icons-v2.ts @@ -9,24 +9,24 @@ const themeKeys: ThemeKey[] = ['themes', 'prefixes', 'suffixes']; */ export function prepareAPIv2IconsList(iconSet: IconifyJSON, iconsList: IconSetIconsListIcons): IconSetAPIv2IconsList { // Prepare data - const rendered: IconSetAPIv2IconsList['rendered'] = { + const result: IconSetAPIv2IconsList = { prefix: iconSet.prefix, total: iconsList.visible.size, }; const info = iconSet.info; if (info) { - rendered.title = info.name; - rendered.info = info; + result.title = info.name; + result.info = info; } if (iconsList.uncategorised.length) { - rendered.uncategorized = iconsList.uncategorised; + result.uncategorized = iconsList.uncategorised; } // Convert categories if (iconsList.tags.length) { - const categories = (rendered.categories = Object.create(null) as Record); + const categories = (result.categories = Object.create(null) as Record); for (let i = 0; i < iconsList.tags.length; i++) { const tag = iconsList.tags[i]; categories[tag.title] = tag.icons; @@ -36,7 +36,7 @@ export function prepareAPIv2IconsList(iconSet: IconifyJSON, iconsList: IconSetIc // Hidden icons const hidden = Array.from(iconsList.hidden).concat(Object.keys(iconsList.hiddenAliases)); if (hidden.length) { - rendered.hidden = hidden; + result.hidden = hidden; } // Add aliases @@ -45,7 +45,7 @@ export function prepareAPIv2IconsList(iconSet: IconifyJSON, iconsList: IconSetIc ...iconsList.hiddenAliases, }; for (const alias in aliases) { - rendered.aliases = aliases; + result.aliases = aliases; break; } @@ -53,19 +53,9 @@ export function prepareAPIv2IconsList(iconSet: IconifyJSON, iconsList: IconSetIc for (let i = 0; i < themeKeys.length; i++) { const key = themeKeys[i] as ThemeKey; if (iconSet[key]) { - rendered[key as 'themes'] = iconSet[key as 'themes']; + result[key as 'themes'] = iconSet[key as 'themes']; } } - // Result - const result: IconSetAPIv2IconsList = { - rendered, - }; - - // Add characters - if (iconSet.chars) { - result.chars = iconSet.chars; - } - return result; } diff --git a/src/data/icon-set/lists/icons.ts b/src/data/icon-set/lists/icons.ts index 7da4072..f288cce 100644 --- a/src/data/icon-set/lists/icons.ts +++ b/src/data/icon-set/lists/icons.ts @@ -145,6 +145,7 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList // Return data return { + names: new Set([...visible, ...hidden, ...Object.keys(visibleAliases), ...Object.keys(hiddenAliases)]), visible, hidden, visibleAliases, @@ -152,5 +153,6 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList failed, tags: tags.filter((tag) => tag.icons.length > 0), uncategorised, + chars: iconSet.chars, }; } diff --git a/src/data/icon-set/lists/validate.ts b/src/data/icon-set/lists/validate.ts index 0417579..bdd33da 100644 --- a/src/data/icon-set/lists/validate.ts +++ b/src/data/icon-set/lists/validate.ts @@ -2,14 +2,26 @@ import type { IconifyJSON } from '@iconify/types'; import type { IconSetIconsListIcons } from '../../../types/icon-set/extra'; /** - * Removes bad aliases + * Removes bad items */ -export function removeBadAliases(data: IconifyJSON, iconsList: IconSetIconsListIcons) { +export function removeBadIconSetItems(data: IconifyJSON, iconsList: IconSetIconsListIcons) { + // Remove bad aliases const aliases = data.aliases; - if (!aliases) { - return; + if (aliases) { + iconsList.failed.forEach((name) => { + delete aliases[name]; + }); + } + + // Remove bad characters + const chars = iconsList.chars; + if (chars) { + for (const key in chars) { + if (iconsList.names.has(key) || !iconsList.names.has(chars[key])) { + // Character matches existing icon or points to missing icon + // Also deletes data.chars[key] because it points to same object + delete chars[key]; + } + } } - iconsList.failed.forEach((name) => { - delete aliases[name]; - }); } diff --git a/src/data/icon-set/store/storage.ts b/src/data/icon-set/store/storage.ts index 0063eb2..39bdf1c 100644 --- a/src/data/icon-set/store/storage.ts +++ b/src/data/icon-set/store/storage.ts @@ -8,7 +8,7 @@ import { createSplitRecordsTree, splitRecords } from '../../storage/split'; import { createStorage, createStoredItem } from '../../storage/create'; import { getIconSetSplitChunksCount, splitIconSetMainData } from './split'; import { generateIconSetIconsTree } from '../lists/icons'; -import { removeBadAliases } from '../lists/validate'; +import { removeBadIconSetItems } from '../lists/validate'; import { prepareAPIv2IconsList } from '../lists/icons-v2'; /** @@ -33,7 +33,7 @@ export function storeLoadedIconSet( ) { // Get icons list and remove bad aliases const icons = generateIconSetIconsTree(iconSet); - removeBadAliases(iconSet, icons); + removeBadIconSetItems(iconSet, icons); // Fix icons counter if (iconSet.info) { diff --git a/src/data/icon-set/utils/get-icon.ts b/src/data/icon-set/utils/get-icon.ts index b4c3576..ddf7513 100644 --- a/src/data/icon-set/utils/get-icon.ts +++ b/src/data/icon-set/utils/get-icon.ts @@ -74,7 +74,7 @@ export function getStoredIconData( name = resolved.name; } else { props = {} as ExtendedIconifyAlias; - const charValue = iconSet.apiV2IconsCache.chars?.[name]; + const charValue = iconSet.icons.chars?.[name]; if (charValue) { // Character const icons = iconSet.icons; diff --git a/src/data/icon-set/utils/get-icons.ts b/src/data/icon-set/utils/get-icons.ts index 5d09e92..6911a0d 100644 --- a/src/data/icon-set/utils/get-icons.ts +++ b/src/data/icon-set/utils/get-icons.ts @@ -10,28 +10,29 @@ import { getStoredItem } from '../../storage/get'; export function getIconsToRetrieve(iconSet: StoredIconSet, names: string[], copyTo?: IconifyAliases): Set { const icons: Set = new Set(); const iconSetData = iconSet.common; + const allNames = iconSet.icons.names; + const chars = iconSet.icons.chars; const aliases = iconSetData.aliases || (Object.create(null) as IconifyAliases); function resolve(name: string, nested: boolean) { - if (!aliases[name]) { - if (!nested) { - // Check for character - const charValue = iconSet.apiV2IconsCache.chars?.[name]; - if (charValue) { - // Character - const icons = iconSet.icons; - if (!icons.visible.has(name) && !icons.hidden.has(name)) { - // Resolve character instead of alias - copyTo && - (copyTo[name] = { - parent: charValue, - }); - resolve(charValue, true); - return; - } - } + if (!allNames.has(name)) { + // No such icon: check for character + const charValue = chars?.[name]; + if (!charValue) { + return; } + // Resolve character instead of alias + copyTo && + (copyTo[name] = { + parent: charValue, + }); + resolve(charValue, true); + return; + } + + // Icon or alias exists + if (!aliases[name]) { // Icon icons.add(name); return; @@ -52,72 +53,6 @@ export function getIconsToRetrieve(iconSet: StoredIconSet, names: string[], copy return icons; } -/** - * Extract icons from chunks of icon data - */ -export function getIconsData( - iconSetData: SplitIconifyJSONMainData, - names: string[], - sourceIcons: IconifyIcons[], - chars?: Record -): IconifyJSON { - const sourceAliases = iconSetData.aliases; - const icons = Object.create(null) as IconifyJSON['icons']; - const aliases = Object.create(null) as IconifyAliases; - - const result: IconifyJSON = { - ...iconSetData, - icons, - aliases, - }; - - function resolve(name: string, nested: boolean): boolean { - if (!sourceAliases[name]) { - // Icon - for (let i = 0; i < sourceIcons.length; i++) { - const item = sourceIcons[i]; - if (name in item) { - icons[name] = item[name]; - return true; - } - } - - // Check for character - if (!nested) { - const charValue = chars?.[name]; - if (charValue) { - aliases[name] = { - parent: charValue, - }; - return resolve(charValue, true); - } - } - } else if (name in sourceAliases) { - // Alias - if (name in aliases) { - // Already resolved - return true; - } - - const item = sourceAliases[name]; - if (resolve(item.parent, true)) { - aliases[name] = item; - return true; - } - } - - // Missing - (result.not_found || (result.not_found = [])).push(name); - return false; - } - - for (let i = 0; i < names.length; i++) { - resolve(names[i], false); - } - - return result; -} - /** * Get icons from stored icon set */ @@ -139,22 +74,22 @@ export function getStoredIconsData(iconSet: StoredIconSet, names: string[], call // Get map of chunks to load const chunks = searchSplitRecordsTreeForSet(iconSet.tree, iconNames); let pending = chunks.size; - let not_found: string[] | undefined; + let missing: Set = new Set(); const icons = Object.create(null) as IconifyIcons; const storage = iconSet.storage; - chunks.forEach((names, storedItem) => { + chunks.forEach((chunkNames, storedItem) => { getStoredItem(storage, storedItem, (data) => { // Copy data from chunk if (!data) { - not_found = names.concat(not_found || []); + missing = new Set([...chunkNames, ...missing]); } else { - for (let i = 0; i < names.length; i++) { - const name = names[i]; + for (let i = 0; i < chunkNames.length; i++) { + const name = chunkNames[i]; if (data[name]) { icons[name] = data[name]; } else { - (not_found || (not_found = [])).push(name); + missing.add(name); } } } @@ -167,8 +102,17 @@ export function getStoredIconsData(iconSet: StoredIconSet, names: string[], call icons, aliases, }; - if (not_found) { - result.not_found = not_found; + + // Add missing icons + for (let i = 0; i < names.length; i++) { + const name = names[i]; + if (!icons[name] && !aliases[name]) { + missing.add(name); + } + } + + if (missing.size) { + result.not_found = Array.from(missing); } callback(result); } diff --git a/src/http/responses/collection-v1.ts b/src/http/responses/collection-v1.ts index b172f48..dacbbb0 100644 --- a/src/http/responses/collection-v1.ts +++ b/src/http/responses/collection-v1.ts @@ -33,35 +33,35 @@ export function generateAPIv1IconsListResponse( } function parse(prefix: string, iconSet: StoredIconSet): APIv1ListIconsResponse | APIv1ListIconsCategorisedResponse { + const icons = iconSet.icons; const v2Cache = iconSet.apiV2IconsCache; - const rendered = v2Cache.rendered; // Generate common data const base: APIv1ListIconsBaseResponse = { prefix, - total: rendered.total, + total: v2Cache.total, }; - if (rendered.title) { - base.title = rendered.title; + if (v2Cache.title) { + base.title = v2Cache.title; } - if (q.info && rendered.info) { - base.info = rendered.info; + if (q.info && v2Cache.info) { + base.info = v2Cache.info; } - if (q.aliases && rendered.aliases) { - base.aliases = rendered.aliases; + if (q.aliases && v2Cache.aliases) { + base.aliases = v2Cache.aliases; } - if (q.chars && v2Cache.chars) { - base.chars = v2Cache.chars; + if (q.chars && icons.chars) { + base.chars = icons.chars; } // Add icons if (categorised) { const result = base as APIv1ListIconsCategorisedResponse; - if (rendered.categories) { - result.categories = rendered.categories; + if (v2Cache.categories) { + result.categories = v2Cache.categories; } - if (rendered.uncategorized) { - result.uncategorized = rendered.uncategorized; + if (v2Cache.uncategorized) { + result.uncategorized = v2Cache.uncategorized; } return result; } diff --git a/src/http/responses/collection-v2.ts b/src/http/responses/collection-v2.ts index 36f35d0..65008ed 100644 --- a/src/http/responses/collection-v2.ts +++ b/src/http/responses/collection-v2.ts @@ -34,16 +34,16 @@ export function generateAPIv2CollectionResponse(query: FastifyRequest['query'], // Filter prefixes const response: APIv2CollectionResponse = { - ...apiV2IconsCache.rendered, + ...apiV2IconsCache, }; if (!q.info) { // Delete info delete response.info; } - if (q.chars && apiV2IconsCache.chars) { + if (q.chars && iconSet.icons.chars) { // Add characters map - response.chars = apiV2IconsCache.chars; + response.chars = iconSet.icons.chars; } sendJSONResponse(response, q, wrap, res); diff --git a/src/http/responses/svg.ts b/src/http/responses/svg.ts index f758e55..a7d48e0 100644 --- a/src/http/responses/svg.ts +++ b/src/http/responses/svg.ts @@ -16,15 +16,22 @@ import { iconSets } from '../../data/icon-sets'; */ export function generateSVGResponse(prefix: string, name: string, query: FastifyRequest['query'], res: FastifyReply) { // Get icon set - const iconSet = iconSets[prefix]; - if (!iconSet) { + const iconSetItem = iconSets[prefix]?.item; + if (!iconSetItem) { // No such icon set res.send(404); return; } + // Check if icon exists + if (!iconSetItem.icons.names.has(name) && !iconSetItem.icons.chars?.[name]) { + // No such icon + res.send(404); + return; + } + // Get icon - getStoredIconData(iconSet.item, name, (data) => { + getStoredIconData(iconSetItem, name, (data) => { if (!data) { // Invalid icon res.send(404); diff --git a/src/types/icon-set/extra.ts b/src/types/icon-set/extra.ts index 32305ce..9087665 100644 --- a/src/types/icon-set/extra.ts +++ b/src/types/icon-set/extra.ts @@ -15,6 +15,9 @@ export interface IconSetIconsListTag { * Icons */ export interface IconSetIconsListIcons { + // All names: icons + aliases + names: Set; + // Visible icons visible: Set; @@ -31,46 +34,43 @@ export interface IconSetIconsListIcons { // Tags tags: IconSetIconsListTag[]; uncategorised: string[]; + + // Characters, key = character, value = icon name + chars?: Record; } /** * Prepared icons list for API v2 response */ export interface IconSetAPIv2IconsList { - // Prepared data - rendered: { - // Icon set prefix - prefix: string; + // Icon set prefix + prefix: string; - // Number of icons (duplicate of info?.total) - total: number; + // Number of icons (duplicate of info?.total) + total: number; - // Icon set title, if available (duplicate of info?.name) - title?: string; + // Icon set title, if available (duplicate of info?.name) + title?: string; - // Icon set info - info?: IconifyInfo; + // Icon set info + info?: IconifyInfo; - // List of icons without categories - uncategorized?: string[]; + // List of icons without categories + uncategorized?: string[]; - // List of icons, sorted by category - categories?: Record; + // List of icons, sorted by category + categories?: Record; - // List of hidden icons - hidden?: string[]; + // List of hidden icons + hidden?: string[]; - // List of aliases, key = alias, value = parent icon - aliases?: Record; + // List of aliases, key = alias, value = parent icon + aliases?: Record; - // Themes - themes?: IconifyJSON['themes']; - prefixes?: IconifyJSON['prefixes']; - suffixes?: IconifyJSON['suffixes']; - }; - - // Characters, key = character, value = icon name - chars?: Record; + // Themes + themes?: IconifyJSON['themes']; + prefixes?: IconifyJSON['prefixes']; + suffixes?: IconifyJSON['suffixes']; } /** diff --git a/tests/icon-set/validate-test.ts b/tests/icon-set/validate-test.ts index 179ef13..b7848f9 100644 --- a/tests/icon-set/validate-test.ts +++ b/tests/icon-set/validate-test.ts @@ -1,5 +1,5 @@ import { generateIconSetIconsTree } from '../../lib/data/icon-set/lists/icons'; -import { removeBadAliases } from '../../lib/data/icon-set/store/validate'; +import { removeBadIconSetItems } from '../../lib/data/icon-set/lists/validate'; describe('Validating icon set', () => { test('Long chain of aliases, bad aliases', () => { @@ -56,7 +56,7 @@ describe('Validating icon set', () => { }, }, }; - removeBadAliases(iconSet, generateIconSetIconsTree(iconSet)); + removeBadIconSetItems(iconSet, generateIconSetIconsTree(iconSet)); // Check aliases expect(Object.keys(iconSet.aliases)).toEqual(['baz', 'baz2', 'baz3', 'baz4', 'baz5', 'baz6', 'bazz5']);