chore: restructure icons list in preparation for search index

This commit is contained in:
Vjacheslav Trushkin 2022-10-20 20:46:32 +03:00
parent 068ba3dc39
commit d27c6e36a8
13 changed files with 242 additions and 142 deletions

View File

@ -1,8 +1,5 @@
import type { IconifyJSON } from '@iconify/types';
import type { IconSetAPIv2IconsList, IconSetIconsListIcons } from '../../../types/icon-set/extra';
type ThemeKey = 'themes' | 'prefixes' | 'suffixes';
const themeKeys: ThemeKey[] = ['themes', 'prefixes', 'suffixes'];
import type { IconSetIconsListIcons, IconSetAPIv2IconsList } from '../../../types/icon-set/extra';
/**
* Prepare data for icons list API v2 response
@ -11,7 +8,7 @@ export function prepareAPIv2IconsList(iconSet: IconifyJSON, iconsList: IconSetIc
// Prepare data
const result: IconSetAPIv2IconsList = {
prefix: iconSet.prefix,
total: iconsList.visible.size,
total: iconsList.total,
};
const info = iconSet.info;
@ -20,40 +17,57 @@ export function prepareAPIv2IconsList(iconSet: IconifyJSON, iconsList: IconSetIc
result.info = info;
}
// Icons without categories
if (iconsList.uncategorised.length) {
result.uncategorized = iconsList.uncategorised;
result.uncategorized = iconsList.uncategorised.map((item) => item[0]);
}
// Convert categories
// Categories
if (iconsList.tags.length) {
const tags = iconsList.tags;
const categories = (result.categories = Object.create(null) as Record<string, string[]>);
for (let i = 0; i < iconsList.tags.length; i++) {
const tag = iconsList.tags[i];
categories[tag.title] = tag.icons;
for (let i = 0; i < tags.length; i++) {
const tag = tags[i];
categories[tag.title] = tag.icons.map((icon) => icon[0]);
}
}
// Aliases
const aliases = Object.create(null) as Record<string, string>;
for (const name in iconsList.visible) {
const item = iconsList.visible[name];
if (item[0] !== name) {
aliases[name] = item[0];
}
}
// Hidden icons
const hidden = Array.from(iconsList.hidden).concat(Object.keys(iconsList.hiddenAliases));
const hidden: string[] = [];
for (const name in iconsList.hidden) {
const item = iconsList.hidden[name];
if (item[0] === name) {
hidden.push(name);
} else {
aliases[name] = item[0];
}
}
if (hidden.length) {
result.hidden = hidden;
}
// Add aliases
const aliases = {
...iconsList.visibleAliases,
...iconsList.hiddenAliases,
};
for (const alias in aliases) {
// Aliases
for (const key in aliases) {
result.aliases = aliases;
break;
}
// Themes
for (let i = 0; i < themeKeys.length; i++) {
const key = themeKeys[i] as ThemeKey;
if (iconSet[key]) {
result[key as 'themes'] = iconSet[key as 'themes'];
if (iconsList.chars) {
// Add characters map
const chars = (result.chars = Object.create(null) as Record<string, string>);
const sourceChars = iconsList.chars;
for (const key in sourceChars) {
chars[key] = sourceChars[key][0];
}
}

View File

@ -1,6 +1,6 @@
import type { IconifyAliases, IconifyJSON, IconifyOptional } from '@iconify/types';
import { defaultIconProps } from '@iconify/utils/lib/icon/defaults';
import type { IconSetIconsListIcons, IconSetIconsListTag } from '../../../types/icon-set/extra';
import type { IconSetIconNames, IconSetIconsListIcons, IconSetIconsListTag } from '../../../types/icon-set/extra';
const customisableProps = Object.keys(defaultIconProps) as (keyof IconifyOptional)[];
@ -12,15 +12,14 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList
const iconSetAliases = iconSet.aliases || (Object.create(null) as IconifyAliases);
const checked: Set<string> = new Set();
const visible: Set<string> = new Set();
const hidden: Set<string> = new Set();
const visible = Object.create(null) as Record<string, IconSetIconNames>;
const hidden = Object.create(null) as Record<string, IconSetIconNames>;
const failed: Set<string> = new Set();
const visibleAliases = Object.create(null) as Record<string, string>;
const hiddenAliases = Object.create(null) as Record<string, string>;
let total = 0;
// Generate list of tags for each icon
const tags: IconSetIconsListTag[] = [];
const uncategorised: string[] = [];
const uncategorised: IconSetIconNames[] = [];
const resolvedTags = Object.create(null) as Record<string, Set<IconSetIconsListTag>>;
const categories = iconSet.categories;
@ -28,9 +27,10 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList
for (const title in categories) {
const items = categories[title];
if (items instanceof Array) {
const icons: IconSetIconNames[] = [];
const tag: IconSetIconsListTag = {
title,
icons: [],
icons,
};
tags.push(tag);
for (let i = 0; i < items.length; i++) {
@ -44,20 +44,23 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList
// Parse all icons
for (const name in iconSetIcons) {
const isVisible = !iconSetIcons[name].hidden;
(isVisible ? visible : hidden).add(name);
const icon: IconSetIconNames = [name];
(isVisible ? visible : hidden)[name] = icon;
checked.add(name);
if (isVisible) {
// Check tag
total++;
// Check tags
const iconTags = resolvedTags[name];
if (iconTags) {
// Add icon to each tag
iconTags.forEach((tag) => {
tag.icons.push(name);
tag.icons.push(icon);
});
} else {
// No tags: uncategorised
uncategorised.push(name);
uncategorised.push(icon);
}
}
}
@ -99,38 +102,54 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList
}
}
// Success
const isVisible =
item.hidden === false || (!item.hidden && (visible.has(parent) || visibleAliases[parent] !== void 0));
failed.delete(name);
// Add tags
let itemTags: Set<IconSetIconsListTag> | undefined = resolvedTags[name];
if (!itemTags) {
// Use tags from parent icon
itemTags = resolvedTags[name] = resolvedTags[parent];
}
if (isVisible && transformed) {
// Icon: add to tags
if (itemTags) {
itemTags.forEach((tag) => {
tag.icons.push(name);
});
} else {
uncategorised.push(name);
}
// Check visibility
const parentVisible = !!visible[parent];
let isVisible: boolean;
if (typeof item.hidden === 'boolean') {
isVisible = !item.hidden;
} else {
// Same visibility as parent icon
isVisible = parentVisible;
}
// Add icon
if (transformed) {
const parentIcon = visible[parent] || hidden[parent];
let icon: IconSetIconNames;
if (transformed || isVisible !== parentVisible) {
// Treat as new icon
(isVisible ? visible : hidden).add(name);
icon = [name];
if (isVisible) {
total++;
// Check for categories
const iconTags = resolvedTags[name];
if (iconTags) {
// Alias has its own categories!
iconTags.forEach((tag) => {
tag.icons.push(icon);
});
} else {
// Copy from parent
const iconTags = resolvedTags[parentIcon[0]];
if (iconTags) {
resolvedTags[name] = iconTags;
iconTags.forEach((tag) => {
tag.icons.push(icon);
});
} else {
uncategorised.push(icon);
}
}
}
} else {
// Treat as alias
const parentName = visibleAliases[parent] || hiddenAliases[parent] || parent;
(isVisible ? visibleAliases : hiddenAliases)[name] = parentName;
// Treat as alias: add to parent icon
icon = parentIcon;
icon.push(name);
}
(isVisible ? visible : hidden)[name] = icon;
// Success
failed.delete(name);
};
for (const name in iconSetAliases) {
@ -139,20 +158,33 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList
// Sort icons in tags
for (let i = 0; i < tags.length; i++) {
tags[i].icons.sort((a, b) => a.localeCompare(b));
tags[i].icons.sort((a, b) => a[0].localeCompare(b[0]));
}
uncategorised.sort((a, b) => a.localeCompare(b));
uncategorised.sort((a, b) => a[0].localeCompare(b[0]));
// Return data
return {
names: new Set([...visible, ...hidden, ...Object.keys(visibleAliases), ...Object.keys(hiddenAliases)]),
// Create data
const result: IconSetIconsListIcons = {
total,
visible,
hidden,
visibleAliases,
hiddenAliases,
failed,
tags: tags.filter((tag) => tag.icons.length > 0),
uncategorised,
chars: iconSet.chars,
};
// Add characters
if (iconSet.chars) {
const sourceChars = iconSet.chars;
const chars = Object.create(null) as Record<string, IconSetIconNames>;
for (const char in sourceChars) {
const name = sourceChars[char];
const item = visible[name] || hidden[name];
if (item) {
chars[char] = item;
}
}
result.chars = chars;
}
return result;
}

View File

@ -16,10 +16,11 @@ export function removeBadIconSetItems(data: IconifyJSON, iconsList: IconSetIcons
// Remove bad characters
const chars = iconsList.chars;
if (chars) {
const visible = iconsList.visible;
const hidden = iconsList.hidden;
for (const key in chars) {
if (iconsList.names.has(key) || !iconsList.names.has(chars[key])) {
// Character matches existing icon or points to missing icon
// Also deletes data.chars[key] because it points to same object
if (visible[key] || hidden[key]) {
// Character matches existing icon
delete chars[key];
}
}

View File

@ -1,21 +1,26 @@
import type { IconifyIcons, IconifyJSON } from '@iconify/types';
import { appConfig, splitIconSetConfig, storageConfig } from '../../../config/app';
import type { SplitIconSetConfig } from '../../../types/config/split';
import type { StoredIconSet, StoredIconSetDone } from '../../../types/icon-set/storage';
import type { StorageIconSetThemes, StoredIconSet, StoredIconSetDone } from '../../../types/icon-set/storage';
import type { SplitRecord } from '../../../types/split';
import type { MemoryStorage, MemoryStorageItem } from '../../../types/storage';
import { createSplitRecordsTree, splitRecords } from '../../storage/split';
import { createStorage, createStoredItem } from '../../storage/create';
import { getIconSetSplitChunksCount, splitIconSetMainData } from './split';
import { generateIconSetIconsTree } from '../lists/icons';
import { removeBadIconSetItems } from '../lists/validate';
import { prepareAPIv2IconsList } from '../lists/icons-v2';
import { generateIconSetIconsTree } from '../lists/icons';
/**
* Storage
*/
export const iconSetsStorage = createStorage<IconifyIcons>(storageConfig);
/**
* Themes to copy
*/
const themeKeys: (keyof StorageIconSetThemes)[] = ['themes', 'prefixes', 'suffixes'];
/**
* Counter for prefixes
*/
@ -31,18 +36,28 @@ export function storeLoadedIconSet(
storage: MemoryStorage<IconifyIcons> = iconSetsStorage,
config: SplitIconSetConfig = splitIconSetConfig
) {
// Get icons list and remove bad aliases
const icons = generateIconSetIconsTree(iconSet);
removeBadIconSetItems(iconSet, icons);
// Fix icons counter
if (iconSet.info) {
iconSet.info.total = icons.visible.size;
iconSet.info.total = icons.total;
}
// Get common items
const common = splitIconSetMainData(iconSet);
// Get themes
const themes: StorageIconSetThemes = {};
if (appConfig.enableIconLists) {
for (let i = 0; i < themeKeys.length; i++) {
const key = themeKeys[i];
if (iconSet[key]) {
themes[key as 'prefixes'] = iconSet[key as 'prefixes'];
}
}
}
// Get number of chunks
const chunksCount = getIconSetSplitChunksCount(iconSet.icons, config);
@ -84,6 +99,10 @@ export function storeLoadedIconSet(
result.info = iconSet.info;
}
if (appConfig.enableIconLists) {
for (const key in themes) {
result.themes = themes;
break;
}
result.apiV2IconsCache = prepareAPIv2IconsList(iconSet, icons);
}
done(result);

View File

@ -74,11 +74,11 @@ export function getStoredIconData(
name = resolved.name;
} else {
props = {} as ExtendedIconifyAlias;
const charValue = iconSet.icons.chars?.[name];
const charValue = iconSet.icons.chars?.[name]?.[0];
if (charValue) {
// Character
const icons = iconSet.icons;
if (!icons.visible.has(name) && !icons.hidden.has(name)) {
if (!icons.visible[name] && !icons.hidden[name]) {
// Resolve character instead of alias
name = charValue;
if (common.aliases[name]) {

View File

@ -1,5 +1,4 @@
import type { IconifyJSON, IconifyAliases, IconifyIcons } from '@iconify/types';
import type { SplitIconifyJSONMainData } from '../../../types/icon-set/split';
import type { StoredIconSet } from '../../../types/icon-set/storage';
import { searchSplitRecordsTreeForSet } from '../../storage/split';
import { getStoredItem } from '../../storage/get';
@ -10,14 +9,14 @@ import { getStoredItem } from '../../storage/get';
export function getIconsToRetrieve(iconSet: StoredIconSet, names: string[], copyTo?: IconifyAliases): Set<string> {
const icons: Set<string> = new Set();
const iconSetData = iconSet.common;
const allNames = iconSet.icons.names;
const chars = iconSet.icons.chars;
const iconsData = iconSet.icons;
const chars = iconsData.chars;
const aliases = iconSetData.aliases || (Object.create(null) as IconifyAliases);
function resolve(name: string, nested: boolean) {
if (!allNames.has(name)) {
if (!iconsData.visible[name] && !iconsData.hidden[name]) {
// No such icon: check for character
const charValue = chars?.[name];
const charValue = chars?.[name]?.[0];
if (!charValue) {
return;
}

View File

@ -1,4 +1,4 @@
import fastify, { FastifyReply } from 'fastify';
import fastify from 'fastify';
import { appConfig } from '../config/app';
import { runWhenLoaded } from '../data/loading';
import { iconNameRoutePartialRegEx, iconNameRouteRegEx, splitIconName } from '../misc/name';

View File

@ -54,8 +54,8 @@ export function generateAPIv1IconsListResponse(
if (q.aliases && v2Cache.aliases) {
base.aliases = v2Cache.aliases;
}
if (q.chars && icons.chars) {
base.chars = icons.chars;
if (q.chars && v2Cache.chars) {
base.chars = v2Cache.chars;
}
// Add icons
@ -71,7 +71,13 @@ export function generateAPIv1IconsListResponse(
}
const result = base as APIv1ListIconsResponse;
result.icons = Array.from(iconSet.icons.visible);
result.icons = [];
const visible = iconSet.icons.visible;
for (const name in visible) {
if (visible[name][0] === name) {
result.icons.push(name);
}
}
return result;
}

View File

@ -37,18 +37,19 @@ export function generateAPIv2CollectionResponse(query: FastifyRequest['query'],
return;
}
// Filter prefixes
// Generate response
const response: APIv2CollectionResponse = {
...apiV2IconsCache,
...iconSet.themes,
};
if (!q.info) {
// Delete info
delete response.info;
}
if (q.chars && iconSet.icons.chars) {
// Add characters map
response.chars = iconSet.icons.chars;
if (!q.chars) {
// Remove characters map
delete response.chars;
}
sendJSONResponse(response, q, wrap, res);

View File

@ -24,7 +24,8 @@ export function generateSVGResponse(prefix: string, name: string, query: Fastify
}
// Check if icon exists
if (!iconSetItem.icons.names.has(name) && !iconSetItem.icons.chars?.[name]) {
const icons = iconSetItem.icons;
if (!(icons.visible[name] || icons.hidden[name]) && !iconSetItem.icons.chars?.[name]) {
// No such icon
res.send(404);
return;

View File

@ -1,4 +1,9 @@
import type { IconifyInfo, IconifyJSON } from '@iconify/types';
import type { IconifyInfo } from '@iconify/types';
/**
* Icon. First entry is main name, other entries are aliases
*/
export type IconSetIconNames = [string, ...string[]];
/**
* Tag
@ -7,36 +12,32 @@ export interface IconSetIconsListTag {
// Title
title: string;
// Names of icons
icons: string[];
// Icons
icons: IconSetIconNames[];
}
/**
* Icons
*/
export interface IconSetIconsListIcons {
// All names: icons + aliases
names: Set<string>;
// Number of visible icons
total: number;
// Visible icons
visible: Set<string>;
visible: Record<string, IconSetIconNames>;
// Hidden icons
hidden: Set<string>;
// Aliases, pointing to parent icon in either `visible` or `hidden` set
visibleAliases: Record<string, string>;
hiddenAliases: Record<string, string>;
hidden: Record<string, IconSetIconNames>;
// Failed aliases
failed: Set<string>;
// Tags
tags: IconSetIconsListTag[];
uncategorised: string[];
uncategorised: IconSetIconNames[];
// Characters, key = character, value = icon name
chars?: Record<string, string>;
// Characters, key = character, value = icon
chars?: Record<string, IconSetIconNames>;
}
/**
@ -67,10 +68,8 @@ export interface IconSetAPIv2IconsList {
// List of aliases, key = alias, value = parent icon
aliases?: Record<string, string>;
// Themes
themes?: IconifyJSON['themes'];
prefixes?: IconifyJSON['prefixes'];
suffixes?: IconifyJSON['suffixes'];
// Characters, key = character, value = icon name
chars?: Record<string, string>;
}
/**

View File

@ -1,9 +1,18 @@
import type { IconifyIcons, IconifyInfo } from '@iconify/types';
import type { IconifyIcons, IconifyInfo, IconifyJSON } from '@iconify/types';
import type { SplitDataTree } from '../split';
import type { MemoryStorage, MemoryStorageItem } from '../storage';
import type { IconSetAPIv2IconsList, IconSetIconsListIcons } from './extra';
import type { IconSetIconsListIcons, IconSetAPIv2IconsList } from './extra';
import type { SplitIconifyJSONMainData } from './split';
/**
* Themes
*/
export interface StorageIconSetThemes {
themes?: IconifyJSON['themes'];
prefixes?: IconifyJSON['prefixes'];
suffixes?: IconifyJSON['suffixes'];
}
/**
* Generated data
*/
@ -25,6 +34,9 @@ export interface StoredIconSet {
icons: IconSetIconsListIcons;
apiV2IconsCache?: IconSetAPIv2IconsList;
// Themes
themes?: StorageIconSetThemes;
// TODO: add properties for search data
}

View File

@ -1,4 +1,5 @@
import { generateIconSetIconsTree } from '../../lib/data/icon-set/lists/icons';
import type { IconSetIconNames, IconSetIconsListTag } from '../../lib/types/icon-set/extra';
describe('Icons tree', () => {
test('Simple icon set', () => {
@ -18,13 +19,17 @@ describe('Icons tree', () => {
});
expect(tree.failed).toEqual(new Set());
expect(tree.visible).toEqual(new Set(['bar', 'baz', 'foo']));
expect(tree.hidden).toEqual(new Set());
expect(tree.visibleAliases).toEqual({});
expect(tree.hiddenAliases).toEqual({});
const expectedVisible: Record<string, IconSetIconNames> = {
bar: ['bar'],
baz: ['baz'],
foo: ['foo'],
};
expect(tree.visible).toEqual(expectedVisible);
expect(tree.hidden).toEqual({});
expect(tree.tags).toEqual([]);
expect(tree.uncategorised).toEqual(['bar', 'baz', 'foo']);
expect(tree.uncategorised).toEqual([expectedVisible.bar, expectedVisible.baz, expectedVisible.foo]);
});
test('Few aliases', () => {
@ -56,18 +61,24 @@ describe('Icons tree', () => {
});
expect(tree.failed).toEqual(new Set(['missing-alias', 'missing-icon']));
expect(tree.visible).toEqual(new Set(['bar', 'bar2', 'foo']));
expect(tree.hidden).toEqual(new Set());
expect(tree.visibleAliases).toEqual({ foo2: 'foo' });
expect(tree.hiddenAliases).toEqual({});
expect(tree.tags).toEqual([
const expectedVisible: Record<string, IconSetIconNames> = {
bar: ['bar'],
bar2: ['bar2'],
foo: ['foo', 'foo2'],
foo2: ['foo', 'foo2'],
};
expect(tree.visible).toEqual(expectedVisible);
expect(tree.hidden).toEqual({});
const expectedTags: IconSetIconsListTag[] = [
{
title: 'Bar',
icons: ['bar', 'foo'],
icons: [expectedVisible.bar, expectedVisible.foo],
},
]);
expect(tree.uncategorised).toEqual(['bar2']);
];
expect(tree.tags).toEqual(expectedTags);
expect(tree.uncategorised).toEqual([expectedVisible.bar2]);
});
test('Many aliases', () => {
@ -154,28 +165,33 @@ describe('Icons tree', () => {
});
expect(tree.failed).toEqual(new Set(['alias3', 'icon3', 'loop1', 'loop2', 'loop3']));
expect(tree.visible).toEqual(new Set(['icon1', 'alias2z']));
expect(tree.hidden).toEqual(new Set(['icon2', 'alias2f', 'alias2a']));
expect(tree.visibleAliases).toEqual({
alias2z3: 'alias2z',
alias2z4: 'alias2z',
alias2z5: 'alias2z',
});
expect(tree.hiddenAliases).toEqual({
alias2z6: 'alias2z',
alias2z7: 'alias2z',
});
expect(tree.tags).toEqual([
const alias2z: IconSetIconNames = ['alias2z', 'alias2z3', 'alias2z4', 'alias2z5'];
const expectedVisible: Record<string, IconSetIconNames> = {
icon1: ['icon1'],
alias2z: alias2z,
alias2z3: alias2z,
alias2z4: alias2z,
alias2z5: alias2z,
};
expect(tree.visible).toEqual(expectedVisible);
const expectedHidden: Record<string, IconSetIconNames> = {
icon2: ['icon2'],
alias2f: ['alias2f'],
alias2a: ['alias2a'],
alias2z6: ['alias2z6', 'alias2z7'],
alias2z7: ['alias2z6', 'alias2z7'],
};
expect(tree.hidden).toEqual(expectedHidden);
const expectedTags: IconSetIconsListTag[] = [
{
title: 'Icon1',
icons: ['icon1'],
icons: [expectedVisible.icon1],
},
{
title: 'Icon2',
icons: ['alias2z'],
},
]);
expect(tree.uncategorised).toEqual([]);
];
expect(tree.tags).toEqual(expectedTags);
expect(tree.uncategorised).toEqual([expectedVisible.alias2z]);
});
});