feat: search engine

This commit is contained in:
Vjacheslav Trushkin 2022-10-30 21:59:47 +02:00
parent df42beb8c3
commit d542004aab
20 changed files with 1941 additions and 10 deletions

View File

@ -192,5 +192,31 @@ export function generateIconSetIconsTree(iconSet: IconifyJSON): IconSetIconsList
result.chars = chars;
}
// Generate keywords for all visible icons if:
// - search engine is enabled
// - icon set has info (cannot search icon set if cannot show it)
// - icon set is not marked as hidden
if (appConfig.enableIconLists && appConfig.enableSearchEngine && iconSet.info && !iconSet.info.hidden) {
const keywords = (result.keywords = Object.create(null) as Record<string, Set<IconSetIconNames>>);
for (const name in visible) {
const icon = visible[name];
if (icon[0] !== name) {
// Alias. Another entry for parent icon should be present in `visible` object
continue;
}
const iconKeywords: Set<string> = new Set();
for (let i = 0; i < icon.length; i++) {
icon[i].split('-').forEach((chunk) => {
if (iconKeywords.has(chunk)) {
return;
}
iconKeywords.add(chunk);
(keywords[chunk] || (keywords[chunk] = new Set())).add(icon);
});
}
}
}
return result;
}

View File

@ -120,6 +120,18 @@ export function asyncStoreLoadedIconSet(
config: SplitIconSetConfig = splitIconSetConfig
): Promise<StoredIconSet> {
return new Promise((fulfill) => {
storeLoadedIconSet(iconSet, fulfill, storage, config);
storeLoadedIconSet(
iconSet,
(data: StoredIconSet) => {
// Purge unused memory if garbage collector global is exposed
try {
global.gc?.();
} catch {}
fulfill(data);
},
storage,
config
);
});
}

View File

@ -1,5 +1,6 @@
import type { StoredIconSet } from '../types/icon-set/storage';
import type { IconSetEntry, Importer } from '../types/importers';
import { updateSearchIndex } from './search';
/**
* All importers
@ -100,6 +101,8 @@ export function updateIconSets(): number {
if (loadedIconSets.size) {
// Got some icon sets to clean up
const cleanup = loadedIconSets;
// TODO: clean up old icon sets
}
loadedIconSets = newLoadedIconSets;
@ -107,6 +110,15 @@ export function updateIconSets(): number {
allPrefixes = Array.from(newPrefixes);
prefixesWithInfo = Array.from(newPrefixesWithInfo);
visiblePrefixes = Array.from(newVisiblePrefixes);
// Update search index
updateSearchIndex(allPrefixes, iconSets);
// Purge unused memory if garbage collector global is exposed
try {
global.gc?.();
} catch {}
return allPrefixes.length;
}

55
src/data/search.ts Normal file
View File

@ -0,0 +1,55 @@
import { appConfig } from '../config/app';
import type { IconSetEntry } from '../types/importers';
import type { SearchIndexData } from '../types/search';
interface SearchIndex {
data?: SearchIndexData;
}
/**
* Search data
*/
export const searchIndex: SearchIndex = {};
/**
* Update search index
*/
export function updateSearchIndex(
prefixes: string[],
iconSets: Record<string, IconSetEntry>
): SearchIndexData | undefined {
if (!appConfig.enableIconLists || !appConfig.enableSearchEngine) {
// Search engine is disabled
delete searchIndex.data;
return;
}
// Parse all icon sets
const sortedPrefixes: string[] = [];
const keywords = Object.create(null) as Record<string, Set<string>>;
for (let i = 0; i < prefixes.length; i++) {
const prefix = prefixes[i];
const iconSet = iconSets[prefix]?.item;
if (!iconSet) {
continue;
}
const iconSetKeywords = iconSet.icons.keywords;
if (!iconSetKeywords) {
continue;
}
sortedPrefixes.push(prefix);
for (const keyword in iconSetKeywords) {
(keywords[keyword] || (keywords[keyword] = new Set())).add(prefix);
}
}
// Set data
return (searchIndex.data = {
sortedPrefixes,
keywords,
partial: Object.create(null),
partialCleanup: Date.now(),
});
}

194
src/data/search/index.ts Normal file
View File

@ -0,0 +1,194 @@
import type { IconSetIconNames } from '../../types/icon-set/extra';
import type { IconSetEntry } from '../../types/importers';
import type { SearchIndexData, SearchKeywordsEntry, SearchParams, SearchResultsData } from '../../types/search';
import { getPartialKeywords } from './partial';
import { filterSearchPrefixes, filterSearchPrefixesList } from './prefixes';
import { splitKeyword } from './split';
/**
* Run search
*/
export function search(
params: SearchParams,
data: SearchIndexData,
iconSets: Record<string, IconSetEntry>
): SearchResultsData | undefined {
// Get keywords
const keywords = splitKeyword(params.keyword, params.partial);
if (!keywords) {
return;
}
// Make sure all keywords exist
keywords.searches = keywords.searches.filter((search) => {
for (let i = 0; i < search.keywords.length; i++) {
if (!data.keywords[search.keywords[i]]) {
// One of required keywords is missing: no point in searching
return false;
}
}
return true;
});
if (!keywords.searches.length) {
return;
}
// Check for partial
const partial = keywords.partial;
let partialKeywords: string[] | undefined;
if (partial) {
// Get all partial keyword matches
const cache = getPartialKeywords(partial, data);
const exists = data.keywords[partial];
if (!cache || !cache.length) {
// No partial matches: check if keyword exists
if (!exists) {
return;
}
partialKeywords = [partial];
} else {
// Partial keywords exist
partialKeywords = exists ? [partial].concat(cache) : cache.slice(0);
}
}
// Get prefixes
const basePrefixes = filterSearchPrefixes(data, iconSets, {
...params,
// Params extracted from query override default params
...keywords.params,
});
// Prepare variables
const addedIcons = Object.create(null) as Record<string, Set<IconSetIconNames>>;
const foundPrefixes: Set<string> = new Set();
const results: string[] = [];
const limit = params.limit;
// Run all searches
const check = (partial?: string) => {
for (let searchIndex = 0; searchIndex < keywords.searches.length; searchIndex++) {
// Add prefixes cache to avoid re-calculating it for every partial keyword
interface ExtendedSearchKeywordsEntry extends SearchKeywordsEntry {
filteredPrefixes?: Readonly<string[]>;
}
const search = keywords.searches[searchIndex] as ExtendedSearchKeywordsEntry;
// Filter prefixes (or get it from cache)
let filteredPrefixes: Readonly<string[]>;
if (search.filteredPrefixes) {
filteredPrefixes = search.filteredPrefixes;
} else {
filteredPrefixes = search.prefixes
? filterSearchPrefixesList(basePrefixes, search.prefixes)
: basePrefixes;
// Filter by required keywords
for (let i = 0; i < search.keywords.length; i++) {
filteredPrefixes = filteredPrefixes.filter((prefix) =>
data.keywords[search.keywords[i]].has(prefix)
);
}
search.filteredPrefixes = filteredPrefixes;
}
if (!filteredPrefixes.length) {
continue;
}
// Get keywords
const testKeywords = partial ? search.keywords.concat([partial]) : search.keywords;
const testMatches = search.test ? search.test.concat(testKeywords) : testKeywords;
// Check for partial keyword if testing for exact match
if (partial) {
filteredPrefixes = filteredPrefixes.filter((prefix) => data.keywords[partial].has(prefix));
}
// Check icons
for (let prefixIndex = 0; prefixIndex < filteredPrefixes.length; prefixIndex++) {
const prefix = filteredPrefixes[prefixIndex];
const prefixAddedIcons = addedIcons[prefix] || (addedIcons[prefix] = new Set());
const iconSet = iconSets[prefix].item;
const iconSetIcons = iconSet.icons;
const iconSetKeywords = iconSetIcons.keywords;
if (!iconSetKeywords) {
// This should not happen!
continue;
}
// Check icons in current prefix
let matches: IconSetIconNames[] | undefined;
let failed = false;
for (let keywordIndex = 0; keywordIndex < testKeywords.length && !failed; keywordIndex++) {
const keyword = testKeywords[keywordIndex];
const keywordMatches = iconSetKeywords[keyword];
if (!keywordMatches) {
failed = true;
break;
}
if (!matches) {
// Copy all matches
matches = Array.from(keywordMatches);
} else {
// Match previous set
matches = matches.filter((item) => keywordMatches.has(item));
}
}
// Test matched icons
if (!failed && matches) {
for (let matchIndex = 0; matchIndex < matches.length; matchIndex++) {
const item = matches[matchIndex];
if (prefixAddedIcons.has(item)) {
// Already added
continue;
}
// Find icon name that matches all keywords
const name = item.find((name) => {
for (let i = 0; i < testMatches.length; i++) {
if (name.indexOf(testMatches[i]) === -1) {
return false;
}
}
return true;
});
if (name) {
// Add icon
prefixAddedIcons.add(item);
results.push(prefix + ':' + name);
if (results.length >= limit) {
return;
}
}
}
}
}
}
};
// Check all keywords
if (!partialKeywords) {
check();
} else {
let partial: string | undefined;
while ((partial = partialKeywords.shift())) {
check(partial);
if (results.length >= limit) {
break;
}
}
}
// Generate results
if (results.length) {
return {
prefixes: Object.keys(addedIcons),
names: results,
hasMore: results.length >= limit,
};
}
}

View File

@ -0,0 +1,50 @@
import type { PartialKeywords, SearchIndexData } from '../../types/search';
import { searchIndex } from '../search';
export const minPartialKeywordLength = 3;
/**
* Find partial keywords for keyword
*/
export function getPartialKeywords(
keyword: string,
data: SearchIndexData | undefined = searchIndex.data
): PartialKeywords | undefined {
// const data = searchIndex.data;
const length = keyword.length;
if (!data || length < minPartialKeywordLength) {
return;
}
if (data.partial[keyword]) {
return data.partial[keyword];
}
// Cache takes a lot of memory, so clean up old cache once every few minutes before generating new item
const time = Date.now();
if (data.partialCleanup < time - 60000) {
data.partial = Object.create(null);
data.partialCleanup = time;
}
// Generate partial list
const prefixMatches: string[] = [];
const suffixMatches: string[] = [];
// Find similar keywords
const keywords = data.keywords;
for (const item in keywords) {
if (item.length > length) {
if (item.slice(0, length) === keyword) {
prefixMatches.push(item);
} else if (item.slice(0 - length) === keyword) {
suffixMatches.push(item);
}
}
}
// Sort: shortest matches first
return (data.partial[keyword] = prefixMatches
.sort((a, b) => (a.length === b.length ? a.localeCompare(b) : a.length - b.length))
.concat(suffixMatches.sort((a, b) => (a.length === b.length ? a.localeCompare(b) : a.length - b.length))));
}

View File

@ -0,0 +1,56 @@
import type { IconSetEntry } from '../../types/importers';
import type { SearchIndexData, SearchParams } from '../../types/search';
/**
* Filter prefixes by keyword
*/
export function filterSearchPrefixesList(prefixes: readonly string[], filters: string[]): string[] {
const set = new Set(filters);
const hasPartial = !!filters.find((item) => item.slice(-1) === '-');
return prefixes.filter((prefix) => {
if (set.has(prefix)) {
return true;
}
if (hasPartial) {
// Check for partial matches
const parts = prefix.split('-');
let test = '';
while (parts.length > 1) {
test += parts.shift() + '-';
if (set.has(test)) {
return true;
}
}
}
return false;
});
}
/**
* Filter prefixes
*/
export function filterSearchPrefixes(
data: SearchIndexData,
iconSets: Record<string, IconSetEntry>,
params: SearchParams
): Readonly<string[]> {
let prefixes: string[] | undefined;
// Filter by prefix
if (params.prefixes) {
prefixes = filterSearchPrefixesList(prefixes || data.sortedPrefixes, params.prefixes);
}
// Filter by palette
const palette = params.palette;
if (typeof palette === 'boolean') {
prefixes = (prefixes || data.sortedPrefixes).filter((prefix) => {
const info = iconSets[prefix].item.info;
return info?.palette === palette;
});
}
// TODO: add more filter options
return prefixes || data.sortedPrefixes;
}

398
src/data/search/split.ts Normal file
View File

@ -0,0 +1,398 @@
import { matchIconName } from '@iconify/utils/lib/icon/name';
import { paramToBoolean } from '../../misc/bool';
import type { SearchKeywords, SearchKeywordsEntry } from '../../types/search';
import { minPartialKeywordLength } from './partial';
interface SplitOptions {
// Can include prefix
prefix: boolean;
// Can be partial
partial: boolean;
}
interface SplitResultItem {
// Icon set prefix
prefix?: string;
// List of exact matches
keywords: string[];
// Strings to test icon name
test?: string[];
}
interface SplitResult {
searches: SplitResultItem[];
// Partial keyword. It is last chunk of last keyword, which cannot be treated
// as prefix, so it is identical to all searches
partial?: string;
}
export function splitKeywordEntries(values: string[], options: SplitOptions): SplitResult | undefined {
const results: SplitResult = {
searches: [],
};
let invalid = false;
// Split each entry
interface Entry {
value: string;
empty: boolean;
}
const splitValues: Entry[][] = [];
values.forEach((item) => {
const entries: Entry[] = [];
let hasValue = false;
const parts = item.split('-');
for (let i = 0; i < parts.length; i++) {
const value = parts[i];
const empty = !value;
if (!empty && !matchIconName.test(value)) {
// Invalid entry
invalid = true;
return;
}
entries.push({
value,
empty,
});
hasValue = hasValue || !empty;
}
splitValues.push(entries);
if (!hasValue) {
invalid = true;
}
});
if (invalid || !splitValues.length) {
// Something went wrong
return;
}
// Convert value to test string, returns undefined if it is a simple keyword
function valuesToString(items: Entry[]): string | undefined {
if (!items.length || (items.length === 1 && !items[0].empty)) {
// Empty or only one keyword
return;
}
return (items[0].empty ? '-' : '') + items.map((item) => item.value).join('-');
}
// Function to add item
function add(items: Entry[], keywords: Set<string>, test: Set<string>, checkPartial: boolean) {
let partial: string | undefined;
// Add keywords
const max = items.length - 1;
for (let i = 0; i <= max; i++) {
const value = items[i];
if (!value.empty) {
if (i === max && checkPartial && value.value.length >= minPartialKeywordLength) {
partial = value.value;
} else {
keywords.add(value.value);
}
}
}
// Get test value
const testValue = valuesToString(items);
if (testValue) {
test.add(testValue);
}
// Validate partial
if (checkPartial) {
if (results.searches.length) {
if (results.partial !== partial) {
// Partial should be identical for all searches. Something went wrong !!!
console.error('Mismatches partials when splitting keywords:', values);
delete results.partial;
}
} else {
results.partial = partial;
}
}
}
// Add items
const lastIndex = splitValues.length - 1;
if (options.prefix) {
const firstItem = splitValues[0];
const maxFirstItemIndex = firstItem.length - 1;
// Add with first keyword as prefix
if (lastIndex) {
// Check for empty item. It can only be present at the end of value
const emptyItem = firstItem.find((item) => item.empty);
if (!emptyItem || (maxFirstItemIndex > 0 && emptyItem === firstItem[maxFirstItemIndex])) {
const prefix = firstItem.length > 1 ? valuesToString(firstItem) : firstItem[0].value;
if (prefix) {
// Valid prefix
const keywords: Set<string> = new Set();
const test: Set<string> = new Set();
for (let i = 1; i <= lastIndex; i++) {
add(splitValues[i], keywords, test, options.partial && i === lastIndex);
}
if (keywords.size || results.partial) {
const item: SplitResultItem = {
keywords: Array.from(keywords),
prefix,
};
if (test.size) {
item.test = Array.from(test);
}
results.searches.push(item);
}
}
}
}
// Add with first part of first keyword as prefix
// First 2 items cannot be empty
if (maxFirstItemIndex && !firstItem[0].empty && !firstItem[1].empty) {
const modifiedFirstItem = firstItem.slice(0);
const prefix = modifiedFirstItem.shift()!.value;
const keywords: Set<string> = new Set();
const test: Set<string> = new Set();
for (let i = 0; i <= lastIndex; i++) {
add(i ? splitValues[i] : modifiedFirstItem, keywords, test, options.partial && i === lastIndex);
}
if (keywords.size || results.partial) {
const item: SplitResultItem = {
keywords: Array.from(keywords),
prefix,
};
if (test.size) {
item.test = Array.from(test);
}
results.searches.push(item);
}
}
}
// Add as is
const keywords: Set<string> = new Set();
const test: Set<string> = new Set();
for (let i = 0; i <= lastIndex; i++) {
add(splitValues[i], keywords, test, options.partial && i === lastIndex);
}
if (keywords.size || results.partial) {
const item: SplitResultItem = {
keywords: Array.from(keywords),
};
if (test.size) {
item.test = Array.from(test);
}
results.searches.push(item);
}
return results;
}
/**
* Handle partial prefix
*/
function addPartialPrefix(prefix: string, set: Set<string>): boolean {
if (prefix.slice(-1) === '*') {
// Wildcard entry
prefix = prefix.slice(0, prefix.length - 1);
if (matchIconName.test(prefix)) {
set.add(prefix);
set.add(prefix + '-');
return true;
}
} else if (prefix.length && matchIconName.test(prefix + 'a')) {
// Add 'a' to allow partial prefixes like 'mdi-'
set.add(prefix);
return true;
}
return false;
}
/**
* Split keyword
*/
export function splitKeyword(keyword: string, allowPartial = true): SearchKeywords | undefined {
const commonPrefixes: Set<string> = new Set();
let palette: boolean | undefined;
// Split by space, check for prefixes and reserved keywords
const keywordChunks = keyword.toLowerCase().trim().split(/\s+/);
const keywords: string[] = [];
let hasPrefixes = false;
let checkPartial = false;
for (let i = 0; i < keywordChunks.length; i++) {
const part = keywordChunks[i];
const prefixChunks = part.split(':') as string[];
if (prefixChunks.length > 2) {
// Too many prefixes: invalidate search query
return;
}
// Check for prefix or reserved keyword
if (prefixChunks.length === 2) {
const keyword = prefixChunks[0];
const value = prefixChunks[1];
switch (keyword) {
case 'palette': {
palette = paramToBoolean(value);
break;
}
case 'prefix':
case 'prefixes': {
// Prefixes
if (hasPrefixes) {
// Already had entry with prefix: invalidate query
return;
}
const values = value.split(',');
let invalid = true;
hasPrefixes = true;
for (let j = 0; j < values.length; j++) {
if (addPartialPrefix(values[j].trim(), commonPrefixes)) {
invalid = false;
}
}
if (invalid) {
// All prefixes are bad: invalidate search query
return;
}
break;
}
default: {
// Icon with prefix
if (hasPrefixes) {
// Already had entry with prefix: invalidate query
return;
}
const values = keyword.split(',');
let invalid = true;
hasPrefixes = true;
for (let j = 0; j < values.length; j++) {
const prefix = values[j].trim();
if (matchIconName.test(prefix)) {
commonPrefixes.add(prefix);
invalid = false;
}
}
if (invalid) {
// All prefixes are bad: invalidate search query
return;
}
if (value.length) {
// Add icon name, unless it is empty: 'mdi:'
// Allow partial if enabled
checkPartial = allowPartial;
keywords.push(value);
}
}
}
continue;
}
// 1 part
// Check for 'key=value' pairs
const paramChunks = part.split('=');
if (paramChunks.length > 2) {
// Bad query
return;
}
if (paramChunks.length === 2) {
const value = paramChunks[1] as string;
switch (paramChunks[0]) {
// 'palette=true', 'palette=false' -> filter icon sets by palette
case 'palette':
palette = paramToBoolean(value);
if (typeof palette !== 'boolean') {
return;
}
break;
// 'prefix=material-symbols', 'prefix=material-'
// 'prefixes=material-symbols,material-'
case 'prefix':
case 'prefixes':
if (hasPrefixes) {
// Already had entry with prefix: invalidate query
return;
}
let invalid = true;
const values = value.split(',');
for (let j = 0; j < values.length; j++) {
if (addPartialPrefix(values[j].trim(), commonPrefixes)) {
invalid = false;
}
}
if (invalid) {
// All prefixes are bad: invalidate search query
return;
}
break;
default: {
// Unknown keyword
return;
}
}
continue;
}
// Simple keyword. Allow partial if enabled
checkPartial = allowPartial;
keywords.push(part);
}
if (!keywords.length) {
// No keywords
return;
}
const entries = splitKeywordEntries(keywords, {
prefix: !hasPrefixes && !commonPrefixes.size,
partial: checkPartial,
});
if (!entries) {
return;
}
const searches: SearchKeywordsEntry[] = entries.searches.map((item) => {
return {
...item,
prefixes: item.prefix
? [...commonPrefixes, item.prefix]
: commonPrefixes.size
? [...commonPrefixes]
: undefined,
};
});
const params: SearchKeywords['params'] = {};
if (typeof palette === 'boolean') {
params.palette = palette;
}
return {
searches,
params,
partial: entries.partial,
};
}

View File

@ -37,11 +37,6 @@ export function cleanupStoredItem<T>(storage: MemoryStorage<T>, storedItem: Memo
stopTimer(storage);
}
// Purge unused memory if garbage collector global is exposed
try {
global.gc?.();
} catch {}
return true;
}

View File

@ -7,6 +7,7 @@ import { generateAPIv2CollectionResponse } from './responses/collection-v2';
import { generateCollectionsListResponse } from './responses/collections';
import { generateIconsDataResponse } from './responses/icons';
import { generateLastModifiedResponse } from './responses/modified';
import { generateAPIv2SearchResponse } from './responses/search';
import { generateSVGResponse } from './responses/svg';
import { generateUpdateResponse } from './responses/update';
import { initVersionResponse, versionResponse } from './responses/version';
@ -135,6 +136,15 @@ export async function startHTTPServer() {
generateAPIv1IconsListResponse(req.query, res, true);
});
});
if (appConfig.enableSearchEngine) {
// Search, currently version 2
server.get('/search', (req, res) => {
runWhenLoaded(() => {
generateAPIv2SearchResponse(req.query, res);
});
});
}
}
// Update icon sets

View File

@ -0,0 +1,116 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { iconSets } from '../../data/icon-sets';
import { searchIndex } from '../../data/search';
import { search } from '../../data/search/index';
import { paramToBoolean } from '../../misc/bool';
import type { SearchParams } from '../../types/search';
import type { APIv2SearchParams, APIv2SearchResponse } from '../../types/server/v2';
import { checkJSONPQuery, sendJSONResponse } from '../helpers/json';
const minSearchLimit = 32;
const maxSearchLimit = 999;
const defaultSearchLimit = maxSearchLimit;
/**
* Send API v2 response
*/
export function generateAPIv2SearchResponse(query: FastifyRequest['query'], res: FastifyReply) {
const q = (query || {}) as Record<string, string>;
const wrap = checkJSONPQuery(q);
if (!wrap) {
// Invalid JSONP callback
res.send(400);
return;
}
// Check if search data is available
const searchIndexData = searchIndex.data;
if (!searchIndexData) {
res.send(404);
return;
}
// Get query
const keyword = q.query;
if (!keyword) {
res.send(400);
return;
}
// Convert to params
const params: SearchParams = {
keyword,
limit: defaultSearchLimit,
};
const v2Query = q as unknown as Record<keyof APIv2SearchParams, string>;
// Get limits
if (v2Query.limit) {
const limit = parseInt(v2Query.limit);
if (!limit) {
res.send(400);
return;
}
params.limit = Math.max(minSearchLimit, Math.min(limit, maxSearchLimit));
}
let start = 0;
if (v2Query.start) {
start = parseInt(v2Query.start);
if (isNaN(start) || start < 0 || start >= params.limit) {
res.send(400);
return;
}
}
// Get prefixes
if (v2Query.prefixes) {
params.prefixes = v2Query.prefixes.split(',');
} else if (v2Query.prefix) {
params.prefixes = [v2Query.prefix];
} else if (v2Query.collection) {
params.prefixes = [v2Query.collection];
}
// Category
if (v2Query.category) {
params.category = v2Query.category;
}
// Disable partial
if (v2Query.similar) {
const similar = paramToBoolean(v2Query.similar);
if (typeof similar === 'boolean') {
params.partial = similar;
}
}
// Run query
const searchResults = search(params, searchIndexData, iconSets);
if (!searchResults) {
res.send(404);
return;
}
// Generate result
const response: APIv2SearchResponse = {
icons: searchResults.names.slice(start),
total: searchResults.names.length,
limit: params.limit,
start,
collections: Object.create(null),
request: v2Query,
};
// Add icon sets
for (let i = 0; i < searchResults.prefixes.length; i++) {
const prefix = searchResults.prefixes[i];
const info = iconSets[prefix]?.item.info;
if (info) {
response.collections[prefix] = info;
}
}
sendJSONResponse(response, q, wrap, res);
}

18
src/misc/bool.ts Normal file
View File

@ -0,0 +1,18 @@
/**
* Convert string to boolean
*/
export function paramToBoolean(value: string, defaultValue?: boolean): boolean | undefined {
switch (value) {
case 'true':
case 'yes':
case '1':
return true;
case 'false':
case 'no':
case '0':
return false;
}
return defaultValue;
}

View File

@ -38,6 +38,9 @@ export interface IconSetIconsListIcons {
// Characters, key = character, value = icon
chars?: Record<string, IconSetIconNames>;
// Keywords, set if search engine is enabled
keywords?: Record<string, Set<IconSetIconNames>>;
}
/**

View File

@ -36,8 +36,6 @@ export interface StoredIconSet {
// Themes
themes?: StorageIconSetThemes;
// TODO: add properties for search data
}
/**

92
src/types/search.ts Normal file
View File

@ -0,0 +1,92 @@
/**
* List of keywords that can be used to autocomplete keyword
*/
export type PartialKeywords = Readonly<string[]>;
/**
* Search data
*/
export interface SearchIndexData {
// List of searchable prefixes
sortedPrefixes: string[];
// List of keywords, value is set of prefixes where keyword is used
// Prefixes are added in set in same order as in `sortedPrefixes`
keywords: Record<string, Set<string>>;
// Partial keywords: ['foo'] = ['foo1', 'foo2', 'foobar', ...]
// Can be used for auto-completion for search results
// Keywords are generated on demand and sorted by length: shortest first
partial: Record<string, PartialKeywords>;
// Last cleanup for old partial lists
partialCleanup: number;
}
/**
* Search parameters
*/
export interface SearchParams {
// List of prefixes to search
prefixes?: string[];
// Icon set category
category?: string;
// Icon set tag
tag?: string;
// Filter icon sets by palette
palette?: boolean;
// Keyword
keyword: string;
// Search results limit
limit: number;
// Toggle partial matches
partial?: boolean;
}
/**
* List of matches
*/
export interface SearchKeywordsEntry {
// List of prefixes, extracted from search query
prefixes?: string[];
// List of keywords icon should match
keywords: string[];
// Strings to test icon value
test?: string[];
}
/**
* Searches
*/
export interface SearchKeywords {
// List of searches
searches: SearchKeywordsEntry[];
// Partial keyword, used in all matches
partial?: string;
// Params extracted from keywords
params: Partial<SearchParams>;
}
/**
* Search results
*/
export interface SearchResultsData {
// Prefixes
prefixes: string[];
// Icon names
names: string[];
// True if has more results
hasMore?: boolean;
}

View File

@ -136,6 +136,6 @@ export interface APIv2SearchResponse {
// Info about icon sets
collections: Record<string, IconifyInfo>;
// Copy of request
request: APIv2SearchParams;
// Copy of request, values are string
request: Record<keyof APIv2SearchParams, string>;
}

View File

@ -0,0 +1,164 @@
import { DirectoryDownloader } from '../../lib/downloaders/directory';
import { createHardcodedCollectionsListImporter } from '../../lib/importers/collections/list';
import { createJSONIconSetImporter } from '../../lib/importers/icon-set/json';
import { updateSearchIndex } from '../../lib/data/search';
import { getPartialKeywords } from '../../lib/data/search/partial';
import { filterSearchPrefixes } from '../../lib/data/search/prefixes';
import type { IconSetImportedData } from '../../lib/types/importers/common';
import type { IconSetEntry } from '../../lib/types/importers';
import type { SearchParams } from '../../lib/types/search';
describe('Creating search index, checking prefixes', () => {
test('One icon set', async () => {
// Create importer
const importer = createHardcodedCollectionsListImporter(['mdi-light'], (prefix) =>
createJSONIconSetImporter(new DirectoryDownloader<IconSetImportedData>(`tests/fixtures/json`), {
prefix,
filename: `/${prefix}.json`,
})
);
await importer.init();
const data = importer.data!;
// Get keywords for mdi-light
const mdiLightData = data.iconSets['mdi-light']!;
const mdiLightKeywords = mdiLightData.icons.keywords!;
expect(mdiLightKeywords).toBeTruthy();
const accountKeyword = mdiLightKeywords['account']!;
expect(accountKeyword).toBeTruthy();
expect(accountKeyword.size).toBe(2);
const xmlKeyword = mdiLightKeywords['xml']!;
expect(xmlKeyword).toBeTruthy();
expect(xmlKeyword.size).toBe(1);
// Create search index
const prefixes = data.prefixes;
expect(prefixes).toEqual(['mdi-light']);
const searchIndex = updateSearchIndex(prefixes, {
'mdi-light': {
importer,
item: mdiLightData,
},
})!;
// Check index
expect(searchIndex).toBeTruthy();
expect(searchIndex!.sortedPrefixes).toEqual(['mdi-light']);
expect(Object.keys(searchIndex.keywords)).toEqual(Object.keys(mdiLightKeywords));
expect(searchIndex.keywords['account']).toEqual(new Set(['mdi-light']));
expect(searchIndex.keywords['xml']).toEqual(new Set(['mdi-light']));
// Check for partial keywords
expect(getPartialKeywords('acc', searchIndex)).toEqual(['account']);
expect(getPartialKeywords('arr', searchIndex)).toEqual(['arrow', 'arrange']);
}, 5000);
test('Two icon sets', async () => {
// Create importer
// Use 'mdi-test-prefix' instead of 'mdi' to test prefix filters
const importer = createHardcodedCollectionsListImporter(['mdi-light', 'mdi-test-prefix'], (prefix) =>
createJSONIconSetImporter(new DirectoryDownloader<IconSetImportedData>(`tests/fixtures/json`), {
prefix,
filename: prefix === 'mdi-test-prefix' ? '/mdi.json' : `/${prefix}.json`,
ignoreInvalidPrefix: true,
})
);
await importer.init();
const data = importer.data!;
// Get keywords
const mdiData = data.iconSets['mdi-test-prefix']!;
const mdiKeywords = mdiData.icons.keywords!;
expect(mdiKeywords).toBeTruthy();
const mdiLightData = data.iconSets['mdi-light']!;
const mdiLightKeywords = mdiLightData.icons.keywords!;
expect(mdiLightKeywords).toBeTruthy();
// Create search index
const prefixes = data.prefixes;
expect(prefixes).toEqual(['mdi-light', 'mdi-test-prefix']);
const iconSets: Record<string, IconSetEntry> = {
'mdi-light': {
importer,
item: mdiLightData,
},
'mdi-test-prefix': {
importer,
item: mdiData,
},
};
const searchIndex = updateSearchIndex(prefixes, iconSets)!;
// Check index
expect(searchIndex).toBeTruthy();
expect(searchIndex!.sortedPrefixes).toEqual(['mdi-light', 'mdi-test-prefix']);
expect(Object.keys(searchIndex.keywords)).not.toEqual(Object.keys(mdiLightKeywords));
expect(Object.keys(searchIndex.keywords)).not.toEqual(Object.keys(mdiKeywords));
expect(new Set(Object.keys(searchIndex.keywords))).toEqual(
new Set([...Object.keys(mdiKeywords), ...Object.keys(mdiLightKeywords)])
);
expect(searchIndex.keywords['account']).toEqual(new Set(['mdi-light', 'mdi-test-prefix']));
expect(searchIndex.keywords['xml']).toEqual(new Set(['mdi-light', 'mdi-test-prefix']));
expect(searchIndex.keywords['alphabetical']).toEqual(new Set(['mdi-test-prefix']));
// Test filter
const baseParams: SearchParams = {
keyword: '',
limit: 0,
};
expect(filterSearchPrefixes(searchIndex, iconSets, baseParams)).toEqual(['mdi-light', 'mdi-test-prefix']);
// Test filter by prefixes
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
prefixes: ['mdi-light', 'whatever'],
})
).toEqual(['mdi-light']);
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
prefixes: ['mdi-'],
})
).toEqual(['mdi-light', 'mdi-test-prefix']);
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
prefixes: ['mdi', 'mdi-test-'],
})
).toEqual(['mdi-test-prefix']);
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
prefixes: ['material'],
})
).toEqual([]);
// Add palette
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
palette: false,
})
).toEqual(['mdi-light', 'mdi-test-prefix']);
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
prefixes: ['mdi-test-prefix'],
palette: false,
})
).toEqual(['mdi-test-prefix']);
expect(
filterSearchPrefixes(searchIndex, iconSets, {
...baseParams,
palette: true,
})
).toEqual([]);
}, 5000);
});

101
tests/search/search-test.ts Normal file
View File

@ -0,0 +1,101 @@
import { DirectoryDownloader } from '../../lib/downloaders/directory';
import { createHardcodedCollectionsListImporter } from '../../lib/importers/collections/list';
import { createJSONIconSetImporter } from '../../lib/importers/icon-set/json';
import { updateSearchIndex } from '../../lib/data/search';
import { search } from '../../lib/data/search/index';
import type { IconSetImportedData } from '../../lib/types/importers/common';
import type { IconSetEntry } from '../../lib/types/importers';
describe('Searching icons', () => {
test('Multiple icon sets', async () => {
// Create importer
const importer = createHardcodedCollectionsListImporter(
['mdi-light', 'mdi-test-prefix', 'emojione-v1'],
(prefix) => {
let filename: string;
switch (prefix) {
case 'mdi-test-prefix':
filename = '/json/mdi.json';
break;
case 'mdi':
case 'mdi-light':
filename = `/json/${prefix}.json`;
break;
default:
filename = `/${prefix}.json`;
}
return createJSONIconSetImporter(new DirectoryDownloader<IconSetImportedData>(`tests/fixtures`), {
prefix,
filename,
ignoreInvalidPrefix: true,
});
}
);
await importer.init();
const data = importer.data!;
// Get keywords
const mdiData = data.iconSets['mdi-test-prefix']!;
const mdiKeywords = mdiData.icons.keywords!;
expect(mdiKeywords).toBeTruthy();
const mdiLightData = data.iconSets['mdi-light']!;
const mdiLightKeywords = mdiLightData.icons.keywords!;
expect(mdiLightKeywords).toBeTruthy();
// Create search index
const prefixes = data.prefixes;
expect(prefixes).toEqual(['mdi-light', 'mdi-test-prefix', 'emojione-v1']);
const iconSets: Record<string, IconSetEntry> = {};
prefixes.forEach((prefix) => {
iconSets[prefix] = {
importer,
item: data.iconSets[prefix]!,
};
});
const searchIndex = updateSearchIndex(prefixes, iconSets)!;
// Check index
expect(searchIndex).toBeTruthy();
expect(searchIndex!.sortedPrefixes).toEqual(['mdi-light', 'mdi-test-prefix', 'emojione-v1']);
// Search
expect(
search(
{
keyword: 'cycle',
limit: 999,
},
searchIndex,
iconSets
)
).toEqual({
prefixes: ['mdi-test-prefix', 'emojione-v1'],
names: [
'mdi-test-prefix:cash-cycle',
'mdi-test-prefix:hand-cycle',
'mdi-test-prefix:power-cycle',
'mdi-test-prefix:bicycle',
'mdi-test-prefix:bicycle-basket',
'mdi-test-prefix:bicycle-cargo',
'mdi-test-prefix:bicycle-electric',
'mdi-test-prefix:bicycle-penny-farthing',
'emojione-v1:bicycle',
'mdi-test-prefix:battery-recycle',
'mdi-test-prefix:battery-recycle-outline',
'mdi-test-prefix:recycle',
'mdi-test-prefix:recycle-variant',
'mdi-test-prefix:water-recycle',
'mdi-test-prefix:unicycle',
'mdi-test-prefix:motorcycle',
'mdi-test-prefix:motorcycle-electric',
'mdi-test-prefix:motorcycle-off',
'emojione-v1:motorcycle',
],
hasMore: false,
});
}, 5000);
});

View File

@ -0,0 +1,313 @@
import { splitKeywordEntries } from '../../lib/data/search/split';
describe('Splitting keywords', () => {
test('Bad entries', () => {
expect(
splitKeywordEntries(['home?'], {
prefix: false,
partial: false,
})
).toBeUndefined();
expect(
splitKeywordEntries(['bad_stuff'], {
prefix: false,
partial: false,
})
).toBeUndefined();
expect(
splitKeywordEntries([], {
prefix: false,
partial: false,
})
).toBeUndefined();
expect(
splitKeywordEntries(['mdi', ''], {
prefix: false,
partial: false,
})
).toBeUndefined();
});
test('Simple entry', () => {
expect(
splitKeywordEntries(['home'], {
prefix: false,
partial: false,
})
).toEqual({
searches: [
{
keywords: ['home'],
},
],
});
expect(
splitKeywordEntries(['home'], {
prefix: true,
partial: false,
})
).toEqual({
searches: [
{
keywords: ['home'],
},
],
});
expect(
splitKeywordEntries(['home'], {
prefix: true,
partial: true,
})
).toEqual({
searches: [
{
keywords: [],
},
],
partial: 'home',
});
});
test('Multiple simple entries', () => {
expect(
splitKeywordEntries(['mdi', 'home'], {
prefix: false,
partial: false,
})
).toEqual({
searches: [
{
keywords: ['mdi', 'home'],
},
],
});
expect(
splitKeywordEntries(['mdi', 'home'], {
prefix: true,
partial: false,
})
).toEqual({
searches: [
{
prefix: 'mdi',
keywords: ['home'],
},
{
keywords: ['mdi', 'home'],
},
],
});
expect(
splitKeywordEntries(['mdi', 'home'], {
prefix: true,
partial: true,
})
).toEqual({
searches: [
{
prefix: 'mdi',
keywords: [],
},
{
keywords: ['mdi'],
},
],
partial: 'home',
});
});
test('Incomplete prefix', () => {
expect(
splitKeywordEntries(['mdi-', 'home'], {
prefix: false,
partial: false,
})
).toEqual({
searches: [
{
keywords: ['mdi', 'home'],
test: ['mdi-'],
},
],
});
expect(
splitKeywordEntries(['mdi-', 'home'], {
prefix: true,
partial: false,
})
).toEqual({
searches: [
{
prefix: 'mdi-',
keywords: ['home'],
},
{
keywords: ['mdi', 'home'],
test: ['mdi-'],
},
],
});
expect(
splitKeywordEntries(['mdi-', 'home'], {
prefix: true,
partial: true,
})
).toEqual({
searches: [
{
prefix: 'mdi-',
keywords: [],
},
{
keywords: ['mdi'],
test: ['mdi-'],
},
],
partial: 'home',
});
});
test('Long entry', () => {
expect(
splitKeywordEntries(['mdi-home-outline'], {
prefix: false,
partial: false,
})
).toEqual({
searches: [
{
keywords: ['mdi', 'home', 'outline'],
test: ['mdi-home-outline'],
},
],
});
expect(
splitKeywordEntries(['mdi-home-outline'], {
prefix: true,
partial: false,
})
).toEqual({
searches: [
{
prefix: 'mdi',
keywords: ['home', 'outline'],
test: ['home-outline'],
},
{
keywords: ['mdi', 'home', 'outline'],
test: ['mdi-home-outline'],
},
],
});
expect(
splitKeywordEntries(['mdi-home-outline'], {
prefix: true,
partial: true,
})
).toEqual({
searches: [
{
prefix: 'mdi',
keywords: ['home'],
test: ['home-outline'],
},
{
keywords: ['mdi', 'home'],
test: ['mdi-home-outline'],
},
],
partial: 'outline',
});
});
test('Complex entries', () => {
expect(
splitKeywordEntries(['mdi-light', 'arrow-left'], {
prefix: false,
partial: false,
})
).toEqual({
searches: [
{
keywords: ['mdi', 'light', 'arrow', 'left'],
test: ['mdi-light', 'arrow-left'],
},
],
});
expect(
splitKeywordEntries(['mdi-light', 'arrow-left'], {
prefix: true,
partial: false,
})
).toEqual({
searches: [
{
prefix: 'mdi-light',
keywords: ['arrow', 'left'],
test: ['arrow-left'],
},
{
prefix: 'mdi',
keywords: ['light', 'arrow', 'left'],
test: ['arrow-left'],
},
{
keywords: ['mdi', 'light', 'arrow', 'left'],
test: ['mdi-light', 'arrow-left'],
},
],
});
expect(
splitKeywordEntries(['mdi-light', 'arrow-left'], {
prefix: false,
partial: true,
})
).toEqual({
searches: [
{
keywords: ['mdi', 'light', 'arrow'],
test: ['mdi-light', 'arrow-left'],
},
],
partial: 'left',
});
expect(
splitKeywordEntries(['mdi-light', 'arrow-left'], {
prefix: true,
partial: true,
})
).toEqual({
searches: [
{
prefix: 'mdi-light',
keywords: ['arrow'],
test: ['arrow-left'],
},
{
prefix: 'mdi',
keywords: ['light', 'arrow'],
test: ['arrow-left'],
},
{
keywords: ['mdi', 'light', 'arrow'],
test: ['mdi-light', 'arrow-left'],
},
],
partial: 'left',
});
});
});

View File

@ -0,0 +1,318 @@
import { splitKeyword } from '../../lib/data/search/split';
describe('Splitting keywords', () => {
test('Bad entries', () => {
expect(splitKeyword('')).toBeUndefined();
expect(splitKeyword('-')).toBeUndefined();
expect(splitKeyword('prefix:mdi')).toBeUndefined();
expect(splitKeyword('palette=true')).toBeUndefined();
expect(splitKeyword('bad,entry')).toBeUndefined();
// Too many prefix entries
expect(splitKeyword('mdi:home mdi-light:home')).toBeUndefined();
});
test('Prefixes', () => {
// 'mdi-home'
expect(splitKeyword('mdi-home')).toEqual({
searches: [
{
prefixes: ['mdi'],
prefix: 'mdi', // leftover from internal function
keywords: [],
},
{
keywords: ['mdi'],
test: ['mdi-home'],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('mdi-home', false)).toEqual({
searches: [
{
prefixes: ['mdi'],
prefix: 'mdi', // leftover from internal function
keywords: ['home'],
},
{
keywords: ['mdi', 'home'],
test: ['mdi-home'],
},
],
params: {},
});
// 'mdi:home'
expect(splitKeyword('mdi:home')).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('mdi:home', false)).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: ['home'],
},
],
params: {},
});
// 'prefix:mdi home'
expect(splitKeyword('prefix:mdi home')).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('prefix:mdi home', false)).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: ['home'],
},
],
params: {},
});
// 'prefix=mdi home'
expect(splitKeyword('prefix=mdi home')).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('prefix=mdi home', false)).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: ['home'],
},
],
params: {},
});
// 'prefixes:mdi home'
expect(splitKeyword('prefixes:mdi home')).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('prefixes:mdi home', false)).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: ['home'],
},
],
params: {},
});
// 'prefixes:fa6-,mdi- home'
expect(splitKeyword('prefixes:fa6-,mdi- home')).toEqual({
searches: [
{
prefixes: ['fa6-', 'mdi-'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('prefixes:fa6-,mdi- home', false)).toEqual({
searches: [
{
prefixes: ['fa6-', 'mdi-'],
keywords: ['home'],
},
],
params: {},
});
// 'prefixes=mdi* home'
expect(splitKeyword('prefixes=mdi* home')).toEqual({
searches: [
{
prefixes: ['mdi', 'mdi-'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('prefixes=mdi* home', false)).toEqual({
searches: [
{
prefixes: ['mdi', 'mdi-'],
keywords: ['home'],
},
],
params: {},
});
// 'mdi-light home'
expect(splitKeyword('mdi-light home')).toEqual({
searches: [
{
prefixes: ['mdi-light'],
prefix: 'mdi-light',
keywords: [],
},
{
prefixes: ['mdi'],
prefix: 'mdi',
keywords: ['light'],
},
{
keywords: ['mdi', 'light'],
test: ['mdi-light'],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('mdi-light home', false)).toEqual({
searches: [
{
prefixes: ['mdi-light'],
prefix: 'mdi-light',
keywords: ['home'],
},
{
prefixes: ['mdi'],
prefix: 'mdi',
keywords: ['light', 'home'],
},
{
keywords: ['mdi', 'light', 'home'],
test: ['mdi-light'],
},
],
params: {},
});
// 'mdi-light home-outline'
expect(splitKeyword('mdi-light home-outline')).toEqual({
searches: [
{
prefixes: ['mdi-light'],
prefix: 'mdi-light',
keywords: ['home'],
test: ['home-outline'],
},
{
prefixes: ['mdi'],
prefix: 'mdi',
keywords: ['light', 'home'],
test: ['home-outline'],
},
{
keywords: ['mdi', 'light', 'home'],
test: ['mdi-light', 'home-outline'],
},
],
partial: 'outline',
params: {},
});
expect(splitKeyword('mdi-light home-outline', false)).toEqual({
searches: [
{
prefixes: ['mdi-light'],
prefix: 'mdi-light',
keywords: ['home', 'outline'],
test: ['home-outline'],
},
{
prefixes: ['mdi'],
prefix: 'mdi',
keywords: ['light', 'home', 'outline'],
test: ['home-outline'],
},
{
keywords: ['mdi', 'light', 'home', 'outline'],
test: ['mdi-light', 'home-outline'],
},
],
params: {},
});
});
test('Keywords', () => {
expect(splitKeyword('home palette:true')).toEqual({
searches: [
{
keywords: [],
},
],
partial: 'home',
params: {
palette: true,
},
});
expect(splitKeyword('home palette=0')).toEqual({
searches: [
{
keywords: [],
},
],
partial: 'home',
params: {
palette: false,
},
});
expect(splitKeyword('home prefixes=mdi*,fa6-')).toEqual({
searches: [
{
prefixes: ['mdi', 'mdi-', 'fa6-'],
keywords: [],
},
],
partial: 'home',
params: {},
});
expect(splitKeyword('home prefix=mdi palette=1', false)).toEqual({
searches: [
{
prefixes: ['mdi'],
keywords: ['home'],
},
],
params: {
palette: true,
},
});
// Too short for partial
expect(splitKeyword('ab')).toEqual({
searches: [
{
keywords: ['ab'],
},
],
params: {},
});
});
});