From 4ea82d2e4df337d8e881b34d2f1f303e623c056c Mon Sep 17 00:00:00 2001 From: Vjacheslav Trushkin Date: Tue, 6 Nov 2018 20:09:52 +0200 Subject: [PATCH] Full rewrite to make API faster and use less memory. Version 2.0.0-beta1 --- .gitignore | 2 - .npmignore | 3 - app.js | 523 ++++++++----------------- config-default.json | 3 +- config.md | 2 +- license.txt | 0 package-lock.json | 99 ++++- package.json | 12 +- src/collections.js | 316 --------------- src/dirs.js | 149 +++++-- src/files.js | 102 +++++ src/json.js | 143 +++++++ src/log.js | 267 ++++++------- src/logger.js | 64 +++ src/mail.js | 50 +++ src/promise.js | 10 +- src/query.js | 97 ----- src/reload.js | 343 ++++++++++++++++ src/request-icons.js | 95 +++++ src/request.js | 80 ++++ src/response.js | 87 +++++ src/startup.js | 82 ++++ src/sync.js | 579 ++++++---------------------- tests/fixtures/test1-optimized.json | 82 ++++ tests/fixtures/test1.json | 115 ++++++ tests/load_json_test.js | 53 +++ tests/log_test.js | 144 +++++++ tests/query_split_test.js | 2 +- tests/query_test.js | 112 ------ tests/request_icons_test.js | 136 +++++++ 30 files changed, 2192 insertions(+), 1560 deletions(-) mode change 100755 => 100644 license.txt delete mode 100644 src/collections.js create mode 100644 src/files.js create mode 100644 src/json.js create mode 100644 src/logger.js create mode 100644 src/mail.js delete mode 100644 src/query.js create mode 100644 src/reload.js create mode 100644 src/request-icons.js create mode 100644 src/request.js create mode 100644 src/response.js create mode 100644 src/startup.js create mode 100644 tests/fixtures/test1-optimized.json create mode 100644 tests/fixtures/test1.json create mode 100644 tests/load_json_test.js create mode 100644 tests/log_test.js delete mode 100644 tests/query_test.js create mode 100644 tests/request_icons_test.js diff --git a/.gitignore b/.gitignore index 5e2f1ca..5efbc3b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,4 @@ node_modules config.json *.log -_debug*.* -.ssl/ssl.* git-repos diff --git a/.npmignore b/.npmignore index ad59ceb..669f6fa 100644 --- a/.npmignore +++ b/.npmignore @@ -1,11 +1,8 @@ .idea .git -.reload .DS_Store config.json node_modules npm-debug.log tests -debug -_debug*.* git-repos diff --git a/app.js b/app.js index bf1fcde..438b84f 100644 --- a/app.js +++ b/app.js @@ -1,422 +1,207 @@ /** - * Main file to run in Node.js + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. */ + "use strict"; -/* - * Main stuff - */ +// Load required modules const fs = require('fs'), util = require('util'), + express = require('express'); - // Express stuff - express = require('express'), - app = express(), +// Log uncaught exceptions to stderr +process.on('uncaughtException', function (err) { + console.error('Uncaught exception:', err); +}); - // Configuration and version - version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, +// Create application +let app = { + root: __dirname +}; - // Included files - Collections = require('./src/collections'), - - // Query parser - parseQuery = require('./src/query'); - -// Configuration -let config = JSON.parse(fs.readFileSync(__dirname + '/config-default.json', 'utf8')); +/** + * Load config.json and config-default.json + */ +app.config = JSON.parse(fs.readFileSync(__dirname + '/config-default.json', 'utf8')); try { let customConfig = fs.readFileSync(__dirname + '/config.json', 'utf8'); if (typeof customConfig === 'string') { - customConfig = JSON.parse(customConfig); - Object.keys(customConfig).forEach(key => { - if (typeof config[key] !== typeof customConfig[key]) { - return; - } + try { + customConfig = JSON.parse(customConfig); + Object.keys(customConfig).forEach(key => { + if (typeof app.config[key] !== typeof customConfig[key]) { + return; + } - if (typeof config[key] === 'object') { - // merge object - Object.assign(config[key], customConfig[key]); - } else { - // overwrite scalar variables - config[key] = customConfig[key]; - } - }); + if (typeof app.config[key] === 'object') { + // merge object + Object.assign(app.config[key], customConfig[key]); + } else { + // overwrite scalar variables + app.config[key] = customConfig[key]; + } + }); + } catch (err) { + console.error('Error parsing config.json', err); + } } } catch (err) { + console.log('Missing config.json. Using default API configuration'); } -config._dir = __dirname; -// Enable logging module -require('./src/log')(config); +// Add logging and mail modules +app.mail = require('./src/mail').bind(this, app); +let log = require('./src/log'); +app.log = log.bind(this, app, false); +app.error = log.bind(this, app, true); + +app.logger = require('./src/logger').bind(this, app); + +/** + * Validate configuration + */ // Port -if (config['env-port'] && process.env.PORT) { - config.port = process.env.PORT; +if (app.config['env-port'] && process.env.PORT) { + app.config.port = process.env.PORT; } // Region file to easy identify server in CDN -if (!config['env-region'] && process.env.region) { - config.region = process.env.region; +if (!app.config['env-region'] && process.env.region) { + app.config.region = process.env.region; } -if (config.region.length > 10 || !config.region.match(/^[a-z0-9_-]+$/i)) { - config.region = ''; - config.log('Invalid value for region config variable.', 'config-region', true); +if (app.config.region.length > 10 || !app.config.region.match(/^[a-z0-9_-]+$/i)) { + app.config.region = ''; + app.error('Invalid value for region config variable.'); } // Reload secret key -if (config['reload-secret'] === '') { +if (app.config['reload-secret'] === '') { // Add reload-secret to config.json to be able to run /reload?key=your-secret-key that will reload collections without restarting server console.log('reload-secret configuration is empty. You will not be able to update all collections without restarting server.'); } -// Collections list -let collections = null, - loading = true, - anotherReload = false; - -// Modules -let dirs = require('./src/dirs')(config), - sync = require('./src/sync')(config); - /** - * Load icons - * - * @param {boolean} firstLoad - * @param {object} [logger] - * @returns {Promise} + * Continue loading modules */ -function loadIcons(firstLoad, logger) { - let newLogger = false; - if (!firstLoad && !logger) { - logger = new config.Logger('Reloading collections at ' + (new Date()).toString(), 90); - newLogger = true; - } +// Get version +app.version = JSON.parse(fs.readFileSync(__dirname + '/package.json', 'utf8')).version; - return new Promise((fulfill, reject) => { - function log(message) { - if (logger) { - logger.log(message, true); - } else { - console.log(message); - } - } - - function getCollections() { - let t = Date.now(), - newCollections = new Collections(config); - - console.log('Loading collections at ' + (new Date()).toString()); - newCollections.reload(dirs.getRepos(), logger).then(() => { - log('Loaded in ' + (Date.now() - t) + 'ms'); - if (newLogger) { - logger.send(); - } - fulfill(newCollections); - }).catch(err => { - log('Error loading collections: ' + util.format(err)); - if (logger) { - logger.send(); - } - reject(err); - }); - } - - if (firstLoad && config.sync && config.sync['sync-on-startup']) { - // Synchronize repositories first - let promises = []; - dirs.keys().forEach(repo => { - if (sync.canSync(repo)) { - switch (config.sync['sync-on-startup']) { - case 'always': - break; - - case 'never': - return; - - case 'missing': - // Check if repository is missing - if (sync.time(repo)) { - return; - } - } - if (!logger) { - logger = new config.Logger('Synchronizing repositories at startup', 120); - } - logger.log('Adding repository "' + repo + '" to queue'); - promises.push(sync.sync(repo, true, logger)); - } - }); - - if (promises.length) { - log('Synchronizing repositories before starting...'); - } - Promise.all(promises).then(() => { - getCollections(); - }).catch(err => { - console.log(err); - getCollections(); - }); - } else { - getCollections(); +// Files helper +app.fs = require('./src/files')(app); + +// JSON loader +app.loadJSON = require('./src/json').bind(this, app); + +// Add directories storage +app.dirs = require('./src/dirs')(app); +if (!app.dirs.getRepos().length) { + console.error('No repositories found. Make sure either Iconify or custom repository is set in configuration.'); + return; +} + +// Collections +app.collections = {}; +app.reload = require('./src/reload').bind(this, app); + +// Sync module +app.sync = require('./src/sync').bind(this, app); + +// API request and response handlers +app.response = require('./src/response').bind(this, app); +app.iconsRequest = require('./src/request-icons').bind(this, app); +app.miscRequest = require('./src/request').bind(this, app); + +// Start application +require('./src/startup')(app).then(() => { + + // Create HTTP server + app.server = express(); + + // Disable X-Powered-By header + app.server.disable('x-powered-by'); + + // CORS + app.server.options('/*', (req, res) => { + if (app.config.cors) { + res.header('Access-Control-Allow-Origin', app.config.cors.origins); + res.header('Access-Control-Allow-Methods', app.config.cors.methods); + res.header('Access-Control-Allow-Headers', app.config.cors.headers); + res.header('Access-Control-Max-Age', app.config.cors.timeout); } + res.send(200); }); -} -function reloadIcons(firstLoad, logger) { - loading = true; - anotherReload = false; - loadIcons(false, logger).then(newCollections => { - collections = newCollections; - loading = false; - if (anotherReload) { - reloadIcons(false, logger); - } - }).catch(err => { - config.log('Fatal error loading collections:\n' + util.format(err), null, true); - if (logger && logger.active) { - logger.log('Fatal error loading collections:\n' + util.format(err)); - } - loading = false; - if (anotherReload) { - reloadIcons(false, logger); - } + // GET 3 part request + app.server.get(/^\/([a-z0-9-]+)\/([a-z0-9-]+)\.(js|json|svg)$/, (req, res) => { + // prefix/icon.svg + // prefix/icons.json + app.iconsRequest(req, res, req.params[0], req.params[1], req.params[2]); }); -} -/** - * Send cache headers - * - * @param req - * @param res - */ -function cacheHeaders(req, res) { - if ( - config.cache && config.cache.timeout && - (req.get('Pragma') === void 0 || req.get('Pragma').indexOf('no-cache') === -1) && - (req.get('Cache-Control') === void 0 || req.get('Cache-Control').indexOf('no-cache') === -1) - ) { - res.set('Cache-Control', (config.cache.private ? 'private' : 'public') + ', max-age=' + config.cache.timeout + ', min-refresh=' + config.cache['min-refresh']); - if (!config.cache.private) { - res.set('Pragma', 'cache'); - } - } -} + // GET 2 part JS/JSON request + app.server.get(/^\/([a-z0-9-]+)\.(js|json)$/, (req, res) => { + // prefix.json + app.iconsRequest(req, res, req.params[0], 'icons', req.params[1]); + }); -/** - * Send result object generated by query parser - * - * @param {object} result - * @param req - * @param res - */ -function sendResult(result, req, res) { - if (typeof result === 'number') { - res.sendStatus(result); - return; - } + // GET 2 part SVG request + app.server.get(/^\/([a-z0-9:-]+)\.svg$/, (req, res) => { + let parts = req.params[0].split(':'); - // Send cache header - cacheHeaders(req, res); - - // Check for download - if (result.filename !== void 0 && (req.query.download === '1' || req.query.download === 'true')) { - res.set('Content-Disposition', 'attachment; filename="' + result.filename + '"'); - } - - // Send data - res.type(result.type).send(result.body); -} - -/** - * Delay response - * - * @param {function} callback - * @param res - */ -function delayResponse(callback, res) { - // Attempt to parse query every 250ms for up to 10 seconds - let attempts = 0, - timer = setInterval(function() { - attempts ++; - if (collections === null) { - if (attempts > 40) { - clearInterval(timer); - res.sendStatus(503); - } - } else { - clearInterval(timer); - callback(); - } - }, 250); -} - -/** - * Parse request - * - * @param {string} prefix - * @param {string} query - * @param {string} ext - * @param {object} req - * @param {object} res - */ -function parseRequest(prefix, query, ext, req, res) { - function parse() { - let result = 404, - collection = collections.find(prefix); - - if (collection !== null) { - result = parseQuery(collection, query, ext, req.query); - } - - sendResult(result, req, res); - } - - // Parse query - if (collections === null) { - // This means script is still loading - delayResponse(parse, res); - } else { - parse(); - } -} - -// Load icons -loadIcons(true).then(newCollections => { - collections = newCollections; - loading = false; - if (anotherReload) { - anotherReload = false; - setTimeout(() => { - reloadIcons(false); - }, 30000); - } -}).catch(err => { - config.log('Fatal error loading collections:\n' + util.format(err), null, true); - loading = false; - reloadIcons(true); -}); - -// Disable X-Powered-By header -app.disable('x-powered-by'); - -// CORS -app.options('/*', (req, res) => { - if (config.cors) { - res.header('Access-Control-Allow-Origin', config.cors.origins); - res.header('Access-Control-Allow-Methods', config.cors.methods); - res.header('Access-Control-Allow-Headers', config.cors.headers); - res.header('Access-Control-Max-Age', config.cors.timeout); - } - res.send(200); -}); - -// GET 3 part request -app.get(/^\/([a-z0-9-]+)\/([a-z0-9-]+)\.(js|json|svg)$/, (req, res) => { - // prefix/icon.svg - // prefix/icons.json - parseRequest(req.params[0], req.params[1], req.params[2], req, res); -}); - -// GET 2 part JS/JSON request -app.get(/^\/([a-z0-9-]+)\.(js|json)$/, (req, res) => { - // prefix.json - parseRequest(req.params[0], 'icons', req.params[1], req, res); -}); - -// GET 2 part SVG request -app.get(/^\/([a-z0-9:-]+)\.svg$/, (req, res) => { - let parts = req.params[0].split(':'); - - if (parts.length === 2) { - // prefix:icon.svg - parseRequest(parts[0], parts[1], 'svg', req, res); - return; - } - - if (parts.length === 1) { - parts = parts[0].split('-'); - if (parts.length > 1) { - // prefix-icon.svg - parseRequest(parts.shift(), parts.join('-'), 'svg', req, res); + if (parts.length === 2) { + // prefix:icon.svg + app.iconsRequest(req, res, parts[0], parts[1], 'svg'); return; } - } - res.sendStatus(404); -}); - -// Disable crawling -app.get('/robots.txt', (req, res) => { - res.type('text/plain').send('User-agent: *\nDisallow: /'); -}); - -// Debug information and AWS health check -app.get('/version', (req, res) => { - let body = 'Iconify API version ' + version + ' (Node'; - if (config.region.length) { - body += ', ' + config.region; - } - body += ')'; - res.send(body); -}); - -// Reload collections without restarting app -app.get('/reload', (req, res) => { - if (config['reload-secret'].length && req.query && req.query.key && req.query.key === config['reload-secret']) { - // Reload collections - process.nextTick(() => { - if (loading) { - anotherReload = true; + if (parts.length === 1) { + parts = parts[0].split('-'); + if (parts.length > 1) { + // prefix-icon.svg + app.iconsRequest(req, res, parts.shift(), parts.join('-'), 'svg'); return; } - reloadIcons(false); - }); - } - - // Send 200 response regardless of reload status, so visitor would not know if secret key was correct - // Testing should be done by checking new icons that should have been added by reload - res.sendStatus(200); -}); - -// Update collection without restarting app -let syncRequest = (req, res) => { - let repo = req.query.repo; - - if (sync.canSync(repo) && sync.validKey(req.query.key)) { - if (config.sync['sync-delay']) { - console.log('Will start synchronizing repository "' + repo + '" in up to ' + config.sync['sync-delay'] + ' seconds...'); } - sync.sync(repo, false).then(result => { - if (result.result) { - // Refresh all icons - if (loading) { - anotherReload = true; - } else { - reloadIcons(false, result.logger); - } - } - }).catch(err => { - config.log('Error synchronizing repository "' + repo + '":\n' + util.format(err), 'sync-' + repo, true); - }); - } - // Send 200 response regardless of reload status, so visitor would not know if secret key was correct - // Testing should be done by checking new icons that should have been added by reload - res.sendStatus(200); -}; -app.get('/sync', syncRequest); -app.post('/sync', syncRequest); + app.response(req, res, 404); + }); -// Redirect home page -app.get('/', (req, res) => { - res.redirect(301, config['index-page']); + // Send robots.txt that disallows everything + app.server.get('/robots.txt', (req, res) => app.miscRequest(req, res, 'robots')); + app.server.post('/robots.txt', (req, res) => app.miscRequest(req, res, 'robots')); + + // API version information + app.server.get('/version', (req, res) => app.miscRequest(req, res, 'version')); + + // Reload collections without restarting app + app.server.get('/reload', (req, res) => app.miscRequest(req, res, 'reload')); + app.server.post('/reload', (req, res) => app.miscRequest(req, res, 'reload')); + + // Get latest collection from Git repository + app.server.get('/sync', (req, res) => app.miscRequest(req, res, 'sync')); + app.server.post('/sync', (req, res) => app.miscRequest(req, res, 'sync')); + + // Redirect home page + app.server.get('/', (req, res) => { + res.redirect(301, app.config['index-page']); + }); + + // Create server + app.server.listen(app.config.port, () => { + app.log('Listening on port ' + app.config.port); + }); + +}).catch(err => { + console.error(err); }); -// Create server -app.listen(config.port, () => { - console.log('Listening on port ' + config.port); -}); -module.exports = app; + diff --git a/config-default.json b/config-default.json index 5096d65..514a6c8 100644 --- a/config-default.json +++ b/config-default.json @@ -7,6 +7,7 @@ "custom-icons-dir": "{dir}/json", "serve-default-icons": true, "index-page": "https://iconify.design/", + "json-loader": "parse", "cache": { "timeout": 604800, "min-refresh": 604800, @@ -21,7 +22,7 @@ "sync": { "sync-on-startup": "missing", "sync-delay": 60, - "repeated-sync-delay": 600, + "repeated-sync-delay": 60, "versions": "{dir}/git-repos/versions.json", "storage": "{dir}/git-repos", "git": "git clone {repo} --depth 1 --no-tags {target}", diff --git a/config.md b/config.md index 72fb5c4..ff113dc 100644 --- a/config.md +++ b/config.md @@ -161,7 +161,7 @@ Set to true to enable logging to email. #### throttle -Number of seconds to delay email sending. +Number of seconds to delay email sending. Default is 30 seconds. All error messages within 30 seconds will be combined to one email instead of sending multiple emails. diff --git a/license.txt b/license.txt old mode 100755 new mode 100644 diff --git a/package-lock.json b/package-lock.json index bb85ac8..9eedbec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,28 @@ { - "version": "1.0.0-rc2", + "version": "2.0.0-beta1", "lockfileVersion": 1, "requires": true, "dependencies": { "@iconify/json": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@iconify/json/-/json-1.0.8.tgz", - "integrity": "sha512-I2953Mf++qOuo2hHVWvv8jiW+tyS8OAHixSr5UjIWAADzrIFLK5Q1FP28fBfmb2cMpeBPiV5oR+3KrkzoX0N6A==" + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@iconify/json/-/json-1.0.10.tgz", + "integrity": "sha512-W7rSBnh7tGlK5LS7IA9McFxm2wVSzx0K6SbT9oFwee3WXtEKRc1knZoAWfybCq+SheH2JTLc/p+QH8a/T+jIAA==", + "optional": true }, "@iconify/json-tools": { - "version": "1.0.0-beta2", - "resolved": "https://registry.npmjs.org/@iconify/json-tools/-/json-tools-1.0.0-beta2.tgz", - "integrity": "sha512-pBCW9h0nEQyCuizYuiEtLsFZo0QeFZxA7ps7MWWFU1AF+ja8apuhjFbVfG2rex7utOriwp56B7VVGT3RiTRvOw==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@iconify/json-tools/-/json-tools-1.0.1.tgz", + "integrity": "sha512-A15I5GRny9gDZIfgu4nutUzdMsG0Bd2Ju6GbochRLu/ZSoMSH22L8mQrQdvdj0U/ydzR68mktUXrPUyG786Zlw==" + }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "optional": true, + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } }, "accepts": { "version": "1.3.5", @@ -162,6 +173,11 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, + "duplexer": { + "version": "0.1.1", + "resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=" + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -188,6 +204,21 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "optional": true, + "requires": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, "express": { "version": "4.16.4", "resolved": "https://registry.npmjs.org/express/-/express-4.16.4.tgz", @@ -249,6 +280,12 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=", + "optional": true + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -332,6 +369,18 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", + "optional": true + }, + "map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=", + "optional": true + }, "media-typer": { "version": "0.3.0", "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -432,7 +481,8 @@ "nodemailer": { "version": "4.6.8", "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-4.6.8.tgz", - "integrity": "sha512-A3s7EM/426OBIZbLHXq2KkgvmKbn2Xga4m4G+ZUA4IaZvG8PcZXrFh+2E4VaS2o+emhuUVRnzKN2YmpkXQ9qwA==" + "integrity": "sha512-A3s7EM/426OBIZbLHXq2KkgvmKbn2Xga4m4G+ZUA4IaZvG8PcZXrFh+2E4VaS2o+emhuUVRnzKN2YmpkXQ9qwA==", + "optional": true }, "on-finished": { "version": "2.3.0", @@ -473,6 +523,15 @@ "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, + "pause-stream": { + "version": "0.0.11", + "resolved": "http://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "optional": true, + "requires": { + "through": "~2.3" + } + }, "proxy-addr": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", @@ -549,11 +608,30 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "optional": true, + "requires": { + "through": "2" + } + }, "statuses": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" }, + "stream-combiner": { + "version": "0.2.2", + "resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", + "optional": true, + "requires": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, "supports-color": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", @@ -563,6 +641,11 @@ "has-flag": "^3.0.0" } }, + "through": { + "version": "2.3.8", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", diff --git a/package.json b/package.json index fc0b625..34079aa 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "1.0.0-rc2", + "version": "2.0.0-beta1", "description": "Node.js version of api.iconify.design", "private": true, "main": "app.js", @@ -16,13 +16,17 @@ "url": "git+ssh://git@github.com/iconify-design/api.js.git" }, "dependencies": { - "@iconify/json": "^1.0.8", "@iconify/json-tools": "^1.0.0-beta2", - "express": "^4.16.4", - "nodemailer": "^4.6.8" + "express": "^4.16.4" }, "devDependencies": { "chai": "^4.2.0", "mocha": "^5.2.0" + }, + "optionalDependencies": { + "@iconify/json": "^1.0.8", + "JSONStream": "^1.3.5", + "event-stream": "^4.0.1", + "nodemailer": "^4.6.8" } } diff --git a/src/collections.js b/src/collections.js deleted file mode 100644 index d8414bb..0000000 --- a/src/collections.js +++ /dev/null @@ -1,316 +0,0 @@ -/** - * This file is part of the @iconify/api package. - * - * (c) Vjacheslav Trushkin - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -"use strict"; - -const fs = require('fs'); -const util = require('util'); -const Collection = require('@iconify/json-tools').Collection; - -/** - * Class to represent collection of collections - */ -class Collections { - /** - * Constructor - * - * @param {object} config Application configuration - */ - constructor(config) { - this._config = config; - - this.items = {}; - this._loadQueue = []; - } - - /** - * Add directory to loading queue - * - * @param {string} dir - * @param {string} repo - * @param {object} [logger] - */ - addDirectory(dir, repo, logger) { - let message = 'Loading collections for repository "' + repo + '" from directory: ' + dir; - if (logger) { - logger.log(message, true) - } else { - console.log(message); - } - this._loadQueue.push({ - type: 'dir', - dir: dir.slice(-1) === '/' ? dir.slice(0, dir.length - 1) : dir, - repo: repo - }); - } - - /** - * Add file to loading queue - * - * @param {string} filename - * @param {string} repo - */ - addFile(filename, repo) { - this._loadQueue.push({ - type: 'file', - filename: filename, - repo: repo - }); - } - - /** - * Find collections - * - * @private - */ - _findCollections(repo, logger) { - return new Promise((fulfill, reject) => { - let config = this._config, - dirs = config._dirs, - iconsDir = dirs.iconsDir(repo); - - if (iconsDir === '') { - // Nothing to add - fulfill(); - return; - } - - switch (repo) { - case 'iconify': - // Get collections.json - let filename = dirs.rootDir(repo) + '/collections.json'; - fs.readFile(filename, 'utf8', (err, data) => { - if (err) { - let message = 'Error locating collections.json for Iconify default icons.\n' + util.format(err); - if (logger) { - logger.log(message); - } - reject(message); - return; - } - - try { - data = JSON.parse(data); - } catch (err) { - let message = 'Error reading contents of' + filename + '\n' + util.format(err); - if (logger) { - logger.log(message); - } - reject(message); - return; - } - - this.addDirectory(iconsDir, repo, logger); - this.info = data; - - fulfill(); - }); - return; - - default: - this.addDirectory(iconsDir, repo, logger); - fulfill(); - } - }); - } - - /** - * Find all collections and loadQueue - * - * @param {Array} repos - * @param {object} [logger] - * @returns {Promise} - */ - reload(repos, logger) { - return new Promise((fulfill, reject) => { - let promises = repos.map(repo => this._findCollections(repo, logger)); - - Promise.all(promises).then(() => { - return this.loadQueue(logger); - }).then(() => { - fulfill(this); - }).catch(err => { - reject(err); - }) - }); - } - - /** - * Load only one repository - * - * @param {string} repo Repository name - * @param {object} [logger] - * @returns {Promise} - */ - loadRepo(repo, logger) { - return new Promise((fulfill, reject) => { - Promise.all(this._findCollections(repo, logger)).then(() => { - return this.loadQueue(logger); - }).then(() => { - fulfill(this); - }).catch(err => { - reject(err); - }) - }); - } - - /** - * Load queue - * - * Promise will never reject because single file should not break app, - * it will log failures instead - * - * @param {object} [logger] - * @returns {Promise} - */ - loadQueue(logger) { - return new Promise((fulfill, reject) => { - let promises = []; - - this._loadQueue.forEach(item => { - switch (item.type) { - case 'dir': - promises.push(this._loadDir(item.dir, item.repo, logger)); - break; - - case 'file': - promises.push(this._loadFile(item.filename, item.repo, logger)); - break; - } - }); - - Promise.all(promises).then(res => { - let total = 0; - res.forEach(count => { - if (typeof count === 'number') { - total += count; - } - }); - let message = 'Loaded ' + total + ' icons'; - if (logger) { - logger.log(message, true); - } else { - console.log(message); - } - fulfill(this); - }).catch(err => { - reject(err); - }); - }); - } - - /** - * Load directory - * - * @param {string} dir - * @param {string} repo - * @param {object} [logger] - * @returns {Promise} - * @private - */ - _loadDir(dir, repo, logger) { - return new Promise((fulfill, reject) => { - fs.readdir(dir, (err, files) => { - if (err) { - this._config.log('Error reading directory: ' + dir + '\n' + util.format(err), 'collections-' + dir, true, logger); - fulfill(false); - } else { - let promises = []; - files.forEach(file => { - if (file.slice(-5) !== '.json') { - return; - } - promises.push(this._loadFile(dir + '/' + file, repo)); - }); - - // Load all promises - Promise.all(promises).then(res => { - let total = 0; - res.forEach(count => { - if (typeof count === 'number') { - total += count; - } - }); - fulfill(total); - }).catch(err => { - fulfill(false); - }); - } - }); - }); - } - - /** - * Load file - * - * @param {string} filename Full filename - * @param {string} repo - * @param {object} [logger] - * @returns {Promise} - */ - _loadFile(filename, repo, logger) { - return new Promise((fulfill, reject) => { - let file = filename.split('/').pop(), - fileParts = file.split('.'); - if (fileParts.length !== 2) { - fulfill(false); - return; - } - - let prefix = fileParts[0], - collection = new Collection(); - - collection.repo = repo; - collection.loadFromFileAsync(filename, prefix).then(result => { - collection = result; - if (collection.prefix() === false) { - this._config.log('Failed to load collection: ' + filename, 'collection-load-' + filename, true, logger); - fulfill(false); - return; - } - - if (collection.prefix() !== prefix) { - this._config.log('Collection prefix does not match: ' + collection.prefix() + ' in file ' + filename, 'collection-prefix-' + filename, true, logger); - fulfill(false); - return; - } - - let count = collection.listIcons(false).length; - if (!count) { - this._config.log('Collection is empty: ' + filename, 'collection-empty-' + filename, true, logger); - fulfill(false); - return; - } - - this.items[prefix] = collection; - let message = 'Loaded collection ' + prefix + ' from ' + file + ' (' + count + ' icons)'; - if (logger) { - logger.log(message, true); - } else { - console.log(message); - } - fulfill(count); - }).catch(err => { - fulfill(false); - }); - }); - } - - /** - * Find collection - * - * @param {string} prefix - * @returns {Collection|null} - */ - find(prefix) { - return this.items[prefix] === void 0 ? null : this.items[prefix]; - } -} - -module.exports = Collections; diff --git a/src/dirs.js b/src/dirs.js index 0e5f5c8..14d1fe0 100644 --- a/src/dirs.js +++ b/src/dirs.js @@ -9,18 +9,37 @@ "use strict"; -let config, _dirs; +const fs = require('fs'); -let repos; +/** + * Directories storage. + * This module is responsible for storing and updating locations of collections + * + * @param app + * @returns {object} + */ +module.exports = app => { + let functions = {}, + dirs = {}, + custom = {}, + repos = [], + storageDir = null, + versionsFile = null; -const functions = { /** * Get root directory of repository * * @param {string} repo * @returns {string} */ - rootDir: repo => _dirs[repo] === void 0 ? '' : _dirs[repo], + functions.rootDir = repo => dirs[repo] === void 0 ? '' : dirs[repo]; + + /** + * Get storage directory + * + * @return {string} + */ + functions.storageDir = () => storageDir; /** * Get icons directory @@ -28,7 +47,7 @@ const functions = { * @param {string} repo * @returns {string} */ - iconsDir: repo => { + functions.iconsDir = repo => { let dir; switch (repo) { @@ -39,18 +58,24 @@ const functions = { default: return functions.rootDir(repo); } - }, + }; /** * Set root directory for repository * - * @param repo - * @param dir + * @param {string} repo + * @param {string} dir */ - setRootDir: (repo, dir) => { - let extraKey = repo + '-dir'; - if (config.sync && config.sync[extraKey] !== void 0 && config.sync[extraKey] !== '') { - let extra = config.sync[extraKey]; + functions.setRootDir = (repo, dir) => { + // Append additional directory from config + let extra; + try { + extra = app.config.sync[repo + '-dir']; + } catch (err) { + extra = ''; + } + + if (extra !== void 0 && extra !== '') { if (extra.slice(0, 1) !== '/') { extra = '/' + extra; } @@ -59,41 +84,107 @@ const functions = { } dir += extra; } - _dirs[repo] = dir; - }, + + // Set directory + dirs[repo] = dir; + }; + + /** + * Set root directory for repository using repository time + * + * @param {string} repo + * @param {number} time + * @param {boolean} [save] True if new versions.json should be saved + */ + functions.setSynchronizedRepoDir = (repo, time, save) => { + let dir = storageDir + '/' + repo + '.' + time; + custom[repo] = time; + functions.setRootDir(repo, dir); + if (save === true) { + fs.writeFileSync(versionsFile, JSON.stringify(custom, null, 4), 'utf8'); + } + }; /** * Get all repositories * * @returns {string[]} */ - keys: () => Object.keys(_dirs), + functions.keys = () => Object.keys(dirs); /** * Get all repositories * * @returns {string[]} */ - getRepos: () => repos, -}; + functions.getRepos = () => repos; -module.exports = appConfig => { - config = appConfig; - _dirs = {}; - repos = []; + /** + * Check if repository has been synchronized + * + * @param {string} repo + * @return {boolean} + */ + functions.synchronized = repo => custom[repo] === true; + + /** + * Initialize + */ + + // Get synchronized repositories + let cached = {}; + app.config.canSync = false; + try { + if (app.config.sync.versions && app.config.sync.storage) { + // Set storage directory and versions.json location + storageDir = app.config.sync.storage.replace('{dir}', app.root); + versionsFile = app.config.sync.versions.replace('{dir}', app.root); + app.config.canSync = true; + + // Try getting latest repositories + cached = fs.readFileSync(versionsFile, 'utf8'); + cached = JSON.parse(cached); + } + } catch (err) { + if (typeof cached !== 'object') { + cached = {}; + } + } + + if (storageDir !== null) { + try { + fs.mkdirSync(storageDir); + } catch (err) { + } + } // Set default directories - if (config['serve-default-icons']) { - let icons = require('@iconify/json'); - repos.push('iconify'); - _dirs['iconify'] = icons.rootDir(); + if (app.config['serve-default-icons']) { + let key = 'iconify'; + if (cached && cached[key]) { + repos.push(key); + functions.setSynchronizedRepoDir(key, cached[key], false); + } else { + let icons; + try { + icons = require('@iconify/json'); + repos.push(key); + dirs[key] = icons.rootDir(); + } catch (err) { + app.error('Cannot load Iconify icons because @iconify/json package is not installed'); + } + } } - if (config['custom-icons-dir']) { - repos.push('custom'); - _dirs['custom'] = config['custom-icons-dir'].replace('{dir}', config._dir); + if (app.config['custom-icons-dir']) { + let key = 'custom'; + repos.push(key); + if (cached[key]) { + functions.setSynchronizedRepoDir(key, cached[key], false); + } else { + dirs[key] = app.config['custom-icons-dir'].replace('{dir}', app.root); + } } - config._dirs = functions; return functions; }; diff --git a/src/files.js b/src/files.js new file mode 100644 index 0000000..61746f5 --- /dev/null +++ b/src/files.js @@ -0,0 +1,102 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +const fs = require('fs'); +const util = require('util'); +const promiseEach = require('./promise'); + +let _app; + +let functions = { + /** + * Remove file + * + * @param file + * @param options + * @return {Promise} + */ + unlink: (file, options) => new Promise((fulfill, reject) => { + fs.unlink(file, err => { + if (err) { + _app.error('Error deleting file ' + file, Object.assign({ + key: 'unlink-' + file + }, typeof options === 'object' ? options : {})); + } + fulfill(); + }) + }), + + /** + * Recursively remove directory + * + * @param dir + * @param options + * @return {Promise} + */ + rmdir: (dir, options) => new Promise((fulfill, reject) => { + options = typeof options === 'object' ? options : {}; + + function done() { + fs.rmdir(dir, err => { + if (err) { + _app.error('Error deleting directory ' + dir, Object.assign({ + key: 'rmdir-' + dir + }, options)); + } + fulfill(); + }); + } + + fs.readdir(dir, (err, files) => { + if (err) { + // fulfill instead of rejecting + fulfill(); + return; + } + + let children = {}; + + files.forEach(file => { + let filename = dir + '/' + file, + stats = fs.lstatSync(filename); + + if (stats.isDirectory()) { + children[filename] = true; + return; + } + + if (stats.isFile() || stats.isSymbolicLink()) { + children[filename] = false; + } + }); + + promiseEach(Object.keys(children), file => { + if (children[file]) { + return functions.rmdir(file, options); + } else { + return functions.unlink(file, options); + } + }).then(() => { + done(); + }).catch(err => { + _app.error('Error recursively removing directory ' + dir + '\n' + util.format(err), Object.assign({ + key: 'rmdir-' + dir + }, options)); + done(); + }); + }); + }) +}; + +module.exports = app => { + _app = app; + return functions; +}; diff --git a/src/json.js b/src/json.js new file mode 100644 index 0000000..4ba2c59 --- /dev/null +++ b/src/json.js @@ -0,0 +1,143 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +const fs = require('fs'); +const util = require('util'); + +// List of imported modules +let imported = {}; + +/** + * Import json file + * + * @param {object|string} app Application object or import method + * @param {string} file File to import + * @param {*} [hash] Hash of previously loaded file. If hashes match, load will be aborted + * @returns {Promise} + */ +module.exports = (app, file, hash) => new Promise((fulfill, reject) => { + let newHash = null, + result; + + /** + * Parse json using JSONStream library + * + * @param JSONStream + * @param es + */ + function parseStream(JSONStream, es) { + let stream = fs.createReadStream(file, 'utf8'), + data; + + stream.on('error', err => { + reject('Error importing ' + file + '\n' + util.format(err)); + }); + stream.on('end', () => { + result.data = data; + fulfill(result); + }); + stream.pipe(JSONStream.parse(true)).pipe(es.mapSync(res => { + data = res; + })); + } + + /** + * Common parser that uses synchronous functions to convert string to object + * + * @param method + */ + function syncParser(method) { + fs.readFile(file, 'utf8', (err, data) => { + if (err) { + reject('Error importing ' + file + '\n' + util.format(err)); + return; + } + try { + switch (method) { + case 'eval': + data = Function('return ' + data)(); + break; + + default: + data = JSON.parse(data); + break; + } + } catch (err) { + reject('Error importing ' + file + '\n' + util.format(err)); + return; + } + + result.data = data; + fulfill(result); + }); + } + + // Get file information + fs.lstat(file, (err, stats) => { + if (!err) { + // Use file size instead of hash for faster loading + // assume json files are same when size is not changed + newHash = stats.size; + } + if (newHash && newHash === hash) { + fulfill({ + changed: false, + hash: newHash + }); + return; + } + + result = { + changed: true, + hash: newHash + }; + + // Figure out which parser to use + // 'eval' is fastest, but its not safe + // 'json' is slower, but might crash when memory limit is low + // 'stream' is + let parser = 'parse'; + try { + parser = typeof app === 'string' ? app : app.config['json-loader']; + } catch(err) { + } + + switch (parser) { + case 'stream': + // use stream + if (imported.JSONStream === void 0) { + try { + imported.JSONStream = require('JSONStream'); + imported.eventStream = require('event-stream'); + } catch (err) { + console.error('Cannot use stream JSON parser because JSONStream or event-stream module is not available. Switching to default parser.'); + imported.JSONStream = null; + } + } + + if (imported.JSONStream === null) { + syncParser('json'); + } else { + parseStream(imported.JSONStream, imported.eventStream); + } + break; + + case 'eval': + // use Function() + syncParser('eval'); + break; + + default: + // use JSON.parse() + syncParser('json'); + } + }); +}); diff --git a/src/log.js b/src/log.js index 76809f8..96ffc85 100644 --- a/src/log.js +++ b/src/log.js @@ -9,167 +9,116 @@ "use strict"; -const nodemailer = require('nodemailer'); +const util = require('util'); + +const defaultOptions = { + // True if message should be copied to stdout or stderr + log: true, + + // Logger object for event logging (combines multiple messages for one big log) + logger: null, + + // Unique key. If set, message with that key will be sent by mail only once. Used to avoid sending too many emails + key: null, + + // Console object + console: console +}; + +// List of notices that are sent only once per session +let logged = {}; + +// List of throttled messages +let throttled = null; /** - * Inject logging function as config.log() + * Send throttled messages * - * @param config + * @param app */ -module.exports = config => { - if (config.mail && config.mail.active) { - let logged = {}, - mailError = false, - throttled = false, - throttledData = [], - repeat = Math.max(config.mail.repeat, 15) * 60 * 1000; // convert minutes to ms, no less than 15 minutes - - /** - * Send message - * - * @param message - */ - let sendMail = message => { - // Create transport - let transporter = nodemailer.createTransport(config.mail.transport); - - // Set data - let mailOptions = { - from: config.mail.from, - to: config.mail.to, - subject: config.mail.subject, - text: message - }; - - // Send email - transporter.sendMail(mailOptions, (err, info) => { - if (err) { - if (mailError === false) { - console.error('Error sending mail (this messages will not show up again on further email errors until app is restarted):'); - console.error(err); - mailError = true; - } - } - }); - }; - - /** - * Send messages queue - */ - let sendQueue = () => { - let mailOptions = throttledData.join('\n\n- - - - - - - - - - -\n\n'); - - throttled = false; - throttledData = []; - - sendMail(mailOptions); - }; - - console.log('Logging to email is active. If you do not receive emails with errors, check configuration options.'); - - /** - * - * @param {string} message - * @param {string} [key] Unique key to identify logging message to avoid sending too many duplicate emails - * @param {boolean} [copyToConsole] True if log should be copied to console - * @param {object} [logger] Logger instance to copy message to - */ - config.log = (message, key, copyToConsole, logger) => { - if (copyToConsole) { - console.error('\x1b[31m' + message + '\x1b[0m'); - } - if (logger) { - logger.log(message); - } - - // Do not send same email more than once within "repeat" minutes - let time = Date.now() / repeat; - if (typeof key === 'string') { - if (logged[key] === time) { - return; - } - logged[key] = time; - } - - // Throttle - throttledData.push(message); - if (config.mail.throttle) { - if (!throttled) { - throttled = true; - setTimeout(sendQueue, config.mail.throttle * 1000); - } - } else { - sendQueue(); - } - }; - - /** - * Class for logging - * - * @type {Logger} - */ - config.Logger = class { - /** - * Create new logger - * - * @param {string} subject - * @param {number} [delay] Automatically send log after "delay" seconds - */ - constructor(subject, delay) { - this.active = true; - this.subject = subject; - this.messages = [subject]; - if (delay) { - setTimeout(() => { - if (this.messages.length) { - this.send(); - } - }, delay * 1000); - } - } - - /** - * Log message - * - * @param {string} message - * @param {boolean} [sendToConsole] - */ - log(message, sendToConsole) { - if (sendToConsole === true) { - console.log(message); - } - this.messages.push(message); - } - - /** - * Send logged messages - */ - send() { - if (!this.messages.length) { - return; - } - - sendMail(this.messages.join("\n")); - this.messages = []; - } - }; - } else { - console.log('Logging to email is not active.'); - config.log = (message, key, copyToConsole, logger) => { - if (copyToConsole) { - console.error('\x1b[35m' + message + '\x1b[0m'); - } - }; - config.Logger = class { - constructor(subject) { - this.active = false; - } - log(message, sendToConsole) { - if (sendToConsole === true) { - console.log(message); - } - } - send() {} - }; - } +const sendQueue = app => { + let text = throttled.join('\n\n- - - - - - - - - - -\n\n'); + throttled = null; + app.mail(text); +}; + +/** + * Log message. This function combines multiple logging methods, so it can be called only once instead of calling + * multiple log() functions. + * + * Message will be sent to console.log or console.error and sent by email. + * + * @param {object} app + * @param {boolean} error + * @param {string} message + * @param {object|boolean} [options] + */ +module.exports = (app, error, message, options) => { + options = Object.assign({}, defaultOptions, options === void 0 ? {} : (typeof options === 'boolean' ? { + log: options + }: options)); + + // Convert to test + if (typeof message !== 'string') { + message = util.format(message); + } + + // Get time stamp + let time = new Date(); + time = (time.getUTCHours() > 10 ? '[' : '[0') + time.getUTCHours() + (time.getUTCMinutes() > 9 ? ':' : ':0') + time.getUTCMinutes() + (time.getUTCSeconds() > 9 ? ':' : ':0') + time.getUTCSeconds() + '] '; + + // Copy message to console + if (options.log || !app.mail) { + if (error) { + options.console.error(time + '\x1b[31m' + message + '\x1b[0m'); + } else { + options.console.log(time + message); + } + } + + if (!app.mail) { + return; + } + message = time + message; + + // Copy to mail logger + if (options.logger) { + options.logger[error ? 'error' : 'log'](message); + return; + } + + // Send email if its a error and has not been sent before + if (!error) { + return; + } + if (options.key) { + let time = Date.now() / 1000, + repeat; + + try { + repeat = app.config.mail.repeat; + } catch (err) { + repeat = 0; + } + + if (logged[options.key]) { + if (!repeat || logged[options.key] > time) { + return; + } + } + logged[options.key] = repeat ? time + repeat : true; + } + + // Add message to throttled data + if (throttled === null) { + throttled = []; + let delay; + try { + delay = app.config.mail.throttle; + } catch (err) { + delay = 60; + } + setTimeout(sendQueue.bind(null, app), delay * 1000) + } + throttled.push(message); }; diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 0000000..b446d88 --- /dev/null +++ b/src/logger.js @@ -0,0 +1,64 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +class Logger { + constructor(app, subject, delay) { + this.app = app; + this.subject = subject; + this.messages = []; + this.delay = typeof delay === 'number' ? Math.min(Math.max(delay, 15), 300) : 60; + this.throttled = false; + } + + send() { + if (this.messages.length) { + this.app.mail((this.subject ? this.subject + '\n\n' : '') + this.messages.join('\n')); + this.messages = []; + } + } + + queue() { + if (!this.throttled) { + this.throttled = true; + setTimeout(() => { + this.send(); + this.throttled = false; + }, this.delay * 1000); + } + } + + log(message) { + this.messages.push(message); + if (!this.throttled) { + this.queue(); + } + } + + error(message) { + this.messages.push(message); + if (!this.throttled) { + this.queue(); + } + } +} + +/** + * Bulk message logger. It combines several messages and puts them in one email. + * + * Usage: logger = app.logger('subject', 60) + * then use it as "logger" parameter in app.log or app.error calls + * + * @param {object} app API application + * @param {string|null} [subject] Log subject + * @param {number} [delay] Delay of first set of messages + * @return {object} + */ +module.exports = (app, subject, delay) => new Logger(app, subject, delay); diff --git a/src/mail.js b/src/mail.js new file mode 100644 index 0000000..8f9afa9 --- /dev/null +++ b/src/mail.js @@ -0,0 +1,50 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +let nodemailer; + +module.exports = (app, message) => { + if (nodemailer === null) { + return; + } + + let config; + try { + config = app.config.mail; + if (!config.active) { + return; + } + + if (nodemailer === void 0) { + nodemailer = require('nodemailer'); + } + } catch (err) { + nodemailer = null; + return; + } + + let transporter = nodemailer.createTransport(config.transport); + + // Set data + let mailOptions = { + from: config.from, + to: config.to, + subject: config.subject, + text: message + }; + + // Send email + transporter.sendMail(mailOptions, (err, info) => { + if (err) { + console.error('Error sending mail:', err); + } + }); +}; diff --git a/src/promise.js b/src/promise.js index e6ab41d..fd29d30 100644 --- a/src/promise.js +++ b/src/promise.js @@ -17,16 +17,18 @@ * @returns {Promise} */ module.exports = (list, callback) => new Promise((fulfill, reject) => { - let results = []; + let results = [], + index = -1, + total = list.length; function next() { - let item = list.shift(); - if (item === void 0) { + index ++; + if (index === total) { fulfill(results); return; } - let promise = callback(item); + let promise = callback(list[index]); if (promise === null) { // skip next(); diff --git a/src/query.js b/src/query.js deleted file mode 100644 index 313b095..0000000 --- a/src/query.js +++ /dev/null @@ -1,97 +0,0 @@ -/** - * This file is part of the @iconify/api package. - * - * (c) Vjacheslav Trushkin - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -"use strict"; - -const SVG = require('@iconify/json-tools').SVG; - -/** - * Generate SVG string - * - * @param {object} icon - * @param {object} [params] - * @returns {string} - */ -function generateSVG(icon, params) { - let svg = new SVG(icon); - return svg.getSVG(params); -} - -/** - * Regexp for checking callback attribute - * - * @type {RegExp} - * @private - */ -const _callbackMatch = /^[a-z0-9_.]+$/i; - -/** - * Generate data for query - * - * @param {Collection} collection - * @param {string} query Query string after last / without extension - * @param {string} ext Extension - * @param {object} params Parameters - * @returns {number|object} - */ -module.exports = (collection, query, ext, params) => { - switch (ext) { - case 'svg': - // Generate SVG - // query = icon name - let icon = collection.getIconData(query); - if (icon === null) { - return 404; - } - return { - filename: query + '.svg', - type: 'image/svg+xml; charset=utf-8', - body: generateSVG(icon, params) - }; - - case 'js': - case 'json': - if (query !== 'icons' || typeof params.icons !== 'string') { - return 404; - } - - let result = collection.getIcons(params.icons.split(',')); - - if (result === null || !Object.keys(result.icons).length) { - return 404; - } - if (result.aliases !== void 0 && !Object.keys(result.aliases).length) { - delete result.aliases; - } - result = JSON.stringify(result); - - if (ext === 'js') { - let callback; - if (params.callback !== void 0) { - callback = params.callback; - if (!callback.match(_callbackMatch)) { - return 400; - } - } else { - callback = 'SimpleSVG._loaderCallback'; - } - return { - type: 'application/javascript; charset=utf-8', - body: callback + '(' + result + ')' - }; - } - return { - type: 'application/json; charset=utf-8', - body: result - }; - - default: - return 404; - } -}; \ No newline at end of file diff --git a/src/reload.js b/src/reload.js new file mode 100644 index 0000000..0bbe39b --- /dev/null +++ b/src/reload.js @@ -0,0 +1,343 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +const fs = require('fs'); +const util = require('util'); +const promiseEach = require('./promise'); +const Collection = require('@iconify/json-tools').Collection; + +const defaultOptions = { + // Logger instance + logger: null +}; + +let repoItems = {}, + collectionRepos = {}, + hashes = {}, + nextReload = 0; + +class Loader { + constructor(app, repos, options) { + this.app = app; + this.repos = repos; + this.options = options; + this.updated = []; + this.start = Date.now(); + this.reloadInfo = false; + } + + /** + * Remove root directory from filename + * + * @param {string} filename + * @return {string} + * @private + */ + _prettyFile(filename) { + return filename.slice(0, this.app.root.length) === this.app.root ? filename.slice(this.app.root.length + 1) : filename; + } + + /** + * Find collections + * + * @return {Promise} + */ + findCollections() { + return new Promise((fulfill, reject) => { + promiseEach(this.repos, repo => new Promise((fulfill, reject) => { + // Get directory + let dir = this.app.dirs.iconsDir(repo); + if (dir === '') { + reject('Missing directory for repository "' + repo + '"'); + return; + } + + // Find all files + fs.readdir(dir, (err, files) => { + let items = []; + if (err) { + reject('Error reading directory: ' + this._prettyFile(dir) + '\n' + util.format(err)); + return; + } + files.forEach(file => { + if (file.slice(-5) !== '.json') { + return; + } + items.push({ + repo: repo, + file: file, + filename: dir + '/' + file, + prefix: file.slice(0, file.length - 5) + }); + }); + fulfill(items); + }); + })).then(results => { + let items = []; + + results.forEach(result => { + result.forEach(item => { + if (collectionRepos[item.prefix] === void 0) { + // New collection. Add it to list + if (repoItems[item.repo] === void 0) { + repoItems[item.repo] = [item.prefix]; + } else { + repoItems[item.repo].push(item.prefix); + } + items.push(item); + return; + } + + if (collectionRepos[item.prefix] !== item.repo) { + // Conflict: same prefix in multiple repositories + this.app.error('Collection "' + item.prefix + '" is found in multiple repositories. Ignoring json file from ' + item.repo + ', using file from ' + collectionRepos[item.prefix], Object.assign({ + key: 'json-duplicate/' + item.repo + '/' + item.prefix + }, this.options)); + return; + } + + // Everything is fine + items.push(item); + }); + + }); + fulfill(items); + }).catch(err => { + reject(err); + }); + }); + } + + /** + * Load collections + * + * @param {Array} items + * @return {Promise} + */ + loadCollections(items) { + return new Promise((fulfill, reject) => { + let total; + + // Load all files + promiseEach(items, item => new Promise((fulfill, reject) => { + let collection; + + // Load JSON file + this.app.loadJSON(item.filename, hashes[item.prefix] === void 0 ? null : hashes[item.prefix]).then(result => { + if (!result.changed) { + // Nothing to do + fulfill(true); + return; + } + + return this.loadCollection(item, result); + }).then(result => { + collection = result; + + // Run post-load function if there is one + if (this.app.postLoadCollection) { + return this.app.postLoadCollection(collection, this.options); + } + }).then(() => { + fulfill(collection); + }).catch(err => { + reject('Error loading json file: ' + this._prettyFile(item.filename) + '\n' + util.format(err)); + }); + + })).then(collections => { + let loaded = 0, + skipped = 0; + + total = 0; + collections.forEach(collection => { + if (collection === true) { + skipped ++; + return; + } + loaded ++; + + let count = Object.keys(collection.items.icons).length, + prefix = collection.prefix(); + + this.app.log('Loaded collection ' + prefix + ' from ' + collection.filename + ' (' + count + ' icons)', this.options); + total += count; + this.app.collections[prefix] = collection; + }); + this.app.log('Loaded ' + total + ' icons from ' + loaded + (loaded > 1 ? ' collections ' : ' collection ') + (skipped ? '(no changes in ' + skipped + (skipped > 1 ? ' collections) ' : ' collection) ') : '') + 'in ' + (Date.now() - this.start) / 1000 + ' seconds.', this.options); + + if (this.reloadInfo) { + return this.getInfo(); + } + + }).then(() => { + fulfill(total); + }).catch(err => { + reject(err); + }); + }); + } + + /** + * Get Iconify collections data + * + * @return {Promise} + */ + getInfo() { + return new Promise((fulfill, reject) => { + let filename = this.app.dirs.rootDir('iconify') + '/collections.json'; + fs.readFile(filename, 'utf8', (err, data) => { + if (err) { + reject('Error locating collections.json for Iconify default icons.\n' + util.format(err)); + return; + } + + try { + data = JSON.parse(data); + } catch (err) { + reject('Error reading contents of' + filename + '\n' + util.format(err)); + return; + } + + this.app.info = data; + fulfill(); + }); + }); + } + + /** + * Load one collection + * + * @param {object} item findCollections() result + * @param {object} data loadJSON() result + * @return {Promise} + */ + loadCollection(item, data) { + return new Promise((fulfill, reject) => { + let collection = new Collection(); + if (!collection.loadJSON(data.data, item.prefix)) { + delete data.data; + reject('Error loading collection "' + item.prefix + '" from repository "' + item.repo + '": error parsing JSON'); + return; + } + delete data.data; + + let prefix = collection.prefix(); + if (prefix !== item.prefix) { + delete collection.items; + reject('Error loading collection "' + item.prefix + '" from repository "' + item.repo + '": invalid prefix in JSON file: ' + prefix); + return; + } + + collection.filename = this._prettyFile(item.filename); + collection.repo = item.repo; + hashes[item.prefix] = data.hash; + this.updated.push(item.prefix); + if (item.repo === 'iconify') { + this.reloadInfo = true; + } + fulfill(collection); + }); + } +} + +/** + * Reload collections + * + * @param {object} app + * @param {Array|string|boolean} [repos] Repositories to reload + * @param {object} [options] + */ +module.exports = (app, repos, options) => new Promise((fulfill, reject) => { + // Options + options = Object.assign({}, defaultOptions, typeof options === 'object' ? options : {}); + + // Get list of repositories to reload + let availableRepos = app.dirs.getRepos(); + // noinspection FallThroughInSwitchStatementJS + switch (typeof repos) { + case 'string': + if (availableRepos.indexOf(repos) === -1) { + reject('Cannot update repository: ' + repos); + return; + } + repos = [repos]; + break; + + case 'object': + if (repos instanceof Array) { + let newList = []; + repos.forEach(repo => { + if (availableRepos.indexOf(repo) !== -1) { + newList.push(repo); + } + }); + repos = newList; + break; + } + + case 'boolean': + if (repos === false) { + // false -> reload was called by /reload url + // limit such reloads to 1 per 30 seconds + if (Date.now() < nextReload) { + fulfill(false); + return; + } + } + + default: + repos = availableRepos.slice(0); + } + + if (!repos.length) { + reject('No available repositories to update.'); + return; + } + + if (app.reloading === true) { + reject('Reload is already in progress.'); + return; + } + + // Create logger if its missing + if (!options.logger) { + options.logger = app.logger('Loading repositories', 30); + } + + // Create loader instance and do stuff + let loader = new Loader(app, repos, options), + count; + + app.reloading = true; + loader.findCollections().then(items => { + return loader.loadCollections(items); + }).then(total => { + count = total; + + // Run post-load function if there is one + if (app.postReload) { + return app.postReload(loader.updated, options); + } + }).then(() => { + // Do not allow /reload for 30 seconds + nextReload = Date.now() + 30000; + + // Done + fulfill({ + icons: count, + updated: loader.updated + }); + app.reloading = false; + }).catch(err => { + reject(err); + app.reloading = false; + }); +}); diff --git a/src/request-icons.js b/src/request-icons.js new file mode 100644 index 0000000..cc36ec9 --- /dev/null +++ b/src/request-icons.js @@ -0,0 +1,95 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +const SVG = require('@iconify/json-tools').SVG; + +/** + * Generate SVG string + * + * @param {object} icon + * @param {object} [params] + * @returns {string} + */ +function generateSVG(icon, params) { + let svg = new SVG(icon); + return svg.getSVG(params); +} + +/** + * Regexp for checking callback attribute + * + * @type {RegExp} + * @private + */ +const _callbackMatch = /^[a-z0-9_.]+$/i; + +/** + * Parse request + * + * @param {object} app + * @param {object} req + * @param {object} res + * @param {string} prefix Collection prefix + * @param {string} query Query + * @param {string} ext Extension + */ +module.exports = (app, req, res, prefix, query, ext) => { + if (app.collections[prefix] === void 0) { + app.response(req, res, 404); + return; + } + + let collection = app.collections[prefix], + params = req.query; + + let parse = () => { + switch (ext) { + case 'svg': + // Generate SVG + // query = icon name + let icon = collection.getIconData(query); + if (icon === null) { + return 404; + } + return { + filename: query + '.svg', + type: 'image/svg+xml; charset=utf-8', + body: generateSVG(icon, params) + }; + + case 'js': + case 'json': + if (query !== 'icons' || typeof params.icons !== 'string') { + return 404; + } + + let result = collection.getIcons(params.icons.split(',')); + + if (result === null || !Object.keys(result.icons).length) { + return 404; + } + if (result.aliases !== void 0 && !Object.keys(result.aliases).length) { + delete result.aliases; + } + + return { + js: ext === 'js', + defaultCallback: 'SimpleSVG._loaderCallback', + data: result + }; + + default: + return 404; + } + }; + + app.response(req, res, parse()); +}; diff --git a/src/request.js b/src/request.js new file mode 100644 index 0000000..2a618b8 --- /dev/null +++ b/src/request.js @@ -0,0 +1,80 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +/** + * Parse request + * + * @param {object} app + * @param {object} req + * @param {object} res + * @param {string} query Query + */ +module.exports = (app, req, res, query) => { + let body; + + switch (query) { + case 'version': + body = 'Iconify API version ' + app.version + ' (Node'; + if (app.config.region.length) { + body += ', ' + app.config.region; + } + body += ')'; + app.response(req, res, { + type: 'text/plain', + body: body + }); + return; + + case 'robots': + app.response(req, res, { + type: 'text/plain', + body: 'User-agent: *\nDisallow: /' + }); + return; + + case 'reload': + // Send 200 response regardless of success to prevent visitors from guessing key + app.response(req, res, 200); + + // Do stuff + if (app.config['reload-secret'].length && req.query && req.query.key && req.query.key === app.config['reload-secret'] && !app.reloading) { + process.nextTick(() => { + app.reload(false).then(() => { + }).catch(err => { + app.error('Error reloading collections:\n' + util.format(err)); + }); + }); + } + return; + + case 'sync': + // Send 200 response regardless of success to prevent visitors from guessing key + app.response(req, res, 200); + + let repo = req.query.repo; + if (!app.config.canSync || !app.config.sync[repo] || !app.config.sync.git || !app.config.sync.secret) { + return; + } + + let key = req.query.key; + if (key !== app.config.sync.secret) { + return; + } + + process.nextTick(() => { + app.sync(repo).then(() => { + }).catch(err => { + app.error(err); + }); + }); + return; + } +}; diff --git a/src/response.js b/src/response.js new file mode 100644 index 0000000..c79fab0 --- /dev/null +++ b/src/response.js @@ -0,0 +1,87 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +/** + * Regexp for checking callback attribute + * + * @type {RegExp} + * @private + */ +const callbackMatch = /^[a-z0-9_.]+$/i; + +/** + * Send response + * + * @param app + * @param req + * @param res + * @param result + */ +module.exports = (app, req, res, result) => { + if (typeof result === 'number') { + // Send error + res.sendStatus(result); + return; + } + + // Convert JSON(P) response + if (result.body === void 0 && result.data !== void 0) { + if (typeof result.data === 'object') { + result.body = (req.query.pretty === '1' || req.query.pretty === 'true') ? JSON.stringify(result.data, null, 4) : JSON.stringify(result.data); + } + + if (result.js === void 0) { + result.js = req.query.callback !== void 0; + } + + if (result.js === true) { + let callback; + if (result.callback === void 0 && req.query.callback !== void 0) { + callback = req.query.callback; + if (!callback.match(callbackMatch)) { + // Invalid callback + res.sendStatus(400); + return; + } + } else { + callback = result.callback === void 0 ? result.defaultCallback : result.callback; + if (callback === void 0) { + res.sendStatus(400); + return; + } + } + result.body = callback + '(' + result.body + ');'; + result.type = 'application/javascript; charset=utf-8'; + } else { + result.type = 'application/json; charset=utf-8'; + } + } + + // Send cache header + if ( + app.config.cache && app.config.cache.timeout && + (req.get('Pragma') === void 0 || req.get('Pragma').indexOf('no-cache') === -1) && + (req.get('Cache-Control') === void 0 || req.get('Cache-Control').indexOf('no-cache') === -1) + ) { + res.set('Cache-Control', (app.config.cache.private ? 'private' : 'public') + ', max-age=' + app.config.cache.timeout + ', min-refresh=' + app.config.cache['min-refresh']); + if (!app.config.cache.private) { + res.set('Pragma', 'cache'); + } + } + + // Check for download + if (result.filename !== void 0 && (req.query.download === '1' || req.query.download === 'true')) { + res.set('Content-Disposition', 'attachment; filename="' + result.filename + '"'); + } + + // Send data + res.type(result.type).send(result.body); +}; diff --git a/src/startup.js b/src/startup.js new file mode 100644 index 0000000..43ca692 --- /dev/null +++ b/src/startup.js @@ -0,0 +1,82 @@ +/** + * This file is part of the @iconify/api package. + * + * (c) Vjacheslav Trushkin + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +"use strict"; + +const util = require('util'); +const promiseEach = require('./promise'); + +module.exports = app => new Promise((fulfill, reject) => { + let actions = [], + logger = app.logger('Starting API...', 60), + start = Date.now(); + + // Check for repositories to synchronize + if (app.config.canSync) { + switch (app.config['sync-on-startup']) { + case 'always': + case 'missing': + app.dirs.getRepos().forEach(repo => { + if (app.sync[repo] && ( + app.config['sync-on-startup'] === 'always' || !app.dirs.synchronized(repo) + )) { + actions.push({ + action: 'sync', + repo: repo + }); + } + }); + break; + } + } + + // Load icons + actions.push({ + action: 'load' + }); + + // Parse each promise + promiseEach(actions, action => new Promise((fulfill, reject) => { + switch (action.action) { + case 'load': + // Load icons + app.reload(null, { + logger: logger + }).then(() => { + if (!Object.keys(app.collections).length) { + reject('No collections were found.'); + } else { + fulfill(); + } + }).catch(err => { + reject('Error loading collections: ' + util.format(err)); + }); + return; + + case 'sync': + // Load icons + app.sync(action.repo, { + noDelay: true, + reload: false, + logger: logger + }).then(res => { + fulfill(); + }).catch(err => { + reject('Error synchronizing repository "' + repo + '": ' + util.format(err)); + }); + return; + } + })).then(() => { + logger.log('\nStart up process completed in ' + (Date.now() - start) / 1000 + ' seconds.'); + fulfill(); + }).catch(err => { + logger.error('\nStart up process failed!\n\n' + util.format(err)); + reject(err); + }); +}); diff --git a/src/sync.js b/src/sync.js index 11c6133..27bcfa4 100644 --- a/src/sync.js +++ b/src/sync.js @@ -9,484 +9,155 @@ "use strict"; -const fs = require('fs'), - util = require('util'), - child_process = require('child_process'), - promiseQueue = require('./promise'); +const fs = require('fs'); +const util = require('util'); +const child_process = require('child_process'); -let synchronized = {}, - active = false, - cleaning = false, - synchronizing = {}, - reSync = {}, - syncQueue = {}; - -let config, dirs, repos, _baseDir, _repoDir, _versionsFile; - -/** - * Start synchronization - * - * @param repo - * @param logger - */ -const startSync = (repo, logger) => { - if (syncQueue[repo] === void 0) { - return; - } - - function done(success) { - if (success) { - logger.log('Saved latest version of repository "' + repo + '" to ' + targetDir, true); - synchronized[repo] = t; - functions.saveVersions(); - dirs.setRootDir(repo, targetDir); - setTimeout(functions.cleanup, 300000); - } - synchronizing[repo] = false; - - syncQueue[repo].forEach((done, index) => { - if (index > 0) { - // Send false to all promises except first one to avoid loading collections several times - done({ - result: false, - logger: logger - }); - } else { - done({ - result: success, - logger: logger - }); - } - }); - } - - logger.log('Synchronizing repository "' + repo + '" ...', true); - synchronizing[repo] = true; - - let t = Date.now(), - targetDir = _baseDir + '/' + repo + '.' + t, - repoURL = config.sync[repo], - cmd = config.sync.git.replace('{target}', '"' + targetDir + '"').replace('{repo}', '"' + repoURL + '"'); - - reSync[repo] = false; - child_process.exec(cmd, { - cwd: _baseDir, - env: process.env, - uid: process.getuid() - }, (error, stdout, stderr) => { - if (error) { - if (logger.active) { - logger.log('Error executing git:' + util.format(error), true); - logger.send(); - } else { - config.log('Error executing git:' + util.format(error), cmd, true); - } - done(false); - return; - } - - if (reSync[repo]) { - // Another sync event was asked while cloning repository. Do it again - startSync(repo); - return; - } - - done(true); - }); +const defaultOptions = { + logger: null, + noDelay: false, + reload: true }; -/** - * Remove file - * - * @param {string} file - * @returns {Promise} - */ -const removeFile = file => new Promise((fulfill, reject) => { - fs.unlink(file, err => { - if (err) { - config.log('Error deleting file ' + file, file, false); - } - fulfill(); - }) -}); +let active = {}, + queued = {}; -/** - * Remove directory with sub-directories and files - * - * @param {string} dir - * @returns {Promise} - */ -const removeDir = dir => new Promise((fulfill, reject) => { - function done() { - fs.rmdir(dir, err => { - if (err) { - config.log('Error deleting directory ' + dir, dir, false); - } - fulfill(); - }); +class Sync { + constructor(app, repo, options) { + this.app = app; + this.repo = repo; + this.options = options; } - fs.readdir(dir, (err, files) => { - if (err) { - // fulfill instead of rejecting - fulfill(); - return; - } + sync() { + return new Promise((fulfill, reject) => { + this.app.log('Synchronizing repository "' + this.repo + '"...', this.options); - let children = {}; + let time = Date.now(), + root = this.app.dirs.storageDir(), + targetDir = root + '/' + this.repo + '.' + time, + repoURL = this.app.config.sync[this.repo], + cmd = this.app.config.sync.git.replace('{target}', '"' + targetDir + '"').replace('{repo}', '"' + repoURL + '"'); - files.forEach(file => { - let filename = dir + '/' + file, - stats = fs.lstatSync(filename); - - if (stats.isDirectory()) { - children[filename] = true; - return; - } - - if (stats.isFile() || stats.isSymbolicLink()) { - children[filename] = false; - } - }); - - promiseQueue(Object.keys(children), file => { - if (children[file]) { - return removeDir(file); - } else { - return removeFile(file); - } - }).then(() => { - done(); - }).catch(err => { - config.log('Error recursively removing directory ' + dir + '\n' + util.format(err), 'rmdir-' + dir, true); - done(); - }); - }); -}); - -/** - * Remove directory with sub-directories and files - * - * @param {string} dir - * @returns {Promise} - */ -const rmDir = dir => new Promise((fulfill, reject) => { - function oldMethod() { - removeDir(dir).then(() => { - fulfill(); - }).catch(err => { - reject(err); - }); - } - - if (!config.sync.rm) { - oldMethod(); - return; - } - - fs.lstat(dir, (err, stats) => { - if (err) { - if (err.code && err.code === 'ENOENT') { - // No such file/directory - fulfill(); - return; - } - // Unknown error - config.log('Error checking directory ' + dir + '\n' + util.format(err), 'rmd-' + dir, true); - fulfill(false); - return; - } - - // Attempt to remove using exec() - let cmd = config.sync.rm.replace('{dir}', '"' + dir + '"'); - child_process.exec(cmd, { - cwd: _baseDir, - env: process.env, - uid: process.getuid() - }, (error, stdout, stderr) => { - if (error) { - // rmdir didn't work? Attempt to remove each file - oldMethod(); - return; - } - - // Make sure directory is removed - fs.lstat(dir, (err, stats) => { - if (err && err.code && err.code === 'ENOENT') { - fulfill(); + child_process.exec(cmd, { + cwd: root, + env: process.env, + uid: process.getuid() + }, (error, stdout, stderr) => { + if (error) { + reject('Error executing git:' + util.format(error)); return; } - oldMethod(); + + // Done. Set new directory and reload collections + this.app.dirs.setSynchronizedRepoDir(this.repo, time, true); + + fulfill(true); }); }); - }); -}); - -/** - * Exported functions - * - * @type {object} - */ -const functions = { - /** - * Get root directory of repository - * - * @param {string} repo - * @returns {string|null} - */ - root: repo => synchronized[repo] ? _baseDir + '/' + repo + '.' + synchronized[repo] : null, - - /** - * Check if repository can be synchronized - * - * @param {string} repo - * @returns {boolean} - */ - canSync: repo => active && synchronized[repo] !== void 0, - - /** - * Get last synchronization time - * - * @param {string} repo - * @returns {number} - */ - time: repo => active && synchronized[repo] !== void 0 ? synchronized[repo] : 0, - - /** - * Check if key is valid - * - * @param {string} key - * @returns {boolean} - */ - validKey: key => typeof key === 'string' && key.length && key === config.sync.secret, - - /** - * Save versions.json - */ - saveVersions: () => { - let data = {}; - Object.keys(synchronized).forEach(repo => { - if (synchronized[repo]) { - data[repo] = synchronized[repo]; - } - }); - - fs.writeFile(_versionsFile, JSON.stringify(data, null, 4), 'utf8', err => { - if (err) { - config.error('Error saving versions.json\n' + util.format(err), 'version-' + _versionsFile, true); - } - }); - }, + } /** * Synchronize repository * - * @param {string} repo - * @param {boolean} [immediate] - * @param {*} [logger] - * @returns {Promise} + * @param app + * @param repo + * @param options + * @param fulfill + * @param reject */ - sync: (repo, immediate, logger) => new Promise((fulfill, reject) => { - let finished = false, - attempts = 0; + static sync(app, repo, options, fulfill, reject) { + active[repo] = true; + queued[repo] = false; - function done(result) { - if (finished) { - return; - } - - finished = true; - fulfill(result); - } - - function nextAttempt() { - if (finished) { - return; - } - - if (synchronizing[repo]) { - // Another repository is still being synchronized? - logger.log('Cannot start repository synchronization because sync is already in progress.', true); - attempts ++; - if (attempts > 3) { - done(false); - } else { - setTimeout(nextAttempt, config.sync['repeated-sync-delay'] * 1000); + let sync = new Sync(app, repo, options); + sync.sync(fulfill, reject).then(() => { + active[repo] = false; + if (queued[repo]) { + // Retry + let retryDelay; + try { + retryDelay = app.config.sync['repeated-sync-delay']; + } catch (err) { + retryDelay = 60; } + app.log('Repository "' + repo + '" has finished synchronizing, but there is another sync request queued. Will do another sync in ' + retryDelay + ' seconds.', options); + + setTimeout(() => { + Sync.sync(app, repo, options, fulfill, reject); + }, retryDelay * 1000); return; } - // Start synchronizing - startSync(repo, logger); - } - - if (!logger) { - logger = new config.Logger('Synchronizing repository "' + repo + '" at ' + (new Date()).toString(), (immediate ? 0 : config.sync['sync-delay']) + 90); - } - - if (!active) { - logger.log('Cannot synchronize repositories.'); - logger.send(); - reject('Cannot synchronize repositories.'); - return; - } - - // Add to queue - reSync[repo] = true; - if (syncQueue[repo] === void 0) { - syncQueue[repo] = [done]; - } else { - syncQueue[repo].push(done); - } - - if (synchronizing[repo]) { - // Wait until previous sync operation is over - logger.log('Another sync is in progress. Waiting for ' + config.sync['repeated-sync-delay'] + ' seconds before next attempt.'); - setTimeout(nextAttempt, config.sync['repeated-sync-delay'] * 1000); - attempts ++; - } else if (immediate === true) { - // Start immediately - nextAttempt(); - } else { - // Wait a bit to avoid multiple synchronizations - logger.log('Waiting for ' + config.sync['sync-delay'] + ' before starting synchronization.'); - setTimeout(nextAttempt, config.sync['sync-delay'] * 1000); - } - }), - - /** - * Remove old files - */ - cleanup: () => { - if (cleaning) { - return; - } - cleaning = true; - - fs.readdir(_baseDir, (err, files) => { - if (err) { - cleaning = false; - return; + // Done + app.log('Completed synchronization of repository "' + repo + '".', options); + if (options.reload && !queued[repo]) { + app.reload(repo, options).then(() => { + fulfill(true); + }).catch(err => { + reject(err); + }); + } else { + fulfill(true); } - - let dirs = []; - files.forEach(file => { - let parts = file.split('.'); - if (parts.length !== 2 || synchronized[parts[0]] === void 0) { - return; - } - - let repo = parts.shift(), - time = parseInt(parts.shift()); - - if (time > (synchronized[repo] - 3600 * 1000)) { - // wait 1 hour before deleting old repository - return; - } - - dirs.push(_baseDir + '/' + file); - }); - - if (!dirs.length) { - cleaning = false; - return; - } - - console.log('Cleaning up old repositories...'); - - // Delete all directories, but only 1 at a time to reduce loadQueue - promiseQueue(dirs, dir => rmDir(dir)).then(() => { - cleaning = false; - }).catch(err => { - config.log('Error cleaning up old files:\n' + util.format(err), 'cleanup', true); - cleaning = false; - }); - }); + }).catch(err => { + reject(err); + }) } -}; - -/** - * Initialize. Find active repositories - */ -function init() { - if (!config.sync || !config.sync.versions || !config.sync.storage) { - // Synchronization is inactive - return; - } - - if (!config.sync.secret) { - // Cannot sync without secret word - console.log('Repositories synchronization is not possible because "secret" is empty. Check config.md for details.'); - return; - } - - // Check active repositories - repos.forEach(repo => { - if (!config.sync[repo]) { - return; - } - - synchronized[repo] = 0; - synchronizing[repo] = false; - }); - - if (!Object.keys(synchronized).length) { - // Nothing was found - console.log('Repositories synchronization is not possible because no active repositories were found. Check config.md for details.'); - return; - } - - // Try to create base directory - _baseDir = config.sync.storage.replace('{dir}', config._dir); - try { - fs.mkdirSync(_baseDir); - } catch (err) { - } - - // Check for versions.json - _versionsFile = config.sync.versions.replace('{dir}', config._dir); - active = true; - - let data; - try { - data = fs.readFileSync(_versionsFile, 'utf8'); - data = JSON.parse(data); - } catch (err) { - // Nothing to parse - return; - } - - Object.keys(data).forEach(key => { - let dir; - - if (synchronized[key] === void 0) { - return; - } - - dir = _baseDir + '/' + key + '.' + data[key]; - try { - let stat = fs.lstatSync(dir); - if (stat && stat.isDirectory()) { - // Found directory - synchronized[key] = data[key]; - dirs.setRootDir(key, dir); - console.log('Icons will be loaded from ' + dir + ' instead of default location.'); - return; - } - } catch (err) { - } - - config.log('Error loading latest collections: directory does not exist: ' + dir, 'missing-' + dir, true); - }); - setTimeout(functions.cleanup, 60000); } -module.exports = appConfig => { - config = appConfig; - dirs = config._dirs; - repos = dirs.getRepos(); - init(); +module.exports = (app, repo, options) => new Promise((fulfill, reject) => { + // Options + options = Object.assign({}, defaultOptions, typeof options !== 'object' ? options : {}); + + // Check if synchronization is disabled + if (!app.config.canSync || !app.config.sync[repo] || !app.config.sync.git) { + reject('Synchronization is disabled.'); + return; + } + + // Check if repository sync is already in queue + if (queued[repo]) { + app.log('Repository "' + repo + '" is already in synchronization queue.', options); + fulfill(false); + return; + } + + let delay, retryDelay; + try { + delay = app.config.sync['sync-delay']; + retryDelay = app.config.sync['repeated-sync-delay']; + } catch (err) { + delay = 60; + retryDelay = 60; + } + if (options.noDelay) { + delay = 0; + } + + // Add to queue + queued[repo] = true; + + // Check if repository is already being synchronized + if (active[repo]) { + app.log('Repository "' + repo + '" is already being synchronized. Will do another sync ' + retryDelay + ' seconds after previous sync completes.', options); + fulfill(false); + return; + } + + // Create logger if its missing + if (!options.logger) { + options.logger = app.logger('Synchronizing repository: ' + repo, delay + 15); + } + + // Start time + if (!delay) { + Sync.sync(app, repo, options, fulfill, reject); + } else { + app.log('Repository "' + repo + '" will start synchronizing in ' + delay + ' seconds.', options); + setTimeout(() => { + Sync.sync(app, repo, options, fulfill, reject); + }, delay * 1000); + } +}); - config._sync = functions; - return functions; -}; diff --git a/tests/fixtures/test1-optimized.json b/tests/fixtures/test1-optimized.json new file mode 100644 index 0000000..a7be45f --- /dev/null +++ b/tests/fixtures/test1-optimized.json @@ -0,0 +1,82 @@ +{ + "prefix": "fa", + "icons": { + "arrow-circle-left": { + "body": "" + }, + "arrow-circle-up": { + "body": "" + }, + "arrow-up": { + "body": "", + "width": 1600, + "height": 1472, + "inlineTop": -192 + }, + "arrow-left": { + "body": "", + "width": 1472, + "height": 1600, + "inlineTop": -160 + }, + "arrows": { + "body": "", + "width": 1792, + "height": 1792, + "inlineTop": 0 + }, + "arrows-alt": { + "body": "" + }, + "arrows-h": { + "body": "", + "width": 1792, + "height": 1280, + "inlineTop": -256 + }, + "arrows-v": { + "body": "", + "width": 640, + "height": 1792, + "inlineTop": 0 + }, + "assistive-listening-systems": { + "body": "", + "width": 1792, + "height": 1792, + "inlineTop": 0 + }, + "asterisk": { + "body": "", + "width": 1472 + }, + "at": { + "body": "" + }, + "audio-description": { + "body": "", + "width": 2304, + "height": 1280, + "inlineTop": -256 + } + }, + "aliases": { + "arrow-circle-right": { + "parent": "arrow-circle-left", + "hFlip": true + }, + "arrow-down": { + "parent": "arrow-up", + "vFlip": true + }, + "arrow-right": { + "parent": "arrow-left", + "hFlip": true + } + }, + "width": 1536, + "height": 1536, + "inlineHeight": 1792, + "inlineTop": -128, + "verticalAlign": -0.143 +} \ No newline at end of file diff --git a/tests/fixtures/test1.json b/tests/fixtures/test1.json new file mode 100644 index 0000000..6a4a051 --- /dev/null +++ b/tests/fixtures/test1.json @@ -0,0 +1,115 @@ +{ + "prefix": "fa", + "icons": { + "arrow-circle-left": { + "body": "", + "width": 1536, + "height": 1536, + "inlineHeight": 1792, + "inlineTop": -128, + "verticalAlign": -0.143 + }, + "arrow-circle-up": { + "body": "", + "width": 1536, + "height": 1536, + "inlineHeight": 1792, + "inlineTop": -128, + "verticalAlign": -0.143 + }, + "arrow-up": { + "body": "", + "width": 1600, + "height": 1472, + "inlineTop": -192, + "inlineHeight": 1792, + "verticalAlign": -0.143 + }, + "arrow-left": { + "body": "", + "width": 1472, + "height": 1600, + "inlineTop": -160, + "inlineHeight": 1792, + "verticalAlign": -0.143 + }, + "arrows": { + "body": "", + "width": 1792, + "height": 1792, + "inlineTop": 0, + "inlineHeight": 1792, + "verticalAlign": -0.143 + }, + "arrows-alt": { + "body": "", + "width": 1536, + "height": 1536, + "inlineHeight": 1792, + "inlineTop": -128, + "verticalAlign": -0.143 + }, + "arrows-h": { + "body": "", + "width": 1792, + "height": 1280, + "inlineTop": -256, + "inlineHeight": 1792, + "verticalAlign": -0.143 + }, + "arrows-v": { + "body": "", + "width": 640, + "height": 1792, + "inlineTop": 0, + "inlineHeight": 1792, + "verticalAlign": -0.143 + }, + "assistive-listening-systems": { + "body": "", + "width": 1792, + "height": 1792, + "inlineTop": 0, + "inlineHeight": 1792, + "verticalAlign": -0.143 + }, + "asterisk": { + "body": "", + "width": 1472, + "height": 1536, + "inlineHeight": 1792, + "inlineTop": -128, + "verticalAlign": -0.143 + }, + "at": { + "body": "", + "width": 1536, + "height": 1536, + "inlineHeight": 1792, + "inlineTop": -128, + "verticalAlign": -0.143 + }, + "audio-description": { + "body": "", + "width": 2304, + "height": 1280, + "inlineTop": -256, + "inlineHeight": 1792, + "verticalAlign": -0.143 + } + }, + "aliases": { + "arrow-circle-right": { + "parent": "arrow-circle-left", + "hFlip": true + }, + "arrow-down": { + "parent": "arrow-up", + "vFlip": true + }, + "arrow-right": { + "parent": "arrow-left", + "hFlip": true + } + } +} \ No newline at end of file diff --git a/tests/load_json_test.js b/tests/load_json_test.js new file mode 100644 index 0000000..c0f6d17 --- /dev/null +++ b/tests/load_json_test.js @@ -0,0 +1,53 @@ +"use strict"; + +(() => { + const loadJSON = require('../src/json'); + + const fs = require('fs'), + chai = require('chai'), + expect = chai.expect, + should = chai.should(); + + describe('Loading JSON file', () => { + const filename = __dirname + '/fixtures/test1.json', + expectedResult = JSON.parse(fs.readFileSync(filename, 'utf8')); + + // Check if stream method is available + let testStream; + try { + require('JSONStream'); + require('event-stream'); + testStream = true; + } catch (err) { + testStream = false; + } + + // Test with each method + ['json', 'eval', 'stream'].forEach(method => { + it(method, function(done) { + if (method === 'stream' && !testStream) { + this.skip(); + return; + } + + // Load file + loadJSON(method, filename).then(result => { + expect(result.changed).to.be.equal(true); + expect(result.data).to.be.eql(expectedResult); + + // Load file with same hash + loadJSON(method, filename, result.hash).then(result2 => { + expect(result2.changed).to.be.equal(false); + expect(result2.hash).to.be.equal(result.hash); + + done(); + }).catch(err => { + done(err); + }); + }).catch(err => { + done(err); + }); + }); + }); + }); +})(); diff --git a/tests/log_test.js b/tests/log_test.js new file mode 100644 index 0000000..5694b67 --- /dev/null +++ b/tests/log_test.js @@ -0,0 +1,144 @@ +"use strict"; + +(() => { + const log = require('../src/log'); + + const chai = require('chai'), + expect = chai.expect, + should = chai.should(); + + describe('Logging messages', () => { + it ('logging error to console and mail', done => { + let logged = { + log: false, + mail: false + }; + let fakeApp = { + mail: message => { + expect(message.indexOf(expectedMessage) !== false).to.be.equal(true); + logged.mail = true; + }, + logger: () => { + done('logger() should not have been called'); + }, + config: { + mail: { + throttle: 0.2 + } + } + }; + + let expectedMessage = 'This is a test'; + log(fakeApp, true, expectedMessage, { + console: { + error: message => { + expect(message.indexOf(expectedMessage) !== false).to.be.equal(true); + logged.log = true; + }, + log: message => { + done('console.log should not have been called'); + } + } + }); + + setTimeout(() => { + expect(logged).to.be.eql({ + log: true, + mail: true + }); + done(); + }, 500); + }); + + it ('logging message to console', done => { + let logged = { + log: false + }; + let fakeApp = { + mail: message => { + done('mail() should not have been called'); + }, + logger: () => { + done('logger() should not have been called'); + }, + config: { + mail: { + throttle: 0.2 + } + } + }; + + let expectedMessage = 'This is a test'; + log(fakeApp, false, expectedMessage, { + console: { + log: message => { + expect(message.indexOf(expectedMessage) !== false).to.be.equal(true); + logged.log = true; + }, + error: message => { + done('console.log should not have been called'); + } + } + }); + + setTimeout(() => { + expect(logged).to.be.eql({ + log: true + }); + done(); + }, 500); + }); + + it ('logging same error only once', done => { + let logged = { + log: false, + mail: false + }; + let fakeApp = { + mail: message => { + if (logged.mail) { + done('mail() was called twice'); + } + expect(message.indexOf(expectedMessage) !== false).to.be.equal(true); + logged.mail = true; + }, + logger: () => { + done('logger() should not have been called'); + }, + config: { + mail: { + throttle: 0.2 + } + } + }; + + let expectedMessage = 'This is a test', + fakeConsole = { + error: message => { + expect(message.indexOf(expectedMessage) !== false).to.be.equal(true); + logged.log = true; + }, + log: message => { + done('console.log should not have been called'); + } + }; + + log(fakeApp, true, expectedMessage, { + console: fakeConsole, + key: 'test' + }); + log(fakeApp, true, expectedMessage, { + console: fakeConsole, + key: 'test' + }); + + setTimeout(() => { + expect(logged).to.be.eql({ + log: true, + mail: true + }); + done(); + }, 500); + }); + }); +})(); diff --git a/tests/query_split_test.js b/tests/query_split_test.js index 336c52a..46cf890 100644 --- a/tests/query_split_test.js +++ b/tests/query_split_test.js @@ -5,7 +5,7 @@ expect = chai.expect, should = chai.should(); - describe('Testing splitting query string', () => { + describe('Splitting query string', () => { it('3 part requests', () => { const exp = /^\/([a-z0-9-]+)\/([a-z0-9-]+)\.(js|json|svg)$/; diff --git a/tests/query_test.js b/tests/query_test.js deleted file mode 100644 index fdffe9e..0000000 --- a/tests/query_test.js +++ /dev/null @@ -1,112 +0,0 @@ -"use strict"; - -(() => { - const chai = require('chai'), - expect = chai.expect, - should = chai.should(); - - const Collection = require('@iconify/json-tools').Collection, - parseQuery = require('../src/query'); - - let collection1 = new Collection('test'); - collection1.loadJSON({ - prefix: 'test', - icons: { - icon1: { - body: '', - width: 30 - }, - icon2: { - body: '' - } - }, - aliases: { - alias1: { - parent: 'icon2', - hFlip: true - } - }, - width: 24, - height: 24 - }); - - let collection2 = new Collection('test2'); - collection2.loadJSON({ - icons: { - 'test2-icon1': { - body: '', - width: 30 - }, - 'test2-icon2': { - body: '' - }, - 'test2-icon3': { - body: '' - } - }, - aliases: { - 'test2-alias1': { - parent: 'test2-icon2', - hFlip: true - } - }, - width: 24, - height: 24 - }); - - describe('Testing requests', () => { - it('icons list', () => { - // Simple query with prefix - expect(parseQuery(collection1, 'icons', 'js', { - icons: 'alias1' - })).to.be.eql({ - type: 'application/javascript; charset=utf-8', - body: 'SimpleSVG._loaderCallback({"prefix":"test","icons":{"icon2":{"body":"","width":24,"height":24}},"aliases":{"alias1":{"parent":"icon2","hFlip":true}}})' - }); - - // Query collection without prefix, json - expect(parseQuery(collection2, 'icons', 'json', { - icons: 'alias1' - })).to.be.eql({ - type: 'application/json; charset=utf-8', - body: '{"prefix":"test2","icons":{"icon2":{"body":"","width":24,"height":24}},"aliases":{"alias1":{"parent":"icon2","hFlip":true}}}' - }); - - // Custom callback - expect(parseQuery(collection1, 'icons', 'js', { - icons: 'icon1,icon2', - callback: 'console.log' - })).to.be.eql({ - type: 'application/javascript; charset=utf-8', - body: 'console.log({"prefix":"test","icons":{"icon1":{"body":"","width":30,"height":24},"icon2":{"body":"","width":24,"height":24}}})' - }); - }); - - it('svg', () => { - // Simple icon - expect(parseQuery(collection1, 'icon1', 'svg', { - })).to.be.eql({ - filename: 'icon1.svg', - type: 'image/svg+xml; charset=utf-8', - body: '' - }); - - // Icon with custom attributes - expect(parseQuery(collection2, 'alias1', 'svg', { - color: 'red' - })).to.be.eql({ - filename: 'alias1.svg', - type: 'image/svg+xml; charset=utf-8', - body: '' - }); - - // Icon with id replacement - let result = parseQuery(collection2, 'icon3', 'svg', { - color: 'red', - rotate: '90deg' - }).body.replace(/IconifyId-[0-9a-f]+-[0-9a-f]+-[0-9]+/g, 'some-id'); - - expect(result).to.be.equal(''); - }); - }); -})(); diff --git a/tests/request_icons_test.js b/tests/request_icons_test.js new file mode 100644 index 0000000..d5478ac --- /dev/null +++ b/tests/request_icons_test.js @@ -0,0 +1,136 @@ +"use strict"; + +(() => { + const chai = require('chai'), + expect = chai.expect, + should = chai.should(); + + const Collection = require('@iconify/json-tools').Collection, + request = require('../src/request-icons'); + + let collection1 = new Collection('test'), + collection2 = new Collection('test2'); + + const parseQuery = (prefix, query, ext, params) => { + let result = null; + request({ + // fake app + response: (req, res, data) => { + result = data; + }, + collections: { + test1: collection1, + test2: collection2 + } + }, { + // fake request + query: params + }, {}, prefix, query, ext); + return result; + }; + + describe('Requests for icons and collections', () => { + before(() => { + collection1.loadJSON({ + prefix: 'test', + icons: { + icon1: { + body: '', + width: 30 + }, + icon2: { + body: '' + } + }, + aliases: { + alias1: { + parent: 'icon2', + hFlip: true + } + }, + width: 24, + height: 24 + }); + + collection2.loadJSON({ + icons: { + 'test2-icon1': { + body: '', + width: 30 + }, + 'test2-icon2': { + body: '' + }, + 'test2-icon3': { + body: '' + } + }, + aliases: { + 'test2-alias1': { + parent: 'test2-icon2', + hFlip: true + } + }, + width: 24, + height: 24 + }); + + expect(collection1.items).to.not.be.equal(null); + expect(collection2.items).to.not.be.equal(null); + }); + + it('icons list', () => { + // Simple query with prefix + expect(parseQuery('test1', 'icons', 'js', { + icons: 'alias1' + })).to.be.eql({ + type: 'application/javascript; charset=utf-8', + body: 'SimpleSVG._loaderCallback({"prefix":"test","icons":{"icon2":{"body":""}},"aliases":{"alias1":{"parent":"icon2","hFlip":true}},"width":24,"height":24})' + }); + + // Query collection without prefix, json + expect(parseQuery('test2', 'icons', 'json', { + icons: 'alias1' + })).to.be.eql({ + type: 'application/json; charset=utf-8', + body: '{"prefix":"test2","icons":{"icon2":{"body":""}},"aliases":{"alias1":{"parent":"icon2","hFlip":true}},"width":24,"height":24}' + }); + + // Custom callback + expect(parseQuery('test1', 'icons', 'js', { + icons: 'icon1,icon2', + callback: 'console.log' + })).to.be.eql({ + type: 'application/javascript; charset=utf-8', + body: 'console.log({"prefix":"test","icons":{"icon1":{"body":"","width":30},"icon2":{"body":""}},"width":24,"height":24})' + }); + }); + + it('svg', () => { + // Simple icon + expect(parseQuery('test1', 'icon1', 'svg', { + })).to.be.eql({ + filename: 'icon1.svg', + type: 'image/svg+xml; charset=utf-8', + body: '' + }); + + // Icon with custom attributes + expect(parseQuery('test2', 'alias1', 'svg', { + color: 'red' + })).to.be.eql({ + filename: 'alias1.svg', + type: 'image/svg+xml; charset=utf-8', + body: '' + }); + + // Icon with id replacement + let result = parseQuery('test2', 'icon3', 'svg', { + color: 'red', + rotate: '90deg' + }).body.replace(/IconifyId-[0-9a-f]+-[0-9a-f]+-[0-9]+/g, 'some-id'); + + expect(result).to.be.equal(''); + }); + }); +})();