diff --git a/lib/cli/cli.js b/lib/cli/cli.js index bfe3b2d51..4bb83130e 100644 --- a/lib/cli/cli.js +++ b/lib/cli/cli.js @@ -1009,6 +1009,8 @@ module.exports.parseCommandLine = function parseCommandLine() { group: 'Crawler' }) .option('crawler.ignoreRobotsTxt', { + type: 'boolean', + default: false, describe: 'Ignore robots.txt rules of the crawled domain.', group: 'Crawler' });