Initalize

node_modules/generate-robotstxt/dist/cli.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+"use strict";
+
+var _path = _interopRequireDefault(require("path"));
+
+var _fsExtra = _interopRequireDefault(require("fs-extra"));
+
+var _meow = _interopRequireDefault(require("meow"));
+
+var _resolveFrom = _interopRequireDefault(require("resolve-from"));
+
+var _standalone = _interopRequireDefault(require("./standalone.js"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const cli = (0, _meow.default)(`
+    Usage generate-robotstxt [options] <dest>
+
+    Options:
+       --config  Path to a specific configuration file.
+`, {
+  flags: {
+    config: {
+      type: "string"
+    }
+  }
+});
+const optionsBase = {};
+
+if (cli.flags.config) {
+  // Should check these possibilities:
+  //   a. name of a node_module
+  //   b. absolute path
+  //   c. relative path relative to `process.cwd()`.
+  // If none of the above work, we'll try a relative path starting
+  // in `process.cwd()`.
+  optionsBase.configFile = (0, _resolveFrom.default)(process.cwd(), cli.flags.config) || _path.default.join(process.cwd(), cli.flags.config);
+}
+
+Promise.resolve().then(() => Object.assign({}, optionsBase)).then(options => (0, _standalone.default)(options)).then(output => {
+  if (cli.input.length === 0) {
+    throw new Error("Require `dest` argument");
+  }
+
+  const dest = _path.default.resolve(cli.input.pop());
+
+  return Promise.resolve().then(() => _fsExtra.default.outputFile(dest, output));
+}).catch(error => {
+  console.log(error); // eslint-disable-line no-console
+
+  process.exit(error.code || 1);
+});

node_modules/generate-robotstxt/dist/index.js

@@ -0,0 +1,15 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _standalone = _interopRequireDefault(require("./standalone.js"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+var _default = _standalone.default;
+exports.default = _default;
+module.exports = exports.default;
+module.exports.default = exports.default;

node_modules/generate-robotstxt/dist/standalone.js

@@ -0,0 +1,211 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+
+var _path = _interopRequireDefault(require("path"));
+
+var _url = _interopRequireDefault(require("url"));
+
+var _ipRegex = _interopRequireDefault(require("ip-regex"));
+
+var _cosmiconfig = require("cosmiconfig");
+
+var _isAbsoluteUrl = _interopRequireDefault(require("is-absolute-url"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+function capitaliseFirstLetter(string) {
+  return string.charAt(0).toUpperCase() + string.slice(1);
+}
+
+function addLine(name, rule) {
+  let contents = "";
+
+  if (rule && Array.isArray(rule) && rule.length > 0) {
+    rule.forEach(item => {
+      contents += addLine(name, item);
+    });
+  } else {
+    const ruleContent = (name === "Allow" || name === "Disallow" ? encodeURI(rule) : rule).toString();
+    contents += `${capitaliseFirstLetter(name.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase())}:${ruleContent.length > 0 ? ` ${ruleContent}` : ""}\n`;
+  }
+
+  return contents;
+}
+
+function generatePoliceItem(item, index) {
+  let contents = "";
+
+  if (index !== 0) {
+    contents += "\n";
+  }
+
+  contents += addLine("User-agent", item.userAgent);
+
+  if (item.allow) {
+    contents += addLine("Allow", item.allow);
+  }
+
+  if (typeof item.disallow === "string" || Array.isArray(item.disallow)) {
+    contents += addLine("Disallow", item.disallow);
+  }
+
+  if (item.crawlDelay) {
+    contents += addLine("Crawl-delay", item.crawlDelay);
+  } // Move from policy for next master version
+  // https://yandex.ru/support/webmaster/controlling-robot/robots-txt.html
+
+
+  if (item.cleanParam && item.cleanParam.length > 0) {
+    contents += addLine("Clean-param", item.cleanParam);
+  }
+
+  return contents;
+}
+
+function buildConfig(configFile = null) {
+  let searchPath = process.cwd();
+  let configPath = null;
+
+  if (configFile) {
+    searchPath = null;
+    configPath = _path.default.resolve(process.cwd(), configFile);
+  }
+
+  const configExplorer = (0, _cosmiconfig.cosmiconfig)("robots-txt");
+  const searchForConfig = configPath ? configExplorer.load(configPath) : configExplorer.search(searchPath);
+  return searchForConfig.then(result => {
+    if (!result) {
+      return {};
+    }
+
+    return result;
+  });
+}
+
+function _default({
+  configFile = null,
+  policy = [{
+    allow: "/",
+    cleanParam: null,
+    crawlDelay: null,
+    userAgent: "*"
+  }],
+  sitemap = null,
+  host = null
+} = {}) {
+  let options = {
+    host,
+    policy,
+    sitemap
+  };
+  return Promise.resolve().then(() => buildConfig(configFile).then(result => {
+    // Need avoid this behaviour in next major release
+    // Load config file when it is passed or options were set
+    options = Object.assign({}, options, result.config);
+    return options;
+  })).then(() => new Promise(resolve => {
+    if (options.policy) {
+      if (!Array.isArray(options.policy)) {
+        throw new Error("Options `policy` must be array");
+      }
+
+      options.policy.forEach(item => {
+        if (!item.userAgent || item.userAgent.length === 0) {
+          throw new Error("Each `policy` should have a single string `userAgent` option");
+        }
+
+        if (item.crawlDelay && typeof item.crawlDelay !== "number" && !Number.isFinite(item.crawlDelay)) {
+          throw new Error("Option `crawlDelay` must be an integer or a float");
+        }
+
+        if (item.cleanParam) {
+          if (typeof item.cleanParam === "string" && item.cleanParam.length > 500) {
+            throw new Error("Option `cleanParam` should have no more than 500 characters");
+          } else if (Array.isArray(item.cleanParam)) {
+            item.cleanParam.forEach(subItem => {
+              if (typeof subItem === "string" && subItem.length > 500) {
+                throw new Error("String in `cleanParam` option should have no more than 500 characters");
+              } else if (typeof subItem !== "string") {
+                throw new Error("String in `cleanParam` option should be a string");
+              }
+            });
+          } else if (typeof item.cleanParam !== "string" && !Array.isArray(item.cleanParam)) {
+            throw new Error("Option `cleanParam` should be a string or an array");
+          }
+        }
+      });
+    } else {
+      throw new Error("Options `policy` should be define");
+    }
+
+    if (options.sitemap) {
+      if (typeof options.sitemap === "string" && !(0, _isAbsoluteUrl.default)(options.sitemap)) {
+        throw new Error("Option `sitemap` should be an absolute URL");
+      } else if (Array.isArray(options.sitemap)) {
+        options.sitemap.forEach(item => {
+          if (typeof item === "string" && !(0, _isAbsoluteUrl.default)(item)) {
+            throw new Error("Item in `sitemap` option should be an absolute URL");
+          } else if (typeof item !== "string") {
+            throw new Error("Item in `sitemap` option should be a string");
+          }
+        });
+      } else if (typeof options.sitemap !== "string" && !Array.isArray(options.sitemap)) {
+        throw new Error("Option `sitemap` should be a string or an array");
+      }
+    }
+
+    if (options.host) {
+      if (typeof options.host !== "string") {
+        throw new Error("Options `host` must be only one string");
+      }
+
+      if ((0, _ipRegex.default)({
+        exact: true
+      }).test(options.host)) {
+        throw new Error("Options `host` should be not an IP address");
+      }
+    }
+
+    let contents = "";
+    options.policy.forEach((item, index) => {
+      contents += generatePoliceItem(item, index);
+    });
+
+    if (options.sitemap) {
+      contents += addLine("Sitemap", options.sitemap);
+    }
+
+    if (options.host) {
+      let normalizeHost = options.host;
+
+      if (normalizeHost.search(/^https?:\/\//) === -1) {
+        normalizeHost = `http://${host}`;
+      } // eslint-disable-next-line node/no-deprecated-api
+
+
+      const parsedURL = _url.default.parse(normalizeHost, false, true);
+
+      if (!parsedURL.host) {
+        throw new Error("Option `host` does not contain correct host");
+      }
+
+      let formattedHost = _url.default.format({
+        host: parsedURL.port && parsedURL.port === "80" ? parsedURL.hostname : parsedURL.host,
+        port: parsedURL.port && parsedURL.port === "80" ? "" : parsedURL.port,
+        protocol: parsedURL.protocol
+      });
+
+      formattedHost = formattedHost.replace(/^http:\/\//, "");
+      contents += addLine("Host", formattedHost);
+    }
+
+    return resolve(contents);
+  }));
+}
+
+module.exports = exports.default;
+module.exports.default = exports.default;