8.0.3 • Published 4 years ago
generate-robotstxt v8.0.3
generate-robotstxt
Awesome generator robots.txt.
Installation
npm install --save-dev generate-robotstxt
Usage
import robotstxt from "generate-robotstxt";
robotstxt({
policy: [
{
userAgent: "Googlebot",
allow: "/",
disallow: "/search",
crawlDelay: 2,
},
{
userAgent: "OtherBot",
allow: ["/allow-for-all-bots", "/allow-only-for-other-bot"],
disallow: ["/admin", "/login"],
crawlDelay: 2,
},
{
userAgent: "*",
allow: "/",
disallow: "/search",
crawlDelay: 10,
cleanParam: "ref /articles/",
},
],
sitemap: "http://example.com/sitemap.xml",
host: "http://example.com",
})
.then((content) => {
console.log(content);
return content;
})
.catch((error) => {
throw error;
});
File based configuration
robots-txt.config.js
module.exports = {
policy: [
{
userAgent: "Googlebot",
allow: "/",
disallow: ["/search"],
crawlDelay: 2,
},
{
userAgent: "OtherBot",
allow: ["/allow-for-all-bots", "/allow-only-for-other-bot"],
disallow: ["/admin", "/login"],
crawlDelay: 2,
},
{
userAgent: "*",
allow: "/",
disallow: "/search",
crawlDelay: 10,
cleanParam: "ref /articles/",
},
],
sitemap: "http://example.com/sitemap.xml",
host: "http://example.com",
};
CLI
Awesome generator robots.txt
Usage generate-robotstxt [options] <dest>
Options:
--config Path to a specific configuration file.
Contribution
Feel free to push your code if you agree with publishing under the MIT license.
Changelog
License
8.0.3
4 years ago
8.0.2
4 years ago
8.0.1
4 years ago
8.0.0
4 years ago
7.1.0
5 years ago
7.0.0
5 years ago
6.0.1
5 years ago
6.0.0
5 years ago
5.0.7
6 years ago
5.0.6
6 years ago
5.0.5
6 years ago
5.0.4
6 years ago
5.0.3
6 years ago
5.0.2
6 years ago
5.0.1
6 years ago
5.0.0
6 years ago
4.0.4
7 years ago
4.0.3
7 years ago
4.0.2
7 years ago
4.0.1
7 years ago
4.0.0
8 years ago
3.0.1
8 years ago
3.0.0
8 years ago
2.0.0
9 years ago
1.0.3
9 years ago
1.0.2
9 years ago
1.0.1
9 years ago
1.0.0
9 years ago