diff --git a/scripts/blog.py b/scripts/blog.py index 6a88554..649a936 100755 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -1475,8 +1475,39 @@ def robots(config: dict[str, typing.Any]) -> int: r.write( f"""User-agent: * Disallow: /{config["assets-dir"]}/* -Allow: * -Sitemap: {config["blog"]}/sitemap.xml""" +Sitemap: {config["blog"]}/sitemap.xml + +# We are not slaves for machines. + +User-agent: Amazonbot +User-agent: anthropic-ai +User-agent: Applebot-Extended +User-agent: Bytespider +User-agent: CCBot +User-agent: ChatGPT-User +User-agent: ClaudeBot +User-agent: Claude-Web +User-agent: cohere-ai +User-agent: Diffbot +User-agent: FacebookBot +User-agent: facebookexternalhit +User-agent: FriendlyCrawler +User-agent: Google-Extended +User-agent: GPTBot +User-agent: ICC-Crawler +User-agent: ImagesiftBot +User-agent: img2dataset +User-agent: meta-externalagent +User-agent: OAI-SearchBot +User-agent: Omgili +User-agent: Omgilibot +User-agent: PerplexityBot +User-agent: PetalBot +User-agent: Scrapy +User-agent: Timpibot +User-agent: VelenPublicWebCrawler +User-agent: YouBot +Disallow: /""" ) lnew(f"generated {r.name!r}")