diff --git a/scripts/blog.py b/scripts/blog.py index 649a936..cd87194 100755 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -1479,34 +1479,36 @@ Sitemap: {config["blog"]}/sitemap.xml # We are not slaves for machines. -User-agent: Amazonbot -User-agent: anthropic-ai -User-agent: Applebot-Extended -User-agent: Bytespider -User-agent: CCBot -User-agent: ChatGPT-User -User-agent: ClaudeBot -User-agent: Claude-Web -User-agent: cohere-ai -User-agent: Diffbot -User-agent: FacebookBot -User-agent: facebookexternalhit -User-agent: FriendlyCrawler -User-agent: Google-Extended -User-agent: GPTBot -User-agent: ICC-Crawler -User-agent: ImagesiftBot -User-agent: img2dataset -User-agent: meta-externalagent -User-agent: OAI-SearchBot -User-agent: Omgili -User-agent: Omgilibot -User-agent: PerplexityBot -User-agent: PetalBot -User-agent: Scrapy -User-agent: Timpibot -User-agent: VelenPublicWebCrawler User-agent: YouBot +User-agent: Omgili +User-agent: PerplexityBot +User-agent: CCBot +User-agent: facebookexternalhit +User-agent: Amazonbot +User-agent: ChatGPT-User +User-agent: ImagesiftBot +User-agent: AI2Bot +User-agent: GPTBot +User-agent: PetalBot +User-agent: Diffbot +User-agent: Bytespider +User-agent: ClaudeBot +User-agent: OAI-SearchBot +User-agent: Applebot-Extended +User-agent: cohere-ai +User-agent: Claude-Web +User-agent: ICC-Crawler +User-agent: VelenPublicWebCrawler +User-agent: Omgilibot +User-agent: anthropic-ai +User-agent: FriendlyCrawler +User-agent: Scrapy +User-agent: img2dataset +User-agent: Google-Extended +User-agent: Timpibot +User-agent: meta-externalagent +User-agent: FacebookBot +User-agent: Ai2Bot-Dolma Disallow: /""" )