From cc08a0f210f4c6af0f765918fee35fb2ebc8579a Mon Sep 17 00:00:00 2001
From: Ari Archer <ari@ari.lt>
Date: Thu, 2 Jan 2025 10:14:52 +0200
Subject: [PATCH] Disable AI crawlers.

Signed-off-by: Ari Archer <ari@ari.lt>
---
 src/aw/__init__.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/aw/__init__.py b/src/aw/__init__.py
index 34ea881..ec9bd19 100644
--- a/src/aw/__init__.py
+++ b/src/aw/__init__.py
@@ -33,7 +33,41 @@ def assign_http(app: flask.Flask) -> flask.Flask:
         """favicon"""
 
         robots: str = (
-            f"User-agent: *\nSitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml\nDisallow: /vote/*/*\n"
+            f"""User-agent: *
+Sitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml
+Disallow: /vote/*/*
+
+# We are not slaves for machines.
+
+User-agent: Amazonbot
+User-agent: anthropic-ai
+User-agent: Applebot-Extended
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: Claude-Web
+User-agent: cohere-ai
+User-agent: Diffbot
+User-agent: FacebookBot
+User-agent: facebookexternalhit
+User-agent: FriendlyCrawler
+User-agent: Google-Extended
+User-agent: GPTBot
+User-agent: ICC-Crawler
+User-agent: ImagesiftBot
+User-agent: img2dataset
+User-agent: meta-externalagent
+User-agent: OAI-SearchBot
+User-agent: Omgili
+User-agent: Omgilibot
+User-agent: PerplexityBot
+User-agent: PetalBot
+User-agent: Scrapy
+User-agent: Timpibot
+User-agent: VelenPublicWebCrawler
+User-agent: YouBot
+Disallow: /"""
         )
 
         return flask.Response(robots, mimetype="text/plain")