From 6dbeff12b3a0066b89e851f7ae3e7eb4f90a8285 Mon Sep 17 00:00:00 2001 From: Ari Archer Date: Sat, 11 May 2024 15:25:15 +0300 Subject: [PATCH] Block AI crawlers Signed-off-by: Ari Archer --- robots.txt | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/robots.txt b/robots.txt index 4b134aa..ed03e15 100644 --- a/robots.txt +++ b/robots.txt @@ -1,5 +1,20 @@ -User-agent: * -Disallow: /content/* -Allow: * -Sitemap: https://ari.lt/sitemap.xml +# Block AI +User-agent: ChatGPT-User +User-agent: CCBot +User-agent: ClaudeBot +User-agent: Claude-Web +User-agent: cohere-ai +User-agent: anthropic-ai +User-agent: Amazonbot +User-agent: Diffbot +User-agent: Google-Extended +User-agent: GPTBot +User-agent: magpie-crawler +User-agent: PerplexityBot +Disallow: / +# Allow everything else :) +User-agent: * +Allow: / + +Sitemap: https://ari.lt/sitemap.xml