diff --git a/src/aw/__init__.py b/src/aw/__init__.py index 34ea881..ec9bd19 100644 --- a/src/aw/__init__.py +++ b/src/aw/__init__.py @@ -33,7 +33,41 @@ def assign_http(app: flask.Flask) -> flask.Flask: """favicon""" robots: str = ( - f"User-agent: *\nSitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml\nDisallow: /vote/*/*\n" + f"""User-agent: * +Sitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml +Disallow: /vote/*/* + +# We are not slaves for machines. + +User-agent: Amazonbot +User-agent: anthropic-ai +User-agent: Applebot-Extended +User-agent: Bytespider +User-agent: CCBot +User-agent: ChatGPT-User +User-agent: ClaudeBot +User-agent: Claude-Web +User-agent: cohere-ai +User-agent: Diffbot +User-agent: FacebookBot +User-agent: facebookexternalhit +User-agent: FriendlyCrawler +User-agent: Google-Extended +User-agent: GPTBot +User-agent: ICC-Crawler +User-agent: ImagesiftBot +User-agent: img2dataset +User-agent: meta-externalagent +User-agent: OAI-SearchBot +User-agent: Omgili +User-agent: Omgilibot +User-agent: PerplexityBot +User-agent: PetalBot +User-agent: Scrapy +User-agent: Timpibot +User-agent: VelenPublicWebCrawler +User-agent: YouBot +Disallow: /""" ) return flask.Response(robots, mimetype="text/plain")