ari/ari.lt
1
0
Fork 0
mirror of https://git.ari.lt/ari.lt/ari.lt.git synced 2025-02-04 09:39:25 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Disable AI crawlers.

Browse source 
Signed-off-by: Ari Archer <ari@ari.lt>
This commit is contained in:
Arija A. 2025-01-02 10:14:52 +02:00
parent de1e13da8d
commit cc08a0f210
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG key ID: A50D5B4B599AF8A2
1 changed files with 35 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/aw/__init__.py
View file

@ -33,7 +33,41 @@ def assign_http(app: flask.Flask) -> flask.Flask:
"""favicon"""
robots: str = (
f"User-agent: *\nSitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml\nDisallow: /vote/*/*\n"
f"""User-agent: *
Sitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml
Disallow: /vote/*/*
# We are not slaves for machines.
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: Diffbot
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: meta-externalagent
User-agent: OAI-SearchBot
User-agent: Omgili
User-agent: Omgilibot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: YouBot
Disallow: /"""
)
return flask.Response(robots, mimetype="text/plain")