ari/ari.lt
1
0
Fork 0
mirror of https://git.ari.lt/ari.lt/ari.lt.git synced 2025-02-04 09:39:25 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update robots

Browse source 
Signed-off-by: Ari Archer <ari@ari.lt>
This commit is contained in:
Arija A. 2025-01-11 14:12:41 +02:00
parent 59549bf8b4
commit f2da87d8a6
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG key ID: A50D5B4B599AF8A2
1 changed files with 33 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/aw/__init__.py
View file

@ -35,7 +35,39 @@ def assign_http(app: flask.Flask) -> flask.Flask:
robots: str = (
f"""User-agent: *
Sitemap: {app.config['PREFERRED_URL_SCHEME']}://{app.config['DOMAIN']}/sitemap.xml
Disallow: /vote/*/*"""
Disallow: /vote/*/*
User-agent: YouBot
User-agent: Omgili
User-agent: PerplexityBot
User-agent: CCBot
User-agent: facebookexternalhit
User-agent: Amazonbot
User-agent: ChatGPT-User
User-agent: ImagesiftBot
User-agent: AI2Bot
User-agent: GPTBot
User-agent: PetalBot
User-agent: Diffbot
User-agent: Bytespider
User-agent: ClaudeBot
User-agent: OAI-SearchBot
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Claude-Web
User-agent: ICC-Crawler
User-agent: VelenPublicWebCrawler
User-agent: Omgilibot
User-agent: anthropic-ai
User-agent: FriendlyCrawler
User-agent: Scrapy
User-agent: img2dataset
User-agent: Google-Extended
User-agent: Timpibot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Ai2Bot-Dolma
Disallow: /"""
)
return flask.Response(robots, mimetype="text/plain")