From d3cee2953230419d9b634dbfe16242293cf7c66c Mon Sep 17 00:00:00 2001
From: Robert Prehn <3952444+prehnRA@users.noreply.github.com>
Date: Fri, 1 Mar 2024 13:21:51 -0600
Subject: [PATCH] feat: Add more scrapers to robots.txt

---
 priv/static/robots.txt | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/priv/static/robots.txt b/priv/static/robots.txt
index 87dfd33..279a891 100644
--- a/priv/static/robots.txt
+++ b/priv/static/robots.txt
@@ -5,5 +5,44 @@ Sitemap: https://pre.hn/sitemap.xml.gz
 User-agent: Google-Extended
 Disallow: /
 
+# AI Data Scraper
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
+User-agent: Bytespider
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
 User-agent: GPTBot
 Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /