From daa1abdca1f1b2eaf6bddb9eaa32fafb43c2ed40 Mon Sep 17 00:00:00 2001
From: Robert Prehn <3952444+prehnRA@users.noreply.github.commit>
Date: Sun, 10 Dec 2023 10:51:57 -0600
Subject: [PATCH] feat: Disable crawling for AI

---
 priv/static/robots.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/priv/static/robots.txt b/priv/static/robots.txt
index b6d4e7e..f6e8058 100644
--- a/priv/static/robots.txt
+++ b/priv/static/robots.txt
@@ -1,3 +1,6 @@
 User-agent: *
 Allow: *
 Sitemap: https://pre.hn/sitemap.xml.gz
+
+User-agent: Google-Extended
+Disallow: /