diff --git a/priv/static/robots.txt b/priv/static/robots.txt index 87dfd33..279a891 100644 --- a/priv/static/robots.txt +++ b/priv/static/robots.txt @@ -5,5 +5,44 @@ Sitemap: https://pre.hn/sitemap.xml.gz User-agent: Google-Extended Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + User-agent: GPTBot Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/omgili + +User-agent: omgili +Disallow: /