diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..22c685b --- /dev/null +++ b/robots.txt @@ -0,0 +1,39 @@ +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: Claude-Web +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: PerplexityBot +Disallow: / + +User-agent: Perplexity-User +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: OAI-SearchBot +Disallow: / + +# Allow non-AI crawlers by default +User-agent: * +Allow: / \ No newline at end of file