# imadethisup.org — robots policy
#
# Default: index everything except utility pages.
# AI crawlers (OpenAI, Anthropic, Google AI, Perplexity, Common Crawl)
# are explicitly opted in — content is CC BY-NC 4.0 and the project's
# purpose is broad public reach.

User-agent: *
Allow: /
Disallow: /404
Disallow: /assets/og/

# ─── OpenAI ──────────────────────────────────────────────────
# https://platform.openai.com/docs/bots/
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ─── Anthropic / Claude ──────────────────────────────────────
# https://docs.anthropic.com/en/docs/agents-and-tools/web-search
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# ─── Google AI / Search ──────────────────────────────────────
# Google-Extended controls inclusion in Gemini/Vertex training & grounding.
User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

# ─── Other AI search and indexing ────────────────────────────
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Bytespider
Allow: /

# ─── Sitemaps & LLM index ────────────────────────────────────
Sitemap: https://imadethisup.org/sitemap.xml
Sitemap: https://imadethisup.org/feed.xml

# llms.txt — site index for LLM consumers (https://llmstxt.org/)
# Indexed by reference; not in the sitemap.xml because it's not a webpage.