# AI search engine crawlers — explicitly allowed
User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

# General crawlers
User-agent: *
Allow: /
Disallow: /admin
Disallow: /api/
# Exception: the public knowledge endpoints are a deliberate discovery surface
# and are advertised in llms.txt. Blanket-disallowing /api/ told strict crawlers
# to ignore exactly the URLs we ask AI agents to call. Allow wins over Disallow
# for a longer path match in Google/Bing; agents that only honour Disallow will
# skip them, which is the same as today.
Allow: /api/v1/public/
Disallow: /settings
Disallow: /mcp

Sitemap: https://awareness.market/sitemap.xml

# AI-readable product documentation
# llms.txt: concise overview | llms-full.txt: complete technical reference
# See: https://awareness.market/llms.txt
# See: https://awareness.market/llms-full.txt