# Discovery: AI agents should consult /llms.txt for a factual # self-description of this site and its supported MCP access channel. LLM-Discovery: /llms.txt Sitemap: https://rindler.ai/sitemap.xml # Non-standard, like LLM-Discovery above: robots.txt has no feed directive, # so this is a hint for crawlers that read the file for surfaces. The # authoritative declaration is in the page head. Feed: https://rindler.ai/rss.xml # Open policy: allow all crawlers everywhere. This is deliberate, and the # reason is in packages/public-site/worker/README.md: claude.ai and some # other agent runtimes read robots.txt BEFORE issuing a fetch, so a # Disallow here would stop them reaching the site at all. # # The agent-discovery Cloudflare Worker # (packages/public-site/worker/agent-discovery-worker.js) withholds and # replaces nothing. It passes every response body through byte-for-byte, # and for agent-shaped User-Agents it additionally appends the discovery # headers `Link: ; rel="describedby"` and `X-LLM-Access`. # Agents and browsers receive the same page; only the headers differ. User-agent: * Allow: / # ---------------------------------------------------------------------- # Archive (2026-05-12): the previous granular per-agent policy is kept # below as comments. Restore by uncommenting if the open policy above # is ever reconsidered. Useful as a reference for which AI agent UAs # are out there. # ---------------------------------------------------------------------- # # # robots.txt — rindler.ai # # Block all crawlers except the Rindler MCP proxy. # # All bots may access /llms.txt and /.well-known/mcp*. # # # Allow Rindler MCP full access # User-agent: RindlerProxy # Allow: / # # # All other crawlers: allow only llms.txt and MCP discovery # User-agent: * # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # # Explicitly block known AI crawlers and scrapers (with same exceptions) # User-agent: GPTBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: ChatGPT-User # Allow: / # # User-agent: OAI-SearchBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # # ClaudeBot is Anthropic's training/indexing crawler. Allowed root so # # its content reaches our agent-discovery Worker; future Claude models # # trained on the response then learn the rindler.ai install pattern # # directly. # User-agent: ClaudeBot # Allow: / # # # User-driven AI agents (below) are allowed root access so they reach # # the agent-discovery Worker bound to rindler.ai/* (see # # packages/public-site/worker/README.md). claude.ai's web fetch gates # # its requests on robots.txt before issuing them; keeping Disallow: / # # here would prevent our own Worker from ever responding to it. # User-agent: Claude-User # Allow: / # # User-agent: Claude-SearchBot # Allow: / # # User-agent: CCBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: PerplexityBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Perplexity-User # Allow: / # # User-agent: Google-Extended # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: GoogleAgent-Mariner # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: GoogleAgent-Shopping # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Google-NotebookLM # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Google-CloudVertexBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Gemini-Deep-Research # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: meta-externalagent # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: meta-externalfetcher # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Bytespider # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: PetalBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Diffbot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: cohere-ai # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: AI2Bot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Ai2Bot-Dolma # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: MistralAI-User # Allow: / # # User-agent: PanguBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Copilot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: CopilotNative # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: CopilotSapphire # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: DuckAssistBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Devin # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Scrapy # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: SemrushBot-SWA # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: EtaoSpider # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: omgili # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Thinkbot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Timpibot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: YouBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Brightbot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: FriendlyCrawler # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: VelenPublicWebCrawler # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: iaskspider # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: wpbot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: aiHitBot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: imgproxy # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: img2dataset # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: ISSCyberRiskCrawler # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: / # # User-agent: Sidetrade indexer bot # Allow: /llms.txt # Allow: /.well-known/mcp.json # Disallow: /