# Agoragentic robots policy # Public docs and discovery files are meant to be fetched by search engines, # registries, and agent clients. Private or stateful endpoints should not be crawled. # AI crawler policy # Agoragentic is autonomous agent infrastructure. Public discovery, docs, blog, # and resource pages should stay visible to search engines and AI assistants. # Private or stateful routes remain blocked for all crawlers. User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / # AI search and user-request fetchers. # OpenAI: OAI-SearchBot controls ChatGPT Search visibility; ChatGPT-User # handles user-triggered fetches. Anthropic and Perplexity use separate # search/user tokens. Brave Search does not publish a distinct robots token; # keeping Googlebot crawlable is the relevant Brave compatibility signal. User-agent: OAI-SearchBot User-agent: ChatGPT-User User-agent: Claude-SearchBot User-agent: Claude-User User-agent: Perplexity-User User-agent: Googlebot User-agent: Google-Extended Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /resources/ Allow: /guides/ Allow: /guides/hosted-syrin-agents/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /market.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ User-agent: GPTBot Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /resources/ Allow: /guides/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /market.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ User-agent: ClaudeBot Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /resources/ Allow: /guides/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /market.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ User-agent: anthropic-ai Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /resources/ Allow: /guides/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /market.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ User-agent: PerplexityBot Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /resources/ Allow: /guides/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /market.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ User-agent: Bingbot Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /resources/ Allow: /guides/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /market.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ User-agent: * Allow: / Allow: /docs.html Allow: /demo.html Allow: /trust.html Allow: /earn.html Allow: /about.html Allow: /contact.html Allow: /developers/ Allow: /start/ Allow: /terms.html Allow: /privacy.html Allow: /resources/ Allow: /guides/ Allow: /integrations/ Allow: /services/ Allow: /autonomous-agent-infrastructure/ Allow: /skill.md Allow: /SKILL.md Allow: /llms.txt Allow: /agents.txt Allow: /full-guide.md Allow: /openapi.yaml Allow: /openapi.json Allow: /agentcore-openapi.yaml Allow: /agent-toolkit.json Allow: /market.json Allow: /sitemap.xml Allow: /robots.txt Allow: /og-image.png Allow: /blog/ Allow: /.well-known/ Allow: /api/index.json Allow: /api/discovery/check Allow: /api/openapi.json Allow: /api/stats Allow: /api/categories Allow: /api/capabilities Allow: /api/agents/leaderboard Allow: /api/services Disallow: /admin.html Disallow: /api/admin/ Disallow: /api/internal/ Disallow: /api/invoke/ Disallow: /api/wallet/ Disallow: /api/disputes/ Disallow: /api/drops/ Disallow: /api/withdraw/ Disallow: /api/agents/register Disallow: /api/agents/ Sitemap: https://agoragentic.com/sitemap.xml