# Content signals: this site sells AI API access, so being cited by AI
# assistants is a customer acquisition channel, not a leak.
User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=full

# AI crawlers — explicitly allowed
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

# General
User-agent: *
Allow: /

# API surfaces: no SEO value, and crawling them burns crawl budget.
Disallow: /api/user/
Disallow: /api/token/

# Auth pages — thin content, and indexing them competes with the landing page.
Disallow: /sign-in
Disallow: /sign-up

# Authenticated / post-conversion surfaces. Indexing /thank-you would publish
# conversion URLs with the top-up amount in the query string.
Disallow: /thank-you
Disallow: /sr/
Disallow: /wallet
Disallow: /profile
Disallow: /console/
Disallow: /users-overview

# Marketplace category pages ARE public and indexable; these sub-paths still
# require sign-in, so keep crawlers off them to avoid soft-404 / redirect errors.
Disallow: /marketplace/my-apis
Disallow: /marketplace/publish
Disallow: /marketplace/community

# Sitemaps — must be the XML sitemap: this is how Bing/Yandex auto-discover it.
# (Google Search Console submissions are manual and bypass this line.)
Sitemap: https://api.jarvisclaw.ai/sitemap.xml

# Agent-readable capability manifests (not sitemaps, listed for discoverability)
# llms.txt:      https://api.jarvisclaw.ai/llms.txt
# x402:          https://api.jarvisclaw.ai/.well-known/x402
# A2A card:      https://api.jarvisclaw.ai/.well-known/agent.json
# AIP:           https://api.jarvisclaw.ai/.well-known/agent-intent-protocol.json
# MCP endpoint:  https://api.jarvisclaw.ai/mcp
# OpenAPI:       https://api.jarvisclaw.ai/openapi.json