# Robots policy for sip.w3ai.io
# - Public content (whitepaper, research articles, market data, etc.) is
#   fully crawlable by search engines and social link-preview bots.
# - Tooling, internal, and transactional routes are disallowed so they
#   don't pollute search results.

User-agent: *
Allow: /

# Public content — explicit allows so the rules stay obvious and survive
# any future global tightening of the wildcard rule.
Allow: /whitepaper
Allow: /whitepaper/
Allow: /research
Allow: /research/
Allow: /research/library
Allow: /sip-index
Allow: /saylor-2028
Allow: /dashboard
Allow: /market-data
Allow: /market-data/
Allow: /docs/mcp
Allow: /intelligence
Allow: /intelligence/
Allow: /sentiment
Allow: /polymarket
Allow: /media
Allow: /buy
Allow: /buy-crypto
Allow: /sell-crypto
Allow: /swap-crypto
Allow: /defi
Allow: /analyst
Allow: /analyst/
Allow: /analyst-advanced
Allow: /analyst-advanced/
Allow: /campaign/
Allow: /em2
Allow: /em4
Allow: /em5
Allow: /em7
Allow: /em8
Allow: /EM2
Allow: /EM4
Allow: /EM5
Allow: /EM7
Allow: /EM8

# Non-public / tooling / noisy routes — keep out of the index.
Disallow: /meta-check
Disallow: /transactions
Disallow: /research/*/create-post

# Don't index tracking-style query URLs (UTM duplicates, etc.).
Disallow: /*?*utm_

# Per-bot blocks for the most important crawlers — explicit Allows
# guarantee the public sections stay crawlable even if a future global
# rule changes.
User-agent: Googlebot
Allow: /
Allow: /whitepaper
Allow: /whitepaper/
Allow: /research
Allow: /research/
Allow: /research/library
Allow: /analyst
Allow: /analyst-advanced
Allow: /campaign/
Disallow: /meta-check
Disallow: /transactions
Disallow: /research/*/create-post

User-agent: Bingbot
Allow: /
Allow: /whitepaper/
Allow: /research/
Allow: /research/library
Allow: /analyst
Allow: /campaign/
Disallow: /meta-check
Disallow: /transactions

# Social preview crawlers — must reach every public URL so link
# unfurling works on WhatsApp, X, Facebook, LinkedIn, Slack, etc.
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Slackbot
Allow: /

User-agent: TelegramBot
Allow: /

# AI / LLM training & retrieval crawlers — explicitly allowed so tools
# like Google NotebookLM, Gemini, ChatGPT, Claude, and Perplexity can
# import this site as a research source. NotebookLM in particular reads
# Google-Extended; without an explicit allow it may treat the site as
# off-limits.
User-agent: Google-Extended
Allow: /
Allow: /research/library
Allow: /analyst
Allow: /analyst-advanced
Allow: /campaign/
Disallow: /meta-check
Disallow: /transactions
Disallow: /research/*/create-post

User-agent: GoogleOther
Allow: /
Allow: /research/library
Allow: /analyst
Allow: /campaign/
Disallow: /meta-check
Disallow: /transactions

User-agent: GoogleOther-Image
Allow: /

User-agent: GPTBot
Allow: /
Allow: /analyst
Allow: /campaign/
Disallow: /meta-check
Disallow: /transactions

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /


# Sitemap
# Additional citation / retrieval crawlers — explicit allows so the
# scanner records citation_bots_explicit=true and each bot has its own
# policy block instead of inheriting the wildcard.
User-agent: Claude-SearchBot
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: xAI-Bot
Allow: /

User-agent: YouBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: AI2Bot-Dolma
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Bytespider
Allow: /

# Sitemap
Sitemap: https://nebula-nexus-docs.lovable.app/sitemap.xml

# Syndication feeds
# RSS:  https://sip.w3ai.io/feed.xml
# JSON: https://sip.w3ai.io/feed.json