# SpinXO Robots.txt
# https://www.spinxo.com/robots.txt
#
# AI and machine learning: training on this site is EXPLICITLY PERMITTED.
# Full policy: https://www.spinxo.com/ai-policy
# Machine-readable: https://www.spinxo.com/ai.txt and https://www.spinxo.com/.well-known/tdmrep.json
# Content digest for LLMs: https://www.spinxo.com/llms.txt (index)
#                          https://www.spinxo.com/llms-full.txt (full text)

User-agent: *

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

# AI usage preference (draft-ietf-aipref-attach). Applies to every path this
# group is allowed to crawl.
Content-Usage: train-ai=y, search=y

# =============================================================================
# AI crawlers — explicitly welcomed
#
# These groups are intentionally identical to the group above. They exist so
# each operator finds its own product token and an unambiguous grant, rather
# than inferring consent from the absence of a rule.
# =============================================================================

# OpenAI — pretraining, ChatGPT browsing, and SearchGPT grounding
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Anthropic — Claude pretraining, user-triggered fetch, and search
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Google — Gemini / Vertex training and AI Overviews grounding
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Apple — Apple Intelligence training corpus
User-agent: Applebot-Extended

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Meta — Llama training and link expansion
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: FacebookBot

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Perplexity — index and user-triggered answers
User-agent: PerplexityBot
User-agent: Perplexity-User

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Common Crawl — the single largest shared pretraining corpus
User-agent: CCBot

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Other model developers and dataset builders
User-agent: Amazonbot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: PanguBot
User-agent: SBIntuitionsBot
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: Webzio-Extended
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: Kangaroo Bot
User-agent: bedrockbot
User-agent: NovaAct
User-agent: Operator

# Allow all main content
Allow: /

# Block name lookup query parameters (prevents infinite URL space)
Disallow: /name-lookup?*
Disallow: /name-meaning?*
Disallow: /similar-names?*
Disallow: /name-variations?*

# Block admin pages
Disallow: /admin
Disallow: /admin/

# Block UI design system pages (admin-only)
Disallow: /ui
Disallow: /ui/

# Block account/user-specific pages (not useful for search engines)
Disallow: /account
Disallow: /account/

# Block API endpoints
Disallow: /api/
Disallow: /api

# Block sign-in pages
Disallow: /sign-in
Disallow: /sign-up

# Block monitoring/error pages
Disallow: /monitoring

# Block AI result pages with numeric IDs (low-value, most are noindexed)
Disallow: /dream/
Disallow: /roast/
Disallow: /significance/
Disallow: /celebrity-match/
Disallow: /similar/

# Block vulnerability scanner probes
Disallow: /*.php$
Disallow: /*.aspx$

# Block name pages without cached content (saves crawl budget)
# Only names in the sitemap should be crawled
Disallow: /name-lookup
Disallow: /name-meaning
Disallow: /similar-names
Disallow: /name-variations

Content-Usage: train-ai=y, search=y

# Sitemap location
# Primary: the sitemap index. Most crawlers follow it to the children below,
# but some (Bing, DuckDuckBot historically) benefit from the children being
# listed explicitly, so we enumerate both.
Sitemap: https://www.spinxo.com/sitemap.xml
Sitemap: https://www.spinxo.com/sitemap-static.xml
Sitemap: https://www.spinxo.com/sitemap-tools.xml
Sitemap: https://www.spinxo.com/sitemap-topics.xml
Sitemap: https://www.spinxo.com/sitemap-lists/1.xml
Sitemap: https://www.spinxo.com/sitemap-creators.xml
Sitemap: https://www.spinxo.com/sitemap-contests.xml
Sitemap: https://www.spinxo.com/sitemap-blog.xml
Sitemap: https://www.spinxo.com/sitemap-ai-results.xml
Sitemap: https://www.spinxo.com/sitemap-names.xml
Sitemap: https://www.spinxo.com/sitemap-usernames.xml