# Graphene-BSM Ltd — robots.txt
# Allow search engines and AI search/retrieval (including user-requested page fetches).
# Opt out of AI model-training crawlers that have their own, separate token.
# Crawlers that serve both purposes under ONE token are allowed, because blocking them
# would also remove the site from search or answers (see the DUAL-PURPOSE note below).
#
# robots.txt is a request to well-behaved crawlers, not access control. The host must also
# not challenge or block these agents.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# ============================================================
# SEARCH ENGINES
# ============================================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

# DUAL-PURPOSE: Amazonbot powers Alexa answers and has no separate training control,
# so it stays allowed.
User-agent: Amazonbot
Allow: /

# ============================================================
# AI SEARCH / ANSWER RETRIEVAL / USER-REQUESTED FETCHES (allowed)
# ============================================================

# OpenAI — ChatGPT Search, user-directed retrieval, ads
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-AdsBot
Allow: /

# Anthropic — user-directed retrieval and search
User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity — search and user-directed retrieval
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ============================================================
# AI TRAINING / MODEL-DEVELOPMENT CRAWLERS (opted out)
# ============================================================

# Google-Extended: opt-out token for Gemini training. Does not affect Google Search.
User-agent: Google-Extended
Disallow: /

# OpenAI model training (OAI-SearchBot and ChatGPT-User above stay allowed)
User-agent: GPTBot
Disallow: /

# Anthropic model training (Claude-User and Claude-SearchBot above stay allowed)
User-agent: ClaudeBot
Disallow: /

# Common Crawl (public dataset widely used for model training)
User-agent: CCBot
Disallow: /

# ByteDance
User-agent: Bytespider
Disallow: /

# Applebot-Extended: opt-out token for Apple model training (Applebot stays allowed)
User-agent: Applebot-Extended
Disallow: /

# Meta model-development crawler
User-agent: Meta-ExternalAgent
Disallow: /

# ============================================================
# SITEMAP
# ============================================================

Sitemap: https://graphene-bsm.com/sitemap.xml