# ============================================================
# AIGrow.gg — robots.txt
# Strategy: maximize AI search citations; AI training crawlers allowed (section 4)
# Last updated: 2026-09-23
# ============================================================

# ===================
# 1. SEARCH ENGINES (direct SEO impact)
# ===================

User-agent: Googlebot
Allow: /
Disallow: /auth/
Disallow: /dashboard/
Disallow: /payments
Disallow: /ref/

Sitemap: https://aigrow.gg/sitemap.xml

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# Terracotta Bot (Qwant search engine)
User-agent: Terracotta Bot
Allow: /

# Social media preview crawlers (OG/Twitter Card rendering)
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# ===================
# 2. AI SEARCH & ASSISTANTS (bring traffic via citations)
# ===================

# OpenAI SearchBot (ChatGPT search results)
User-agent: OAI-SearchBot
Allow: /

# ChatGPT browsing mode (user-initiated, cites sources)
User-agent: ChatGPT-User
Allow: /

# Perplexity search (cites sources with links)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Anthropic search (Claude search, cites sources)
User-agent: Claude-SearchBot
Allow: /

# DuckDuckGo AI assistant (cites sources)
User-agent: DuckAssistBot
Allow: /

# Manus AI agent (cites sources)
User-agent: Manus Bot
Allow: /

# Mistral AI assistant (cites sources)
User-agent: MistralAI-User
Allow: /

# Google Extended (Gemini/Bard search — opt-in for AI Overview citations)
User-agent: Google-Extended
Allow: /

# ===================
# 3. ARCHIVER (good for SEO credibility)
# ===================

User-agent: archive.org_bot
Allow: /

# ===================
# 4. AI TRAINING CRAWLERS (opened 2026-08-28, review 2027-08)
# ===================
# Reversed from a blanket block. The old policy assumed the site had a name
# worth protecting from the datasets. It does not yet: at least four unrelated
# companies answer to "AiGrow", and on our own brand query we sit last. A model
# that never reads us cannot learn which AiGrow we are, and live search alone
# does not fix that. Being in the training corpus is how an unknown entity
# becomes a known one, so the trade is worth taking while we are unknown.
# Revisit once we hold position 1 on the brand query.

# OpenAI model training
User-agent: GPTBot
Allow: /

# Anthropic model training
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# Meta/Facebook training
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon AI training
User-agent: Amazonbot
Allow: /

# Apple AI training (separate from Applebot search)
User-agent: Applebot-Extended
Allow: /

# Google Vertex AI training
User-agent: Google-CloudVertexBot
Allow: /

# Common Crawl (feeds most AI training datasets, so the widest single win)
User-agent: CCBot
Allow: /

# Cohere AI training
User-agent: cohere-ai
Allow: /

# ===================
# 4b. STILL BLOCKED: extraction with no citation and no dataset reach
# ===================
# These neither cite us in an answer nor carry us into a model people use.

# ByteDance/TikTok: aggressive crawler, no citation surface we can reach
User-agent: Bytespider
Disallow: /

# Huawei PetalBot
User-agent: PetalBot
Disallow: /

# Diffbot scraper
User-agent: Diffbot
Disallow: /

# ProRata AI
User-agent: ProRataInc
Disallow: /

# Novellfm AI
User-agent: Novellfm AI Crawl
Disallow: /

# Timpi AI
User-agent: Timpibot
Disallow: /

# Anchor Browser AI
User-agent: Anchor Browser
Disallow: /

# img2dataset (image scraping for AI)
User-agent: img2dataset
Disallow: /

# Omgili scraper
User-agent: OmgiliBot
Disallow: /

# ===================
# 5. SEO CRAWLERS
# AhrefsBot, SemrushBot, MJ12bot (Majestic) and DotBot (Moz) compute the
# authority scores partners and press check, and Ahrefs Webmaster Tools needs
# AhrefsBot, so they have no group here and follow the default rules (section 6).
# Data resellers with no audience of their own stay out.
# ===================

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ===================
# 6. DEFAULT (unknown bots — allow with limits)
# ===================

User-agent: *
Allow: /
Disallow: /auth/
Disallow: /dashboard/
Disallow: /api/
Disallow: /ref/
Disallow: /*.json$
Crawl-delay: 5

# ===================
# 7. SITEMAP
# ===================

Sitemap: https://aigrow.gg/sitemap.xml