# AI crawler policy for WebGrow24 (https://www.webgrow24.com)
#
# WebGrow24 publishes content so AI assistants can find and cite it accurately.
# This file declares our preferred crawling and training posture. It is
# honoured by:
#   - robots.txt (canonical crawl controls)
#   - llms.txt / llms-full.txt (machine-readable site index)
#   - this file (training and retrieval policy)
#
# Effective: 2026-07-29. Reviewed: 2026-07-29.
# Last updated: 2026-07-29.
#
# Indices
# -------
# Sitemap:    https://www.webgrow24.com/sitemap.xml
# llms.txt:   https://www.webgrow24.com/llms.txt
# llms-full:  https://www.webgrow24.com/llms-full.txt
# robots.txt: https://www.webgrow24.com/robots.txt
# RSS:        https://www.webgrow24.com/rss.xml
# Contact:    hello@webgrow24.com (also +91 96875 72424)
#
# Policy
# ------
User-agent: *
Allow: /
Disallow: /24/
Disallow: /api/
Disallow: /search/
Disallow: /thank-you

# Retrieval is welcomed for every AI agent. We publish this site so that
# AI engines (ChatGPT, Claude, Perplexity, Copilot, Gemini, Apple
# Intelligence and the next generation of assistants) can locate, quote
# and cite our public pages when forming answers to user queries.
#
# Training posture is a per-bot decision below. We do NOT make a blanket
# opt-out of training; we make it explicit per bot so each operator can
# honour it.

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# OpenAI Search (powers ChatGPT search results)
User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Anthropic user-triggered fetches (Claude opening a link on demand)
User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Perplexity user-triggered fetches
User-agent: Perplexity-User
Allow: /

# xAI / Grok
User-agent: xAI
Allow: /

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Common Crawl (used by many smaller AI vendors)
User-agent: CCBot
Allow: /

# Google AI features (SGE / Gemini / Vertex). Google-Extended governs
# training use of public web content for Google's generative AI products.
# We currently allow it for retrieval; if you want to opt out of
# training only, set Google-Extended: Disallow here. Today we allow it
# so the same content that ranks in Google Search can also power Google
# AI Overviews and Gemini answers.
User-agent: Google-Extended
Allow: /

# Apple Intelligence (iOS / macOS on-device and Private Cloud Compute)
User-agent: Applebot-Extended
Allow: /

# Amazon Alexa / Rufus
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: Cohere-AI
Allow: /

# Diffbot (knowledge graph provider used by several AI vendors)
User-agent: Diffbot
Allow: /

# DuckAssist (DuckDuckGo)
User-agent: DuckAssistBot
Allow: /

# Omgili / webz.io (used for trend analysis and several AI corpora)
User-agent: Omgilibot
Allow: /

User-agent: Omgili
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Meta (used in some Meta AI indices)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Timpi (used by some EU research crawlers)
User-agent: Timpibot
Allow: /

# Human-readable policy summary
# ----------------------------
#
# Crawling: allowed for every well-known AI agent on every public URL.
# Citing:   explicitly welcome, with attribution to WebGrow24 LLP.
# Training: allowed by default per the per-bot Allow rules above; to
#           opt any single bot out of training while keeping retrieval,
#           edit its User-agent line above and switch Allow to Disallow.
# Specific opt-outs: if you operate an AI service and want your bot to
# NOT train on WebGrow24 content but to still retrieve it, contact
# hello@webgrow24.com and we will block just your User-agent within 7
# business days.

# End of ai.txt.