# ============================
# robots.txt for siilostomy.com

# ============================

# --- Default rules for all crawlers ---
User-agent: *
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
# Block only non-public / low-value system areas to optimize crawl budget
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=  # Blocks internal search results (duplicates)
Disallow: /*add-to-cart=*  # Blocks dynamic cart additions
# NOTE: Only use the next line if you are 100% sure no important content relies on query params.
# Disallow: /*?*  # Blocks all query params (e.g., filters/sorting)

# Allow WordPress AJAX endpoint (needed for normal WP operation)
Allow: /wp-admin/admin-ajax.php


# ============================
# Major Search Engine Crawlers
# ============================
User-agent: Googlebot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Bingbot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Baiduspider
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: YandexBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: DuckDuckBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Slurp  # Yahoo
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Sogou Spider
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Yeti  # Naver
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# OpenAI crawlers (ChatGPT / SearchGPT / training)
# ============================
# Training crawler for OpenAI models (ChatGPT, etc.)
User-agent: GPTBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Proactive search index crawler for ChatGPT search / SearchGPT
User-agent: OAI-SearchBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# On-demand fetcher when ChatGPT users open/click your site
User-agent: ChatGPT-User
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# Google AI-related crawling
# ============================
# Controls whether Google can use crawled content for training Gemini and for grounding.
User-agent: Google-Extended
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: GoogleOther  # Vertex AI / general AI
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Gemini-Deep-Research
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Google-NotebookLM
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# Anthropic / Claude
# ============================
User-agent: ClaudeBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Claude-Web
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# Perplexity AI (search-style + user-triggered crawlers)
# ============================
User-agent: PerplexityBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Perplexity-User
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# Other Major AI Crawlers (from 2025 lists)
# ============================
# Common Crawl (feeds many LLM datasets)
User-agent: CCBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Apple search + AI (Apple Intelligence, Siri)
User-agent: Applebot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Applebot-Extended
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Amazon AI (Alexa, AWS)
User-agent: Amazonbot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# ByteDance/TikTok AI
User-agent: Bytespider
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Cohere AI
User-agent: cohere-ai
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Diffbot (structured data for AI)
User-agent: Diffbot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Mistral AI
User-agent: MistralAI-User
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Groq AI
User-agent: Groq-Bot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Hugging Face
User-agent: HuggingFace-Bot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /

# xAI (Grok)
User-agent: xAI-Bot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# Social and Preview Bots (for sharing visibility)
# ============================
User-agent: facebookexternalhit
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Facebot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: LinkedInBot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /
User-agent: Twitterbot
Disallow: /*order-pay/  # SIIL 2026-07-28 crawl-budget (bots hitting stale pay links)
Disallow: /*checkout_*/  # SIIL 2026-07-28 localized checkout/cart/account slugs
Disallow: /*my-account*/
Disallow: /mi-cuenta/
Disallow: /cesta/
Disallow: /en/cart/
Disallow: /de/cart_de/
Disallow: /fr/basket/
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Allow: /


# ============================
# Sitemaps
# ============================
Sitemap: https://www.siilostomy.com/sitemap_index.xml
# Add more sitemap lines here if you ever split them:
# Sitemap: https://www.siilostomy.com/sitemap-products.xml
# Sitemap: https://www.siilostomy.com/sitemap-blog.xml
# MCP (Model Context Protocol) AI Agent Discovery
# AI agents can connect to our store via MCP
# Discovery: https://siilostomy.com/.well-known/mcp.json
# Endpoint: https://mcp.siilostomy.com/mcp