# =========================================
# Main rules
# =========================================

User-agent: *

# --- Service sections ---
Disallow: /bitrix/admin/
Disallow: /bitrix/tmp/
Disallow: /bitrix/cache/
Disallow: /personal/
Disallow: /auth/
Disallow: /search/
Disallow: /cart/
Disallow: /order/
Disallow: /api/
Disallow: /test/
Disallow: /404.php

# --- Technical and duplicate pages ---
Disallow: /*index.php$
Disallow: /*?print=
Disallow: /*?action=
Disallow: /*?login=
Disallow: /*?logout=
Disallow: /*?register=
Disallow: /*?forgot_password=
Disallow: /*?change_password=
Disallow: /*?auth=
Disallow: /*?backurl=
Disallow: /*?back_url=
Disallow: /*?preview=
Disallow: /*?new=
Disallow: /*?edit=

# --- Filter and sorting duplicates ---
Disallow: /*?sort=
Disallow: /*?order=
Disallow: /*?view=
Disallow: /*?display=
Disallow: /*?filter=
Disallow: /*?set_filter=
Disallow: */filter/
Disallow: */apply/

# --- Pagination duplicates ---
Disallow: /*?PAGEN_
Disallow: /*?page=

# --- Tracking parameters ---
Disallow: /*?utm
Disallow: /*&utm
Disallow: /*?gclid=
Disallow: /*?yclid=
Disallow: /*?fbclid=
Disallow: /*?ymclid=
Disallow: /*?roistat=
Disallow: /*?rb_clickid=

# =========================================
# Allow important assets
# =========================================

Allow: /bitrix/js/
Allow: /bitrix/templates/
Allow: /bitrix/components/

Allow: /local/js/
Allow: /local/templates/
Allow: /local/components/

Allow: *.css
Allow: *.js
Allow: *.jpg
Allow: *.jpeg
Allow: *.png
Allow: *.gif
Allow: *.svg
Allow: *.webp

# =========================================
# AI Crawlers
# =========================================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# Amazon AI
User-agent: Amazonbot
Allow: /

# Apple
User-agent: Applebot
Allow: /

# =========================================
# Block aggressive SEO crawlers
# =========================================

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MegaIndex
Disallow: /

# =========================================
# Sitemap
# =========================================

Sitemap: https://bar-street.ru/sitemap.xml
LLMS: https://bar-street.ru/llms.txt