User-agent: *
Allow: /

# Block admin and private areas
Disallow: /admin/
Disallow: /.netlify/
Disallow: /api/
Disallow: /*?*utm_*
Disallow: /*?*ref=*
Disallow: /*?*source=*

# Block sensitive config files only
Disallow: /config.yml
Disallow: /netlify.toml
Disallow: /package.json
Disallow: /tsconfig.json

# Allow important files for LLMs and indexing
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /favicon.ico
Allow: /llm.txt

# Explicitly allow LLM-friendly content files
Allow: /ai-data.md
Allow: /company-data.json

# Explicitly allow AI and LLM bots
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# Sitemaps
Sitemap: https://synthreo.ai/sitemap.xml