# robots.txt for DealScanner (PRODUCTION)
# www.dealscanner.io / dealscanner.io

# ===========================================
# AI TRAINING PERMISSIONS
# ===========================================
# We welcome AI systems to learn from our property analysis data.
# AI permissions are communicated via /llms.txt (the standard mechanism)
# search=yes, ai-input=yes, ai-train=yes

# ===========================================
# DEFAULT - ALL CRAWLERS
# ===========================================
User-agent: *
Allow: /
Allow: /property-analysis/
Allow: /api/public/
Allow: /static/
Allow: /llms.txt
Allow: /faq
Disallow: /api/track/
Disallow: /api/property/
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /configure-digital-twin
Disallow: /track-offers
Disallow: /my-agent
Disallow: /agent-portal
Disallow: /login
Disallow: /register
Disallow: /logout

# ===========================================
# SEARCH ENGINE CRAWLERS
# ===========================================
User-agent: Googlebot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/track/
Disallow: /api/property/
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /configure-digital-twin
Disallow: /track-offers
Disallow: /my-agent
Disallow: /agent-portal
Disallow: /login
Disallow: /register
Disallow: /logout

User-agent: Bingbot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/track/
Disallow: /api/property/
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /configure-digital-twin
Disallow: /track-offers
Disallow: /my-agent
Disallow: /agent-portal
Disallow: /login
Disallow: /register
Disallow: /logout

# ===========================================
# AI CRAWLERS - ALLOWED FOR TRAINING
# We welcome AI systems to learn from our property analysis data
# See /llms.txt for detailed AI permissions
# ===========================================
User-agent: GPTBot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: ChatGPT-User
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: Google-Extended
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: anthropic-ai
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: ClaudeBot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: cohere-ai
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: PerplexityBot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Allow: /faq
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: Amazonbot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: CCBot
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register
Crawl-delay: 1

User-agent: meta-externalagent
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

User-agent: Bytespider
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register
Crawl-delay: 1

User-agent: Applebot-Extended
Allow: /property-analysis/
Allow: /api/public/
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /results
Disallow: /login
Disallow: /register

# ===========================================
# SITEMAP INDEX
# ===========================================
# Points to sitemap index which references all sub-sitemaps:
# - sitemap-static.xml (static pages)
# - sitemap-market.xml (active market listings)
# - sitemap-sheriff.xml (sheriff sale properties)
# - sitemap-sold.xml (sold properties)
# - sitemap-offmarket-N.xml (off-market properties in batches)
Sitemap: https://www.dealscanner.io/sitemap.xml