# Robots.txt for AI-powered document summarization app
# Generated by automated SEO system

User-agent: *
Allow: /
Disallow: /api/
Disallow: /debug/
Disallow: /admin/
Disallow: /_next/
Disallow: /auth/

# Allow search engines to access important pages
Allow: /pricing
Allow: /use-cases
Allow: /login
Allow: /signup

# Important files
# Sitemap: https://yourapp.com/sitemap.xml

# Crawling considerations
Crawl-delay: 1

# Block AI training crawlers (optional - uncomment if desired)
# User-agent: GPTBot
# Disallow: /
# 
# User-agent: ChatGPT-User
# Disallow: /
# 
# User-agent: CCBot
# Disallow: /
# 
# User-agent: anthropic-ai
# Disallow: /