# Python Programming Blog robots.txt
User-agent: *
Allow: /

# Allow crawling of all content
Allow: /blog/
Allow: /blog/*
Allow: /about
Allow: /contact
Allow: /sitemap
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /cookie-policy
Allow: /disclosure
Allow: /disclaimer

# Point to sitemap (update domain in production)
Sitemap: http://localhost:8000/sitemap.xml

# Crawl delay for respectful crawling
Crawl-delay: 1

# Block admin/auth pages
Disallow: /login
Disallow: /register
Disallow: /dashboard
Disallow: /admin
Disallow: /api/
Disallow: /forgot-password
Disallow: /reset-password

# Allow search engines to index images
User-agent: Googlebot-Image
Allow: /*

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1