# https://www.robotstxt.org/robotstxt.html
# AllDevTool.com - Advanced Robots.txt for SEO Optimization

# Default rules for all crawlers
User-agent: *
Allow: /

# Crawl delay for polite crawling (1 second)
Crawl-delay: 1

# Allow important directories for SEO
Allow: /tools/
Allow: /discover/
Allow: /category/
Allow: /collections/
Allow: /guides/
Allow: /api/sitemap
Allow: /_next/static/
Allow: /images/
Allow: /screenshots/

# Prevent crawling of redirected URLs (these redirect to final URLs)
Disallow: /tools/html-formatter
Disallow: /tools/yaml-to-json
Disallow: /tools/csv-to-json
Disallow: /tools/json-to-csv
Disallow: /tools/xml-to-json

# Don't crawl non-existent pages
Disallow: /tools/websocket-tester

# Disallow paths that should not be indexed
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /_next/
Disallow: /ads-debug/
Disallow: /ad-test/
Disallow: /temp/
Disallow: /draft/
Disallow: /test/
Disallow: /*.json$
Disallow: /*?print=*
Disallow: /*?debug=*
Disallow: /*&print=*
Disallow: /*&debug=*

# Block specific file types that shouldn't be indexed
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.xls$
Disallow: /*.xlsx$
Disallow: /*.ppt$
Disallow: /*.pptx$

# Specific instructions for Googlebot (most important)
User-agent: Googlebot
Allow: /
Crawl-delay: 1
# Allow Google to index images
Allow: /images/
Allow: /screenshots/
Allow: /_next/static/images/

# Specific instructions for Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Specific instructions for other major search engines
User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 1

# Allow social media crawlers for better sharing
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# Allow SEO and analytics tools
User-agent: AhrefsBot
Allow: /
Crawl-delay: 1

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Allow: /
Crawl-delay: 3

User-agent: DotBot
Allow: /
Crawl-delay: 2

# Allow Google's various specialized bots
User-agent: Googlebot-Image
Allow: /images/
Allow: /screenshots/
Allow: /_next/static/images/

User-agent: Googlebot-Video
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: GoogleOther
Allow: /

# Block aggressive crawlers that can harm performance
User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Block malicious bots and scrapers
User-agent: SiteAuditBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: spbot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: dotbot
Disallow: /

# Sitemaps (multiple formats for comprehensive coverage)
Sitemap: https://www.alldevtool.com/sitemap.xml
Sitemap: https://www.alldevtool.com/api/sitemap
# HTML sitemap for crawlers
Sitemap: https://www.alldevtool.com/sitemap-page
# Text sitemap with all URLs listed
Sitemap: https://www.alldevtool.com/sitemap-text.txt
# Static HTML index with all tools for emergency crawling
Sitemap: https://www.alldevtool.com/tool-index.html

# Additional sitemap for images (if implemented)
# Sitemap: https://www.alldevtool.com/image-sitemap.xml

# Host directive (helps with canonical domain)
Host: https://www.alldevtool.com