# Robots.txt for askmitoto.com
# Updated: January 2026
# SEO & AI-Optimized Configuration

# Default rule for all crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow sensitive and duplicate content
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /dashboard/
Disallow: /api/
Disallow: /private/
Disallow: /user/
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /search?
Disallow: /*?*utm_source=
Disallow: /*?*session_id=
Disallow: /thank-you/
Disallow: /temp/
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /cgi-bin/
Disallow: /thank-you-for-contact-us/

# Allow important pages for indexing
Allow: /blog/
Allow: /features/
Allow: /faq/
Allow: /contact-us/
Allow: /pricing/
Allow: /in/
Allow: /*.css$
Allow: /*.js$
Allow: /images/
Allow: /assets/

# Google Crawlers (Prioritized)
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Googlebot-Image
Allow: /images/
Allow: /assets/

# Bing Crawler
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# AI Training Crawlers - Selective Access
# OpenAI GPT Crawler
User-agent: GPTBot
Allow: /blog/
Allow: /faq/
Allow: /features/
Disallow: /

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /blog/
Allow: /faq/
Disallow: /

# Anthropic Claude Crawler
User-agent: anthropic-ai
Allow: /blog/
Allow: /faq/
Allow: /features/
Disallow: /

# Common Crawlers AI
User-agent: CCBot
Allow: /blog/
Allow: /faq/
Disallow: /

# Meta AI Crawler
User-agent: FacebookBot
Allow: /blog/
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /blog/
Allow: /faq/
Disallow: /

# Applebot (Siri, Spotlight)
User-agent: Applebot
Allow: /
Crawl-delay: 1

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# Block aggressive/problematic crawlers
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

# Block malicious bots
User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Crawl-delay: 10

# Sitemap location
Sitemap: https://askmitoto.com/sitemap_index.xml

# LLMs.txt for AI optimization 
# Reference: https://askmitoto.com/llms.txt

# Host preference (optional - for Google)
Host: askmitoto.com