# Robots.txt for arjankc.com.np - Digital Marketing Expert Nepal
# Optimized for Maximum SEO and Visibility
# Last updated: March 18, 2026

# General search engines
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /test/
Disallow: /dev/
Disallow: /backup/
Disallow: /search
Disallow: /404
Disallow: /_site/
Disallow: /_jekyll-cache/
Disallow: /*.bak$
Disallow: /*.tmp$
Disallow: /*.log$

# Block third-party AI training crawlers
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

# Block aggressive SEO scrapers to save crawl budget
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Allow social media/messaging crawlers for rich link previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# XML Sitemap location
Sitemap: https://www.arjankc.com.np/sitemap.xml