# ============================
#   MAIN SEARCH ENGINES (ALLOWED)
# ============================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /


# ============================
#   SEO CRAWLERS (ALLOWED)
# ============================

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /


# ============================
#   AI CRAWLERS (ALLOWED)
# ============================

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: ByteSpider
Allow: /

User-agent: ImagesiftBot
Allow: /


# ============================
#   SOCIAL PREVIEW BOTS (ALLOWED)
# ============================

User-agent: facebookexternalhit
Allow: /

User-agent: Meta-externalagent
Allow: /


# ============================
#   BAD / SCANNER BOTS (BLOCKED)
#   (They will likely ignore this, but it's still good practice)
# ============================

User-agent: sqlmap
Disallow: /

User-agent: masscan
Disallow: /

User-agent: ZmEu
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Download Demon
Disallow: /

User-agent: SiteSucker
Disallow: /

User-agent: Nutch
Disallow: /


# ============================
#   DEFAULT RULES FOR EVERYONE
#   Only protect sensitive areas
# ============================

User-agent: *
Disallow: /admin/
Disallow: /config/
Disallow: /data/
Disallow: /logs/
Disallow: /private/
Disallow: /temp/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php