# 1. Standard Wildcard Block (For everyone else)
User-agent: *
Disallow: /a/downloads/-/*
Disallow: /admin
Sitemap: https://allysec.com/sitemap.xml

# 2. Google Ads Bots (Explicit Allow - NO Crawl Delay)
# We isolate these to ensure 100% crawlability for ads.
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Allow: /

# 3. Google Search Bot (Standard organic search)
User-agent: Googlebot
Allow: /

# 4. Other Bots (With Crawl Delays)
User-agent: Nutch
User-agent: AhrefsBot
Crawl-delay: 10
Allow: /
Allow: /products
Allow: /collections

User-agent: AhrefsSiteAudit
User-agent: MJ12bot
User-agent: Pinterest
Crawl-delay: 1
Allow: /
Allow: /products
Allow: /collections