User-agent: ia_archiver
Disallow: /

# disallow all 
User-agent: *
Disallow: /
Crawl-delay: 600

# but allow only important bots
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: bingbot
User-agent: msnbot
User-agent: msnbot-media
User-agent: Slurp
User-agent: Yahoo-Blogs
User-agent: Yahoo-MMCrawler
# Directories
Disallow: /awstats/
Disallow: /blog/
Disallow: /cgi-bin/
Disallow: /cp/
Disallow: /downloads/
Disallow: /images/
Disallow: /images1/
Disallow: /rewards/
Disallow: /tools/