# robots.txt for http://www.bestwebbuys.com/
User-Agent: dotbot
Crawl-delay: 5

User-Agent: Charlotte
Crawl-delay: 10 

User-Agent: AportWorm
Disallow: /

User-Agent: discobot
Disallow: / 

User-agent: Slurp
Crawl-delay: 2 
Disallow: /admin/
Disallow: /ads/
Disallow: /archive/
Disallow: /books/addtolist/
Disallow: /cf/
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /includes/
Disallow: /members/
Disallow: /ratings/
Disallow: /Java/
Disallow: /v3/
Disallow: /books/cgi-bin/
Disallow: /books/mtry.php
Disallow: /music/cgi-bin/
Disallow: /video/cgi-bin/
Disallow: /bikes/cgi-bin/
Disallow: /bikes/redirect
Disallow: /electronics/cgi-bin/
Disallow: /electronics/sbuy

User-agent: Twiceler
Disallow: /

# http://www.kototoi.org/zao/
User-agent: Zao
Disallow: /

# Alexa disallow from everywhere
User-agent: ia_archiver
Disallow: /

# Openfind disallow from everywhere
User-agent: Openbot
Disallow: /

User-agent: BecomeBot
Disallow: /

# Restrict all agents from these directories
User-agent: *
Disallow: /admin/
Disallow: /ads/
Disallow: /archive/
Disallow: /books/addtolist/
Disallow: /cf/
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /includes/
Disallow: /members/
Disallow: /ratings/
Disallow: /Java/
Disallow: /v3/
Disallow: /books/cgi-bin/
Disallow: /books/mtry.php
Disallow: /music/cgi-bin/
Disallow: /video/cgi-bin/
Disallow: /bikes/cgi-bin/
Disallow: /bikes/redirect
Disallow: /electronics/cgi-bin/
Disallow: /electronics/sbuy