# robots.txt generated at www.mcanerin.com

User-agent: googlebot
Crawl-delay: 5
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-Agent: MediaPartners-Google
Crawl-delay: 1
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: Slurp
Crawl-delay: 5
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: msnbot
Crawl-delay: 5
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: lycos
Crawl-delay: 10
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/


User-agent: whatuseek
Crawl-delay: 10
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: wisenutbot
Crawl-delay: 10
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: Yandex
Crawl-delay: 10
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: baiduspider
Crawl-delay: 5
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

# try to restrict access frequency

User-agent: Gigabot
Crawl-delay: 120
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: GigaBlast
Crawl-delay: 120
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: Teoma
Crawl-delay: 360
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: VoilaBot
Crawl-delay: 1200
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: sogou spider
Crawl-delay: 360
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: YodaoBot
Crawl-delay: 3600
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: YodaoBot-Image
Crawl-delay: 3600
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: yetibot
Crawl-delay: 360
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: QihooBot
Crawl-delay: 3600
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: sosospider
Crawl-delay: 3600
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: sosoimagespider
Crawl-delay: 3600
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/

User-agent: daumoa
Crawl-delay: 6000
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/


#deny

# http://irl.cs.tamu.edu/crawler/
User-agent: IRLbot
Crawl-delay: 18000
Disallow: /

User-agent: twiceler
Crawl-delay: 18000
Disallow: /


#all the rest of the robots

User-agent: *
Crawl-delay: 600
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /bot-trap/