User-agent: *
Disallow: /admin/
Disallow: /test/
Disallow: /cgi-bin/
Disallow: /hidden/
Disallow: /rfc/txt


#
# yahoo ausbremsen
#
User-agent: Slurp
Crawl-delay: 10

#
# msn ausbremsen
#
User-agent: msnbot
Crawl-delay: 10

#
# viele bots, die keiner braucht
#
User-agent: laycat
User-agent: jobs.de-Robot
User-agent: Eurobot
User-agent: Gigabot
User-agent: Speedy
User-agent: SurveyBot_IgnoreIP
User-agent: SurveyBot
User-agent: heise-IT-Markt-Crawler
User-agent: sitecheck.internetseer.com
User-agent: Zealbot
User-agent: MSIECrawler
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebZIP
User-agent: linko
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: Xenu
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Download Ninja
User-agent: wget
User-agent: grub-client
User-agent: WebReaper
User-agent: ScoutJet
User-agent: JobRoboter
User-agent: gonzo*
User-agent: envolk
User-agent: psbot
User-agent: crawly
User-agent: Baiduspider
User-agent: BlogPulseLive
User-agent: searchlink
User-agent: woriobot
User-agent: Snapbot
User-agent: flatlandbot
User-agent: infometrics-bot
User-agent: Exabot
User-agent: MJ12bot
User-agent: YoudaoBot
User-agent: Touche
User-agent: Sosospider
User-agent: Tasapspider
User-agent: Euripbot
User-agent: iearthworm
Disallow: /