# COGENT 1.1
# robot/spider exclusion 
# http://info.webcrawler.com/mak/projects/robots/robots.html
User-agent: EmailSiphon
User-agent: Kilroy             # OCLC
User-agent: Whacker            # WebWhacker
User-agent: FlashSite
User-agent: NetJet
User-agent: Snarf
User-Agent: TestApp/1.0 libwww/4.1b5
User-Agent: GETWWW-ROBOT/2.0
User-Agent: MemoWeb
User-Agent: net.Sweep/v1.2e-T1-CHI
User-Agent: FFiNet32.DLL/3.1 libwww/3.1
User-Agent: Javachapman:05/19/19-15:35
User-Agent: LaSCoW-trace/1.0
User-Agent: ExtractorPro
User-Agent: Rainspider/1.0
User-Agent: /0.5 libwww-perl/0.40
User-Agent: EmailWolf 1.00
User-Agent: EmailCollector/1.0
User-Agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
User-Agent: CherryPickerSE/1.0
User-Agent: CherryPickerElite/1.0
User-Agent: .
Disallow: /

User-agent: *
Disallow: /cgi-bin/
Disallow: /stats/
Disallow: /webtrends/
Disallow: /secure/
Disallow: /nickf/
Disallow: /media/
Disallow: /images/
Disallow: /samples/
Disallow: /classifieds/images/
Disallow: /digisect/images/
Disallow: /features/images/
Disallow: /interviews/images/
Disallow: /classifieds/images/
Disallow: /classifieds/images/
Disallow: /new/images/
Disallow: /portland/images/
Disallow: /classifieds/images/
Disallow: /prodotcon/images/
Disallow: /profiles/images/
Disallow: /resources/images/
Disallow: /tutorials/images/

