# Crawl only the main pages

User-agent: *
Disallow:   /ssi
Disallow:   /images
Disallow:   /brand
Disallow:   /autorespond
Disallow:   /error
Disallow:   /java
Disallow:   /stats
Disallow:   /hidden.shtml

# Alexa Robot

User-agent: ia_archiver
Disallow:   /cgi-bin
Disallow:   /ssi
Disallow:   /images
Disallow:   /brand
Disallow:   /autorespond
Disallow:   /error
Disallow:   /java
Disallow:   /stats
Disallow:   /hidden.shtml