# /robots.txt file for http://webcrawler.com/
# mail david@koochikoo.co.uk for constructive criticism

# All robots will spider the domain ...
User-agent: *
Disallow:

# ... but not these directories
User-agent: *
Disallow: /Templates
Disallow: /Scripts
Disallow: /Library
Disallow: /customerimages/
Disallow: /main/actions
Disallow: /main/admin
Disallow: /main/classes
Disallow: /main/content
Disallow: /main/helpfiles
Disallow: /main/js
Disallow: /main/scripts
Disallow: /main/themes