# robots.txt for http://www.comlab.ox.ac.uk

User-agent: *
Disallow: /teaching/internal/
Disallow: /internal/
Disallow: /personal/teaching/materials08-09/
Disallow: /webalizer/
Disallow: /files/268/
Crawl-delay: 1 # Might slow down the crawlers
Request-rate: 1/1