# /robots.txt file for http://www-cs101.ai.mit.edu/
# mail cs101-webmaster@ai.mit.edu for constructive criticism

#
# ferd produces about 200K/day of access and error logs.  We don't need
# to be indexed that often and I'm tired of grepping through error logs.
# Perhaps an email to search@mit or athena-rcc@mit would be appropiate
# too. -Todd (8 July 98)
#
User-agent: ferd.mit.edu
Disallow: /

User-agent: *
Disallow: /analog
Disallow: /archives
Disallow: /courses
Disallow: /cs101-staff
Disallow: /cs101-staff-old
Disallow: /psets
Disallow: /videos
Disallow: /REMOVED
Disallow: /applets