# robots.txt for http://library.stanford.edu/

User-agent: *
Disallow: /collect/ejourns.html # A redirect that Google seems to index
Disallow: /sulair: # Old ejournal redirect string
Disallow: /rssfeeds/  # for feeds that are not part of our site
Disallow: /rssfeeds/law/  # for feeds that are not part of our site
Disallow: /Templates/ 
Disallow: /Library/ 
Disallow: /outage/ 
Disallow: /outage/sulweb/ 
Disallow: /inv/ 
Disallow: /pers_feed/
Disallow: /error_db/ 
Disallow: /search/ 
Disallow: /site_search/ 
Disallow: /scripts/ 
Disallow: /suatest/ 
Disallow: /test/ 
Disallow: /test2/spider_check/staff/
Disallow: /stats/ #Statistics pages
Disallow: /stats/africa/ #Statistics pages
Disallow: /stats/new/ #Statistics pages
Disallow: /stats/inst21/ #Statistics pages
Disallow: /stats/proxy/ #Statistics pages
Disallow: /stats/highwire1/ #Statistics pages
Disallow: /stats/pls/ #Statistics pages
Disallow: /stats/rsv/ #Statistics pages
Disallow: /stats/romnet/ #Statistics pages
Disallow: /stats/sunset/ #Statistics pages
Disallow: /filemove/ #Filemove form
#Disallow: /weiser/ # Mark Weiser memorial site - allowing this now
#Disallow: /mac/ # Mac history site - allowing; not sure why it was excluded
Disallow: /hanna/ # Hanna House site
Disallow: /gils/ # old GILS mirror site
Disallow: /mirrors/eleccion/ # Mexican election mirror site
Disallow: /depts/ssrg/africa/RCS/
Disallow: /depts/catdept/numgen/
Disallow: /staff/csreport/
Disallow: /staff/csreport/presentation/
Disallow: /prelectur/ # Presidential lecture series site 
Disallow: /hwmg/ # HighWire Management Group
Disallow: /hwmg/italian/ # HighWire Management Group
Disallow: /hwmg/spanish/ # HighWire Management Group
Disallow: /sunset/ # Sunset magazine site
Disallow: /sunset/assets/ # Sunset magazine site
Disallow: /sunset/assets/auto_generated_images # Sunset magazine site
Disallow: /sunset/assets/duplicate1/ # Sunset magazine site
Disallow: /sunset/assets/images/ # Sunset magazine site
Disallow: /sunset/html/ # Sunset magazine site
Disallow: /DROP/ # drop-off files
Disallow: /LOGS/ # 
Disallow: /OLD/ # old files
Disallow: /OBS/  # for obsolete pages with links to new pages
Disallow: /RCS/ # 
Disallow: /SUA/ # Stanford access only
Disallow: /jump/ # Stanford access only
Disallow: /restricted/ # Stanford access only
Disallow: /work/ # Stanford access only
Disallow: /search/ # Don't index this
Disallow: /test/ # Stanford access only
Disallow: /staff/zenunicorn/ # Stanford access only
Disallow: /staff/webprocs/ # Stanford access only
Disallow: /stats/ # Stanford access only
Disallow: /status/ # Sys dept access only (by password)
Disallow: /server/ # Stanford access only
Disallow: /depts/music/mla/ # 
Disallow: /depts/ils/gul/ # SU - Goettingen forms
Disallow: /icons/ # 
Disallow: /depts/ssrg/africa/RCS/ # Need to do this for all RCS dirs 
Disallow: /cgi-bin/ # 
Disallow: /ejournals/ # 
Disallow: /mirrors/ #  Mirror sites, so not really part of ours
Disallow: /mirrors/romnet/ # Previous line doesn't seem to exclude this 
Disallow: /depts/swain/edcinf/mo/ # Password access
Disallow: /depts/jonsson/SUA/
Disallow: /depts/swain/SUA/
Disallow: /depts/ssrg/linguist/SUA/
Disallow: /catdb/help/  # exclude help files since these are fragments
Disallow: /opachelp/ # more .shtml

User-agent: MOMspider
Disallow: /DROP/ # drop-off files
Disallow: /LOGS/ # 
Disallow: /RCS/ # 
Disallow: /jump/ # Stanford access only
Disallow: /work/ # Stanford access only
Disallow: /test/ # Stanford access only
Disallow: /stats/ # Stanford access only
Disallow: /server/ # Stanford access only
Disallow: /icons/ # 
Disallow: /cgi-bin/ #