User-agent: *   # directed to all spiders
Disallow: /dev
Disallow: /cgi
Disallow: /cgi-bin
Disallow: /toc.gifs
Disallow: /intern
Disallow: /cont.html