# All robots will spider the domain 

User-agent: *

Disallow: /cgi-bin/pseek/modify.cgi
Disallow: /cgi-bin/pseek/rprtb.cgi
Disallow: /cgi-bin/pseek/addurl.cgi
Disallow: /html/
Disallow: /stats/