User-agent: *

Disallow: /cgi-bin/ #Keine Scripts lesen!

User-agent: Slurp

Crawl-delay: 10