# /robots.txt file for http://webcrawler.com/
# http://www.robotstxt.org/wc/faq.html#robotstxt
# http://www.google.com/support/webmasters/bin/topic.py?topic=8843

User-agent: Slurp
Crawl-delay: 10 

User-agent: Googlebot
Crawl-delay: 10 

User-agent: *
Disallow: /links_report_broken.asp