# this page was used as a guideline to build our robots.txt file: http://en.wikipedia.org/wiki/Robots_exclusion_standard User-agent: * Allow: /index.html Allow: /services/ Allow: /Handcrafted/ Allow: /refretting/ Allow: /faq/ Allow: /vintage/ Allow: /gallery/ Allow: /contact/ Allow: /articles/ Allow: /letters/ Allow: /gripes/ Disallow: / Crawl-delay: 600 # Several major crawlers support a Crawl-delay parameter, set to the number of seconds to wait between successive requests to the same server Request-rate: 1/5 # maximum rate is one page every 5 seconds Visit-time: 0900-1145 # only visit between 06:00 and 08:45 UTC (GMT)