# robots.txt for http://www.overskrift.dk/

User-agent: *
Disallow: /admin
Disallow: /api
Disallow: /auto
Disallow: /cache
Disallow: /modules
Crawl-Delay:10
Request-rate: 1/15         # maximum rate is one page every 8 seconds
Visit-time: 0100-0600     # only visit between 6:00 AM and 8:45 AM UT (GMT)
Sitemap: http://overskrift.dk/sitemap.xml.gz

User-agent: spbot
Disallow: /