# robots.txt for http://www.overskrift.dk/

User-agent: *
Disallow: /admin
Disallow: /api
Disallow: /auto
Disallow: /cache
Disallow: /modules
Crawl-Delay:10
Request-rate: 15         # maximum rate is one page every 8 seconds
Visit-time: 0100-0600     # only visit between 6:00 AM and 8:45 AM UT (GMT)

User-agent: spbot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /