# robots.txt for erikberg.com
# $Id: robots.txt,v 1.12 2010/02/01 11:09:20 eb Exp $
#

User-agent: MSIECrawler
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: SBIder
Disallow: /

User-agent: msnbot
Disallow:
Crawl-delay: 20

User-agent: OmniExplorer_Bot
Disallow:
Crawl-delay: 20

User-agent: SurveyBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: archive_crawler
Disallow: /mlb
Disallow: /nba
Disallow: /ncaabb

User-Agent: msrbot
Disallow: /nba
Disallow: /mlb
Disallow: /ncaabb

User-Agent: turnitinbot
Disallow: /nba
Disallow: /mlb
Disallow: /ncaabb

# crawl way too fast
User-agent: Yandex
Disallow: /ncaabb
Disallow: /nba
Disallow: /mlb

User-agent: ia_archiver
Disallow: /

User-agent: dotbot
Disallow: /

# too aggressive
User-agent: Linguee
Disallow: /