# Define access-restrictions for robots/spiders
# http://www.robotstxt.org/wc/norobots.html

# By default we allow robots to access all areas of our site
# already accessible to anonymous users

User-agent: *
Disallow: /admin
Crawl-delay: 10


# Add Googlebot-specific syntax extension to exclude forms
# that are repeated for each piece of content in the site
# the wildcard is only supported by Googlebot
# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&ctx=sibling

User-Agent: Googlebot
Disallow: /*sendto_form$
Disallow: /*folder_factories$


User-agent: BLEXBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider+
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: proximic
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: Crowsnest
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: wotbox
Disallow: /

User-agent: dotbot 
Disallow: /

User-agent: sistrix 
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: IDentity
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: WBSearchBot
Disallow: /

User-agent: moget
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: findLinks
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Flipboard
Disallow: / 

User-agent: FlipboardProxy
Disallow: / 

User-agent: UnwindFetchor
Disallow: /

User-agent: PaperLiBot
Disallow: /

User-agent: Page2RSS
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: plukkie 
Disallow: /

User-agent: discoverybot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: Embedly
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Acoon
Disallow: /

User-agent: backlink rastreador
Disallow: /

User-agent: www.integromedb.org/Crawler
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Arquivo-web-crawler 
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: WebTarantula.com Crawler
Disallow: /