User-agent: *

Disallow: /login/
Disallow: /users /
Disallow: /sessions /
Disallow: /logout /
Disallow: /signup /

User-agent: Googlebot-Image 
Disallow: /scalable_captcha/ 
Disallow: /scalable_captcha/scalable_captcha 



User-agent: 008 
Disallow: / 
 
User-agent: Mediapartners-Google 
Allow: /search/  

User-agent: Slurp 
Crawl-delay: 300  
# Spiders added on date 2014-06-17 
# Baidu 
# http://www.baidu.com/search/spider.html 
User-agent: Baiduspider 
Disallow: /  

# EasouSpider 
# http://www.easou.com/search/spider.html 
User-agent: EasouSpider 
Disallow: /  

# Exabot 
# http://www.exabot.com/go/robot 


User-agent: Exabot 
Disallow: /  

User-agent: BLEXBot 
Disallow: /  

User-agent: trovitbot 
Disallow: /  
  

User-agent: Baiduspider+ 
Disallow: /  

User-agent: yandexbot 
Disallow: /  

User-agent: MJ12bot 
Disallow: /  

User-agent: sitecheck.internetseer.com 
Disallow: /  

User-agent: Zealbot 
Disallow: /  

User-agent: MSIECrawler 
Disallow: /  

User-agent: SiteSnagger 
Disallow: /  

User-agent: WebStripper 
Disallow: /  

User-agent: WebCopier 
Disallow: /  

User-agent: Fetch 
Disallow: /  

User-agent: Offline Explorer 
Disallow: /  

User-agent: Teleport 
Disallow: /  

User-agent: TeleportPro 
Disallow: /  

User-agent: WebZIP 
Disallow: /  

User-agent: linko 
Disallow: /  

User-agent: HTTrack 
Disallow: /  

User-agent: Microsoft.URL.Control 
Disallow: /  

User-agent: Xenu 
Disallow: /  

User-agent: larbin 
Disallow: /  

User-agent: libwww 
Disallow: /  

User-agent: ZyBORG 
Disallow: /  

User-agent: Download Ninja 
Disallow: /  

User-agent: WebReaper 
Disallow: /  


User-agent: Screaming Frog SEO Spider 
Disallow: /  

User-agent: ZumBot 
Disallow: /