User-agent: *
Disallow: /admin/
Allow: /

User-agent: *
Disallow: */search/*

User-agent: Scrapy
Disallow: /

User-agent: Scrapy/1.0.3 
Disallow: /

User-agent: Scrapy/1.0.3 (+http://scrapy.org)
Disallow: /

User-Agent: JikeSpider
Disallow: /

User-Agent: discoverybot
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-agent: 008
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: ZumBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: citeseerxbot
Disallow: /

User-agent: ccbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: CCBot/2.0
Disallow: /

User-agent: YYSpider
Disallow: /

User-agent: psbot
Disallow: /

User-agent: OpenWebSpider
Disallow: /

User-agent: Scrapy/0.22.2
Disallow: /

User-agent: OpenWebSpider/0.1.4
Disallow: /

User-agent: msnbot
Crawl-delay: 10

User-agent: bingbot
Crawl-delay: 10