User-agent: *
Allow: /

User-agent: *
Disallow: /proxy/
Disallow: /cta/

User-agent: ia_archiver
Disallow: /

User-agent: SemrushBot
Disallow: / 

User-agent: SemrushBot-BA
Disallow: / 

User-agent: Yandex
Crawl-delay: 10

User-agent: Barkrowler
Disallow: /

User-agent: Barkrowler/0.9
Disallow: /

User-agent: rogerbot
Crawl-Delay: 10

User-agent: Barkrowler/0.9
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: megaindex.com
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: exabot
Disallow: /

User-agent: ezooms
Disallow: /

User-agent: fatbot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: jikespider
Disallow: /

User-agent: meanpathbot
Disallow: /

User-agent: psbot
Disallow: /

User-agent: sitebot
Disallow: /

User-agent: sosospider
Disallow: /

User-agent: turnitinbot
Disallow: /

User-agent: voilabot
Disallow: /

User-agent: willybot
Disallow: /

User-agent: yodaobot
Disallow: /

User-agent: ZumBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Spinn3r
Disallow: /

User-agent: Storebot
Disallow: /

User-agent: TwengaBot-2.0
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: CrazyWebCrawler-Spider
Disallow: /

User-agent: deepcrawl
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: ExtLinksBot
Disallow: /

User-agent: Genieo
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Pigafetta
Disallow: /

User-agent: PetalBot
Disallow: / 

User-agent: SeekportBot
Disallow: / 

User-agent: DataForSeoBot
Disallow: /