User-agent: *
Crawl-delay: 20
Crawl-limit:5

User-agent: MJ12bot
User-agent: AhrefsBot
Disallow: /


User-agent: BLEXBot
Disallow: /

User-agent: Adsbot/3.1 
Disallow: /

User-agent: Adsbot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent:Googlebot/2.1
Crawl-delay: 60

User-agent: msnbot
Crawl-delay: 10

User-Agent: rogerbot
Crawl-limit:5

User-Agent: SemrushBot
Crawl-limit:5

User-Agent: SemrushBot-BA
Crawl-limit:5


User-agent: dotbot
Crawl-Delay: 30
Disallow: /sites/default/files/

User-agent: BomboraBot
Crawl-Delay: 60

User-agent: Y!J-ASR/1.0
Crawl-Delay: 60

User-agent: Gigabot
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /


User-agent: MegaIndex.ru
Disallow: /

User-agent: AlphaSeoBot
Disallow: /

User-agent: AlphaSeoBot-SA
Disallow: /

User-agent: spbot
Crawl-delay: 40

User-Agent: The Knowledge AI
Disallow: /

User-agent: SemrushBot
Disallow: /

User-Agent: panscient.com
Disallow: /

User-agent: G-i-g-a-b-o-t
Disallow: /

User-agent: Barkrowler
Disallow: /

User-Agent: panscient
Disallow: /

User-Agent: coccocbot-web/1.0
Disallow: /

User-Agent: coccocbot-web
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: CCBot
Crawl-Delay: 30