User-agent: *
Disallow: /cgi-bin/
Disallow: /rotary/

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: discobot
Disallow: / 

User-agent: AhrefsBot
Disallow: / 

User-agent: sistrix
Disallow: / 

User-agent: ahrefs
Disallow: / 

User-agent: SemrushBot
Disallow: / 

User-agent: PetalBot
Disallow: /

# brandwatch.net
User-agent: magpie-crawler
Disallow: /