User-agent: *
Disallow: /admin_
Disallow: /add_
Disallow: /change_
Disallow: /delete_
Disallow: /edit_
Disallow: /login.php
Disallow: /logout.php
Disallow: /users_full.php
Disallow: /includes/
Disallow: /components/

# Amazon's crawler — too aggressive, all 500 errors
User-agent: Amazonbot
Disallow: /

# Chinese search engines — not relevant audience, high server load
User-agent: SogouSpider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Baiduspider
Disallow: /

# SEO crawlers with no value for this audience
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# SEO crawlers with no value for this audience
User-agent: DotBot
Disallow: /

# Amazon crawlers — too aggressive
User-agent: Amzn-SearchBot
Disallow: /

# Junk crawlers seen in logs
User-agent: ReyilBot
Disallow: /

User-agent: SleepBot
Disallow: /

Sitemap: https://www.doingthe92.com/sitemap.xml