# Global crawl settings for all user-agents
User-agent: *
Crawl-delay: 5
Allow: /wp-includes/js/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /profile
Disallow: /cgi-bin/
Disallow: /wp-content/cache/
Disallow: /trackback/
Disallow: /comments/
Disallow: /administrator/
Disallow: */trackback/
Disallow: */comments/
Disallow: /license.txt
Disallow: /*.php$
Disallow: *?filter
Disallow: /wp-content/themes/ionMag
Disallow: /readme.html

# Blocking compressed sitemap
Disallow: /sitemap.xml.gz

# Allow Scrapy
User-agent: Scrapy
Allow: /

# Blocking unwanted bots that consume bandwidth
User-agent: AdIdxBot
User-agent: AhrefsBot
User-agent: AiHitBot
User-agent: AlphaSeoBot
User-agent: AlphaSeoBot-SA
User-agent: Baiduspider
User-agent: BingPreview
User-agent: BLEXBot
User-agent: CareerBot
User-agent: Cliqzbot
User-agent: dotbot
User-agent: grapeshot
User-agent: ichiro
User-agent: iCjobs
User-agent: linkdexbot
User-agent: magpie-crawler
User-agent: MegaIndex
User-agent: MJ12bot
User-agent: moget
User-agent: NaverBot
User-agent: Owlin
User-agent: Owlin Bot
User-agent: Owlin Bot v. 3.0
User-agent: proximic
User-agent: QuerySeekerSpider
User-agent: scrapy
User-agent: scrapybot
User-agent: Semrush
User-agent: SemrushBot
User-agent: sentibot
User-agent: SEOkicks-Robot
User-agent: Sogou
User-agent: sogou spider
User-agent: TkBot
User-agent: trendkite-akashic-crawler
User-agent: Vagabondo
User-agent: WBSearchBot
User-agent: Yandex
User-agent: YandexBot
User-agent: Yeti
User-agent: YoudaoBot
Disallow: /

# Additional bots to be blocked
User-agent: SiteAuditBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SemrushBot-BM
User-agent: SplitSignalBot
Disallow: /