# Global User-agent Settings (All bots)
User-agent: *
# Block access to core WordPress files and folders
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html
Disallow: /license.txt

# Block access to specific files and URLs
Disallow: /cgi-bin/
Disallow: /comments/feed/
# Disallow: /*attachment_id=
# Disallow: /*.pdf$
Disallow: /feed/
Disallow: /feeds/
Disallow: /page/
# Disallow: /*//  # Prevents any URL with double slashes
Disallow: /import-feed/
Disallow: /video_categories/*
Disallow: /canada-denmark-agree-on-a-landmark-deal-overdisputed-
Disallow: /auto-draft/
Disallow: /*?s=*
Disallow: /*?action=
Disallow: /*?print=
Disallow: /*?fbclid=
Disallow: /*?utm_

# Allow bots to index audio and video files
Allow: /*.mp3$
Allow: /*.mp4$

# Block external feed content from being indexed
Disallow: /feed/atom/
Disallow: /feed/rss/
Disallow: /feed/rss2/

# Block specific bots
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MedialogiaBot
Disallow: /

User-agent: MSIE
Disallow: /

User-agent: Pinterestbot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Qwantbot-prod94241
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AwarioBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# Special Disallow Rules
Disallow: /wp-admin/admin-ajax.php
Disallow: /wp-content/uploads/wp-import-export-lite/

# **Allow Twitterbot to access uploads for OG images**
User-agent: Twitterbot
Allow: /wp-content/uploads/

# Sitemap location
Sitemap: https://www.arctictoday.com/sitemap_index.xml