# we use WordPress as our CMS platform

# User-agent: * directives apply to all web crawlers.
User-agent: *

# --- SEO & Content Optimization ---
# Disallow URLs with specific parameters to prevent indexing redundant pages.
Disallow: /*?lang=*
Disallow: /*?gtranslate=*
Disallow: /*?s=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?utm_*
Disallow: /*?replytocom=*

# Disallow common WordPress administration and search pages.
# This is standard practice for improving SEO by focusing crawlers on valuable content.
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /search/
Disallow: /tag/
Disallow: /author/
Disallow: /page/
Disallow: /feed/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# --- Performance & Security ---
# Disallow common WordPress core directories.
Disallow: /wp-content/
Disallow: /wp-includes/

# Explicitly allow necessary files.
Allow: /wp-admin/admin-ajax.php

# --- Crawl Management for Specific Bots ---
# Google adsbot ignores robots.txt unless specifically named!
User-agent: AdsBot-Google
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/
Disallow: /wp-includes/

# Set a crawl delay for aggressive crawlers to reduce server load.
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# --- Sitemap Declaration ---
Sitemap: https://www.sungoldsolar.com/sitemap.xml