# =========================
# DEFAULT: BLOCK EVERYTHING
# =========================
User-agent: *
Disallow: /

# Optional crawl pacing (non-Google bots)
Crawl-delay: 15

# Block noisy/internal endpoints
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /scripts/
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*
Disallow: /*act.ma?*

# =========================
# ALLOW MAJOR SEARCH ENGINES
# =========================

# act.ma stays crawlable for Googlebot: pages such as /competition/ and
# /correlations/ load their content through it, and blocking it left the
# rendering crawler with empty 68-word pages sitewide.
User-agent: Googlebot
Allow: /
Disallow: /*topSuggest*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

# Google News uses the same validated v2 publication URLs as Search.
User-agent: Googlebot-News
Allow: /
Disallow: /*topSuggest*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: Bingbot
Allow: /
Crawl-delay: 15
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 15
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: Slurp
Allow: /
Crawl-delay: 15
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: Applebot
Allow: /
Crawl-delay: 15
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: YandexBot
Allow: /
Crawl-delay: 15
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

# =========================
# ALLOW AI SEARCH / RETRIEVAL AGENTS
# (user-triggered fetchers and AI-search indexers - they cite and
# link pages in AI answers; training crawlers stay blocked below)
# =========================

User-agent: OAI-SearchBot
Allow: /
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: ChatGPT-User
Allow: /
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: PerplexityBot
Allow: /
Crawl-delay: 15
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: Perplexity-User
Allow: /
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: Claude-User
Allow: /
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

User-agent: Claude-SearchBot
Allow: /
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

# Google-Extended: lets Google use content for Gemini / Vertex AI grounding and citation
# (does not affect Search ranking, which is governed by Googlebot above).
User-agent: Google-Extended
Allow: /
Disallow: /*topSuggest*
Disallow: /*act.ma*
Disallow: /*.php*
Disallow: /*.info*
Disallow: /wp-content/
Disallow: /*modern-ticker*

# =========================
# BLOCK AI TRAINING BOTS
# =========================

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

# =========================
# BLOCK SEO / SCRAPERS
# =========================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# =========================
# BLOCK BAIDU
# =========================

User-agent: Baiduspider
Disallow: /

# =========================
# LEGACY BAD BOTS
# =========================

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: wget
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

# =========================
# SITEMAPS
# =========================

Sitemap: https://www.macroaxis.com/sitemap-stocks-us.xml
Sitemap: https://www.macroaxis.com/sitemap-stocks.xml
Sitemap: https://www.macroaxis.com/sitemap-funds.xml
Sitemap: https://www.macroaxis.com/sitemap-etfs.xml
Sitemap: https://www.macroaxis.com/sitemap-crypto.xml
Sitemap: https://www.macroaxis.com/sitemap-commodities.xml
Sitemap: https://www.macroaxis.com/sitemap-indices.xml
Sitemap: https://www.macroaxis.com/sitemap-price-history.xml
Sitemap: https://www.macroaxis.com/sitemap-signals.xml
Sitemap: https://www.macroaxis.com/sitemap-options.xml
Sitemap: https://www.macroaxis.com/sitemap-pages.xml
Sitemap: https://www.macroaxis.com/sitemap-stories.xml
Sitemap: https://www.macroaxis.com/sitemap.xml