# ============================================================

# robots.txt — thetreetop.com

# Last updated: 2026-06-24

# Strategy:

# - Allow legitimate search engines

# - Allow AI search, retrieval, and training crawlers

# - Keep WordPress admin areas out of crawl paths

# - Block known low-value scrapers, spam crawlers, and vulnerability scanners

# ============================================================

# ============================================================

# GLOBAL DEFAULT

# Allow all crawlers unless explicitly blocked below.

# This includes search engines, AI answer engines, AI retrieval bots,

# and AI training crawlers.

# ============================================================

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# ============================================================

# AI + SEARCH VISIBILITY

# Intentionally allowed by the global default above:

#

# Search engines:

# - Googlebot

# - Bingbot

# - DuckDuckBot

# - Applebot

# - AdsBot-Google

# - AdsBot-Google-Mobile

#

# AI search / retrieval:

# - OAI-SearchBot

# - ChatGPT-User

# - Claude-SearchBot

# - Claude-User

# - Perplexity-User

# - GoogleOther

# - Amazonbot

# - DuckAssistBot

#

# AI training / model improvement:

# - GPTBot

# - ClaudeBot

# - Google-Extended

# - CCBot

# - cohere-ai

# - Applebot-Extended

# - Meta-ExternalAgent

# - FacebookBot

#

# These are not blocked because the current strategy is to maximize

# AI visibility, citation potential, and future model inclusion.

# ============================================================

# ============================================================

# KNOWN LOW-VALUE SCRAPERS, SPAM CRAWLERS, AND SCANNERS

# ============================================================

User-agent: Bytespider
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: rogerbot
Disallow: /

# ============================================================

# SITEMAP

# ============================================================

Sitemap: https://thetreetop.com/sitemap_index.xml