# robots.txt - Emergys.com
# SEO + Security + Bot-Attack Hardened
# AI Crawler: 3-Tier Strategy (Allow Mainstream / Rate-Limit / Block Unknown)
# Last Updated: July 2026
# =============================================================

# -------------------------------------------------------------
# SECTION 1: SEARCH ENGINES (Full Access)
# These are your core SEO crawlers. They get full access to all
# public content. Named blocks include WP internal Disallows
# because per the robots.txt spec a named User-agent block
# operates as a CLEAN SLATE - it ignores User-agent: * entirely.
# NOTE: Googlebot does NOT support Crawl-delay (use Google
# Search Console crawl rate settings instead).
# -------------------------------------------------------------

# Google - Primary SEO index (does NOT support Crawl-delay)
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /private/
Disallow: /internal/
Disallow: /cgi-bin/
Disallow: /?s=
Disallow: /?p=
Disallow: /author/
Allow: /wp-admin/admin-ajax.php

# Bing - Secondary SEO index (lowercase 'bingbot' is the official token)
User-agent: bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /private/
Disallow: /internal/
Disallow: /cgi-bin/
Disallow: /?s=
Disallow: /?p=
Disallow: /author/
Allow: /wp-admin/admin-ajax.php

# DuckDuckGo - Respects robots.txt, important for privacy-focused users
User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /private/
Disallow: /internal/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 2

# Yandex - Supports Crawl-delay, relevant for international reach
User-agent: Yandex
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /private/
Disallow: /internal/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 3

# Apple - Applebot powers Siri and Spotlight search
User-agent: Applebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /private/
Disallow: /internal/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 3

# -------------------------------------------------------------
# SECTION 2: SEO AUDIT TOOLS (Explicitly Whitelisted)
# Required for SEMrush Site Audit and SEO analysis.
# Must include WP Disallows per clean-slate rule.
# -------------------------------------------------------------

User-agent: SemrushBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 1

User-agent: SiteAuditBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5

# -------------------------------------------------------------
# SECTION 3A: MAINSTREAM AI TRAINING CRAWLERS (Allowed - Heavy Rate Limit)
# Official crawlers from OpenAI, Anthropic, Meta, Google, Apple, Amazon.
# Allowed to crawl public marketing content with heavy Crawl-delay
# to prevent CPU spikes. WordPress internals blocked.
# NOTE: Crawl-delay is advisory; enforce hard rate limits at
# WAF / .htaccess / Cloudflare level for full protection.
# -------------------------------------------------------------

# OpenAI - GPT model training
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /author/
Crawl-delay: 5

# Anthropic - Claude model training
User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /author/
Crawl-delay: 5

# Meta AI - Llama model training
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /author/
Crawl-delay: 5

# Meta AI - Search quality indexing
User-agent: Meta-WebIndexer
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/
Crawl-delay: 5

# Google AI - Gemini / Vertex AI model training (separate from Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/

# Google - Vertex AI agent requests
User-agent: Google-CloudVertexBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /wp-includes/
Crawl-delay: 5

# Apple - Apple Intelligence / Foundation Models
User-agent: Applebot-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/
Crawl-delay: 5

# Amazon - Alexa / Amazon AI search indexing
User-agent: Amazonbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/
Crawl-delay: 5

# -------------------------------------------------------------
# SECTION 3B: AGGRESSIVE / NON-MAINSTREAM SCRAPERS (Blocked)
# SEO scrapers, data brokers, and unidentified bots that
# cause CPU attacks. Blocked here + enforce at WAF / .htaccess.
# -------------------------------------------------------------

# ByteDance / TikTok - ignores Crawl-delay and robots.txt
User-agent: Bytespider
Disallow: /

# SEO data scrapers - no value for Emergys
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# CCBot - Common Crawl (feeds most LLM training data; allowed for long-term brand presence)
# Rate-limit at CDN / WAF if needed
User-agent: CCBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/

# Unidentified data scrapers
User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# Diffbot - knowledge graphs used by AI tools (allowed; rate-limit at CDN / WAF if needed)
User-agent: Diffbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/

# -------------------------------------------------------------
# SECTION 4: AI USER-FACING & SEARCH BOTS (Allowed - Light Rate Limit)
# These serve real users in AI assistants and drive referral traffic.
# WordPress internals blocked for all. Must include full Disallow
# set per clean-slate rule.
# NOTE: Claude-Web and Anthropic-AI are deprecated Anthropic user-agent
# strings, replaced by ClaudeBot / Claude-User / Claude-SearchBot.
# Removed here since they are no longer active and do nothing.
# -------------------------------------------------------------

# OpenAI - ChatGPT live browsing (user-triggered)
User-agent: ChatGPT-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /private/
Disallow: /internal/
Disallow: /wp-includes/
Disallow: /author/

# OpenAI - SearchGPT web search indexing
User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /author/

# Anthropic - Claude user-triggered browsing
User-agent: Claude-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /author/

# Anthropic - Claude search bot
User-agent: Claude-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /author/

# Perplexity - AI search indexing
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /author/

# Perplexity - User-triggered research
User-agent: Perplexity-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /feed/
Disallow: /wp-includes/
Disallow: /author/

# -------------------------------------------------------------
# SECTION 5: GLOBAL RULES (All Other Crawlers / Unknown Bots)
# Catch-all for any bot NOT named above. Named bots IGNORE
# this section entirely (clean-slate rule per RFC 9309).
# This protects against unknown/unidentified crawlers.
# -------------------------------------------------------------

User-agent: *

# - WordPress Core Attack Vectors -
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /wp-config.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wflogs/

# - WordPress REST API (unauthenticated hits trigger full WP bootstrap) -
Disallow: /wp-json/

# - WordPress RSS/Atom Feeds (bot hammering causes repeated DB queries) -
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/
Disallow: /*/feed/rss/
Disallow: /*/feed/atom/

# - WordPress Trackback (legacy pingback amplification vector) -
Disallow: /trackback/
Disallow: /*/trackback/

# - WordPress Author Archives (user enumeration attack vector) -
Disallow: /author/

# - Internal & Private Directories -
Disallow: /private/
Disallow: /internal/
Disallow: /cgi-bin/

# - Search Results (duplicate content + uncached DB hit per query) -
Disallow: /?s=
Disallow: /?paged=

# - Raw WordPress Query Vars (bypass page caching) -
Disallow: /?p=
Disallow: /?page_id=
Disallow: /?cat=
Disallow: /?tag=
Disallow: /?author=
Disallow: /?feed=
Disallow: /?attachment_id=
Disallow: /?preview=

# - PDF Files (end-anchored $ - blocks .pdf URLs only, not /pdf-viewer/ etc.) -
Allow: /*.pdf$

# Allow admin-ajax (required for WP frontend forms and AJAX requests)
Allow: /wp-admin/admin-ajax.php

# Conservative crawl delay for all unrecognized bots
Crawl-delay: 10

# =============================================================
# SITEMAP
# =============================================================

Sitemap: https://www.emergys.com/sitemap.xml