# ============================================================
# nufinishpro.com — robots.txt
# Replaces the current file. Reviewed 2026-10-21.
#
# Changes from the live version:
#   - Explicit Allow for AI search and assistant crawlers
#   - Removed "Disallow: /tag/*" (see note below)
#   - Corrected and completed the sitemap list
#
# NOTE ON TAGS: the live file disallows /tag/* while post_tag-sitemap.xml
# and the auto-generated llms.txt both advertise those same tag URLs.
# Fix the cause, not the symptom: delete the orphaned ad-ops tags created
# by the internal training posts, then let the remaining (if any) tags be
# crawlable, or noindex them in Yoast. A robots Disallow does not remove a
# URL from search — a linked, disallowed URL can still be listed.
# ============================================================

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Allow: /wp-includes/images/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /?s
Disallow: /thanks/

# ---- AI search, answer engines and assistant crawlers: allowed ----
# These are the agents that build and cite AI answers. We want to be read.

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google — Gemini / Vertex grounding. Allowing this is a deliberate choice:
# it governs Gemini and Vertex AI use, separate from Search indexing.
User-agent: Google-Extended
Allow: /

# Microsoft / Copilot
User-agent: Bingbot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Others that feed AI answers
User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# ---- Sitemaps ----
Sitemap: https://nufinishpro.com/sitemap_index.xml

# ============================================================
# ALSO VERIFY (robots.txt alone is not enough):
#
# 1. CDN / WAF: Cloudflare's "Block AI Scrapers and Crawlers" toggle, and
#    the AI-bot rules in Wordfence / Sucuri / All-in-One WP Security, block
#    GPTBot and ClaudeBot at the edge no matter what this file says.
#    Check those settings and confirm in access logs that these user agents
#    receive HTTP 200.
#
# 2. post-sitemap.xml currently returns 404 — the 33 refinishing posts are in
#    no sitemap at all. Enable it in Yoast after the internal training posts
#    are removed from the public blog.
#
# 3. The 94 internal franchisee training posts must be handled with noindex
#    (or moved behind a login), NOT with a robots Disallow.
#
# 4. Test with: curl -A "GPTBot" -I https://nufinishpro.com/
#               curl -A "ClaudeBot" -I https://nufinishpro.com/
#               curl -A "PerplexityBot" -I https://nufinishpro.com/
# ============================================================