# =============================================================
# robots.txt
# Static HTML5 site — 2026 SEO Best Practice from Local Media Solutions
# All AI bots explicitly allowed for maximum AI search visibility
# Last updated: 2026
# =============================================================


# -------------------------------------------------------------
# DEFAULT — All crawlers
# -------------------------------------------------------------
# Allow all crawlers to access the entire site by default.
# Then block only the paths that don't help SEO or expose data.
User-agent: *
Allow: /

# Common non-content directories to keep out of search results.
# Delete any lines below that don't apply to your site.
Disallow: /cgi-bin/
Disallow: /old/

# NOTE: Do NOT block these — search engines and AI bots need them:
# - /css/, /styles/  → CSS is required for page rendering
# - /js/, /scripts/  → JavaScript is required for page rendering
# - /images/, /img/  → Images should be indexed for image search
# - /fonts/          → Fonts affect layout rendering


# =============================================================
# AI BOTS — All explicitly allowed for maximum AI visibility
# =============================================================

# -------------------------------------------------------------
# OPENAI / CHATGPT
# GPTBot        → training crawler
# OAI-SearchBot → SearchGPT indexing
# ChatGPT-User  → user-initiated fetches (URL paste, browse)
# -------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /


# -------------------------------------------------------------
# ANTHROPIC / CLAUDE
# ClaudeBot         → primary training crawler
# Claude-SearchBot  → Claude search/citation fetches
# Claude-User       → user-initiated Claude fetches
# anthropic-ai      → legacy UA, still seen in logs
# -------------------------------------------------------------
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /


# -------------------------------------------------------------
# GOOGLE GEMINI
# Google-Extended controls whether your content can be used for
# Gemini and Vertex AI. Not a separate crawler — Googlebot fetches
# your site normally; this token controls AI training use.
# -------------------------------------------------------------
User-agent: Google-Extended
Allow: /


# -------------------------------------------------------------
# APPLE INTELLIGENCE
# Same pattern as Google-Extended. Applebot (crawler) already
# indexes for Siri/Spotlight; this token controls AI training use.
# -------------------------------------------------------------
User-agent: Applebot-Extended
Allow: /


# -------------------------------------------------------------
# PERPLEXITY
# PerplexityBot    → indexing crawler for Perplexity search
# Perplexity-User  → user-initiated fetches during answers
# -------------------------------------------------------------
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /


# -------------------------------------------------------------
# META (Facebook / Instagram / Threads)
# Meta-ExternalAgent    → Meta AI / Llama training crawler
# Meta-ExternalFetcher  → user-initiated Meta link fetches
# FacebookBot           → Facebook link previews and AI
# NOTE: Meta's crawlers do not always honor robots.txt.
# For enforcement you need CDN/WAF rules, not just this file.
# -------------------------------------------------------------
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /


# -------------------------------------------------------------
# AMAZON
# Alexa AI / Alexa for Shopping (formerly Rufus)
# -------------------------------------------------------------
User-agent: Amazonbot
Allow: /


# -------------------------------------------------------------
# BYTEDANCE / TIKTOK
# Trains ByteDance's AI (Doubao and TikTok recommendations).
# NOTE: Bytespider does not reliably honor robots.txt.
# For enforcement you need CDN/WAF rules.
# -------------------------------------------------------------
User-agent: Bytespider
Allow: /


# -------------------------------------------------------------
# DUCKDUCKGO AI ASSISTANT
# -------------------------------------------------------------
User-agent: DuckAssistBot
Allow: /


# -------------------------------------------------------------
# COMMON CRAWL
# Powers training data for most open-source LLMs (Llama, Mistral,
# and many others). Also historically used by GPT models.
# -------------------------------------------------------------
User-agent: CCBot
Allow: /


# =============================================================
# SITEMAP — REPLACE with your actual domain before uploading
# If you don't have a sitemap.xml yet, tools like xml-sitemaps.com
# can generate one from your site for free.
# =============================================================

Sitemap: https://localmediasolutions.net/sitemap.xml