# robots.txt for rfourenergy.com
# Last updated: 2026-08-26
#
# Structure: one default group, then one explicit group per AI crawler.
#
# A crawler that matches a named group ignores the "User-agent: *" group
# completely. The technical Disallow list is therefore repeated verbatim in
# every group. It is generated from a single list — do not edit one group by
# hand, regenerate the file.
#
# The AI crawlers are allowed deliberately: the published articles and tool
# documentation are the intended answer surface for retrieval systems, and the
# restricted paths below are excluded from every group equally.
#
# PENDING (do not remove this note): the legacy-path Disallow rules
# (/project-*, /portfolio/*, /case-study/*, /work/*, /references/*, /clients/*,
# /experience/*) were lifted on 2026-06-18 so that Google could recrawl those
# URLs and observe the 410 Gone response from the Pages middleware. They must be
# reinstated ONLY after deindexing is confirmed in Search Console. Reinstating
# them while the URLs are still indexed would block the recrawl and freeze the
# stale entries in place permanently.

# Default — applies to any crawler without its own group below
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /
# OpenAI — training and ChatGPT retrieval
User-agent: GPTBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# OpenAI — search index
User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# OpenAI — user-initiated page fetch
User-agent: ChatGPT-User
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Anthropic — crawler
User-agent: ClaudeBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Anthropic — user-initiated page fetch
User-agent: Claude-User
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Anthropic — search index
User-agent: Claude-SearchBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Perplexity — crawler
User-agent: PerplexityBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Perplexity — user-initiated page fetch
User-agent: Perplexity-User
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Google — Gemini grounding and AI Overviews
User-agent: Google-Extended
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Apple — AI training
User-agent: Applebot-Extended
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Microsoft — Bing index, feeds Copilot
User-agent: Bingbot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /

# Common Crawl — corpus used by many models
User-agent: CCBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /__static-content/
Disallow: /functions/
Allow: /
# Sitemap location
Sitemap: https://rfourenergy.com/sitemap.xml