# robots.txt for https://aibeginner.net/
# Last updated: 2026-07-23
# Purpose: Allow full crawl of public content; restrict admin/private and internal search.
# Note: Per RFC 9309, a crawler obeys only the FIRST matching user-agent group and does
#       NOT inherit rules from the wildcard group, so each named AI crawler group below
#       repeats the full Allow/Disallow set intentionally.

User-agent: *
Allow: /

# Priority pages — core
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq

# AI Career Confidence hub and pillar cluster (added 2026-06-11)
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work

# Disallow administrative or private directories
Disallow: /admin/
Disallow: /private/

# Block internal search results (low-value / thin)
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# ─────────────────────────────────────────────────────────────
# Named AI crawler groups (added 2026-07-22).
# Each group repeats the full rule set — named groups do not
# inherit from User-agent: * (RFC 9309).
# ─────────────────────────────────────────────────────────────

# OpenAI (ChatGPT training/search crawler)
User-agent: GPTBot
Allow: /
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work
Disallow: /admin/
Disallow: /private/
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# Anthropic (Claude crawler)
User-agent: ClaudeBot
Allow: /
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work
Disallow: /admin/
Disallow: /private/
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# Anthropic (legacy user-agent token)
User-agent: anthropic-ai
Allow: /
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work
Disallow: /admin/
Disallow: /private/
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# Google (Gemini / Vertex AI training control)
User-agent: Google-Extended
Allow: /
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work
Disallow: /admin/
Disallow: /private/
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# Perplexity
User-agent: PerplexityBot
Allow: /
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work
Disallow: /admin/
Disallow: /private/
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# Common Crawl (feeds many downstream LLM datasets)
User-agent: CCBot
Allow: /
Allow: /start
Allow: /academy
Allow: /free-ai-resources
Allow: /faq
Allow: /ai-career-confidence
Allow: /will-ai-replace-entry-level-jobs
Allow: /ai-skills-for-resume
Allow: /ai-interview-questions-entry-level
Allow: /best-ai-tools-to-learn-for-work
Disallow: /admin/
Disallow: /private/
Disallow: /search
Disallow: /search/
Disallow: /*?q=

# Sitemap
Sitemap: https://aibeginner.net/sitemap.xml