# =====================================================
# robots.txt — The Maths Society (mathsociety.org.uk)
# Strategy: Allow AI Search bots (citations/traffic)
#            Block AI Training bots (scraping only)
# =====================================================

# --- DEFAULT RULE: Allow all standard crawlers ---
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /privatewhatsapp
Disallow: /admin/whatsapp
Disallow: /survey/
Disallow: /confirmation/
Disallow: /tuition-thank-you
Disallow: /tuitionregistration
Disallow: /codingregistration
Disallow: /homework
Disallow: /review/
Disallow: /portal/
Disallow: /portal
Disallow: /auth/
Disallow: /reset-password
Disallow: /verify-email
Disallow: /parent-access/
Disallow: /signup
Disallow: /pupilsignup
Disallow: /login
Disallow: /magic-link
Disallow: /forgot-password
Disallow: /teacher
Disallow: /teacher/

# --- 1. ALLOW: AI Search & Citation Bots ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# --- 2. BLOCK: AI Model Training Bots ---
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

# --- 3. SITEMAP ---
Sitemap: https://www.mathsociety.org.uk/sitemap.xml