# Robots.txt for Virtual Consultant
# Optimised for public SEO content while reducing crawl waste from private and tracking URLs

User-agent: *
Allow: /
Crawl-delay: 5

# --- Private SaaS areas ---
Disallow: /dashboard
Disallow: /patients
Disallow: /companies
Disallow: /practices
Disallow: /manage
Disallow: /admin

# --- Auth & account routes ---
Disallow: /account/
Allow: /account/register
Disallow: /checkout
Disallow: /quotes
Disallow: /consultations
Allow: /evaluations/quote
Disallow: /evaluations/quote/
Disallow: /evaluations
Disallow: /support-hub

# --- Sensitive or internal routes ---
Disallow: /files
Disallow: /api
Disallow: /postcodes

# --- Framework / system routes ---
Disallow: /_framework
Disallow: /_blazor

# --- PWA / runtime files ---
Disallow: /service-worker.js
Disallow: /manifest.json

# --- Query-parameterised pages that render noindex (save crawl budget) ---
Disallow: /blog?*
Disallow: /find?*

# --- Tracking / marketing parameters ---
Disallow: /*utm_
Disallow: /*referrer_url=
Disallow: /*landing_page=
Disallow: /*channel=
Disallow: /*source_platform=
Disallow: /*campaign_id=
Disallow: /*first_visit_date=
Disallow: /*conversion_date=

# --- Named AI crawlers — explicitly allowed for full public content access ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: DuckAssistant
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

# Sitemap
Sitemap: https://virtualconsultant.online/sitemap.xml

# --- AI content usage preferences (Content Signals spec) ---
Content-Signal: ai-train=no, search=yes, ai-input=no

# LLM-readable source maps (informational — not valid robots.txt directives, listed as comments)
# llms.txt: https://virtualconsultant.online/llms.txt
# llms-full.txt: https://virtualconsultant.online/llms-full.txt
# ai-sources: https://virtualconsultant.online/ai-sources.json
# orthodontic-faqs: https://virtualconsultant.online/orthodontic-faqs
# citation-metadata: https://virtualconsultant.online/citation.json
# ai-plugin: https://virtualconsultant.online/.well-known/ai-plugin.json
# mcp-server-card: https://virtualconsultant.online/.well-known/mcp/server-card.json
# agent-card: https://virtualconsultant.online/.well-known/agent-card.json
# agent-skills: https://virtualconsultant.online/.well-known/agent-skills/index.json
# api-catalog: https://virtualconsultant.online/.well-known/api-catalog