# Mediora.AI — robots.txt
# Last updated: 2026-10-13
#
# Mediora.AI is a medical-explanation service: it turns a lab report into a
# plain-language reading of every marker, in English, Russian or Hebrew.
# Patient data, doctor cabinets and the back-office API are off-limits to all
# crawlers (they require authentication anyway). The public marketing,
# article and glossary surfaces are open to everyone.
#
# AI search engines are EXPLICITLY welcomed. Content is written to be citable:
# every page carries a canonical URL, hreflang for all three languages,
# Schema.org types, and the whole corpus is also served as plain text — see
# /llms.txt, /llms-full.txt and /.well-known/ai-plugin.json for structured
# discovery hints, plus a live MCP server at https://mcp.mediora.ai/mcp.
#
# On physician review, stated exactly: a page emits Schema.org `reviewedBy`
# and `lastReviewed` ONLY after a physician has signed off on that page.
# As of 2026-10-13 no page carries either property, and nothing in this file
# should be read as implying it does. Read the page's own markup, not this
# comment, to know whether a given article has been reviewed.
#
# (This file is served verbatim, so these comments reach crawlers too. The
# history of what this paragraph used to say is in the commit log, not here:
# quoting a withdrawn claim inside a public file re-publishes it.)
#
# IMPORTANT (2026-07-01 audit): per the robots spec a crawler obeys ONLY
# its most-specific matching User-agent group and ignores `*`. So every
# named bot MUST carry the full Disallow set itself, or it would inherit
# nothing and crawl the private routes. The named AI/search bots are
# stacked under one shared group below (consecutive User-agent lines form
# a single group), so the Disallow list is maintained in just two places:
# the `*` group and the shared named-bot group.

# ─── Default group (humans, legacy + unknown crawlers) ─────────────────
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /en/admin
Disallow: /ru/admin
Disallow: /he/admin
Disallow: /doctor/cabinet
Disallow: /en/doctor/cabinet
Disallow: /ru/doctor/cabinet
Disallow: /he/doctor/cabinet
Disallow: /en/doctor/articles
Disallow: /ru/doctor/articles
Disallow: /he/doctor/articles
Disallow: /en/doctor/assignments
Disallow: /ru/doctor/assignments
Disallow: /he/doctor/assignments
Disallow: /en/doctor/marketplace
Disallow: /ru/doctor/marketplace
Disallow: /he/doctor/marketplace
Disallow: /en/doctor/earnings
Disallow: /ru/doctor/earnings
Disallow: /he/doctor/earnings
Disallow: /en/doctor/availability
Disallow: /ru/doctor/availability
Disallow: /he/doctor/availability
Disallow: /en/report
Disallow: /ru/report
Disallow: /he/report
Disallow: /en/history
Disallow: /ru/history
Disallow: /he/history
Disallow: /en/history-v2
Disallow: /ru/history-v2
Disallow: /he/history-v2
Disallow: /en/dashboard
Disallow: /ru/dashboard
Disallow: /he/dashboard
Disallow: /en/profile
Disallow: /ru/profile
Disallow: /he/profile
Disallow: /api/
# Public catalog feeds, linked from /llms.txt (2026-10-29). Longest match wins, so
# these four stay open while the rest of /api/ stays closed.
Allow: /api/markers
Allow: /api/conditions
Allow: /api/symptoms
Allow: /api/panels
Disallow: /_blazor
Disallow: /_framework
Disallow: /_content

# ─── AI + search crawlers: EXPLICITLY welcomed to public content ───────
# Stacked User-agents share one rule block. Same Disallow set as `*` so
# these bots (which ignore `*`) still can't crawl the private routes.
# OpenAI:
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic:
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Claude-SearchBot
User-agent: Claude-User
# Perplexity:
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google AI / Gemini:
User-agent: Google-Extended
User-agent: GoogleOther
# Meta / Apple / Mistral:
User-agent: Meta-ExternalAgent
User-agent: Applebot-Extended
User-agent: Applebot
# Common Crawl (a source for many model corpora) and Amazon:
User-agent: CCBot
User-agent: Amazonbot
User-agent: MistralAI-User
# Bytedance / Cohere / You / DuckAssist:
User-agent: Bytespider
User-agent: cohere-ai
User-agent: YouBot
User-agent: DuckAssistBot
# Classic indexing bots:
User-agent: Googlebot
User-agent: Bingbot
User-agent: YandexBot
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /en/admin
Disallow: /ru/admin
Disallow: /he/admin
Disallow: /doctor/cabinet
Disallow: /en/doctor/cabinet
Disallow: /ru/doctor/cabinet
Disallow: /he/doctor/cabinet
Disallow: /en/doctor/articles
Disallow: /ru/doctor/articles
Disallow: /he/doctor/articles
Disallow: /en/doctor/assignments
Disallow: /ru/doctor/assignments
Disallow: /he/doctor/assignments
Disallow: /en/doctor/marketplace
Disallow: /ru/doctor/marketplace
Disallow: /he/doctor/marketplace
Disallow: /en/doctor/earnings
Disallow: /ru/doctor/earnings
Disallow: /he/doctor/earnings
Disallow: /en/doctor/availability
Disallow: /ru/doctor/availability
Disallow: /he/doctor/availability
Disallow: /en/report
Disallow: /ru/report
Disallow: /he/report
Disallow: /en/history
Disallow: /ru/history
Disallow: /he/history
Disallow: /en/history-v2
Disallow: /ru/history-v2
Disallow: /he/history-v2
Disallow: /en/dashboard
Disallow: /ru/dashboard
Disallow: /he/dashboard
Disallow: /en/profile
Disallow: /ru/profile
Disallow: /he/profile
Disallow: /api/
# Public catalog feeds, linked from /llms.txt (2026-10-29). Longest match wins, so
# these four stay open while the rest of /api/ stays closed.
Allow: /api/markers
Allow: /api/conditions
Allow: /api/symptoms
Allow: /api/panels
Disallow: /_blazor
Disallow: /_framework
Disallow: /_content

# ─── Discovery hints ──────────────────────────────────────────────────
# /sitemap.xml is a sitemap INDEX (since 2026-08-25), not a list of URLs. It lists
# one sitemap per kind of page: core / catalog / landings / articles / doctors, so
# that coverage is reported for each kind separately. The child sitemaps are not
# repeated here; crawlers take them from the index.
Sitemap: https://www.mediora.ai/sitemap.xml
# Student section (L4) kept as its own sitemap: it is AI-generated educational
# copy with no physician review, kept apart from the patient-facing catalogue and articles.
# Separate files mean Search Console reports coverage for each independently.
Sitemap: https://www.mediora.ai/sitemap-students.xml

# Markdown firehose for AI crawlers (full article corpus, plain text).
# Equivalent of the HTML pages but stripped of layout / scripts / ads.
# See https://llmstxt.org for the convention.
# Mediora.AI llms.txt:    https://www.mediora.ai/llms.txt
# Mediora.AI llms-full:   https://www.mediora.ai/llms-full.txt