# workers-rights.com robots.txt
# Canonical URL: https://www.workers-rights.com
#
# AI search engines and LLM crawlers are stacked onto the same group as `*`
# so they share one rule set. Per the robots spec a crawler obeys only its
# most-specific matching group — earlier versions gave each AI bot its own
# `Allow: /`-only group, which silently dropped every Disallow below for
# those bots. Stacking the User-agent lines keeps parity permanently.
# Crawlers not named here (e.g. Bytespider, Meta-ExternalAgent) fall through
# to `*` and get the same rules.

User-agent: *
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: CCBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended

# Public content pages — allowed
Allow: /
Allow: /new-check
# /get-started now redirects to /new-check
Allow: /rights
Allow: /rights/
Allow: /learn
Allow: /find-attorney
Allow: /misclassified
Allow: /law-updates
# /whats-new carries a noindex meta tag as of 2026-08-14. It is intentionally
# NOT Disallowed — blocking the crawl would stop Google ever reading the
# noindex, which is how pages get stuck in the index as URL-only entries.
# It stays crawlable under the `Allow: /` above.
Allow: /rulings
Allow: /employers
Allow: /privacy
Allow: /terms
Allow: /disclaimer
Allow: /about

# MCP discovery catalog (advertised in llms.txt) — must stay fetchable
# despite the /api/ Disallow below. Longest-match wins: /api/mcp beats /api/.
Allow: /api/mcp

# Render resources — JS/CSS/optimized images MUST stay crawlable so a crawler
# can render pages (blocking them = render-blocked, pages look broken).
#
# There is deliberately NO "Disallow: /_next/" here. This group carried one
# from March 2026; the two Allow lines below were added as exceptions that
# June. That is correct for a crawler that applies longest-match precedence
# (Google, Bing),
# but a parser that applies any matching Disallow, or that ignores Allow, read
# it as "all of /_next/ is blocked" and could not fetch the CSS and JS needed
# to render a page. Nothing else lives under /_next/ on this site (the App
# Router has no /_next/data/ routes), so the broad rule protected nothing.
Allow: /_next/static/
Allow: /_next/image

# React Server Component prefetch payloads.
#
# Next.js App Router appends ?_rsc=<hash> when a <Link> prefetches, so EVERY
# internal link produces a parallel URL for each distinct route-tree hash. Any
# crawler that executes JS discovers them and the crawl explodes: two Screaming
# Frog runs on 2026-08-19 reached 601,836 URLs against a real corpus of 26,444
# sitemap URLs. Only 9,628 of 519,240 (1.75%) were HTML — the crawlers spent
# 2.5 hours almost entirely on prefetch payloads and never covered the site.
#
# Fetched WITHOUT the RSC request header the server cannot tell it is a prefetch
# and returns the full page: /rights and /rights?_rsc=... are byte-identical at
# 153,904 bytes. The duplication is self-correcting — that body carries
# rel=canonical to the clean URL — so this rule is about crawl budget, not a
# canonicalisation bug.
#
# Googlebot does not client-side navigate and so never fires these — zero _rsc
# URLs appear in Search Console. This rule is therefore preventive for Google
# and corrective for every other crawler. It cannot affect real users: browsers
# do not read robots.txt, so client-side navigation is untouched.
Disallow: /*?_rsc=
Disallow: /*&_rsc=

# API — not crawlable (except /api/mcp above)
Disallow: /api/

# Correction-request links: 38K parameterized variants (one per employer/
# attorney) are linked from each profile page. They're noindex+canonical,
# but Google still has to crawl every one to learn that — ~25% of crawl
# budget on pages that should never be indexed. Disallow stops the crawl
# entirely. The bare /corrections page stays crawlable.
Disallow: /corrections?

# Auth-gated app pages — do not crawl
Disallow: /dashboard
Disallow: /my-timeline
Disallow: /attorney-packet
Disallow: /case-timeline
Disallow: /timeline
Disallow: /evidence
Disallow: /journal
Disallow: /mitigation-log
Disallow: /get-help
Disallow: /deadlines
Disallow: /documents
Disallow: /letters
Disallow: /career
Disallow: /case/
Disallow: /settings
Disallow: /activity
Disallow: /login
Disallow: /eeoc-center
Disallow: /where-things-stand
Disallow: /tell-your-story
Disallow: /investigation
Disallow: /people
Disallow: /response
Disallow: /build-case
Disallow: /research

Sitemap: https://www.workers-rights.com/sitemap-index.xml