# MyFreud — https://www.myfreud.com
#
# Search engines: everything is open.
User-agent: *
Allow: /

Disallow: /preview/
Disallow: /drafts/

# Search Console showed both of these being crawled on 4 August 2026, and
# neither can ever be indexed, so every fetch is crawl budget taken from a page
# that could be.
#
# /search/ already renders `noindex, nofollow`. It is disallowed as well because
# the WebSite JSON-LD advertises a SearchAction whose urlTemplate ends
# `?q={query}`, and Googlebot was fetching that literal placeholder as a URL.
# A search results page is thin, duplicative and not meant to be indexed, so
# there is nothing to lose by keeping crawlers out of it entirely.
Disallow: /search

# Astro's on-demand image endpoint. It exists only in server-rendered output;
# this site is pure static, so the path 404s. Nothing in the built site
# references it.
Disallow: /_image

# ---------------------------------------------------------------------------
# AI crawlers
#
# These were already allowed by the wildcard above. This section states the
# decision rather than leaving it to a default nobody chose. Behaviour is
# unchanged.
#
# The policy is ALLOW, deliberately. Being quotable in AI answers is how a small
# publisher gets discovered now, and every article carries an author, a
# published date and real citations precisely so an answer engine can attribute
# it. Blocking these agents forfeits that and does nothing to stop anyone who
# ignores robots.txt.
#
# This is a business decision, not a technical one. To reverse it, change the
# relevant `Allow: /` below to `Disallow: /`.
#
# Note the distinctions, because they are easy to get wrong:
#   - Google-Extended governs Gemini training and grounding ONLY. It does not
#     affect Googlebot, Search ranking, or AI Overviews eligibility.
#   - GPTBot is OpenAI's training crawler. OAI-SearchBot and ChatGPT-User serve
#     ChatGPT browsing and citation, so blocking GPTBot alone still permits
#     citation.
#   - CCBot is Common Crawl, which many downstream models train from.

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

# The index is the URL Search Console has on file; the three children stay
# listed so nothing depends on a crawler resolving the index first.
Sitemap: https://www.myfreud.com/sitemap.xml
Sitemap: https://www.myfreud.com/sitemap-myfreud.xml
Sitemap: https://www.myfreud.com/sitemap-news.xml
Sitemap: https://www.myfreud.com/sitemap-images.xml