# wallstreetexplained.com
#
# One group per user agent, protective rules first, sitemap last.
#
# The previous version declared "User-agent: *" twice and left the protective
# disallows in the second group, after the Sitemap line. Compliant parsers
# merge groups, so Google honoured them, but less careful crawlers read only
# the first group, which said nothing but "Allow: /". Those disallows are not
# optional, so they now sit in the group that is always read.
#
# POLICY ON AI SYSTEMS
#
# Retrieval yes, training no. These are different things and the distinction is
# deliberate.
#
#   ai-input=yes   A model may read this site to ground an answer and cite it.
#                  Citation is a distribution channel for research of this kind,
#                  and refusing it costs readers rather than protecting them.
#   ai-train=no    The content may not be used to train or fine tune a model.
#   search=yes     Ordinary search indexing, unchanged.
#
# The crawler groups below allow retrieval. The Content-Signal line above them
# reserves the training right. A crawler that honours one and not the other is
# choosing to, and the reservation is expressed either way.
#
# If the live file at wallstreetexplained.com/robots.txt does not match this one,
# Cloudflare is prepending a managed block, which overrides everything here.
# That is a zone setting and cannot be changed by deploying this file.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /sftp-login.md
Disallow: /youtube-setup.md
Disallow: /v2/
Disallow: /v3/
Allow: /

# Answer engines and their retrieval agents, allowed explicitly so that intent
# is recorded in the file rather than inferred from the absence of a rule.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

# Scrapers with no retrieval or citation benefit to this site.
User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://wallstreetexplained.com/sitemap.xml