# robots.txt for ScaleFirst — https://scalefirst.org
#
# Site:      ScaleFirst — independent technical publication
# Publisher: Selwyn Theogarajan Natarajan, United States
# Content:   Original long-form engineering articles. Text and diagrams only;
#            no downloads, no user-generated content, no advertising.
# Category:  Computer & Internet Info (a.k.a. Information Technology /
#            Computers and Internet / Technology)
# Contact:   hello@scalefirst.org — https://scalefirst.org/contact/
# Security:  https://scalefirst.org/.well-known/security.txt
#
# Policy: all content is open to search engines AND to AI answer engines
# and model crawlers. The posts are meant to be cited, summarised and
# surfaced in AI answers, so nothing below is opted out.

User-agent: *
Allow: /
Allow: /.well-known/
Disallow: /admin/
Disallow: /api/

# ---- Search engines -------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ---- AI answer engines (live retrieval / citation) -------------------
# These fetch pages to answer a user's question and link back.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# ---- AI model crawlers (indexing / training corpora) -----------------
# Opt-in: these tokens are how Google, OpenAI, Anthropic, Meta and Apple
# gate whether a site may be used for model grounding and training.
User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

# ---- Machine-readable indexes ---------------------------------------
# llms.txt      — post index for LLM consumers
# llms-full.txt — full text of every post in one file
# humans.txt    — who runs the site and what it is built with
# security.txt  — RFC 9116 security contact
Sitemap: https://scalefirst.org/sitemap.xml