# robots.txt for https://www.startengine.com
# Generated by generate-sitemap.mjs
# Last updated: 2026-02-03T00:22:09.190Z

# ===========================================
# Default rules for all crawlers
# ===========================================
User-Agent: *
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /

# ===========================================
# AI/LLM Bot Rules
# Explicitly allowing AI bots to crawl content
# while respecting blocked paths
# ===========================================

User-Agent: GPTBot
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: ChatGPT-User
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: Google-Extended
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: Claude-Web
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: anthropic-ai
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: Applebot-Extended
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: CCBot
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: PerplexityBot
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: Bytespider
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: Amazonbot
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: FacebookBot
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /


User-Agent: cohere-ai
Disallow: /admin
Disallow: /api
Disallow: /account
Allow: /

# ===========================================
# Sitemap location
# ===========================================
Sitemap: https://www.startengine.com/sitemap.xml