# robots.txt for celestetic.com # Declares crawl policy for classic search and AI crawlers. # See https://contentsignals.org/ and https://www.rfc-editor.org/rfc/rfc9309 # --- AI training crawlers: blocked --- User-agent: GPTBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: anthropic-ai Disallow: / User-agent: ClaudeBot Disallow: / User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Omgilibot Disallow: / User-agent: FacebookBot Disallow: / # --- AI answer / search crawlers: allowed (excluding admin) --- User-agent: OAI-SearchBot Allow: / Disallow: /my-admin/ Allow: /template/assets/ Disallow: /template/ Disallow: /files/cache/ Disallow: /sw-cache/ Disallow: /sw-db/ Disallow: /sw-logs/ Disallow: /vendor/ User-agent: Claude-Web Allow: / Disallow: /my-admin/ Allow: /template/assets/ Disallow: /template/ Disallow: /files/cache/ Disallow: /sw-cache/ Disallow: /sw-db/ Disallow: /sw-logs/ Disallow: /vendor/ User-agent: PerplexityBot Allow: / Disallow: /my-admin/ Allow: /template/assets/ Disallow: /template/ Disallow: /files/cache/ Disallow: /sw-cache/ Disallow: /sw-db/ Disallow: /sw-logs/ Disallow: /vendor/ # --- Default (classic search engines, etc.) --- User-agent: * Disallow: /my-admin/ Allow: /template/assets/ Disallow: /template/ Disallow: /files/cache/ Disallow: /sw-cache/ Disallow: /sw-db/ Disallow: /sw-logs/ Disallow: /vendor/ Disallow: /qr/ Disallow: /404 Disallow: /cdn-cgi/ Disallow: /search/ Disallow: /*?*utm_ Disallow: /*?*session Disallow: /*?*orderby= Disallow: /*?*filter= Disallow: /*?*sort= Disallow: /*?*cb= Disallow: /*?*cachebust= Disallow: /*?*fbclid= Disallow: /*?*gclid= # Content Signals (https://contentsignals.org/) # search=yes : classic search indexing allowed # ai-train=no : content not allowed for AI model training # ai-input=yes: content may be used as input for AI answers (RAG / grounded responses) Content-Signal: search=yes, ai-train=no, ai-input=yes Sitemap: https://celestetic.com/sitemap.xml