# robots.txt for https://creabyte.com
# Generated from the site facts file (crawler policy per purpose, private paths).
# The Disallow lines are a hint only: those paths are protected on the server.
# A named group replaces the * group for that crawler, so every group repeats the Disallow lines.
# Agent entry points: /llms.txt, /agents.md

User-agent: *
Allow: /
Disallow: /deploy.php
Disallow: /csrf.php
Disallow: /contact.php
Disallow: /teams.php
Disallow: /storage/
Disallow: /config.php
Disallow: /config.local.php
Disallow: /deploy.local.php

# AI search crawlers: build search indexes, cite and link (allowed)
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Allow: /
Disallow: /deploy.php
Disallow: /csrf.php
Disallow: /contact.php
Disallow: /teams.php
Disallow: /storage/
Disallow: /config.php
Disallow: /config.local.php
Disallow: /deploy.local.php

# User-triggered AI fetches: an assistant loads a page because a person asked (allowed)
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Allow: /
Disallow: /deploy.php
Disallow: /csrf.php
Disallow: /contact.php
Disallow: /teams.php
Disallow: /storage/
Disallow: /config.php
Disallow: /config.local.php
Disallow: /deploy.local.php

# AI training crawlers: collect content for model training (allowed)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
Allow: /
Disallow: /deploy.php
Disallow: /csrf.php
Disallow: /contact.php
Disallow: /teams.php
Disallow: /storage/
Disallow: /config.php
Disallow: /config.local.php
Disallow: /deploy.local.php

Sitemap: https://creabyte.com/sitemap.xml