User-agent: *
Allow: /
Disallow: /admin
Disallow: /seo/
Disallow: /db/
Disallow: /nodes/

# Raw data files back the site UI; crawler-readable page summaries are served
# on the HTML pages themselves. /data/ stays open for rendering search engines.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: PetalBot
Disallow: /admin
Disallow: /seo/
Disallow: /db/
Disallow: /nodes/
Disallow: /data/

Sitemap: https://theaggregate.ai/sitemap.xml