# minemarket.ai
# Content signals (carried over from the live site's Cloudflare-managed robots):
# search=yes, ai-train=no, use=reference.

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /
# Auth + account areas are deindexed via an X-Robots-Tag: noindex header
# (vercel.json). Do NOT also Disallow them here: robots-blocked pages can
# still be indexed from external links, and Google then can't see the
# noindex (GSC "Indexed, though blocked by robots.txt").
# API endpoints have no SEO value
Disallow: /api/
# Fixes GSC "alternate page with proper canonical tag" warning (from live robots)
Disallow: /*?search={search_term_string}

# AI-training crawler blocks (mirrored from the live site's robots; on the old
# stack Cloudflare injected these — Vercel has no equivalent, so they live here)
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://minemarket.ai/sitemap.xml