User-agent: Googlebot
Allow: /
Allow: /shopping/
Allow: /shopping/ja
Allow: /shopping/us
Disallow: /admin/
Disallow: /workflow/
Disallow: /internal/
Disallow: /api/
Disallow: /shopping/ja/s
Disallow: /shopping/us/s

User-agent: Google-InspectionTool
Allow: /
Allow: /shopping/
Allow: /shopping/ja
Allow: /shopping/us
Disallow: /admin/
Disallow: /workflow/
Disallow: /internal/
Disallow: /api/
Disallow: /shopping/ja/s
Disallow: /shopping/us/s

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: KeenableBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: *
Allow: /
Allow: /shopping/
Allow: /shopping/ja
Allow: /shopping/us
Allow: /sitemap.xml
Allow: /sitemap/
Allow: /build/
Allow: /assets/

# Restricted admin and internal backend routes
Disallow: /admin/
Disallow: /workflow/
Disallow: /internal/
Disallow: /api/

# Outbound tracking and affiliate redirects
Disallow: /url*
Disallow: /ex/url*
Disallow: /shopping/url*
Disallow: /shopping/ex/url*
Disallow: /shopping/*/url*
Disallow: /shopping/*/ex/url*
Disallow: /shopping/*/pd/*/shops
Disallow: /*?ref=*
Disallow: /index.php/

# Internal search pages and queries (prevent aggressive bot crawling)
Disallow: /s$
Disallow: /s?
Disallow: /shopping/ja/s$
Disallow: /shopping/ja/s?
Disallow: /shopping/us/s$
Disallow: /shopping/us/s?
Disallow: /*?*keyword=*
Disallow: /*?*query=*

# XML Sitemaps
Sitemap: https://www.bestpeek.com/sitemap.xml