# PinCatch.com robots.txt
# Public pages and all supported language routes are crawlable.

# Standard search engines and other crawlers
User-agent: *
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# OpenAI search, user-requested access, and model crawler
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Anthropic search, user-requested access, and model crawler
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Perplexity search and user-requested access
User-agent: PerplexityBot
User-agent: Perplexity-User
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Google AI and non-search product controls
User-agent: Google-Extended
User-agent: GoogleOther
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Apple search and generative AI controls
User-agent: Applebot
User-agent: Applebot-Extended
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Amazon search, user access, and AI products
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Common Crawl
User-agent: CCBot
Disallow: /admin/
Disallow: /rosetta/
Disallow: /ckeditor/
Disallow: /i18n/
Disallow: /pin/
Disallow: /media/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# LLM-friendly content files (llmstxt.org specification)
# https://pincatch.com/llms.txt      - Curated site overview for AI systems
# https://pincatch.com/llms-full.txt - Extended content for AI ingestion

# Primary XML sitemap
Sitemap: https://pincatch.com/sitemap.xml