# AI-First Robots.txt Configuration
# Domain: www.tryaivo.com
# Last Updated: 2025-10-03
# Strategy: AI Contract Implementation
# Auto-Generated Sitemap References: 2025-10-03T23:49:46.815Z
# Auto-Generated AI Endpoint References: 2025-10-03T23:49:46.815Z

# === PRIMARY AI CRAWLERS (ALLOW WITH CONDITIONS) ===
# CRITICAL: OAI-SearchBot for ChatGPT search results (HIGHEST PRIORITY)
User-agent: OAI-SearchBot
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 1
# AI Contract: Search results in ChatGPT - CRITICAL for discovery
# IP Ranges: https://openai.com/searchbot.json

User-agent: GPTBot
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 2
# AI Contract: See /llms.txt for usage terms
# IP Ranges: https://openai.com/gptbot.json

User-agent: ClaudeBot  
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 2
# AI Contract: Attribution required

User-agent: PerplexityBot
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 3
# AI Contract: Commercial usage restricted

User-agent: Google-Extended
Allow: /
Disallow: /private/
Crawl-delay: 1
# AI Contract: Training data excluded per /llms.txt

User-agent: BingBot
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 1
# AI Contract: Enhanced for Copilot

User-agent: Claude-Web
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 2

User-agent: Anthropic-AI
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Allow: /api/ai-endpoints/
Crawl-delay: 2
# IP Ranges: https://openai.com/chatgpt-user.json

User-agent: Bard-Bot
Allow: /
Crawl-delay: 1

User-agent: GeminiBot
Allow: /
Crawl-delay: 1

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 3

# === TRADITIONAL SEARCH ENGINES ===
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /api/internal/

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /private/

# === CONTENT PROTECTION ===
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /analytics/
Disallow: /internal/
Disallow: /staging/
Disallow: /dev/
Disallow: /backup/
Disallow: /temp/
Disallow: /cache/
Disallow: /logs/
Disallow: /_next/static/
Disallow: /api/internal/

# === AI-SPECIFIC ENDPOINTS ===
# AI contract and licensing
Allow: /api/llms
Allow: /.well-known/ai-contracts
Allow: /ai-license.txt
# AI discovery and crawler guidance
Allow: /api/ai-endpoints
Allow: /api/robots
# AI-optimized content
Allow: /ai-content

# === PERFORMANCE OPTIMIZATION ===
Crawl-delay: 1

# === AI CONTRACT REFERENCE ===
# AI Usage Terms: See /llms.txt
# Content License: /ai-license.txt
# Structured Data: /api/ai-endpoints/
# AI Contracts: /.well-known/ai-contracts

# === SITEMAPS ===
# Main site sitemap
Sitemap: https://www.tryaivo.com/sitemap.xml
# AI-optimized sitemap
Sitemap: https://www.tryaivo.com/ai-sitemap.xml
# Blog content sitemap
Sitemap: https://www.tryaivo.com/blog/sitemap.xml

# === CONTACT ===
# Organization: AI Visibility Agency (AIVO)
# Technical Contact: team@tryaivo.com
# AI Partnership: team@tryaivo.com
# Priority Response: team@tryaivo.com