# ==============================================================================
# 2026 AI DISCOVERY & AGENTIC GOVERNANCE ROBOTS.TXT
# Domain: futr.work (Modular AI Pods for the Future of Work)
# Policy: Open Ingestion for Answer Engines (AEO) and Autonomous Agents (AgEO)
# ==============================================================================

User-agent: *
Allow: /
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /agents.txt
Disallow: /api/checkout
Disallow: /admin/
Disallow: /private/

# 1. Standard Web Search Crawlers (Full Access)
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Google-InspectionTool
User-agent: GoogleOther
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: YandexBot
Allow: /

# 2. AI Direct Answer & Citation Engines (AEO - Full Access)
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-Web
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
Allow: /

# 3. Foundation Model Training & Dataset Crawlers (Full Ingestion Allowed for Brand Training)
User-agent: GPTBot
User-agent: CCBot
User-agent: Bytespider
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: FacebookBot
Allow: /

# 4. Sitemaps & Machine Directives
Sitemap: https://futr.work/sitemap.xml