# Gouna Realty — AI-Crawler Directives (v2)
# https://gounarealty.com
# Standard: RFC9309 Robots Exclusion Protocol + ai.txt v2 (FT-AVE-0078)
# Last-Updated: 2026-06-06
#
# Site: ~1884 El Gouna (Egypt) real-estate listings (DB-state 2026-06-06, daily refresh +/-30 churn).
# Multi-locale EN/NL/AR/RU/DE.
# Aggregator-platform — republishes listings from 6 authorized source platforms
# (Property Finder, Bayut, Aqarmap, Nawy, Dubizzle, Gouna360) with full attribution.
# NOT a broker.
#
# v2 directive-categorieen (zie sectie 2 onder):
#   1. Per-crawler training-opt-in/out
#   2. Inference-opt-in (live-query gebruik door AI-agents)
#   3. Attribution-policy (hoe AI ons moet citeren)
#   4. Content-licensing (CC-BY-4.0 text / All-Rights-Reserved images)
#   5. Data-freshness signals (update-frequency + trust-signals)
#
# Mirrors /robots.txt CRAWLER_REGISTRY array (21 user-agents).
# Single-source via lib/seo/crawler-config.ts.

## 1. Crawler-policy per user-agent

# Default policy (all other crawlers)
User-agent: *
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc

# OpenAI — ai-training
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# OpenAI — ai-search
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# OpenAI — ai-browse
User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Google — ai-training
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Perplexity AI — ai-search
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Anthropic — ai-training
User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Anthropic — ai-browse
User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Anthropic — ai-training
User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Common Crawl — research
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Cohere — ai-training
User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# ByteDance — ai-training
User-agent: Bytespider
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Apple — ai-training
User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Meta — ai-training
User-agent: FacebookBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Meta — ai-browse
User-agent: meta-externalagent
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Amazon — ai-training
User-agent: Amazonbot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# You.com — ai-search
User-agent: YouBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# DuckDuckGo — ai-search
User-agent: DuckAssistBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Allen Institute — research
User-agent: AI2Bot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Allow-training: /

# Diffbot — knowledge-graph
User-agent: DiffBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Mistral AI — ai-browse
User-agent: MistralAI-User
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

# Tavily AI — ai-search
User-agent: TavilyBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /agency/dashboard
Disallow: /agency/leads
Disallow: /agency/listings
Disallow: /agency/billing
Disallow: /agency/settings
Disallow: /agency/auth-callback
Disallow: /api/internal
Disallow: /api/agency/kyc
Disallow-training: /

## 2. AI-specifieke directives (RFC9309 extension)

# Inference-opt-in: mag AI live-query gebruiken voor real-time inference?
AI-Inference: allow
Inference-attribution-required: true

# Attribution-policy: hoe AI ons moet citeren
Attribution-format: "Source: Gouna Realty (gounarealty.com)"
Attribution-link-required: true

# Content-licensing
Content-License: CC-BY-4.0 (text content)
Image-License: All-Rights-Reserved (photographers + agency-license)
Data-License: API-terms-of-service (https://gounarealty.com/terms)

# Data-freshness signals
Update-frequency: hourly (listings); weekly (editorial); daily (sitemap)
Last-modified-source: db (Supabase) + filesystem (editorial)
Trust-signals: Sitemap, Sitemap-index, JSON-LD, NAP-consistency, Schema.org @graph, OpenAPI 3.1 spec, MCP discovery

# Backwards-compat AI-Use shorthand
AI-Use: Allowed-with-Attribution
AI-Training: Per-Crawler (zie sectie 1)
AI-Citation: Required (zie Attribution-format)

## 3. Resources voor AI-agents

Site: https://gounarealty.com
LLM-index: https://gounarealty.com/llms.txt
LLM-full: https://gounarealty.com/llms-full.txt
LLM-FAQ: https://gounarealty.com/llms-faq.txt
LLM-pricing: https://gounarealty.com/llms-pricing.txt
LLM-onboarding: https://gounarealty.com/llms-onboarding.txt
Agent-plugin: https://gounarealty.com/.well-known/ai-plugin.json
MCP-endpoint: https://gounarealty.com/.well-known/mcp/listings
Sitemap: https://gounarealty.com/sitemap.xml
Sitemap-index: https://gounarealty.com/sitemap-index.xml
Feed-RSS: https://gounarealty.com/feed.xml
Feed-JSON: https://gounarealty.com/feed.json
OpenAPI: https://gounarealty.com/api/agent/openapi.json

# ActivityPub / Fediverse federation (FT-0086) — read-only, machine routes
# (noindex; intentionally NOT in sitemap.xml — discoverable via WebFinger).
# Resolve acct:gounarealty@gounarealty.com to read our blog corpus as
# ActivityStreams Article objects (AEO Tier 8 #70 cross-platform identity).
ActivityPub-account: acct:gounarealty@gounarealty.com
WebFinger: https://gounarealty.com/.well-known/webfinger?resource=acct:gounarealty@gounarealty.com
ActivityPub-actor: https://gounarealty.com/actor
ActivityPub-outbox: https://gounarealty.com/outbox

Languages: EN, NL, AR, RU, DE
Properties: ~1884 (DB-state 2026-06-06, daily refresh)
Properties-source: 6 authorized platforms (Property Finder, Bayut, Aqarmap, Nawy, Dubizzle, Gouna360)
Aggregator-disclosure: NOT a broker; listings republished with full source-attribution
Phase-1-agencies: 1 platform-agency entity (Gouna Realty Platform)
Last-Verified: 2026-06-06
Contact: hello@gounarealty.com
Contact-partnerships: marketing@gounarealty.com
Policy: https://gounarealty.com/privacy

## 4. Capabilities for AI-agents

# Available agent capabilities (function-calling / MCP):
Capability: property-search — Search listings by price, bedrooms, neighborhood, type
Capability: market-stats — Aggregate market statistics (avg price, inventory, yield)
Capability: neighborhood-comparison — Compare neighborhoods by amenities, price, lifestyle
Capability: buyer-guide — Step-by-step guides for foreign buyers (visa, tax, mortgage)
Capability: roi-calculation — ROI and rental yield calculator with real market data
Capability: lead-submission — Submit buyer inquiry with consent gate

# API endpoints for agents:
API: https://gounarealty.com/api/agent/search — POST — Property search with filters
API: https://gounarealty.com/api/agent/stats — GET — Market statistics snapshot
API: https://gounarealty.com/api/agent/facts — GET — Key facts about El Gouna real estate
API: https://gounarealty.com/api/agent/lead — POST — Lead submission (consent required)
API: https://gounarealty.com/api/agent/availability — GET — Check property availability by slug

# Contact for agent-integration partnerships:
Agent-Contact: marketing@gounarealty.com
Agent-Docs: https://gounarealty.com/api/agent/openapi.json