# ============================================ # 911uk.com robots.txt # Policy: search engines and user-triggered AI # fetchers allowed; AI training crawlers blocked # ============================================ # --- AI: search and user-triggered fetchers allowed --- User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-User Allow: / # --- AI: training and bulk-scrape crawlers blocked --- User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Google-Extended Disallow: / User-agent: PerplexityBot Disallow: / User-agent: Perplexity-User Disallow: / User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: Meta-ExternalAgent Disallow: / User-agent: cohere-ai Disallow: / User-agent: Diffbot Disallow: / User-agent: Amazonbot Disallow: / User-agent: Omgilibot Disallow: / User-agent: webzio-extended Disallow: / # xAI / Grok: declared tokens only. # Grok traffic may not reliably identify itself, # so enforce via Cloudflare/WAF as well. User-agent: GrokBot Disallow: / User-agent: xAI-Grok Disallow: / User-agent: Grok-DeepSearch Disallow: / # --- Everyone else, including Googlebot and bingbot --- User-agent: * # Cloudflare Disallow: /cdn-cgi/ # XenForo: no-value and crawl-wasting paths Disallow: /admin.php Disallow: /account/ Disallow: /login/ Disallow: /register/ Disallow: /goto/ Disallow: /misc/ Disallow: /search/ Disallow: /whats-new/ # WordPress integration (mounted at /porsche-news) Disallow: /porsche-news/wp-admin/ Disallow: /porsche-news/wp-login.php Allow: /porsche-news/wp-admin/admin-ajax.php # --- Sitemaps --- Sitemap: https://911uk.com/sitemap.xml # Uncomment at WordPress launch; confirm path per SEO # plugin (Yoast/Rank Math: sitemap_index.xml, native # WP: wp-sitemap.xml) # Sitemap: https://911uk.com/porsche-news/sitemap_index.xml