# +------------------------------------------------------------------------------+
# |  JP Roofing                                                                  |
# |  jproofingandmetalbuildings.com                                              |
# |  Family & Locally Owned, Proudly Serving the Pittsburgh Area for 43 Years    |
# +------------------------------------------------------------------------------+
 
# robots.txt — JP Roofing  
# Last updated: May 2026
# Goal: Maximum search + AI search visibility for a local roofing company.
#       Allow crawling of all content, blog, and service pages.
#       Protect WordPress backend. Block training scrapers.
 
# ============================================================
# CONTENT SIGNALS
# ============================================================
# search=yes    → Standard search indexing (Google, Bing, etc.)
# ai-input=yes  → AI-powered search answers ALLOWED
#                 (ChatGPT, Gemini, Claude, Perplexity, etc.)
# ai-train=no   → AI model training NOT permitted
#
# ANY RESTRICTIONS VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS
# OF RIGHTS UNDER EU DIRECTIVE 2019/790 ARTICLE 4.
# ============================================================
 
User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
 
# Required for WordPress AJAX functionality
Allow: /wp-admin/admin-ajax.php
 
# Allow uploads (project photos, roofing/metal building galleries)
Allow: /wp-content/uploads/
 
# ============================================================
# SITEMAP
# ============================================================
Sitemap: https://jproofingandmetalbuildings.com/sitemap_index.xml
 
# ============================================================
# MAJOR SEARCH ENGINES
# Crawl-delay removed — Googlebot and Bingbot ignore it anyway.
# ============================================================
 
User-agent: Googlebot
Allow: /
 
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
 
User-agent: Bingbot
Allow: /
 
User-agent: DuckDuckBot
Allow: /
 
User-agent: Slurp
Allow: /
 
# ============================================================
# AI SEARCH CRAWLERS — EXPLICITLY ALLOWED
# Being cited by AI tools when homeowners ask about roofing
# contractors in the Pittsburgh area is valuable visibility
# and a source of referral traffic.
# Crawl-delay removed — these bots crawl infrequently by default.
# ============================================================
 
User-agent: GPTBot
Allow: /
 
User-agent: ChatGPT-User
Allow: /
 
User-agent: ClaudeBot
Allow: /
 
User-agent: Google-Extended
Allow: /
 
User-agent: PerplexityBot
Allow: /
 
User-agent: Amazonbot
Allow: /
 
User-agent: meta-externalagent
Allow: /
 
User-agent: Applebot-Extended
Allow: /
 
User-agent: YouBot
Allow: /
 
# ============================================================
# SOCIAL MEDIA CRAWLERS
# Enables rich previews when project photos, reviews, or
# service pages are shared on Facebook, X, or LinkedIn.
# Crawl-delay removed — these are quick one-time fetches.
# ============================================================
 
User-agent: facebookexternalhit
Allow: /
 
User-agent: Twitterbot
Allow: /
 
User-agent: LinkedInBot
Allow: /
 
# ============================================================
# SEO ANALYTICS TOOLS
# Crawl-delay kept — these tools are aggressive and the
# delay is actually respected, protecting server load.
# ============================================================
 
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5
 
User-agent: SemrushBot
Allow: /
Crawl-delay: 5
 
User-agent: MJ12bot
Allow: /
Crawl-delay: 5
 
# ============================================================
# BLOCKED: AI TRAINING SCRAPERS
# ============================================================
 
User-agent: CCBot
Disallow: /
 
User-agent: Bytespider
Disallow: /
 
User-agent: DataForSeoBot
Disallow: /
 
User-agent: Diffbot
Disallow: /
 
# ============================================================
# BLOCKED: AGGRESSIVE / JUNK BOTS
# ============================================================
 
User-agent: SiteAuditBot
Disallow: /
 
User-agent: Ezooms
Disallow: /
 
User-agent: MauiBot
Disallow: /
 
User-agent: dotbot
Disallow: /
 
User-agent: PetalBot
Disallow: /
 
User-agent: Baiduspider
Disallow: /
 
User-agent: YandexBot
Disallow: /