User-agent: * Allow: / # Google and Bing specific optimizations User-agent: Googlebot Allow: / Crawl-delay: 1 User-agent: Bingbot Allow: / Crawl-delay: 1 # Social Media Crawlers User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / # Security and System Protection Disallow: /cdn-cgi/ Disallow: /_astro/ Disallow: /_image/ Disallow: /admin/ Disallow: /draft/ Disallow: /tmp/ Disallow: /*.json$ Disallow: /*.config$ Disallow: /*.env$ # Prevent duplicate content from tracking parameters Disallow: /*?*utm_* Disallow: /*?*fbclid=* Disallow: /*?*gclid=* # Block conversion-only pages from direct search entry Disallow: /*-confirmation/ Disallow: /thank-you/ # Explicitly allow High-Value Content Paths Allow: /programs/ Allow: /programs/* Allow: /fleet/ Allow: /fleet/* Allow: /services/ Allow: /services/* Allow: /blog/ Allow: /blog/* Allow: /about-us/ Allow: /team/ Allow: /contact/ Allow: /contact-mh-fuels/ Allow: /enroll/ Allow: /rates/ Allow: /new-to-flying/ Allow: /future-for-pilots/ Allow: /faqs/ # Asset indexing Allow: /*.css$ Allow: /*.js$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.png$ Allow: /*.webp$ Allow: /*.svg$ # AI Scraper Protection User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: Claude-Web Allow: / User-agent: PerplexityBot Allow: / User-agent: CCBot Disallow: / # Block SEO Spiders User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / # Sitemaps and Host Sitemap: https://mhaviation.com/sitemap-index.xml Sitemap: https://mhaviation.com/rss.xml Host: mhaviation.com