# webspeed.pro — apex content/SEO site
# Allow all well-behaved crawlers; sitemap generated by @astrojs/sitemap.

User-agent: *
Allow: /
# Internal chrome shell for the PHP audit renderer (holds raw placeholder tokens; the real
# audit pages are /en/audit/{host}/). Keep it out of the index.
# Both spellings: the section moved under the locale prefix and the old path still 301s, so
# a crawler can arrive at either — and the shell is reachable through both.
Disallow: /en/audit/frame
Disallow: /audit/frame

# Block aggressive SEO/backlink scrapers (competitive recon), like the competitors do.
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: BLEXBot
Disallow: /

Sitemap: https://webspeed.pro/sitemap-index.xml
# Published audit pages. Separate file because it's rebuilt hourly on the server by
# audit/cron.php as snapshots are published and pruned — the Astro index is built locally and
# would go stale. The hub (/audit/) is noindex, so without this the only crawl path to these
# pages is that one page's links.
Sitemap: https://webspeed.pro/sitemap-audit.xml