User-agent: *
Disallow: /cache/
Disallow: /old/
Disallow: /data/pending/
Allow: /
Sitemap: https://hostingratepro.site/sitemap.xml
Sitemap: https://hostingratepro.site/sitemap.xml.php
Host: hostingratepro.site

# AI/ML crawlers
User-agent: GPTBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: CCBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: PerplexityBot
Allow: /

User-agent: Yandex
Allow: /
Clean-param: lang /reviews.php
Clean-param: page /reviews.php