User-agent: * Allow: / Disallow: /about-us/annual-reports Disallow: /about-us/annual-reports/ Disallow: /documents/yearly-reports/ Disallow: /documents/yearly-reports/* # Explicitly block major crawlers from the reports page and PDF folder User-agent: Googlebot Disallow: /about-us/annual-reports Disallow: /documents/yearly-reports/ User-agent: Googlebot-Image Disallow: /about-us/annual-reports Disallow: /documents/yearly-reports/ User-agent: Bingbot Disallow: /about-us/annual-reports Disallow: /documents/yearly-reports/ User-agent: YandexBot Disallow: /about-us/annual-reports Disallow: /documents/yearly-reports/ User-agent: DuckDuckBot Disallow: /about-us/annual-reports Disallow: /documents/yearly-reports/ # Block access to admin or private areas (if any) # Disallow: /admin/ # Disallow: /private/ # Sitemap location Sitemap: https://icta.az/sitemap.xml # Crawl delay (optional - helps with server load) Crawl-delay: 1