# robots.txt for https://www.nts.org.uk/ sitemap: https://www.nts.org.uk/sitemaps-1-sitemap.xml sitemap: https://www.nts.org.uk/sco-gb/sitemaps-1-sitemap.xml sitemap: https://www.nts.org.uk/gd-gb/sitemaps-1-sitemap.xml # live - don't allow web crawlers to index cpresources/ or vendor/ User-agent: * Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ crawl-delay: 10 User-agent: Amazonbot Disallow: / User-agent: Applebot Disallow: / User-agent: Baiduspider Disallow: / User-agent: barkrowler Disallow: / User-agent: Bingbot Disallow: /*search* User-agent: ClaudeBot Disallow: / User-agent: Claude-SearchBot Disallow: / User-agent: dotbot Disallow: / User-agent: GPTBot Disallow: / User-agent: meta-externalagent Disallow: / User-agent: MJ12bot Disallow: / User-agent: Twitterbot Disallow: / User-agent: serpstatbot Disallow: / User-agent: PetalBot Disallow: / User-agent: YandexBot Disallow: / User-agent: Yandex Disallow: / User-agent: YisouSpider Disallow: /