robots.txt
Raw
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: facebookexternalhit
User-agent: iaskspider/2.0
User-agent: img2dataset
User-agent: omgili
User-agent: omgilibot
Disallow: /
| 1 | User-agent: AI2Bot |
| 2 | User-agent: Ai2Bot-Dolma |
| 3 | User-agent: Amazonbot |
| 4 | User-agent: Applebot |
| 5 | User-agent: Applebot-Extended |
| 6 | User-agent: Bytespider |
| 7 | User-agent: CCBot |
| 8 | User-agent: ChatGPT-User |
| 9 | User-agent: Claude-Web |
| 10 | User-agent: ClaudeBot |
| 11 | User-agent: Diffbot |
| 12 | User-agent: FacebookBot |
| 13 | User-agent: FriendlyCrawler |
| 14 | User-agent: GPTBot |
| 15 | User-agent: Google-Extended |
| 16 | User-agent: GoogleOther |
| 17 | User-agent: GoogleOther-Image |
| 18 | User-agent: GoogleOther-Video |
| 19 | User-agent: ICC-Crawler |
| 20 | User-agent: ImagesiftBot |
| 21 | User-agent: Meta-ExternalAgent |
| 22 | User-agent: Meta-ExternalFetcher |
| 23 | User-agent: OAI-SearchBot |
| 24 | User-agent: PerplexityBot |
| 25 | User-agent: PetalBot |
| 26 | User-agent: Scrapy |
| 27 | User-agent: Timpibot |
| 28 | User-agent: VelenPublicWebCrawler |
| 29 | User-agent: Webzio-Extended |
| 30 | User-agent: YouBot |
| 31 | User-agent: anthropic-ai |
| 32 | User-agent: cohere-ai |
| 33 | User-agent: facebookexternalhit |
| 34 | User-agent: iaskspider/2.0 |
| 35 | User-agent: img2dataset |
| 36 | User-agent: omgili |
| 37 | User-agent: omgilibot |
| 38 | Disallow: / |