# Hashlytics.io Robots.txt - Optimized for SEO, Tools & AI # Allow all crawlers by default User-agent: * Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/plugins/ Disallow: /wp-content/themes/ Disallow: /wp-content/cache/ Disallow: /wp-json/ Disallow: /xmlrpc.php Disallow: /readme.html Disallow: /license.txt Disallow: /?s= Disallow: /search/ Disallow: *?replytocom= Disallow: */trackback/ Disallow: */embed/ Disallow: /cgi-bin/ # Allow important resources Allow: /wp-admin/admin-ajax.php Allow: /wp-content/uploads/ Allow: /wp-content/*.css Allow: /wp-content/*.js Allow: /llms.txt # === A-ADS Crawler Access === User-agent: A-ADS Allow: / # === GOOGLE BOTS - Full Access === User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / User-agent: Google-InspectionTool Allow: / User-agent: GoogleOther Allow: / User-agent: Google-Extended Allow: / User-agent: Google-Read-Aloud Allow: / User-agent: Mediapartners-Google Allow: / User-agent: AdsBot-Google Allow: / User-agent: AdsBot-Google-Mobile Allow: / User-agent: Feedfetcher-Google Allow: / User-agent: APIs-Google Allow: / User-agent: Storebot-Google Allow: / # === BING BOTS - Full Access === User-agent: Bingbot Allow: / User-agent: BingPreview Allow: / User-agent: msnbot Allow: / User-agent: msnbot-media Allow: / User-agent: adidxbot Allow: / # === YANDEX (Russian Search) === User-agent: Yandex Allow: / User-agent: YandexBot Allow: / User-agent: YandexImages Allow: / # === BAIDU (Chinese Search) === User-agent: Baiduspider Allow: / User-agent: Baiduspider-image Allow: / User-agent: Baiduspider-video Allow: / # === DUCKDUCKGO === User-agent: DuckDuckBot Allow: / # === YAHOO (Uses Bing) === User-agent: Slurp Allow: / # Allow SEO & Analytics Tools (You're Paying For These!) User-agent: AhrefsBot Allow: / User-agent: SemrushBot Allow: / User-agent: MJ12bot Allow: / # Allow AI Crawlers (For llms.txt Integration) User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: anthropic-ai Allow: / User-agent: CCBot Allow: / User-agent: Claude-Web Allow: / User-agent: Bytespider Allow: / User-agent: Applebot-Extended Allow: / User-agent: FacebookBot Allow: / User-agent: Diffbot Allow: / User-agent: cohere-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: ImagesiftBot Allow: / User-agent: Omgilibot Allow: / User-agent: Omgili Allow: / # Block genuinely bad/aggressive bots User-agent: MauiBot Disallow: / User-agent: AlphaBot Disallow: / User-agent: DotBot Disallow: / User-agent: 008 Disallow: / User-agent: SurveyBot Disallow: / User-agent: asterias Disallow: / User-agent: BLEXBot Disallow: / User-agent: BuiltWith Disallow: / User-agent: BullseyeBot Disallow: / User-agent: Cliqzbot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: DomainStatsBot Disallow: / User-agent: Ezooms Disallow: / User-agent: MegaIndex Disallow: / User-agent: PetalBot Disallow: / User-agent: SiteSnagger Disallow: / User-agent: Turnitin Disallow: / User-agent: Vagabondo Disallow: / User-agent: WebReaper Disallow: / User-agent: WebStripper Disallow: / # Sitemaps Sitemap: https://hashlytics.io/sitemap_index.xml