# HonestDog Robots.txt # Primary markets: DE (Germany), EN (International) # Content available in 14 languages User-agent: * Allow: / # Block admin and private areas Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render # Block API endpoints except sitemaps and the RSS feed Disallow: /api/ Allow: /api/sitemaps/ Allow: /api/feed # Allow static assets for proper rendering Allow: /_next/static/ Allow: /_next/image/ Allow: /assets/ Allow: /public/ # Block Next.js internal routes Disallow: /_next/webpack-hmr* Disallow: /_next/server/ # Block authentication and user account pages Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ # Honeypot traps for malicious bots Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ # Googlebot-specific (Google ignores Crawl-delay) User-agent: Googlebot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ # Bing and Yandex respect Crawl-delay User-agent: Bingbot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 1 User-agent: Yandex Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 2 # DuckDuckGo (uses Bing's index but crawls its own Instant Answers cache) User-agent: DuckDuckBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ # Apple (Siri + Spotlight) — honors Crawl-delay; ~74k req/day on the 14-locale # long tail (2026-07), pace it without blocking User-agent: Applebot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 2 # Seznam (primary CZ engine — ~25% of Czech search) User-agent: SeznamBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 2 # Baidu (ZH locale) User-agent: Baiduspider Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 2 # AI answer-engine crawlers — explicitly opted in (we want to be cited). # Mirrors the WAF AllowAICrawlers rule + the /llms.txt manifest. # Bulk TRAINING crawlers additionally get a Crawl-delay: they re-crawl the # 14-locale long tail so aggressively that they dominate origin (SSR Lambda) # load — 2026-07 CloudFront logs: GPTBot ~100k req/day, meta-externalagent # ~90k/day, Applebot ~74k/day, Amazonbot ~32k/day. Pacing keeps them fully # allowed (AEO policy unchanged) while capping the daily crawl budget. # User-triggered / answer-engine agents (ChatGPT-User, OAI-SearchBot, # Perplexity-*) stay un-delayed — they fire on real user queries. User-agent: GPTBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 2 User-agent: ChatGPT-User Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: OAI-SearchBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: ClaudeBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: anthropic-ai Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: Claude-Web Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: PerplexityBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: Perplexity-User Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: Google-Extended Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: Applebot-Extended Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: CCBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 10 User-agent: Amazonbot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 5 User-agent: meta-externalagent Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 5 User-agent: Bytespider Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ Crawl-delay: 10 User-agent: cohere-ai Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: DuckAssistBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ User-agent: YouBot Allow: / Disallow: /admin/ Disallow: /private/ Disallow: /wp-admin/ Disallow: /.env Disallow: /config/ Disallow: /backup/ Disallow: /database/ Disallow: /render Allow: /_next/static/ Allow: /_next/image/ Disallow: /_next/webpack-hmr* Disallow: /_next/server/ Disallow: /*/login Disallow: /*/signup Disallow: /*/reset-password Disallow: /*/forgot-password Disallow: /*/dashboard Disallow: /*/profile Disallow: /*/breeder-dashboard Disallow: /*/breeder-onboarding Disallow: /*/partner-dashboard Disallow: /*/demo/ Disallow: /api/honeypot/ Disallow: /secret/ Disallow: /hidden/ # SEO-tool crawlers — no search/AEO/user value, pure origin load. Not part of # the AI allow-list policy. (AhrefsBot deliberately NOT blocked — the free-tier # Ahrefs dashboard still feeds the in-house SEO audit with backlink data.) User-agent: SemrushBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: PetalBot Disallow: / User-agent: BLEXBot Disallow: / # Web archives — blocked by owner decision (2026-07-17): no public historical # snapshots of the site; archive.org exclusion additionally requested by email. User-agent: ia_archiver Disallow: / User-agent: archive.org_bot Disallow: / User-agent: special_archiver Disallow: / # AI manifest (crawler allow-list + structured Data-Source feeds) # https://www.honestdog.de/llms.txt # Sitemap location Sitemap: https://www.honestdog.de/sitemap.xml