# robots.txt for maliktimes.in
# Policy: AI READ/SEARCH = allow, AI TRAINING = block
# Note: this does not override Yoast/RankMath generated rules.
# If your current robots.txt already has WordPress-specific
# Disallow rules or a different Sitemap line, merge them in below.

# ---------------------------------------------------------
# Standard rules (applies to everything not matched above)
# ---------------------------------------------------------
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /my-account-free_news-2/
Disallow: /login-register-free_news-2/
Disallow: /*?s=
Allow: /wp-admin/admin-ajax.php
Allow: /

# ---------------------------------------------------------
# AI TRAINING crawlers — BLOCK
# These scrape content to train future model weights.
# ---------------------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: img2dataset
Disallow: /

# ---------------------------------------------------------
# AI READ / live-retrieval / search crawlers — ALLOW
# These fetch a page only when an AI assistant answers a
# real user query, and they typically cite/link back.
# ---------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# ---------------------------------------------------------
# Regular search engines — ALLOW (independent of AI rules)
# ---------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ---------------------------------------------------------
# Sitemap — replace with your actual sitemap URL
# (check your SEO plugin, e.g. /sitemap_index.xml for Yoast/RankMath)
# ---------------------------------------------------------
Sitemap: https://maliktimes.in/sitemap_index.xml