# Robots.txt for WindowsNews.ai
# Allow all web crawlers

User-agent: *
Allow: /
Crawl-delay: 1

# Specific directives for Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Specific directives for Google News
User-agent: Googlebot-News
Allow: /
Crawl-delay: 0

# Specific directives for Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Disallow admin and internal directories
User-agent: *
Disallow: /test/
Disallow: /includes/
Disallow: /templates/
Disallow: /admin/
Disallow: /src/
Disallow: /vendor/
Disallow: /node_modules/
Disallow: /audio/
Disallow: /podcast-audio/

# Allow specific includes for API endpoints
User-agent: *
Allow: /includes/api-proxy.php
Allow: /includes/ai-chat-proxy.php

# Sitemap locations
Sitemap: https://windowsnews.ai/sitemap.php
Sitemap: https://windowsnews.ai/sitemap.xml
Sitemap: https://windowsnews.ai/sitemap-news.xml
Sitemap: https://windowsnews.ai/sitemaps/sitemap_index.xml