# CryptoAnalyzer Pro - Professional Cryptocurrency Trading Analysis
# Website: https://cryptoanalyzer.pro
# Global Search Engine Optimization

# Allow all crawlers by default
User-agent: *
Allow: /
Crawl-delay: 1

# MAJOR SEARCH ENGINES WORLDWIDE

# Google (Global)
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /
Allow: /images/
Allow: /charts/

User-agent: Googlebot-News
Allow: /
Allow: /blog
Allow: /news

# Microsoft Bing (Global)
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo (Global)
User-agent: Slurp
Allow: /
Crawl-delay: 1

# Yandex (Russia, CIS countries)
User-agent: YandexBot
Allow: /
Crawl-delay: 1

# Baidu (China)
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# DuckDuckGo (Privacy-focused global)
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Sogou (China)
User-agent: Sogou
Allow: /
Crawl-delay: 2

# Naver (South Korea)
User-agent: NaverBot
Allow: /
Crawl-delay: 2

# Seznam (Czech Republic)
User-agent: SeznamBot
Allow: /
Crawl-delay: 2

# Qwant (France)
User-agent: Qwantify
Allow: /
Crawl-delay: 1

# Startpage (Privacy-focused)
User-agent: StartpageBot
Allow: /
Crawl-delay: 1

# SOCIAL MEDIA CRAWLERS
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: SkypeUriPreview
Allow: /

User-agent: Applebot
Allow: /

# SEO AND ANALYTICS TOOLS
User-agent: SemrushBot
Allow: /
Crawl-delay: 3

User-agent: AhrefsBot
Allow: /
Crawl-delay: 3

User-agent: MJ12bot
Allow: /
Crawl-delay: 5

User-agent: DotBot
Allow: /
Crawl-delay: 3

# ARCHIVE CRAWLERS
User-agent: ia_archiver
Allow: /

User-agent: Wayback
Allow: /

User-agent: archive.org_bot
Allow: /

# NEWS AGGREGATORS
User-agent: NewsNow
Allow: /
Allow: /blog
Allow: /news

User-agent: Moreover
Allow: /
Allow: /blog

# REGIONAL AND SPECIALIZED CRAWLERS

# Europe
User-agent: Findxbot
Allow: /
Crawl-delay: 2

# India
User-agent: JikeSpider
Allow: /
Crawl-delay: 2

# Japan
User-agent: livedoor
Allow: /
Crawl-delay: 2

# Brazil
User-agent: UOL
Allow: /
Crawl-delay: 2

# SECURITY AND DISALLOWED AREAS
Disallow: /admin/
Disallow: /api/internal/
Disallow: /.env*
Disallow: /config/
Disallow: /server/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /build/
Disallow: /dist/
Disallow: /tmp/
Disallow: /temp/
Disallow: /private/
Disallow: /secret/
Disallow: /hidden/

# BLOCK MALICIOUS BOTS
User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SemrushBot-SI
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: serpstatbot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MegaIndex
Disallow: /

# SITEMAPS FOR COMPREHENSIVE CRAWLING
Sitemap: https://cryptoanalyzer.pro/sitemap.xml

# ADDITIONAL SPECIFIC SITEMAPS FOR BETTER INDEXING
# Main application sitemap
Sitemap: https://cryptoanalyzer.pro/sitemap.xml

# RSS Feed for Blog Content
Sitemap: https://cryptoanalyzer.pro/rss.xml

# CANONICAL HOST
Host: https://cryptoanalyzer.pro

# SPECIFIC CRAWLING DIRECTIVES FOR CRYPTO CONTENT
Allow: /bitcoin-analysis
Allow: /ethereum-analysis
Allow: /crypto-trading-signals
Allow: /technical-analysis
Allow: /crypto-news-sentiment
Allow: /free-crypto-analyzer

# API ENDPOINTS - ALLOW PUBLIC ACCESS
Allow: /api/sitemap
Allow: /api/ping
Disallow: /api/internal/
Disallow: /api/admin/

# GLOBAL CRAWLING OPTIMIZATION
# - Optimized for cryptocurrency and financial content
# - Balanced crawl delays for performance and server resources
# - Comprehensive global search engine coverage
# - Enhanced for international markets and languages