User-agent: Scrapy
Allow: /

# Block Meta AI Training Crawlers
User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

# Allow link preview crawler (optional - comment out to block everything)
User-agent: facebookexternalhit
Disallow: /

User-agent: *
Crawl-delay: 60
Disallow: