User-agent: *

# Allow important pages
Allow: /
Allow: /docs
Allow: /pricing
Allow: /scraper-api-pricing
Allow: /post
Allow: /services
Allow: /datasets
Allow: /legal
Allow: /serpapi-alternative
Allow: /serper-alternative
Allow: /scraping-api-no-subscription
Allow: /scrapingbee-alternative
Allow: /bright-data-alternative
Allow: /apify-alternative
Allow: /scrapingdog-alternative
Allow: /scraperapi-alternative
Allow: /firecrawl-alternative
Allow: /oxylabs-alternative
Allow: /compare

# Disallow admin/internal/auth routes
Disallow: /admin
Disallow: /admin/*
Disallow: /filament
Disallow: /filament/*
Disallow: /livewire/*
Disallow: /api/*

# Sitemap index
Sitemap: https://scrappa.co/sitemap.xml

# LLM-readable API documentation is exposed through HTML link tags.