# Socioverse Robots.txt
# Website: https://www.socioverse.io
# Last Updated: 2026-07-29
# Public creator storefronts ("Planets") live at /planet/@username and are crawlable.
# Old /biolink/* URLs 301-redirect to /planet/* and are kept crawlable so link equity transfers.

# ============================================
# Default rules for all crawlers
# ============================================
User-agent: *
Allow: /

# Explicit public product landings - keep crawlable
Allow: /instagram-dm-automation
Allow: /ai-instagram-content
Allow: /instagram-scheduler
Allow: /instagram-analytics-reports
Allow: /lead-magnet-generator
Allow: /link-in-bio-store

# Block authenticated/private areas
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /onboarding/
Disallow: /offer/
Disallow: /invite/
Disallow: /instagram/callback
Disallow: /admin/

# Block internal endpoints (actual API is on api.socioverse.io)
Disallow: /.well-known/

# NOTE: /assets/ must stay crawlable. It holds the JS and CSS bundles, so
# blocking it stops Googlebot rendering the app at all - Google treats blocked
# JS/CSS as a rendering fault, not a saving. Do not re-add Disallow: /assets/*.js.
Allow: /assets/

# Block redirect shortcuts (prevent duplicate content indexing)
Disallow: /content-studio/
Disallow: /automations/
Disallow: /messages/
Disallow: /templates/
Disallow: /billing/
Disallow: /settings/

# Block utility pages (no SEO value)
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /unsubscribe

# Block private buyer/token pages (per-purchase, no SEO value)
Disallow: /purchases/
Disallow: /learn/
Disallow: /booking/
Disallow: /lp/

# Block query parameters that create duplicate content
Disallow: /*?ref=
Disallow: /*?utm_

# Crawl delay for polite crawling
Crawl-delay: 1

# ============================================
# Googlebot (no crawl-delay, Google ignores it)
# ============================================
User-agent: Googlebot
Allow: /
Allow: /instagram-dm-automation
Allow: /ai-instagram-content
Allow: /instagram-scheduler
Allow: /instagram-analytics-reports
Allow: /lead-magnet-generator
Allow: /link-in-bio-store
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /onboarding/
Disallow: /instagram/callback
Disallow: /admin/
Disallow: /.well-known/
Allow: /assets/
Disallow: /content-studio/
Disallow: /automations/
Disallow: /messages/
Disallow: /templates/
Disallow: /billing/
Disallow: /settings/
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /unsubscribe
Disallow: /purchases/
Disallow: /learn/
Disallow: /booking/
Disallow: /lp/

# Googlebot-Image
User-agent: Googlebot-Image
Disallow: /lp/
Allow: /
Allow: /assets/
Allow: /images/
Disallow: /dashboard/
Disallow: /dashboard


# ============================================
# Bingbot
# ============================================
User-agent: Bingbot
Allow: /
Allow: /instagram-dm-automation
Allow: /ai-instagram-content
Allow: /instagram-scheduler
Allow: /instagram-analytics-reports
Allow: /lead-magnet-generator
Allow: /link-in-bio-store
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /onboarding/
Disallow: /instagram/callback
Disallow: /admin/
Disallow: /.well-known/
Allow: /assets/
Disallow: /content-studio/
Disallow: /automations/
Disallow: /messages/
Disallow: /templates/
Disallow: /billing/
Disallow: /settings/
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /unsubscribe
Disallow: /purchases/
Disallow: /learn/
Disallow: /booking/
Disallow: /lp/
Crawl-delay: 2

# ============================================
# Yandex
# ============================================
User-agent: Yandex
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /onboarding/

Disallow: /admin/
Crawl-delay: 3

# ============================================
# Social media crawlers - allow for rich previews
# ============================================
User-agent: facebookexternalhit
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: Twitterbot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: LinkedInBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: WhatsApp
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: Slackbot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: Discordbot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: TelegramBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: Pinterest
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


# ============================================
# AI crawlers - allow public content for LLM indexing
# ============================================
User-agent: GPTBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /onboarding/

User-agent: ChatGPT-User
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard
Disallow: /onboarding/

User-agent: Google-Extended
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: anthropic-ai
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: CCBot
Disallow: /lp/
Disallow: /

User-agent: PerplexityBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: ClaudeBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


User-agent: Bytespider
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard


# ============================================
# SEO tool crawlers - rate limited
# ============================================
User-agent: SemrushBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard

Crawl-delay: 10

User-agent: AhrefsBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard

Crawl-delay: 10

User-agent: rogerbot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard

Crawl-delay: 5

User-agent: DataForSeoBot
Disallow: /lp/
Allow: /
Disallow: /dashboard/
Disallow: /dashboard

Crawl-delay: 10

# ============================================
# Block aggressive/malicious bots
# ============================================
User-agent: MJ12bot
Disallow: /lp/
Disallow: /

User-agent: DotBot
Disallow: /lp/
Disallow: /

User-agent: BLEXBot
Disallow: /lp/
Disallow: /

User-agent: SearchmetricsBot
Disallow: /lp/
Disallow: /

User-agent: MegaIndex
Disallow: /lp/
Disallow: /

User-agent: ltx71
Disallow: /lp/
Disallow: /

User-agent: Riddler
Disallow: /lp/
Disallow: /

User-agent: Gigabot
Disallow: /lp/
Disallow: /

User-agent: SeznamBot
Disallow: /lp/
Disallow: /

User-agent: PetalBot
Disallow: /lp/
Disallow: /

User-agent: Sogou
Disallow: /lp/
Disallow: /

# ============================================
# Sitemap & Host
# ============================================
Sitemap: https://www.socioverse.io/sitemap.xml
Sitemap: https://www.socioverse.io/sitemap-blog.xml
Host: https://www.socioverse.io