# Allow all crawlers
User-agent: *
Allow: /

# Disallow admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /auth/
Disallow: /*?_rsc=

# Disallow lang dir
Disallow: /en
Disallow: /en/
Disallow: /ja
Disallow: /ja/
Disallow: /ko
Disallow: /ko/
Disallow: /th
Disallow: /th/
Disallow: /zh-TW
Disallow: /zh-TW/

# Sitemap location
Sitemap: https://www.readink.app/sitemap.xml

# Crawl-delay for all bots
# Crawl-delay: 1

# Block specific bots that might cause issues
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Allow: /

# Block image crawlers from specific directories
User-agent: Googlebot-Image
Disallow: /images/private/
Allow: /public/