feat(seo): add production robots.txt with AI crawler visibility

.github/workflows/deploy.yml

@@ -30,13 +30,13 @@ jobs:
           node-version: 18
           # cache: yarn
 
-      - name: If repo is website-prod replace text website.wopee.io to wopee.io and remove robots.txt
+      - name: If repo is website-prod replace text website.wopee.io to wopee.io and write production robots.txt
         if: github.repository == 'autonomous-testing/website-prod'
         run: |
           sed -i 's/website.wopee.io/wopee.io/g' ./CNAME
           sed -i 's/website.wopee.io/wopee.io/g' ./docusaurus.config.js
           sed -i 's/dev.wopee.io/wopee.io/g' ./cmdBaseUrl.js
-          rm -rf ./static/robots.txt
+          cp ./static/robots.prod.txt ./static/robots.txt
 
       - name: If repo is website-prod replace text website.wopee.io to wopee.io and remove robots.txt
         if: github.repository == 'autonomous-testing/website-demo'

static/robots.prod.txt

@@ -0,0 +1,29 @@
+User-agent: *
+Allow: /
+Disallow: /billing/
+
+User-agent: GPTBot
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
+User-agent: Amazonbot
+Allow: /
+
+User-agent: YouBot
+Allow: /
+
+User-agent: cohere-ai
+Allow: /
+
+Sitemap: https://wopee.io/sitemap.xml

static/robots.txt

@@ -1 +1,3 @@
-User-agent: * Disallow: /
+# Staging — block all crawlers (production robots.txt is written by deploy pipeline)
+User-agent: *
+Disallow: /