diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b1ae987..c232ab5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,13 +30,13 @@ jobs: node-version: 18 # cache: yarn - - name: If repo is website-prod replace text website.wopee.io to wopee.io and remove robots.txt + - name: If repo is website-prod replace text website.wopee.io to wopee.io and write production robots.txt if: github.repository == 'autonomous-testing/website-prod' run: | sed -i 's/website.wopee.io/wopee.io/g' ./CNAME sed -i 's/website.wopee.io/wopee.io/g' ./docusaurus.config.js sed -i 's/dev.wopee.io/wopee.io/g' ./cmdBaseUrl.js - rm -rf ./static/robots.txt + cp ./static/robots.prod.txt ./static/robots.txt - name: If repo is website-prod replace text website.wopee.io to wopee.io and remove robots.txt if: github.repository == 'autonomous-testing/website-demo' diff --git a/static/robots.prod.txt b/static/robots.prod.txt new file mode 100644 index 0000000..cf3dbad --- /dev/null +++ b/static/robots.prod.txt @@ -0,0 +1,29 @@ +User-agent: * +Allow: / +Disallow: /billing/ + +User-agent: GPTBot +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Applebot-Extended +Allow: / + +User-agent: Amazonbot +Allow: / + +User-agent: YouBot +Allow: / + +User-agent: cohere-ai +Allow: / + +Sitemap: https://wopee.io/sitemap.xml diff --git a/static/robots.txt b/static/robots.txt index f677fae..89cadf6 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1 +1,3 @@ -User-agent: * Disallow: / \ No newline at end of file +# Staging — block all crawlers (production robots.txt is written by deploy pipeline) +User-agent: * +Disallow: /