Merge branch 'feature/global-semantic-search-PLM' into semantic-search

Signed-off-by: Zhenxing Shen <[email protected]>

Author: s-zx

.github/CODEOWNERS

@@ -1 +1 @@
-*   @ananzh @kavilla @AMoo-Miki @ashwin-pc @joshuarrrr @abbyhu2000 @zengyan-amazon @zhongnansu @manasvinibs @ZilongX @Flyingliuhub @curq @bandinib-amzn @SuZhou-Joe @ruanyl @BionIT @xinruiba @zhyuanqi @mengweieric @LDrago27 @virajsanghvi @sejli @joshuali925 @huyaboo @angle943 @Maosaic @wanglam @d-buckner
+*   @ananzh @kavilla @AMoo-Miki @ashwin-pc @joshuarrrr @abbyhu2000 @zengyan-amazon @zhongnansu @manasvinibs @ZilongX @Flyingliuhub @curq @bandinib-amzn @SuZhou-Joe @ruanyl @BionIT @xinruiba @zhyuanqi @mengweieric @LDrago27 @virajsanghvi @sejli @joshuali925 @huyaboo @angle943 @Maosaic @wanglam @d-buckner @raintygao @ps48

.github/workflows/cypress_workflow.yml

@@ -37,18 +37,18 @@ env:
   CYPRESS_BROWSER: 'chromium'
   CYPRESS_VISBUILDER_ENABLED: true
   CYPRESS_DATASOURCE_MANAGEMENT_ENABLED: false
-  CYPRESS_MEMORY_LIMIT: '10240'
+  CYPRESS_MEMORY_LIMIT: '12288'
   CYPRESS_CACHE_FOLDER: '${{ github.workspace }}/.cypress-cache'
   # Optimize Chrome
   CYPRESS_CHROME_ARGS: >-
     --disable-dev-shm-usage
     --no-sandbox
     --disable-gpu
     --disable-software-rasterizer
-    --js-flags="--max-old-space-size=10240"
+    --js-flags="--max-old-space-size=12288"
   OSD_SNAPSHOT_SKIP_VERIFY_CHECKSUM: true
-  OSD_OPTIMIZER_MAX_WORKERS: 8
-  NODE_OPTIONS: '--max-old-space-size=10240 --max-semi-space-size=64 --dns-result-order=ipv4first'
+  OSD_OPTIMIZER_MAX_WORKERS: 12
+  NODE_OPTIONS: '--max-old-space-size=12288 --max-semi-space-size=128 --dns-result-order=ipv4first'
   COMMENT_TAG: '[MANUAL CYPRESS TEST RUN RESULTS]'
   COMMENT_SUCCESS_MSG: ':white_check_mark: Cypress test run succeeded!'
   COMMENT_FAILURE_MSG: ':x: Cypress test run failed!'
@@ -134,12 +134,16 @@ jobs:
             config: explore
             test_location: source
           # Explore tests - group 5
-          - group: 15
+          - group: 15Explore
+            config: explore
+            test_location: source
+
+          - group: 16Explore
             config: explore
             test_location: source
     container:
       image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
-      options: '--user 1001 --shm-size=2g'
+      options: '--user 1001 --shm-size=4g'
     env:
       START_CMD: ${{ matrix.config == 'query_enhanced' &&
         'node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=true --uiSettings.overrides[''home:useNewHomePage'']=true --data_source.enabled=true --workspace.enabled=true --opensearch.ignoreVersionMismatch=true --data.savedQueriesNewUI.enabled=true' ||
@@ -148,7 +152,8 @@ jobs:
         matrix.config == 'dashboard' &&
         'node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true' ||
         'node ../scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=false' }}
-      OPENSEARCH_SNAPSHOT_CMD: ${{ (matrix.config == 'query_enhanced' || matrix.config == 'explore') &&
+      OPENSEARCH_SNAPSHOT_CMD:
+        ${{ (matrix.config == 'query_enhanced' || matrix.config == 'explore') &&
         '/bin/bash -c "./opensearch-${{ env.LATEST_VERSION }}/opensearch-tar-install.sh &"' ||
         matrix.config == 'dashboard' &&
         'node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false -E search.concurrent_segment_search.mode=none' ||
@@ -193,15 +198,16 @@ jobs:
 
       - name: Setup Yarn
         run: |
-          npm uninstall -g yarn
-          npm i -g yarn@^1.22.10
+            npm uninstall -g yarn
+            npm i -g yarn@^1.22.10
+            yarn config set network-timeout 1000000 -g
 
       - name: Run bootstrap
         run: yarn osd bootstrap
 
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12
-
+    
       - name: Checkout FT repo
         uses: actions/checkout@v4
         with:
@@ -229,8 +235,8 @@ jobs:
           echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV
           echo "SPEC=${FORMATTED_SPEC}"
 
-      # Setup spec files for Dashboards in-house cypress tests
-      - name: Setup spec files for Dashboards tests
+      # Setup spec files for OSD in-house cypress tests
+      - name: Setup spec files for OSD tests
         if: ${{ inputs.specs == '' && matrix.test_location == 'source' }}
         shell: bash
         run: |
@@ -290,13 +296,13 @@ jobs:
           sleep 30
         shell: bash
 
-      # Clear Cypress Cache before running Dashboards tests
+      # Clear Cypress Cache before running OSD tests
       - name: Clear Cypress Cache
         if: matrix.test_location == 'source'
         run: npx cypress cache clear
 
-      # Run Dashboards Cypress tests within the source repo
-      - name: Run Dashboards Cypress tests with query enhancements
+      # Run OSD Cypress tests within the source repo
+      - name: Run OSD Cypress tests with query enhancements
         if: matrix.test_location == 'source' && (matrix.config == 'query_enhanced' || matrix.config == 'explore')
         uses: cypress-io/github-action@v6
         with:
@@ -305,7 +311,7 @@ jobs:
           wait-on: 'http://localhost:9200, http://localhost:5601'
           command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
 
-      - name: Run Dashboards Cypress tests without query enhancements
+      - name: Run OSD Cypress tests without query enhancements
         if: matrix.test_location == 'source' && matrix.config == 'dashboard'
         uses: cypress-io/github-action@v6
         with:
@@ -314,7 +320,6 @@ jobs:
           wait-on: 'http://localhost:9200, http://localhost:5601'
           command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
 
-      # Compress and upload artifacts for FT repo
       - name: Compress FT repo artifacts
         if: always() && matrix.test_location == 'ftr'
         run: |
@@ -335,25 +340,24 @@ jobs:
           path: ftr-cypress-artifacts-${{ matrix.group }}.tar.gz
           retention-days: 1
 
-      # Compress and upload artifacts for Dashboards repo
-      - name: Compress Dashboards repo artifacts
+      - name: Compress OSD repo artifacts
         if: always() && matrix.test_location == 'source'
         run: |
-          mkdir -p dashboard-artifacts
+          mkdir -p osd-artifacts
             for dir in screenshots videos results; do
               if [ -d "cypress/$dir" ]; then
-                cp -r "cypress/$dir" dashboard-artifacts/
+                cp -r "cypress/$dir" osd-artifacts/
               fi
           done
-          tar -czf dashboards-cypress-artifacts-${{ matrix.group }}.tar.gz -C dashboard-artifacts .
+          tar -czf osd-cypress-artifacts-${{ matrix.group }}.tar.gz -C osd-artifacts .
         shell: bash
 
-      - name: Upload Dashboards repo artifacts
+      - name: Upload OSD repo artifacts
         if: always() && matrix.test_location == 'source'
         uses: actions/upload-artifact@v4
         with:
-          name: dashboards-cypress-artifacts-${{ matrix.group }}
-          path: dashboards-cypress-artifacts-${{ matrix.group }}.tar.gz
+          name: osd-cypress-artifacts-${{ matrix.group }}
+          path: osd-cypress-artifacts-${{ matrix.group }}.tar.gz
           retention-days: 1
           overwrite: true
 

.github/workflows/links_checker.yml

@@ -18,6 +18,6 @@ jobs:
         uses: lycheeverse/lychee-action@v1
         with:
           fail: true
-          args: --accept=200,403,429 --base . --retry-wait-time=15 --max-retries=5 --exclude-path cypress/fixtures "**/*.html" "**/*.md" "**/*.txt" "**/*.json" "**/*.js" "**/*.ts" "**/*.tsx"
+          args: --accept=200,403,429 --base . --retry-wait-time=15 --max-retries=5 --exclude-path cypress/fixtures --exclude-path src/plugins/explore/public/components/data_table/data_table.mocks.ts "**/*.html" "**/*.md" "**/*.txt" "**/*.json" "**/*.js" "**/*.ts" "**/*.tsx"
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/version-increment.yml

@@ -0,0 +1,109 @@
+name: Auto Version Bump
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  version-bump:
+    if: github.repository == 'opensearch-project/OpenSearch-Dashboards'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Setup Node
+      uses: actions/setup-node@v4
+      with:
+        node-version-file: '.nvmrc'
+        registry-url: 'https://registry.npmjs.org'
+
+    - name: Parse tag and determine version bump
+      id: parse_tag
+      run: |
+        # Extract tag name without refs/tags/ prefix
+        TAG=${GITHUB_REF#refs/tags/}
+        VERSION=$TAG
+        
+        # Validate version format (x.y.z)
+        if [[ ! $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          echo "Invalid version format: $VERSION"
+          exit 1
+        fi
+        
+        # Extract major, minor, patch
+        IFS='.' read -r MAJOR MINOR PATCH <<< "$VERSION"
+        
+        echo "current_version=$VERSION" >> $GITHUB_OUTPUT
+        echo "major=$MAJOR" >> $GITHUB_OUTPUT
+        echo "minor=$MINOR" >> $GITHUB_OUTPUT
+        echo "patch=$PATCH" >> $GITHUB_OUTPUT
+        
+        # Determine version type and next version
+        if [ "$PATCH" = "0" ] && [ "$MINOR" = "0" ]; then
+          # Major release (x.0.0) - bump to next major
+          NEXT_MAJOR=$((MAJOR + 1))
+          NEXT_VERSION="${NEXT_MAJOR}.0.0"
+          VERSION_TYPE="major"
+          TARGET_BRANCH="main"
+        elif [ "$PATCH" = "0" ]; then
+          # Minor release (x.y.0) - bump to next minor
+          NEXT_MINOR=$((MINOR + 1))
+          NEXT_VERSION="${MAJOR}.${NEXT_MINOR}.0"
+          VERSION_TYPE="minor"
+          TARGET_BRANCH="main"
+        else
+          # Patch release (x.y.z) - bump to next patch
+          NEXT_PATCH=$((PATCH + 1))
+          NEXT_VERSION="${MAJOR}.${MINOR}.${NEXT_PATCH}"
+          VERSION_TYPE="patch"
+          TARGET_BRANCH="${MAJOR}.${MINOR}"
+        fi
+        
+        echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
+        echo "version_type=$VERSION_TYPE" >> $GITHUB_OUTPUT
+        echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT
+        
+        echo "Current version: $VERSION"
+        echo "Version type: $VERSION_TYPE"
+        echo "Next version: $NEXT_VERSION"
+        echo "Target branch: $TARGET_BRANCH"
+
+    - name: Checkout target branch
+      uses: actions/checkout@v4
+      with:
+        ref: "${{ steps.parse_tag.outputs.target_branch }}"
+
+    - name: Set environment variables for PR
+      run: |
+        TAG=${GITHUB_REF#refs/tags/}
+        
+        echo "TAG=$TAG" >> $GITHUB_ENV
+        echo "NEXT_VERSION=${{ steps.parse_tag.outputs.next_version }}" >> $GITHUB_ENV
+        echo "BASE_BRANCH=${{ steps.parse_tag.outputs.target_branch }}" >> $GITHUB_ENV
+
+    - name: Update package.json version
+      run: |
+        NEXT_VERSION="${{ steps.parse_tag.outputs.next_version }}"
+        
+        # Update version in package.json
+        npm version $NEXT_VERSION --no-git-tag-version
+        
+        echo "Updated package.json version to $NEXT_VERSION"
+
+    - name: Create PR for target branch
+      uses: peter-evans/create-pull-request@v7
+      with:
+        base: ${{ env.BASE_BRANCH }}
+        branch: 'create-pull-request/patch-${{ env.BASE_BRANCH }}'
+        commit-message: Increment version to ${{ env.NEXT_VERSION }}
+        signoff: true
+        delete-branch: true
+        title: '[AUTO] Increment version to ${{ env.NEXT_VERSION }}'
+        body: |
+          I've noticed that a new tag ${{ env.TAG }} was pushed, and incremented the version from ${{ env.CURRENT_VERSION }} to ${{ env.NEXT_VERSION }}.

.gitignore

@@ -4,6 +4,7 @@
 /.opensearch
 /.chromium
 /package.json.bak
+.claude/
 .DS_Store
 .node_binaries
 .native_modules
@@ -79,3 +80,4 @@ snapshots.js
 # Ignore the lighthousereport generatated
 .lighthouserc.js
 .lhci_output.json
+/.lighthouseci/

.lycheeignore

@@ -1,6 +1,11 @@
 # Copyright OpenSearch Contributors
 # SPDX-License-Identifier: Apache-2.0
 
+node_modules/
+dist/
+build/
+.git/
+
 # Local or predefined end points
 localhost
 //opensearch
@@ -96,3 +101,4 @@ http://dev.w3.org/
 https://qntm.org/cmd
 http://fakenews.co/
 http://blond-winner.biz/
+https://www.iso.org/*

.vscode/launch.json

@@ -1,11 +1,62 @@
 {
   "version": "0.2.0",
   "configurations": [
-      {
-          "type": "pwa-node",
-          "request": "attach",
-          "name": "attach-to-service",
-          "port": 9229
-      }
+    /**
+     * Attaches a debugger to an already running OSD instance
+     */
+    {
+      "name": "Attach debugger to running Dashboards",
+      "type": "pwa-node",
+      "request": "attach",
+      "port": 9229
+    },
+    /**
+     * Starts the OSD instance only
+     */
+    {
+      "name": "Start Dashboards",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "yarn",
+      "args": ["start", "--no-base-path"],
+      "cwd": "${workspaceFolder}",
+      "runtimeArgs": [],
+      "env": {
+        "NODE_ENV": "development"
+      },
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "openOnSessionStart"
+    },
+    /**
+     * Starts the OpenSearch snapshot server
+     * - Everytime this launch configuration runs, it overwrites your previous data. Thus, if you need to persist data, you can add a configuration in your VSCode workspace to run '${workspaceFolder}/.opensearch/{version}/bin/opensearch'
+     */
+    {
+      "name": "Start OpenSearch cluster snapshot",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "yarn",
+      "args": ["opensearch", "snapshot"],
+      "cwd": "${workspaceFolder}",
+      "runtimeArgs": [],
+      "env": {
+        "NODE_ENV": "development"
+      },
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "openOnSessionStart"
+    }
+  ],
+  "compounds": [
+    /**
+     * This is the run configuration to startup OSD end-to-end
+     * 1. Starts up OpenSearch Snapshot server
+     * 2. Starts up Dashboards server
+     */
+    {
+      "name": "Start Dashboards with OpenSearch Cluster Snapshot",
+      "configurations": ["Start OpenSearch cluster snapshot", "Start Dashboards"],
+      // If either Dashboards or OpenSearch server is stopped, both servers will be stopped (set this to false to stop an individual server without stopping all servers)
+      "stopAll": true
+    }
   ]
 }