Fix disk no space issue in bwc-test

.github/workflows/sql-test-and-build-workflow.yml

@@ -179,7 +179,7 @@ jobs:
             doctest/build/testclusters/docTestCluster-0/logs/*
             integ-test/build/testclusters/*/logs/*
 
-  bwc-tests:
+  bwc-tests-rolling-upgrade:
     needs: Get-CI-Image-Tag
     runs-on: ubuntu-latest
     strategy:
@@ -190,37 +190,83 @@ jobs:
       options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - name: Run start commands
-      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
-
-    - uses: actions/checkout@v4
-
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v4
-      with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java }}
-
-    - name: Run backward compatibility tests
-      run: |
-        chown -R 1000:1000 `pwd`
-        su `id -un 1000` -c "./scripts/bwctest.sh"
-
-    - name: Upload test reports
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v4
-      continue-on-error: true
-      with:
-        name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
-        path: |
-          sql/build/reports/**
-          ppl/build/reports/**
-          core/build/reports/**
-          common/build/reports/**
-          opensearch/build/reports/**
-          integ-test/build/reports/**
-          protocol/build/reports/**
-          legacy/build/reports/**
-          plugin/build/reports/**
-          doctest/build/testclusters/docTestCluster-0/logs/*
-          integ-test/build/testclusters/*/logs/*
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in rolling upgrade
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-rolling-upgrade.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*
+
+  bwc-tests-full-restart:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21, 24]
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in full restart
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-full-restart.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*

integ-test/build.gradle

@@ -592,101 +592,96 @@ task comparisonTest(type: RestIntegTestTask) {
     systemProperty "queries", System.getProperty("queries")
 }
 
-2.times { i ->
-    testClusters {
-        "${baseName}$i" {
-            testDistribution = "ARCHIVE"
-            versions = [baseVersion, opensearch_version]
-            numberOfNodes = 3
-            plugin(provider { (RegularFile) (() -> {
-                if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
-                    project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
-                }
-                project.mkdir bwcJobSchedulerPath + bwcVersion
-                ant.get(src: bwcOpenSearchJSDownload,
-                        dest: bwcJobSchedulerPath + bwcVersion,
-                        httpusecaches: false)
-                return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
-            })})
-            plugin(provider { (RegularFile) (() -> {
-                return configurations.zipArchive.asFileTree.matching {
-                    include '**/opensearch-sql-plugin*'
-                }.singleFile
-            })})
-            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-            setting 'http.content_type.required', 'true'
-        }
+testClusters {
+    "${baseName}" {
+        testDistribution = "ARCHIVE"
+        versions = [baseVersion, opensearch_version]
+        numberOfNodes = 3
+        plugin(provider { (RegularFile) (() -> {
+            if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
+                project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
+            }
+            project.mkdir bwcJobSchedulerPath + bwcVersion
+            ant.get(src: bwcOpenSearchJSDownload,
+                    dest: bwcJobSchedulerPath + bwcVersion,
+                    httpusecaches: false)
+            return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
+        })})
+        plugin(provider { (RegularFile) (() -> {
+            return configurations.zipArchive.asFileTree.matching {
+                include '**/opensearch-sql-plugin*'
+            }.singleFile
+        })})
+        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+        setting 'http.content_type.required', 'true'
     }
 }
-
 List<Provider<RegularFile>> plugins = [
         getJobSchedulerPlugin(),
         provider { (RegularFile) (() ->
                 fileTree(bwcFilePath + project.version).getSingleFile())
         }
 ]
 
-// Creates 2 test clusters with 3 nodes of the old version.
-2.times { i ->
-    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
-        useCluster testClusters."${baseName}$i"
-        filter {
-            includeTestsMatching "org.opensearch.sql.bwc.*IT"
-        }
-        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
-        systemProperty 'tests.rest.bwcsuite_round', 'old'
-        systemProperty 'tests.plugin_bwc_version', bwcVersion
-        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
-        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
+// Creates test cluster with 3 nodes of the old version.
+task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) {
+    useCluster testClusters."${baseName}"
+    filter {
+        includeTestsMatching "org.opensearch.sql.bwc.*IT"
     }
+    systemProperty 'tests.rest.bwcsuite', 'old_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'old'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version.
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 // This is also used as a one third upgraded cluster for a rolling upgrade.
 task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
-    useCluster testClusters."${baseName}0"
-    dependsOn "${baseName}#oldVersionClusterTask0"
+    useCluster testClusters."${baseName}"
+    dependsOn "${baseName}#oldVersionClusterTask"
     doFirst {
-        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.sql.bwc.*IT"
     }
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'first'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
 // This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
 // This is used for rolling upgrade.
 task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#mixedClusterTask"
-    useCluster testClusters."${baseName}0"
+    useCluster testClusters."${baseName}"
     doFirst {
-        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.sql.bwc.*IT"
     }
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'second'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
 // This results in a fully upgraded cluster.
 // This is used for rolling upgrade.
 task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
-    useCluster testClusters."${baseName}0"
+    useCluster testClusters."${baseName}"
     doFirst {
-        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.sql.bwc.*IT"
@@ -695,36 +690,45 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'third'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
 // Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
 // at the same time resulting in a fully upgraded cluster.
 task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
-    dependsOn "${baseName}#oldVersionClusterTask1"
-    useCluster testClusters."${baseName}1"
+    dependsOn "${baseName}#oldVersionClusterTask"
+    useCluster testClusters."${baseName}"
     doFirst {
-        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+        testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins)
     }
     filter {
         includeTestsMatching "org.opensearch.sql.bwc.*IT"
     }
     systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
-    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
 }
 
-// A bwc test suite which runs all the bwc tasks combined
-task bwcTestSuite(type: StandaloneRestIntegTestTask) {
+// A bwc test suite which runs all the bwc tasks in rolling upgrade
+task bwcTestRollingUpgradeSuite(type: StandaloneRestIntegTestTask) {
     testLogging {
         events "passed", "skipped", "failed"
     }
     exclude '**/*Test*'
     exclude '**/*IT*'
     dependsOn tasks.named("${baseName}#mixedClusterTask")
     dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+}
+
+// A bwc test suite which runs all the bwc tasks in full restart
+task bwcTestFullRestartSuite(type: StandaloneRestIntegTestTask) {
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+    exclude '**/*Test*'
+    exclude '**/*IT*'
     dependsOn tasks.named("${baseName}#fullRestartClusterTask")
 }
 

scripts/bwctest.sh → scripts/bwctest-full-restart.sh

@@ -54,5 +54,5 @@ function setup_bwc_artifact() {
 }
 
 setup_bwc_artifact
-./gradlew bwcTestSuite -Dtests.security.manager=false
+./gradlew bwcTestFullRestartSuite -Dtests.security.manager=false
 

scripts/bwctest-rolling-upgrade.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -e
+
+function usage() {
+    echo ""
+    echo "This script is used to run Backwards Compatibility tests"
+    echo "--------------------------------------------------------------------------"
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Required arguments:"
+    echo "None"
+    echo ""
+    echo -e "-h\tPrint this message."
+    echo "--------------------------------------------------------------------------"
+}
+
+while getopts ":h" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${OPTARG}"
+            exit 1
+            ;;
+    esac
+done
+
+# Place SQL artifact for the current version for bwc
+function setup_bwc_artifact() {
+    # This gets opensearch version from build.gradle (e.g. 1.2.0-SNAPSHOT),
+    # then converts to plugin version by appending ".0" (e.g. 1.2.0.0-SNAPSHOT),
+    # assuming one line in build.gradle is 'opensearch_version = System.getProperty("opensearch.version", "<opensearch_version>")'.
+    plugin_version=$(grep 'opensearch_version = System.getProperty' build.gradle | \
+        grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+[^"]*' | sed -e 's/\(.*\)\(\.[0-9]\)/\1\2.0/')
+    plugin_artifact="./plugin/build/distributions/opensearch-sql-$plugin_version.zip"
+    bwc_artifact_dir="./integ-test/src/test/resources/bwc/$plugin_version"
+
+    if [ -z "${plugin_version// }" ]; then
+        echo "Error: failed to retrieve plugin version from build.gradle." >&2
+        exit 1
+    fi
+
+    # copy current artifact to bwc artifact directory if it's not there
+    if [ ! -f "$bwc_artifact_dir/opensearch-sql-$plugin_version.zip" ]; then
+        if [ ! -f "$plugin_artifact" ]; then
+            ./gradlew assemble
+        fi
+        mkdir -p "$bwc_artifact_dir"
+        cp "$plugin_artifact" "$bwc_artifact_dir"
+    fi
+}
+
+setup_bwc_artifact
+./gradlew bwcTestRollingUpgradeSuite -Dtests.security.manager=false
+