Merge remote-tracking branch 'origin/main' into issues/399

Signed-off-by: Yuanchun Shen <[email protected]>

Author: yuancu

.github/workflows/sql-test-and-build-workflow.yml

@@ -179,7 +179,7 @@ jobs:
             doctest/build/testclusters/docTestCluster-0/logs/*
             integ-test/build/testclusters/*/logs/*
 
-  bwc-tests:
+  bwc-tests-rolling-upgrade:
     needs: Get-CI-Image-Tag
     runs-on: ubuntu-latest
     strategy:
@@ -190,37 +190,83 @@ jobs:
       options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - name: Run start commands
-      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
-
-    - uses: actions/checkout@v4
-
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v4
-      with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java }}
-
-    - name: Run backward compatibility tests
-      run: |
-        chown -R 1000:1000 `pwd`
-        su `id -un 1000` -c "./scripts/bwctest.sh"
-
-    - name: Upload test reports
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v4
-      continue-on-error: true
-      with:
-        name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
-        path: |
-          sql/build/reports/**
-          ppl/build/reports/**
-          core/build/reports/**
-          common/build/reports/**
-          opensearch/build/reports/**
-          integ-test/build/reports/**
-          protocol/build/reports/**
-          legacy/build/reports/**
-          plugin/build/reports/**
-          doctest/build/testclusters/docTestCluster-0/logs/*
-          integ-test/build/testclusters/*/logs/*
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in rolling upgrade
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-rolling-upgrade.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*
+
+  bwc-tests-full-restart:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21, 24]
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in full restart
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-full-restart.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*

api/build.gradle

@@ -15,7 +15,7 @@ dependencies {
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
-    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.38.0'
+    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0'
 }
 
 spotless {

core/build.gradle

@@ -58,19 +58,19 @@ dependencies {
     api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
     api group: 'com.tdunning', name: 't-digest', version: '3.3'
     api "net.minidev:json-smart:${versions.json_smart}"
-    api('org.apache.calcite:calcite-core:1.38.0') {
+    api('org.apache.calcite:calcite-core:1.41.0') {
         exclude group: 'net.minidev', module: 'json-smart'
-        exclude group: 'commons-lang', module: 'commons-lang'
     }
-    api 'org.apache.calcite:calcite-linq4j:1.38.0'
+    api 'org.apache.calcite:calcite-linq4j:1.41.0'
     api project(':common')
     implementation "com.github.seancfoley:ipaddress:5.4.2"
     implementation "com.jayway.jsonpath:json-path:2.9.0"
 
     annotationProcessor('org.immutables:value:2.8.8')
-    compileOnly('org.immutables:value-annotations:2.8.8')
+    compileOnly 'org.immutables:value-annotations:2.8.8'
+    compileOnlyApi 'com.google.code.findbugs:jsr305:3.0.2'
 
-    testImplementation('org.junit.jupiter:junit-jupiter:5.9.3')
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
     testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}"

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -93,6 +93,7 @@
 import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.Sort.SortOption;
+import org.opensearch.sql.ast.tree.StreamWindow;
 import org.opensearch.sql.ast.tree.SubqueryAlias;
 import org.opensearch.sql.ast.tree.TableFunction;
 import org.opensearch.sql.ast.tree.Timechart;
@@ -749,6 +750,11 @@ public LogicalPlan visitTrendline(Trendline node, AnalysisContext context) {
         computationsAndTypes.build());
   }
 
+  @Override
+  public LogicalPlan visitStreamWindow(StreamWindow node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Streamstats");
+  }
+
   @Override
   public LogicalPlan visitFlatten(Flatten node, AnalysisContext context) {
     throw getOnlyForCalciteException("Flatten");

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -80,6 +80,7 @@
 import org.opensearch.sql.ast.tree.SPath;
 import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
+import org.opensearch.sql.ast.tree.StreamWindow;
 import org.opensearch.sql.ast.tree.SubqueryAlias;
 import org.opensearch.sql.ast.tree.TableFunction;
 import org.opensearch.sql.ast.tree.Timechart;
@@ -415,6 +416,10 @@ public T visitWindow(Window window, C context) {
     return visitChildren(window, context);
   }
 
+  public T visitStreamWindow(StreamWindow node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitJoin(Join node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java

@@ -18,7 +18,7 @@
 public class AllFields extends UnresolvedExpression {
   public static final AllFields INSTANCE = new AllFields();
 
-  public AllFields() {}
+  protected AllFields() {}
 
   public static AllFields of() {
     return INSTANCE;

core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java

@@ -22,6 +22,7 @@
 @Getter
 @EqualsAndHashCode(callSuper = false)
 public class QualifiedName extends UnresolvedExpression {
+  public static final String DELIMITER = ".";
   private final List<String> parts;
 
   public QualifiedName(String name) {
@@ -94,7 +95,7 @@ public QualifiedName rest() {
   }
 
   public String toString() {
-    return String.join(".", this.parts);
+    return String.join(DELIMITER, this.parts);
   }
 
   @Override

core/src/main/java/org/opensearch/sql/ast/tree/StreamWindow.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+
+@Getter
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class StreamWindow extends UnresolvedPlan {
+
+  private final List<UnresolvedExpression> windowFunctionList;
+  private final List<UnresolvedExpression> groupList;
+  private final boolean current;
+  private final int window;
+  private final boolean global;
+  private final UnresolvedExpression resetBefore;
+  private final UnresolvedExpression resetAfter;
+  @ToString.Exclude private UnresolvedPlan child;
+
+  /** StreamWindow Constructor. */
+  public StreamWindow(
+      List<UnresolvedExpression> windowFunctionList,
+      List<UnresolvedExpression> groupList,
+      boolean current,
+      int window,
+      boolean global,
+      UnresolvedExpression resetBefore,
+      UnresolvedExpression resetAfter) {
+    this.windowFunctionList = windowFunctionList;
+    this.groupList = groupList;
+    this.current = current;
+    this.window = window;
+    this.global = global;
+    this.resetBefore = resetBefore;
+    this.resetAfter = resetAfter;
+  }
+
+  public boolean isCurrent() {
+    return current;
+  }
+
+  public boolean isGlobal() {
+    return global;
+  }
+
+  @Override
+  public StreamWindow attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitStreamWindow(this, context);
+  }
+}