Merge remote-tracking branch 'origin/main' into issues/4582

Signed-off-by: Yuanchun Shen <[email protected]>

Author: yuancu

.github/workflows/maven-publish.yml

@@ -5,11 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
-
-env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:
@@ -35,9 +32,13 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
-
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
       - name: publish snapshots to maven
         run: |
-          ./gradlew publishPluginZipPublicationToSnapshotsRepository
+          ./gradlew publishPluginZipPublicationToSnapshotsRepository

.github/workflows/publish-async-query-core.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'async-query-core/**'
       - '.github/workflows/publish-async-query-core.yml'
@@ -18,7 +18,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-async-query-core.json
 
 jobs:
@@ -47,8 +46,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set commit ID
         id: set_commit
@@ -83,4 +93,4 @@ jobs:
           source ./.github/maven-publish-utils.sh
           
           # Call the main function for async-query-core
-          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"
+          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"

.github/workflows/publish-grammar-files.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'language-grammar/src/main/antlr4/**'
       - 'language-grammar/build.gradle'
@@ -19,7 +19,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-language-grammar.json
 
 jobs:
@@ -51,8 +50,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set version
         id: set_version

async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java

@@ -72,7 +72,7 @@ public class SparkConstants {
   public static final String PPL_STANDALONE_PACKAGE =
       "org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT";
   public static final String AWS_SNAPSHOT_REPOSITORY =
-      "https://central.sonatype.com/repository/maven-snapshots/";
+      "https://ci.opensearch.org/ci/dbc/snapshots/maven/";
   public static final String GLUE_HIVE_CATALOG_FACTORY_CLASS =
       "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory";
   public static final String FLINT_DELEGATE_CATALOG =

async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java

@@ -981,7 +981,7 @@ private String constructExpectedSparkSubmitParameterString(
             "spark.hadoop.fs.s3.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider",
             "spark.hadoop.aws.catalog.credentials.provider.factory.class=com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory",
             "spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT",
-            "spark.jars.repositories=https://central.sonatype.com/repository/maven-snapshots/",
+            "spark.jars.repositories=https://ci.opensearch.org/ci/dbc/snapshots/maven/",
             "spark.emr-serverless.driverEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/",
             "spark.executorEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/",
             "spark.emr-serverless.driverEnv.FLINT_CLUSTER_NAME=TEST_CLUSTER",

build.gradle

@@ -6,7 +6,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.3.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.4.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -67,8 +67,7 @@ buildscript {
     repositories {
         mavenLocal()
         mavenCentral()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
     }
 
     dependencies {
@@ -93,9 +92,8 @@ apply plugin: 'opensearch.java-agent'
 repositories {
     mavenLocal()
     mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
-    maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
+    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
     maven { url 'https://jitpack.io' }
-    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
 }
 
 spotless {
@@ -158,9 +156,9 @@ subprojects {
     repositories {
         mavenLocal()
         mavenCentral()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
         maven { url 'https://jitpack.io' }
-        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
     }
 }
 

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -80,6 +80,7 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /** Class of static methods to create specific node instances. */
 @UtilityClass
@@ -491,15 +492,19 @@ public static Span spanFromSpanLengthLiteral(
       UnresolvedExpression field, Literal spanLengthLiteral) {
     if (spanLengthLiteral.getType() == DataType.STRING) {
       String spanText = spanLengthLiteral.getValue().toString();
-      String valueStr = spanText.replaceAll("[^0-9]", "");
-      String unitStr = spanText.replaceAll("[0-9]", "");
+      String valueStr = spanText.replaceAll("[^0-9-]", "");
+      String unitStr = spanText.replaceAll("[0-9-]", "");
 
       if (valueStr.isEmpty()) {
         // No numeric value found, use the literal as-is
         return new Span(field, spanLengthLiteral, SpanUnit.NONE);
       } else {
         // Parse numeric value and unit
         Integer value = Integer.parseInt(valueStr);
+        if (value <= 0) {
+          throw new IllegalArgumentException(
+              String.format("Zero or negative time interval not supported: %s", spanText));
+        }
         SpanUnit unit = unitStr.isEmpty() ? SpanUnit.NONE : SpanUnit.of(unitStr);
         return span(field, intLiteral(value), unit);
       }
@@ -713,4 +718,9 @@ public static Bin bin(UnresolvedExpression field, Argument... arguments) {
       return DefaultBin.builder().field(field).alias(alias).build();
     }
   }
+
+  /** Get a reference to the implicit timestamp field {@code @timestamp} */
+  public static Field referImplicitTimestampField() {
+    return AstDSL.field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP);
+  }
 }

core/src/main/java/org/opensearch/sql/ast/statement/Explain.java

@@ -39,7 +39,9 @@ public enum ExplainFormat {
     SIMPLE,
     STANDARD,
     EXTENDED,
-    COST
+    COST,
+    /** Formats explain output in yaml format. */
+    YAML
   }
 
   public static ExplainFormat format(String format) {

core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java

@@ -109,6 +109,22 @@ public boolean equals(Object o) {
     public int hashCode() {
       return Objects.hash(root, calcite);
     }
+
+    public static ExplainResponse normalizeLf(ExplainResponse response) {
+      ExecutionEngine.ExplainResponseNodeV2 calcite = response.getCalcite();
+      if (calcite != null) {
+        return new ExplainResponse(
+            new ExecutionEngine.ExplainResponseNodeV2(
+                normalizeLf(calcite.getLogical()),
+                normalizeLf(calcite.getPhysical()),
+                normalizeLf(calcite.getExtended())));
+      }
+      return response;
+    }
+
+    private static String normalizeLf(String value) {
+      return value == null ? null : value.replace("\r\n", "\n");
+    }
   }
 
   @AllArgsConstructor

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -201,7 +201,8 @@ public void explainWithLegacy(
       Explain.ExplainFormat format,
       Optional<Throwable> calciteFailure) {
     try {
-      if (format != null && format != Explain.ExplainFormat.STANDARD) {
+      if (format != null
+          && (format != Explain.ExplainFormat.STANDARD && format != Explain.ExplainFormat.YAML)) {
         throw new UnsupportedOperationException(
             "Explain mode " + format.name() + " is not supported in v2 engine");
       }