Merge branch 'main' into feature/headtop-limit

Signed-off-by: Simeon Widdis <[email protected]>

Author: Swiddis

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -30,6 +30,7 @@ public enum Key {
     PATTERN_MAX_SAMPLE_COUNT("plugins.ppl.pattern.max.sample.count"),
     PATTERN_BUFFER_LIMIT("plugins.ppl.pattern.buffer.limit"),
     PPL_REX_MAX_MATCH_LIMIT("plugins.ppl.rex.max_match.limit"),
+    PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"),
 
     /** Enable Calcite as execution engine */
     CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"),

core/build.gradle

@@ -54,6 +54,7 @@ dependencies {
     api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
     api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
     api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
     api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
     api group: 'com.tdunning', name: 't-digest', version: '3.3'
     api "net.minidev:json-smart:${versions.json_smart}"

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -86,6 +86,7 @@
 import org.opensearch.sql.ast.tree.Rename;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
+import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.Sort.SortOption;
 import org.opensearch.sql.ast.tree.SubqueryAlias;
@@ -278,6 +279,18 @@ public LogicalPlan visitLimit(Limit node, AnalysisContext context) {
     return new LogicalLimit(child, node.getLimit(), node.getOffset());
   }
 
+  @Override
+  public LogicalPlan visitSearch(Search node, AnalysisContext context) {
+    LogicalPlan child = node.getChild().get(0).accept(this, context);
+    Function queryStringFunc =
+        AstDSL.function(
+            "query_string",
+            AstDSL.unresolvedArg("query", AstDSL.stringLiteral(node.getQueryString())));
+
+    Expression analyzed = expressionAnalyzer.analyze(queryStringFunc, context);
+    return new LogicalFilter(child, analyzed);
+  }
+
   @Override
   public LogicalPlan visitFilter(Filter node, AnalysisContext context) {
     LogicalPlan child = node.getChild().get(0).accept(this, context);
@@ -880,7 +893,11 @@ private LogicalAggregation analyzeAggregation(
     groupBys.forEach(
         group ->
             newEnv.define(new Symbol(Namespace.FIELD_NAME, group.getNameOrAlias()), group.type()));
-    return new LogicalAggregation(child, aggregators, groupBys);
+
+    Argument.ArgumentMap statsArgs = Argument.ArgumentMap.of(node.getArgExprList());
+    boolean bucketNullable =
+        (Boolean) statsArgs.getOrDefault(Argument.BUCKET_NULLABLE, Literal.TRUE).getValue();
+    return new LogicalAggregation(child, aggregators, groupBys, bucketNullable);
   }
 
   private Aggregation analyzePatternsAgg(Patterns node) {

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -17,6 +17,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.Getter;
 import org.opensearch.sql.analysis.symbol.Namespace;
@@ -53,6 +54,7 @@
 import org.opensearch.sql.ast.expression.subquery.ExistsSubquery;
 import org.opensearch.sql.ast.expression.subquery.InSubquery;
 import org.opensearch.sql.ast.expression.subquery.ScalarSubquery;
+import org.opensearch.sql.calcite.utils.CalciteUtils;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
@@ -191,6 +193,7 @@ public Expression visitRelevanceFieldList(RelevanceFieldList node, AnalysisConte
   @Override
   public Expression visitFunction(Function node, AnalysisContext context) {
     FunctionName functionName = FunctionName.of(node.getFuncName());
+    validateCalciteOnlyFunction(functionName);
     List<Expression> arguments =
         node.getFuncArgs().stream()
             .map(
@@ -208,6 +211,34 @@ public Expression visitFunction(Function node, AnalysisContext context) {
         repository.compile(context.getFunctionProperties(), functionName, arguments);
   }
 
+  /**
+   * Validates that functions requiring Calcite engine are not used without it.
+   *
+   * @param functionName The function name to validate
+   */
+  private void validateCalciteOnlyFunction(FunctionName functionName) {
+    if (isCalciteOnlyFunction(functionName)) {
+      throw CalciteUtils.getOnlyForCalciteException(functionName.getFunctionName().toUpperCase());
+    }
+  }
+
+  /**
+   * Checks if a function requires Calcite engine to be enabled.
+   *
+   * @param functionName The function name to check
+   * @return true if the function requires Calcite, false otherwise
+   */
+  private boolean isCalciteOnlyFunction(FunctionName functionName) {
+    // Set of functions that are only supported with Calcite engine
+    Set<String> calciteOnlyFunctions =
+        ImmutableSet.of(
+            BuiltinFunctionName.REGEX_MATCH.getName().getFunctionName(),
+            BuiltinFunctionName.STRFTIME.getName().getFunctionName());
+
+    return calciteOnlyFunctions.stream()
+        .anyMatch(f -> f.equalsIgnoreCase(functionName.getFunctionName()));
+  }
+
   @SuppressWarnings("unchecked")
   @Override
   public Expression visitWindowFunction(WindowFunction node, AnalysisContext context) {

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -75,6 +75,7 @@
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
 import org.opensearch.sql.ast.tree.SPath;
+import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.SubqueryAlias;
 import org.opensearch.sql.ast.tree.TableFunction;
@@ -131,6 +132,10 @@ public T visitTableFunction(TableFunction node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitSearch(Search node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitFilter(Filter node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -71,6 +71,7 @@
 import org.opensearch.sql.ast.tree.RelationSubquery;
 import org.opensearch.sql.ast.tree.Rename;
 import org.opensearch.sql.ast.tree.SPath;
+import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.Sort.SortOption;
 import org.opensearch.sql.ast.tree.SpanBin;
@@ -109,6 +110,10 @@ public UnresolvedPlan describe(String tableName) {
     return new DescribeRelation(qualifiedName(tableName));
   }
 
+  public static UnresolvedPlan search(UnresolvedPlan input, String queryString) {
+    return new Search(input, queryString);
+  }
+
   public UnresolvedPlan subqueryAlias(UnresolvedPlan child, String alias) {
     return new SubqueryAlias(child, alias);
   }
@@ -449,6 +454,7 @@ public static List<Argument> defaultStatsArgs() {
         argument("partitions", intLiteral(1)),
         argument("allnum", booleanLiteral(false)),
         argument("delim", stringLiteral(" ")),
+        argument(Argument.BUCKET_NULLABLE, booleanLiteral(true)),
         argument("dedupsplit", booleanLiteral(false)));
   }
 

core/src/main/java/org/opensearch/sql/ast/expression/Argument.java

@@ -20,6 +20,8 @@
 @RequiredArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 public class Argument extends UnresolvedExpression {
+  public static final String BUCKET_NULLABLE = "bucket_nullable";
+
   private final String argName;
   private final Literal value;
 
@@ -66,5 +68,9 @@ public static ArgumentMap empty() {
     public Literal get(String name) {
       return map.get(name);
     }
+
+    public Literal getOrDefault(String name, Literal literal) {
+      return map.getOrDefault(name, literal);
+    }
   }
 }

core/src/main/java/org/opensearch/sql/ast/expression/Literal.java

@@ -49,5 +49,6 @@ public String toString() {
 
   public static Literal TRUE = new Literal(true, DataType.BOOLEAN);
   public static Literal FALSE = new Literal(false, DataType.BOOLEAN);
-  public static Literal ZERO = new Literal(Integer.valueOf("0"), DataType.INTEGER);
+  public static Literal ZERO = new Literal(0, DataType.INTEGER);
+  public static Literal ONE = new Literal(1, DataType.INTEGER);
 }

core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.Arrays;
+import java.util.List;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+
+/** Search expression for AND operator. */
+@Getter
+@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@ToString
+public class SearchAnd extends SearchExpression {
+
+  private final SearchExpression left;
+  private final SearchExpression right;
+
+  @Override
+  public String toQueryString() {
+    return left.toQueryString() + " AND " + right.toQueryString();
+  }
+
+  @Override
+  public List<? extends UnresolvedExpression> getChild() {
+    return Arrays.asList(left, right);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.Arrays;
+import java.util.List;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+import org.opensearch.sql.utils.QueryStringUtils;
+
+/** Search expression for field comparisons. */
+@Getter
+@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@ToString
+public class SearchComparison extends SearchExpression {
+
+  public enum Operator {
+    EQUALS("="),
+    NOT_EQUALS("!="),
+    LESS_THAN("<"),
+    LESS_OR_EQUAL("<="),
+    GREATER_THAN(">"),
+    GREATER_OR_EQUAL(">=");
+
+    private final String symbol;
+
+    Operator(String symbol) {
+      this.symbol = symbol;
+    }
+
+    public String getSymbol() {
+      return symbol;
+    }
+  }
+
+  private final Field field;
+  private final Operator operator;
+  private final SearchLiteral value;
+
+  @Override
+  public String toQueryString() {
+    String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString());
+    String valueStr = value.toQueryString();
+    switch (operator) {
+      case NOT_EQUALS:
+        return "( _exists_:" + fieldName + " AND NOT " + fieldName + ":" + valueStr + " )";
+      case GREATER_THAN:
+        return fieldName + ":>" + valueStr;
+      case GREATER_OR_EQUAL:
+        return fieldName + ":>=" + valueStr;
+      case LESS_THAN:
+        return fieldName + ":<" + valueStr;
+      case LESS_OR_EQUAL:
+        return fieldName + ":<=" + valueStr;
+      default:
+        return fieldName + ":" + valueStr;
+    }
+  }
+
+  @Override
+  public List<? extends UnresolvedExpression> getChild() {
+    return Arrays.asList(field, value);
+  }
+}