From 0ed23defed4e13a8485669279718de3f33f90ceb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 15 Oct 2025 22:08:58 -0700 Subject: [PATCH 01/24] PPL tostring() implementation issue #4492 Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 7 + .../function/PPLBuiltinOperators.java | 2 + .../expression/function/PPLFuncImpTable.java | 3 + .../function/udf/ToStringFunction.java | 162 +++++++++++++++ .../function/udf/ToStringFunctionTest.java | 149 ++++++++++++++ docs/user/ppl/functions/conversion.rst | 92 +++++++++ docs/user/ppl/functions/string.rst | 3 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 6 +- .../sql/ppl/parser/AstExpressionBuilder.java | 11 +- .../calcite/CalcitePPLStringFunctionTest.java | 185 ++++++++++++++++++ sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + 13 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 12a9a297542..69e1492538c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -112,6 +112,13 @@ private PPLOperandTypes() {} SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)); + public static final UDFOperandMetadata BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING = + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.family(SqlTypeFamily.BOOLEAN) + .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 76f625ebd38..984f398b1ee 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -66,6 +66,7 @@ import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction; import org.opensearch.sql.expression.function.udf.RexOffsetFunction; import org.opensearch.sql.expression.function.udf.SpanFunction; +import org.opensearch.sql.expression.function.udf.ToStringFunction; import org.opensearch.sql.expression.function.udf.condition.EarliestFunction; import org.opensearch.sql.expression.function.udf.condition.EnhancedCoalesceFunction; import org.opensearch.sql.expression.function.udf.condition.LatestFunction; @@ -411,6 +412,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match", false); public static final SqlOperator NUMBER_TO_STRING = new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); + public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); public static final SqlOperator WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() .toUDF("WIDTH_BUCKET"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 8297ecf73ce..de89df383d2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -210,6 +210,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TOSTRING; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; @@ -887,6 +888,7 @@ void populate() { registerOperator(WEEKOFYEAR, PPLBuiltinOperators.WEEK); registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER); + registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING); // Register MVJOIN to use Calcite's ARRAY_JOIN register( @@ -1058,6 +1060,7 @@ void populate() { SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), false)); + register( LOG, (FunctionImp2) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java new file mode 100644 index 00000000000..70d2a82b13c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.text.NumberFormat; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.function.Strict; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * A custom implementation of number/boolean to string . + * + *

This operator is necessary because tostring has following requirements "binary" Converts a + * number to a binary value. "hex" Converts the number to a hexadecimal value. "commas" Formats the + * number with commas. If the number includes a decimal, the function rounds the number to nearest + * two decimal places. "duration" Converts the value in seconds to the readable time format + * HH:MM:SS. if not format parameter provided, then consider value as boolean + */ +public class ToStringFunction extends ImplementorUDF { + public ToStringFunction() { + super( + new org.opensearch.sql.expression.function.udf.ToStringFunction.ToStringImplementor(), + NullPolicy.ANY); + } + + public static final String DURATION_FORMAT = "duration"; + public static final String HEX_FORMAT = "hex"; + public static final String COMMAS_FORMAT = "commas"; + public static final String BINARY_FORMAT = "binary"; + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return PPLReturnTypes.STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING; + } + + public static class ToStringImplementor implements NotNullImplementor { + + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + Expression fieldValue = translatedOperands.get(0); + if (translatedOperands.size() > 1) { + Expression format = translatedOperands.get(1); + return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); + } else { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } + } + } + + @Strict + public static String toString(boolean fieldValue) { + if (fieldValue) { + return "True"; + } else { + return "False"; + } + } + + @Strict + public static String toString(String fieldValue) { + return toString(Boolean.parseBoolean(fieldValue)); + } + + @Strict + public static String toString(BigDecimal num, String format) { + if (format.equals(DURATION_FORMAT)) { + Duration d = Duration.ofSeconds(num.toBigInteger().longValue()); + long hours = d.toHours(); + int minutes = d.toMinutesPart(); + int remainingSeconds = d.toSecondsPart(); + + String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return num.toBigInteger().toString(16); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + nf.setMinimumFractionDigits(0); + nf.setMaximumFractionDigits(2); + return nf.format(num); + + } else if (format.equals(BINARY_FORMAT)) { + BigInteger integerPart = num.toBigInteger(); // 42 + return integerPart.toString(2); + } + return num.toString(); + } + + @Strict + public static String toString(double num, String format) { + if (format.equals(DURATION_FORMAT)) { + Duration d = Duration.ofSeconds(Math.round(num)); + long hours = d.toHours(); + int minutes = d.toMinutesPart(); + int remainingSeconds = d.toSecondsPart(); + String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return Double.toHexString(num); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + return nf.format(num); + } else if (format.equals(BINARY_FORMAT)) { + return Long.toBinaryString(Double.doubleToLongBits(num)); + } + return Double.toString(num); + } + + @Strict + public static String toString(int num, String format) { + + if (format.equals(DURATION_FORMAT)) { + + int hours = num / 3600; + int minutes = (num % 3600) / 60; + int seconds = num % 60; + + String time_str = String.format("%02d:%02d:%02d", hours, minutes, seconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return Integer.toHexString(num); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + return nf.format(num); + } else if (format.equals(BINARY_FORMAT)) { + return Integer.toBinaryString(num); + } + return Integer.toString(num); + } + + @Strict + public static String toString(String str, String format) { + if (str.contains(".") || (str.length() > 10)) { + return toString(Double.parseDouble(str), format); + } else { + return toString(Integer.parseInt(str), format); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java new file mode 100644 index 00000000000..cfd25796b23 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.*; + +public class ToStringFunctionTest { + + private final ToStringFunction function = new ToStringFunction(); + + @Test + void testBooleanToString() { + assertEquals("True", ToStringFunction.toString(true)); + assertEquals("False", ToStringFunction.toString(false)); + } + + @Test + void testStringBooleanToString() { + assertEquals("True", ToStringFunction.toString("true")); + assertEquals("False", ToStringFunction.toString("false")); + assertEquals("False", ToStringFunction.toString("anythingElse")); + } + + @Test + void testBigDecimalToStringDurationFormat() { + BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testBigDecimalToStringHexFormat() { + BigDecimal num = new BigDecimal("255"); + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertEquals("ff", result); + } + + @Test + void testBigDecimalToStringCommasFormat() { + Locale.setDefault(Locale.US); // Ensure predictable comma placement + BigDecimal num = new BigDecimal("1234567.891"); + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testBigDecimalToStringBinaryFormat() { + BigDecimal num = new BigDecimal("10"); + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertEquals("1010", result); + } + + @Test + void testBigDecimalToStringDefault() { + BigDecimal num = new BigDecimal("123.45"); + assertEquals("123.45", ToStringFunction.toString(num, "unknown")); + } + + @Test + void testDoubleToStringDurationFormat() { + double num = 3661.4; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testDoubleToStringHexFormat() { + double num = 10.5; + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertTrue(result.startsWith("0x")); + } + + @Test + void testDoubleToStringCommasFormat() { + Locale.setDefault(Locale.US); + double num = 12345.678; + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testDoubleToStringBinaryFormat() { + double num = 10.0; + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testDoubleToStringDefault() { + assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); + } + + @Test + void testIntToStringDurationFormat() { + int num = 3661; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testIntToStringHexFormat() { + assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); + } + + @Test + void testIntToStringCommasFormat() { + Locale.setDefault(Locale.US); + String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testIntToStringBinaryFormat() { + assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); + } + + @Test + void testIntToStringDefault() { + assertEquals("123", ToStringFunction.toString(123, "unknown")); + } + + @Test + void testStringNumericToStringIntFormat() { + String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); + assertEquals("2a", result); + } + + @Test + void testStringNumericToStringDoubleFormat() { + String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains("42")); + } + + @Test + void testStringLargeNumberAsDouble() { + String largeNum = "1234567890123"; + String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + } +} diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index dbe4403540c..21124c27edc 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -78,3 +78,95 @@ Cast function can be chained:: |-------| | True | +-------+ + +TOSTRING +----- + +Description +>>>>>>>>>>> +There are two available usage based on paraemter types and number of parameters. +Usage with format type: tostring(number|string, string) converts the number in first argument to provided format type string in second argument. + Return type: string +Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. + Return type: string + +You can use this function with the eval commands and as part of eval expressions. +The first argument can be a number, number as string or boolean. +If first argument is a a number or number as string , second argument need to be format name. +If first argument is boolean, then second argument is not needed. + +format types: +a) "binary" Converts a number to a binary value. +b) "hex" Converts the number to a hexadecimal value. +c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. +d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. + +Binary conversion +You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: +eval result = tostring(9, "binary") + +For information about bitwise functions that you can use with the tostring function, see Bitwise functions. + +Basic examples +The following example returns "True 0xF 12,345.68". +... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") +The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. + +... | eval foo=615 | eval foo2 = tostring(foo, "duration") +The following example formats the column totalSales to display values with a currency symbol and commas. You must use a period between the currency value and the tostring function. + +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + +---------------------+ + | boolean_str | + |---------------------+ + | True | + +---------------------+ + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_binary | SAL | + |---------------+------------------+------------+ + | SMITH | 1001110001000000 | 80000.00 | + +---------------+------------------+------------+ + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_hex | SAL | + |---------------+------------------+------------+ + | SMITH | 13880 | 80000.00 | + +---------------+---------------+---------------+ + + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_commas | SAL | + |---------------+------------------+------------+ + | SMITH | 80,000 | 80000.00 | + +---------------+------------------+------------+ + + + duration + + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" + fetched rows / total rows = 1/1 + +---------------+-------------+ + | ENAME | duration | + |---------------+-------------+ + | SMITH | 01:48:20 | + +---------------+-------------+ + +Usage for boolean parameter without format type:: + +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + +---------------------+ + | boolean_str | + |---------------------+ + | True | + +---------------------+ \ No newline at end of file diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 24efa1434f5..43efed3c470 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -397,3 +397,6 @@ Example:: |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | +---------------------+---------------------+ + + + diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ba1e4960bb2..e528bb553ab 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -392,6 +392,7 @@ STRFTIME: 'STRFTIME'; // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; +TOSTRING: 'TOSTRING'; LTRIM: 'LTRIM'; RTRIM: 'RTRIM'; TRIM: 'TRIM'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index e13447b68e9..852c55c863c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -847,11 +847,14 @@ evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS ; -// cast function + +// cast, tostring function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS + | TOSTRING LT_PRTHS functionArgs RT_PRTHS ; + convertedDataType : typeName = DATE | typeName = TIME @@ -1434,6 +1437,7 @@ searchableKeyWord | USING | VALUE | CAST + | TOSTRING | GET_FORMAT | EXTRACT | INTERVAL diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f037376f5c2..1460a3efc9a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -414,7 +414,16 @@ private Function buildFunction( /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + ParseTree rootNode = ctx.getChild(0); + String functionName = rootNode.getText(); + final String mappedName = + FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); + + if (mappedName.equals("cast")) { + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + } else { + return buildFunction(mappedName, ctx.functionArgs().functionArg()); + } } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index 1e97052dea0..d41b2c22453 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -46,6 +46,191 @@ public void testLower() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testToStringBoolean() { + String ppl = "source=EMP | eval boolean_value = tostring(1==1) | fields boolean_value |head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(boolean_value=[TOSTRING(true)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "boolean_value=True\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TOSTRING`(TRUE) `boolean_value`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringBin() { + String ppl = + "source=EMP | eval salary_binary = tostring(SAL, \"binary\") | fields ENAME," + + " salary_binary, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_binary=[TOSTRING($5, 'binary':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_binary=1100100000; SAL=800.00\n" + + "ENAME=ALLEN; salary_binary=11001000000; SAL=1600.00\n" + + "ENAME=WARD; salary_binary=10011100010; SAL=1250.00\n" + + "ENAME=JONES; salary_binary=101110011111; SAL=2975.00\n" + + "ENAME=MARTIN; salary_binary=10011100010; SAL=1250.00\n" + + "ENAME=BLAKE; salary_binary=101100100010; SAL=2850.00\n" + + "ENAME=CLARK; salary_binary=100110010010; SAL=2450.00\n" + + "ENAME=SCOTT; salary_binary=101110111000; SAL=3000.00\n" + + "ENAME=KING; salary_binary=1001110001000; SAL=5000.00\n" + + "ENAME=TURNER; salary_binary=10111011100; SAL=1500.00\n" + + "ENAME=ADAMS; salary_binary=10001001100; SAL=1100.00\n" + + "ENAME=JAMES; salary_binary=1110110110; SAL=950.00\n" + + "ENAME=FORD; salary_binary=101110111000; SAL=3000.00\n" + + "ENAME=MILLER; salary_binary=10100010100; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'binary') `salary_binary`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringHex() { + String ppl = + "source=EMP | eval salary_hex = tostring(SAL, \"hex\") | fields ENAME, salary_hex, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_hex=[TOSTRING($5, 'hex':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_hex=320; SAL=800.00\n" + + "ENAME=ALLEN; salary_hex=640; SAL=1600.00\n" + + "ENAME=WARD; salary_hex=4e2; SAL=1250.00\n" + + "ENAME=JONES; salary_hex=b9f; SAL=2975.00\n" + + "ENAME=MARTIN; salary_hex=4e2; SAL=1250.00\n" + + "ENAME=BLAKE; salary_hex=b22; SAL=2850.00\n" + + "ENAME=CLARK; salary_hex=992; SAL=2450.00\n" + + "ENAME=SCOTT; salary_hex=bb8; SAL=3000.00\n" + + "ENAME=KING; salary_hex=1388; SAL=5000.00\n" + + "ENAME=TURNER; salary_hex=5dc; SAL=1500.00\n" + + "ENAME=ADAMS; salary_hex=44c; SAL=1100.00\n" + + "ENAME=JAMES; salary_hex=3b6; SAL=950.00\n" + + "ENAME=FORD; salary_hex=bb8; SAL=3000.00\n" + + "ENAME=MILLER; salary_hex=514; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'hex') `salary_hex`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringHexFromNumberAsString() { + String ppl = + "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_hex=640\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringCommaFromNumberAsString() { + String ppl = + "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME, salary_comma| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR, 'commas':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_comma=160,040,222\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test + public void testToStringBinaryFromNumberAsString() { + String ppl = + "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME, salary_binary| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR, 'binary':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test + public void testToStringCommas() { + String ppl = + "source=EMP | eval salary_commas = tostring(SAL, \"commas\") | fields ENAME," + + " salary_commas, SAL"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_commas=[TOSTRING($5, 'commas':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_commas=800; SAL=800.00\n" + + "ENAME=ALLEN; salary_commas=1,600; SAL=1600.00\n" + + "ENAME=WARD; salary_commas=1,250; SAL=1250.00\n" + + "ENAME=JONES; salary_commas=2,975; SAL=2975.00\n" + + "ENAME=MARTIN; salary_commas=1,250; SAL=1250.00\n" + + "ENAME=BLAKE; salary_commas=2,850; SAL=2850.00\n" + + "ENAME=CLARK; salary_commas=2,450; SAL=2450.00\n" + + "ENAME=SCOTT; salary_commas=3,000; SAL=3000.00\n" + + "ENAME=KING; salary_commas=5,000; SAL=5000.00\n" + + "ENAME=TURNER; salary_commas=1,500; SAL=1500.00\n" + + "ENAME=ADAMS; salary_commas=1,100; SAL=1100.00\n" + + "ENAME=JAMES; salary_commas=950; SAL=950.00\n" + + "ENAME=FORD; salary_commas=3,000; SAL=3000.00\n" + + "ENAME=MILLER; salary_commas=1,300; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'commas') `salary_commas`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringDuration() { + String ppl = + "source=EMP | eval duration_commas = tostring(6500, \"duration\") | fields ENAME," + + " duration_commas|HEAD 1"; + + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], duration_commas=[TOSTRING(6500, 'duration':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; duration_commas=01:48:20\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(6500, 'duration') `duration_commas`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testLike() { String ppl = "source=EMP | where like(JOB, 'SALE%') | stats count() as cnt"; diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index ba7c5be85ab..6465c692da3 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -133,6 +133,7 @@ STDDEV_SAMP: 'STDDEV_SAMP'; SUBSTRING: 'SUBSTRING'; TRIM: 'TRIM'; +TOSTRING: 'TOSTRING'; // Keywords, but can be ID // Common Keywords, but can be ID diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5f7361160b3..fbaef12fb98 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -417,6 +417,7 @@ specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CAST '(' expression AS convertedDataType ')' # dataTypeFunctionCall + | TOSTRING '(' functionArg ')' # dataTypeFunctionCall ; relevanceFunction From 5a5b778cdf83230ff8a68a259a628e3ee0b3f318 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:33:26 -0700 Subject: [PATCH 02/24] removed sql changes Signed-off-by: Asif Bashar --- sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 - sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 - 2 files changed, 2 deletions(-) diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 6465c692da3..ba7c5be85ab 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -133,7 +133,6 @@ STDDEV_SAMP: 'STDDEV_SAMP'; SUBSTRING: 'SUBSTRING'; TRIM: 'TRIM'; -TOSTRING: 'TOSTRING'; // Keywords, but can be ID // Common Keywords, but can be ID diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index fbaef12fb98..5f7361160b3 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -417,7 +417,6 @@ specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CAST '(' expression AS convertedDataType ')' # dataTypeFunctionCall - | TOSTRING '(' functionArg ')' # dataTypeFunctionCall ; relevanceFunction From 9d71a95f8487a7c5fa0e4cea8d11b4a3893efcaf Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:50:53 -0700 Subject: [PATCH 03/24] doc changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 4 ++-- docs/user/ppl/functions/string.rst | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 21124c27edc..33af0075431 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -95,14 +95,14 @@ The first argument can be a number, number as string or boolean. If first argument is a a number or number as string , second argument need to be format name. If first argument is boolean, then second argument is not needed. -format types: +Format types: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -Binary conversion +Binary conversion: You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: eval result = tostring(9, "binary") diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 43efed3c470..c1dd52a5d89 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -397,6 +397,4 @@ Example:: |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | +---------------------+---------------------+ - - - + From be2c2e264a8cc337940f3f1ae2152a15672ce2cb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:52:09 -0700 Subject: [PATCH 04/24] docs changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 12 ++++++------ docs/user/ppl/functions/string.rst | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 33af0075431..9d0bf6eab1b 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -85,15 +85,15 @@ TOSTRING Description >>>>>>>>>>> There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(number|string, string) converts the number in first argument to provided format type string in second argument. - Return type: string +Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. + Return type: string Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. - Return type: string + Return type: string You can use this function with the eval commands and as part of eval expressions. The first argument can be a number, number as string or boolean. -If first argument is a a number or number as string , second argument need to be format name. -If first argument is boolean, then second argument is not needed. +If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. +If first argument is boolean, then second argument is not used even if its provided. Format types: a) "binary" Converts a number to a binary value. @@ -108,7 +108,7 @@ eval result = tostring(9, "binary") For information about bitwise functions that you can use with the tostring function, see Bitwise functions. -Basic examples +Basic examples: The following example returns "True 0xF 12,345.68". ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index c1dd52a5d89..01b9b85b882 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -396,5 +396,4 @@ Example:: | UPPER('helloworld') | UPPER('HELLOWORLD') | |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | - +---------------------+---------------------+ - + +---------------------+---------------------+ \ No newline at end of file From fc763a431cbbdcad3fe35ab3f0e70f334d0b12a6 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:00:16 -0700 Subject: [PATCH 05/24] reverted string doc changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 01b9b85b882..24efa1434f5 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -396,4 +396,4 @@ Example:: | UPPER('helloworld') | UPPER('HELLOWORLD') | |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | - +---------------------+---------------------+ \ No newline at end of file + +---------------------+---------------------+ From a04eb14529029cf51d020e75ba5e7e6f921fb07c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:06:46 -0700 Subject: [PATCH 06/24] removed extra word Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 9d0bf6eab1b..58171d7cc72 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -91,7 +91,7 @@ Usage for boolean parameter without format type: tostring(boolean) converts the Return type: string You can use this function with the eval commands and as part of eval expressions. -The first argument can be a number, number as string or boolean. +The first argument can be a number, number as string or boolean. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. If first argument is boolean, then second argument is not used even if its provided. @@ -148,9 +148,6 @@ Example:: | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ - - duration - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ From 590a8e63add13b75fa90047cd2eca1ab4d802936 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:24:25 -0700 Subject: [PATCH 07/24] added any type Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 38 +++++++++++--------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 58171d7cc72..c4ebe0505de 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -85,27 +85,22 @@ TOSTRING Description >>>>>>>>>>> There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. +Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. Return type: string Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. Return type: string - You can use this function with the eval commands and as part of eval expressions. -The first argument can be a number, number as string or boolean. -If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. + +If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. -Format types: +Format types:: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -Binary conversion: -You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: -eval result = tostring(9, "binary") - For information about bitwise functions that you can use with the tostring function, see Bitwise functions. Basic examples: @@ -114,17 +109,11 @@ The following example returns "True 0xF 12,345.68". The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. ... | eval foo=615 | eval foo2 = tostring(foo, "duration") -The following example formats the column totalSales to display values with a currency symbol and commas. You must use a period between the currency value and the tostring function. -Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` - fetched rows / total rows = 1/1 - +---------------------+ - | boolean_str | - |---------------------+ - | True | - +---------------------+ + +You can use this function to convert a number to a string of its binary representation. +Example:: os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -132,6 +121,10 @@ Example:: |---------------+------------------+------------+ | SMITH | 1001110001000000 | 80000.00 | +---------------+------------------+------------+ + + +You can use this function to convert a number to a string of its hex representation. +Example:: os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -140,6 +133,8 @@ Example:: | SMITH | 13880 | 80000.00 | +---------------+---------------+---------------+ +The following example formats the column totalSales to display values with commas. +Example:: os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -147,7 +142,8 @@ Example:: |---------------+------------------+------------+ | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ - +The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. +Example:: os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ @@ -156,10 +152,8 @@ Example:: | SMITH | 01:48:20 | +---------------+-------------+ -Usage for boolean parameter without format type:: - +Example for boolean parameter. Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 +---------------------+ From 6938e2c08dfef04119f1e37802d0a3a9554a0d21 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 11:00:43 -0700 Subject: [PATCH 08/24] doc formatting fixes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 38 ++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index c4ebe0505de..280befee0aa 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -80,40 +80,44 @@ Cast function can be chained:: +-------+ TOSTRING ------ +----------- Description >>>>>>>>>>> -There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. - Return type: string -Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. - Return type: string -You can use this function with the eval commands and as part of eval expressions. +The following usage options are available, depending on the parameter types and the number of parameters. -If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. -If first argument is boolean, then second argument is not used even if its provided. +Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. +Return type: string + +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'True' or 'False'. +Return type: string + +You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. + +Format types: -Format types:: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -For information about bitwise functions that you can use with the tostring function, see Bitwise functions. +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. Basic examples: + The following example returns "True 0xF 12,345.68". -... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") + + ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") + The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. -... | eval foo=615 | eval foo2 = tostring(foo, "duration") + ... | eval foo=615 | eval foo2 = tostring(foo, "duration") You can use this function to convert a number to a string of its binary representation. Example:: + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -125,6 +129,7 @@ Example:: You can use this function to convert a number to a string of its hex representation. Example:: + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -135,6 +140,7 @@ Example:: The following example formats the column totalSales to display values with commas. Example:: + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -142,8 +148,10 @@ Example:: |---------------+------------------+------------+ | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ + The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ @@ -154,8 +162,10 @@ Example:: Example for boolean parameter. Example:: + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 + +---------------------+ | boolean_str | |---------------------+ From 314fccdaee8ab28ede0eab2f1da4e2b83cc24e96 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 11:04:06 -0700 Subject: [PATCH 09/24] description for boolean example Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 280befee0aa..036a89daf00 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -160,7 +160,7 @@ Example:: | SMITH | 01:48:20 | +---------------+-------------+ -Example for boolean parameter. +The following example for converts boolean parameter to string. Example:: os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` From 0ee17b9a4a262a1d6070b98ae8a1162f6556f517 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:21:49 -0700 Subject: [PATCH 10/24] added format_time call from calcite , added duration_millis as splunk default duration is in seconds which will be used for duration format , added cast call for tostring with 1 argument Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 2 +- .../function/udf/ToStringFunction.java | 53 ++-- .../function/udf/ToStringFunctionTest.java | 265 +++++++++--------- .../sql/ppl/parser/AstExpressionBuilder.java | 52 +++- .../calcite/CalcitePPLStringFunctionTest.java | 157 +++++++---- 5 files changed, 312 insertions(+), 217 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 69e1492538c..20811f1af48 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -117,7 +117,7 @@ private PPLOperandTypes() {} (CompositeOperandTypeChecker) OperandTypes.family(SqlTypeFamily.BOOLEAN) .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) - .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = UDFOperandMetadata.wrap( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index 70d2a82b13c..be76100f38d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -8,7 +8,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.text.NumberFormat; -import java.time.Duration; import java.util.List; import java.util.Locale; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -18,6 +17,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -41,9 +41,13 @@ public ToStringFunction() { } public static final String DURATION_FORMAT = "duration"; + public static final String DURATION_MILLIS_FORMAT = "duration_millis"; public static final String HEX_FORMAT = "hex"; public static final String COMMAS_FORMAT = "commas"; public static final String BINARY_FORMAT = "binary"; + public static final SqlFunctions.DateFormatFunction dateTimeFormatter = + new SqlFunctions.DateFormatFunction(); + public static final String format24hour = "%H:%M:%S"; // 24-hour format @Override public SqlReturnTypeInference getReturnTypeInference() { @@ -65,7 +69,13 @@ public Expression implement( Expression format = translatedOperands.get(1); return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } else { - return Expressions.call(ToStringFunction.class, "toString", fieldValue); + // autoboxes to Boolean + + if (!fieldValue.getType().getTypeName().equals("Boolean")) { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } else { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } } } } @@ -87,13 +97,13 @@ public static String toString(String fieldValue) { @Strict public static String toString(BigDecimal num, String format) { if (format.equals(DURATION_FORMAT)) { - Duration d = Duration.ofSeconds(num.toBigInteger().longValue()); - long hours = d.toHours(); - int minutes = d.toMinutesPart(); - int remainingSeconds = d.toSecondsPart(); - String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue() * 1000); + + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + + return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue()); + } else if (format.equals(HEX_FORMAT)) { return num.toBigInteger().toString(16); } else if (format.equals(COMMAS_FORMAT)) { @@ -112,12 +122,11 @@ public static String toString(BigDecimal num, String format) { @Strict public static String toString(double num, String format) { if (format.equals(DURATION_FORMAT)) { - Duration d = Duration.ofSeconds(Math.round(num)); - long hours = d.toHours(); - int minutes = d.toMinutesPart(); - int remainingSeconds = d.toSecondsPart(); - String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num)) * 1000); + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + + return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num))); + } else if (format.equals(HEX_FORMAT)) { return Double.toHexString(num); } else if (format.equals(COMMAS_FORMAT)) { @@ -129,17 +138,19 @@ public static String toString(double num, String format) { return Double.toString(num); } + @Strict + public static String toString(short num, String format) { + int i = (int) num; + return toString(i, format); + } + @Strict public static String toString(int num, String format) { if (format.equals(DURATION_FORMAT)) { - - int hours = num / 3600; - int minutes = (num % 3600) / 60; - int seconds = num % 60; - - String time_str = String.format("%02d:%02d:%02d", hours, minutes, seconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, num * 1000); + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + return dateTimeFormatter.formatTime(format24hour, num); } else if (format.equals(HEX_FORMAT)) { return Integer.toHexString(num); } else if (format.equals(COMMAS_FORMAT)) { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java index cfd25796b23..f0867f93327 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -5,145 +5,144 @@ package org.opensearch.sql.expression.function.udf; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; import java.math.BigDecimal; import java.util.Locale; - -import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; public class ToStringFunctionTest { private final ToStringFunction function = new ToStringFunction(); - @Test - void testBooleanToString() { - assertEquals("True", ToStringFunction.toString(true)); - assertEquals("False", ToStringFunction.toString(false)); - } - - @Test - void testStringBooleanToString() { - assertEquals("True", ToStringFunction.toString("true")); - assertEquals("False", ToStringFunction.toString("false")); - assertEquals("False", ToStringFunction.toString("anythingElse")); - } - - @Test - void testBigDecimalToStringDurationFormat() { - BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testBigDecimalToStringHexFormat() { - BigDecimal num = new BigDecimal("255"); - String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertEquals("ff", result); - } - - @Test - void testBigDecimalToStringCommasFormat() { - Locale.setDefault(Locale.US); // Ensure predictable comma placement - BigDecimal num = new BigDecimal("1234567.891"); - String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testBigDecimalToStringBinaryFormat() { - BigDecimal num = new BigDecimal("10"); - String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); - assertEquals("1010", result); - } - - @Test - void testBigDecimalToStringDefault() { - BigDecimal num = new BigDecimal("123.45"); - assertEquals("123.45", ToStringFunction.toString(num, "unknown")); - } - - @Test - void testDoubleToStringDurationFormat() { - double num = 3661.4; - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testDoubleToStringHexFormat() { - double num = 10.5; - String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertTrue(result.startsWith("0x")); - } - - @Test - void testDoubleToStringCommasFormat() { - Locale.setDefault(Locale.US); - double num = 12345.678; - String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testDoubleToStringBinaryFormat() { - double num = 10.0; - String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); - assertNotNull(result); - assertFalse(result.isEmpty()); - } - - @Test - void testDoubleToStringDefault() { - assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); - } - - @Test - void testIntToStringDurationFormat() { - int num = 3661; - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testIntToStringHexFormat() { - assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); - } - - @Test - void testIntToStringCommasFormat() { - Locale.setDefault(Locale.US); - String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testIntToStringBinaryFormat() { - assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); - } - - @Test - void testIntToStringDefault() { - assertEquals("123", ToStringFunction.toString(123, "unknown")); - } - - @Test - void testStringNumericToStringIntFormat() { - String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); - assertEquals("2a", result); - } - - @Test - void testStringNumericToStringDoubleFormat() { - String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains("42")); - } - - @Test - void testStringLargeNumberAsDouble() { - String largeNum = "1234567890123"; - String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); - assertNotNull(result); - } + @Test + void testBooleanToString() { + assertEquals("True", ToStringFunction.toString(true)); + assertEquals("False", ToStringFunction.toString(false)); + } + + @Test + void testStringBooleanToString() { + assertEquals("True", ToStringFunction.toString("true")); + assertEquals("False", ToStringFunction.toString("false")); + assertEquals("False", ToStringFunction.toString("anythingElse")); + } + + @Test + void testBigDecimalToStringDurationFormat() { + BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testBigDecimalToStringHexFormat() { + BigDecimal num = new BigDecimal("255"); + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertEquals("ff", result); + } + + @Test + void testBigDecimalToStringCommasFormat() { + Locale.setDefault(Locale.US); // Ensure predictable comma placement + BigDecimal num = new BigDecimal("1234567.891"); + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testBigDecimalToStringBinaryFormat() { + BigDecimal num = new BigDecimal("10"); + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertEquals("1010", result); + } + + @Test + void testBigDecimalToStringDefault() { + BigDecimal num = new BigDecimal("123.45"); + assertEquals("123.45", ToStringFunction.toString(num, "unknown")); + } + + @Test + void testDoubleToStringDurationFormat() { + double num = 3661.4; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testDoubleToStringHexFormat() { + double num = 10.5; + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertTrue(result.startsWith("0x")); + } + + @Test + void testDoubleToStringCommasFormat() { + Locale.setDefault(Locale.US); + double num = 12345.678; + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testDoubleToStringBinaryFormat() { + double num = 10.0; + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testDoubleToStringDefault() { + assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); + } + + @Test + void testIntToStringDurationFormat() { + int num = 3661; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testIntToStringHexFormat() { + assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); + } + + @Test + void testIntToStringCommasFormat() { + Locale.setDefault(Locale.US); + String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testIntToStringBinaryFormat() { + assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); + } + + @Test + void testIntToStringDefault() { + assertEquals("123", ToStringFunction.toString(123, "unknown")); + } + + @Test + void testStringNumericToStringIntFormat() { + String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); + assertEquals("2a", result); + } + + @Test + void testStringNumericToStringDoubleFormat() { + String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains("42")); + } + + @Test + void testStringLargeNumberAsDouble() { + String largeNum = "1234567890123"; + String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 1460a3efc9a..2112282f728 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -29,8 +30,10 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; @@ -411,18 +414,51 @@ private Function buildFunction( functionName, args.stream().map(this::visitFunctionArg).collect(Collectors.toList())); } + public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String castExpression) { + // Create a case-insensitive character stream from the input + CaseInsensitiveCharStream charStream = new CaseInsensitiveCharStream(castExpression); + + // Create lexer and parser + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(charStream); + CommonTokenStream tokens = new CommonTokenStream(lexer); + OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); + + // Parse the expression - cast is part of evalFunctionCall + DataTypeFunctionCallContext evalContext = parser.dataTypeFunctionCall(); + return evalContext; + } + /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - ParseTree rootNode = ctx.getChild(0); - String functionName = rootNode.getText(); - final String mappedName = - FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); - - if (mappedName.equals("cast")) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + if (ctx.functionArgs() != null) { + + ParseTree rootNode = ctx.getChild(0); + String functionName = rootNode.getText(); + final String mappedName = + FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); + System.out.println(mappedName); + if (mappedName != null && mappedName.equals("tostring")) { + if (ctx.functionArgs().functionArg().size() == 1) { + List functionArgs = + ctx.functionArgs().functionArg(); + + String castExpresstion = + String.format("cast( %s as String)", functionArgs.getFirst().getText()); + DataTypeFunctionCallContext toStringDataTypeConversionContext = + this.createDataTypeFunctionCallContext(castExpresstion); + return new Cast( + visit(toStringDataTypeConversionContext.logicalExpression()), + visit(toStringDataTypeConversionContext.convertedDataType())); + // + } else { + return buildFunction(mappedName, ctx.functionArgs().functionArg()); + } + } else { + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + } } else { - return buildFunction(mappedName, ctx.functionArgs().functionArg()); + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index d41b2c22453..b32ecd04e3f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -46,20 +46,55 @@ public void testLower() { verifyPPLToSparkSQL(root, expectedSparkSql); } + // This test evalutes tostring where it gets converted to cast call + + @Test + public void testToStringFormatNotSpecified() { + String ppl = + "source=EMP | eval string_value = tostring(MGR) | eval cast_value = cast(MGR as string)|" + + " fields string_value, cast_value"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(string_value=[SAFE_CAST($3)], cast_value=[SAFE_CAST($3)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "string_value=7902; cast_value=7902\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7566; cast_value=7566\n" + + "string_value=null; cast_value=null\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7788; cast_value=7788\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7566; cast_value=7566\n" + + "string_value=7782; cast_value=7782\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT SAFE_CAST(`MGR` AS STRING) `string_value`, SAFE_CAST(`MGR` AS STRING)" + + " `cast_value`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testToStringBoolean() { String ppl = "source=EMP | eval boolean_value = tostring(1==1) | fields boolean_value |head 1"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(boolean_value=[TOSTRING(true)])\n" + + " LogicalProject(boolean_value=['TRUE':VARCHAR])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = "boolean_value=True\n"; + String expectedResult = "boolean_value=TRUE\n"; verifyLogical(root, expectedLogical); verifyResult(root, expectedResult); - String expectedSparkSql = - "SELECT `TOSTRING`(TRUE) `boolean_value`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + String expectedSparkSql = "SELECT 'TRUE' `boolean_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -126,56 +161,70 @@ public void testToStringHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } - @Test - public void testToStringHexFromNumberAsString() { - String ppl = - "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_hex=640\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - - @Test - public void testToStringCommaFromNumberAsString() { - String ppl = - "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME, salary_comma| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR, 'commas':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_comma=160,040,222\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - @Test - public void testToStringBinaryFromNumberAsString() { - String ppl = - "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME, salary_binary| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR, 'binary':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - @Test + @Test + public void testToStringHexFromNumberAsString() { + String ppl = + "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex|" + + " head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_hex=640\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringCommaFromNumberAsString() { + String ppl = + "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME," + + " salary_comma| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR," + + " 'commas':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_comma=160,040,222\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringBinaryFromNumberAsString() { + String ppl = + "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME," + + " salary_binary| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR," + + " 'binary':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test public void testToStringCommas() { String ppl = "source=EMP | eval salary_commas = tostring(SAL, \"commas\") | fields ENAME," From 6e24aa3c4b16f6a5734db18f210b8b5c1f5f245f Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:32:18 -0700 Subject: [PATCH 11/24] added doc update to specifically set 2nd argument as optional Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 036a89daf00..9f9646efef4 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,10 +86,10 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. +Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. Return type: string -Usage for boolean parameter without format type tostring(boolean): Converts the string to 'True' or 'False'. +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. Return type: string You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. @@ -100,12 +100,13 @@ a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +5) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. Basic examples: -The following example returns "True 0xF 12,345.68". +The following example returns "TRUE 0xF 12,345.68". ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") @@ -169,5 +170,5 @@ Example:: +---------------------+ | boolean_str | |---------------------+ - | True | + | TRUE | +---------------------+ \ No newline at end of file From 454cfc8db324866376f2452290b0fd0739008754 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:33:59 -0700 Subject: [PATCH 12/24] mentioned as value instead of number specifically Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 9f9646efef4..4f2d432c71a 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,7 +86,7 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. +Usage with format type: tostring(ANY, [format]): Converts the value in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. Return type: string Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. From b221e8c4f583b947198253ee90b5edb6f15b382a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:46:56 -0700 Subject: [PATCH 13/24] fixed wrong bullet point Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 4f2d432c71a..708bde1fa11 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -100,7 +100,7 @@ a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -5) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. +e) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. From 1ed88e1dc190096617571796a14d9b8d76ba338e Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 20 Oct 2025 09:38:56 -0700 Subject: [PATCH 14/24] removed extra quote Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 708bde1fa11..6e2149494c3 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -119,7 +119,7 @@ The following example returns foo=615 and foo2=00:10:15. The 615 seconds is conv You can use this function to convert a number to a string of its binary representation. Example:: - os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL fetched rows / total rows = 1/1 +---------------+------------------+------------+ | ENAME | salary_binary | SAL | @@ -131,7 +131,7 @@ Example:: You can use this function to convert a number to a string of its hex representation. Example:: - os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL fetched rows / total rows = 1/1 +---------------+------------------+------------+ | ENAME | salary_hex | SAL | @@ -142,7 +142,7 @@ Example:: The following example formats the column totalSales to display values with commas. Example:: - os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL fetched rows / total rows = 1/1 +---------------+------------------+------------+ | ENAME | salary_commas | SAL | @@ -153,7 +153,7 @@ Example:: The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration fetched rows / total rows = 1/1 +---------------+-------------+ | ENAME | duration | From 5cbedccd57c6862925fce874720def1afea82892 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 20 Oct 2025 17:28:04 -0700 Subject: [PATCH 15/24] we have duplicated example Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 6e2149494c3..61be5b64362 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -106,16 +106,6 @@ The format argument is optional and is only used when the value argument is a nu Basic examples: -The following example returns "TRUE 0xF 12,345.68". - - ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") - -The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. - - ... | eval foo=615 | eval foo2 = tostring(foo, "duration") - - - You can use this function to convert a number to a string of its binary representation. Example:: From 682eb9d8d57a72ee30570744e7ba428a6dfcfbe7 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 20 Oct 2025 21:28:05 -0700 Subject: [PATCH 16/24] applied recommended changes Signed-off-by: Asif Bashar --- .../sql/calcite/CalciteRexNodeVisitor.java | 10 +++- .../function/udf/ToStringFunction.java | 10 ++-- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 +- .../sql/ppl/parser/AstExpressionBuilder.java | 47 +------------------ .../calcite/CalcitePPLStringFunctionTest.java | 5 +- 5 files changed, 17 insertions(+), 59 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index bb6f16b2e8a..b1be49582ec 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -417,11 +417,19 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) { context.setInCoalesceFunction(false); } } - + if (node.getFuncName().equalsIgnoreCase("tostring") && args.size() == 1) { + RelDataType targetType = + context.relBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + // Get the source expression from arguments + RexNode sourceExpression = arguments.get(0); + // Create cast RexNode using ExtendedRexBuilder + return context.rexBuilder.makeCast(targetType, sourceExpression); + } RexNode resolvedNode = PPLFuncImpTable.INSTANCE.resolve( context.rexBuilder, node.getFuncName(), arguments.toArray(new RexNode[0])); if (resolvedNode != null) { + return resolvedNode; } throw new IllegalArgumentException("Unsupported operator: " + node.getFuncName()); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index be76100f38d..1052a8f5f11 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -18,6 +18,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.SqlFunctions; +import org.apache.calcite.sql.*; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -65,17 +66,12 @@ public static class ToStringImplementor implements NotNullImplementor { public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression fieldValue = translatedOperands.get(0); + if (translatedOperands.size() > 1) { Expression format = translatedOperands.get(1); return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } else { - // autoboxes to Boolean - - if (!fieldValue.getType().getTypeName().equals("Boolean")) { - return Expressions.call(ToStringFunction.class, "toString", fieldValue); - } else { - return Expressions.call(ToStringFunction.class, "toString", fieldValue); - } + return Expressions.call(ToStringFunction.class, "toString", fieldValue); } } } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 852c55c863c..4c4fa3bbac6 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -848,10 +848,9 @@ evalFunctionCall ; -// cast, tostring function +// cast function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS - | TOSTRING LT_PRTHS functionArgs RT_PRTHS ; @@ -1219,6 +1218,7 @@ systemFunctionName textFunctionName : SUBSTR | SUBSTRING + | TOSTRING | TRIM | LTRIM | RTRIM diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 2112282f728..f037376f5c2 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -19,7 +19,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -30,10 +29,8 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; -import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; @@ -414,52 +411,10 @@ private Function buildFunction( functionName, args.stream().map(this::visitFunctionArg).collect(Collectors.toList())); } - public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String castExpression) { - // Create a case-insensitive character stream from the input - CaseInsensitiveCharStream charStream = new CaseInsensitiveCharStream(castExpression); - - // Create lexer and parser - OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(charStream); - CommonTokenStream tokens = new CommonTokenStream(lexer); - OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); - - // Parse the expression - cast is part of evalFunctionCall - DataTypeFunctionCallContext evalContext = parser.dataTypeFunctionCall(); - return evalContext; - } - /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - if (ctx.functionArgs() != null) { - - ParseTree rootNode = ctx.getChild(0); - String functionName = rootNode.getText(); - final String mappedName = - FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); - System.out.println(mappedName); - if (mappedName != null && mappedName.equals("tostring")) { - if (ctx.functionArgs().functionArg().size() == 1) { - List functionArgs = - ctx.functionArgs().functionArg(); - - String castExpresstion = - String.format("cast( %s as String)", functionArgs.getFirst().getText()); - DataTypeFunctionCallContext toStringDataTypeConversionContext = - this.createDataTypeFunctionCallContext(castExpresstion); - return new Cast( - visit(toStringDataTypeConversionContext.logicalExpression()), - visit(toStringDataTypeConversionContext.convertedDataType())); - // - } else { - return buildFunction(mappedName, ctx.functionArgs().functionArg()); - } - } else { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); - } - } else { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); - } + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index b32ecd04e3f..c06c536919c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -55,7 +55,7 @@ public void testToStringFormatNotSpecified() { + " fields string_value, cast_value"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(string_value=[SAFE_CAST($3)], cast_value=[SAFE_CAST($3)])\n" + "LogicalProject(string_value=[CAST($3):VARCHAR NOT NULL], cast_value=[SAFE_CAST($3)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; String expectedResult = "string_value=7902; cast_value=7902\n" @@ -76,8 +76,7 @@ public void testToStringFormatNotSpecified() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT SAFE_CAST(`MGR` AS STRING) `string_value`, SAFE_CAST(`MGR` AS STRING)" - + " `cast_value`\n" + "SELECT CAST(`MGR` AS STRING) `string_value`, SAFE_CAST(`MGR` AS STRING) `cast_value`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } From d0b809a2db65599be273a1600a14f224d415f679 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 25 Oct 2025 20:14:25 -0700 Subject: [PATCH 17/24] recommended changes Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 5 +- .../expression/function/PPLFuncImpTable.java | 6 ++ .../function/udf/ToStringFunction.java | 75 +++---------------- .../function/udf/ToStringFunctionTest.java | 15 +--- docs/user/ppl/functions/conversion.rst | 14 ++-- 5 files changed, 26 insertions(+), 89 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 20811f1af48..e2bdc2f02e3 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -112,11 +112,10 @@ private PPLOperandTypes() {} SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)); - public static final UDFOperandMetadata BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING = + public static final UDFOperandMetadata NUMERIC_STRING_OR_STRING_STRING = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.BOOLEAN) - .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) + (OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index de89df383d2..9a26569e493 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -889,6 +889,12 @@ void populate() { registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER); registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING); + register( + TOSTRING, + (FunctionImp1) + (builder, source) -> + builder.makeCast(TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true), source), + PPLTypeChecker.family(SqlTypeFamily.ANY)); // Register MVJOIN to use Calcite's ARRAY_JOIN register( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index 1052a8f5f11..46d0ada1419 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -48,7 +48,7 @@ public ToStringFunction() { public static final String BINARY_FORMAT = "binary"; public static final SqlFunctions.DateFormatFunction dateTimeFormatter = new SqlFunctions.DateFormatFunction(); - public static final String format24hour = "%H:%M:%S"; // 24-hour format + public static final String FORMAT_24_HOUR = "%H:%M:%S"; @Override public SqlReturnTypeInference getReturnTypeInference() { @@ -57,7 +57,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING; + return PPLOperandTypes.NUMERIC_STRING_OR_STRING_STRING; } public static class ToStringImplementor implements NotNullImplementor { @@ -66,39 +66,20 @@ public static class ToStringImplementor implements NotNullImplementor { public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression fieldValue = translatedOperands.get(0); - - if (translatedOperands.size() > 1) { - Expression format = translatedOperands.get(1); - return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); - } else { - return Expressions.call(ToStringFunction.class, "toString", fieldValue); - } - } - } - - @Strict - public static String toString(boolean fieldValue) { - if (fieldValue) { - return "True"; - } else { - return "False"; + Expression format = translatedOperands.get(1); + return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } } - @Strict - public static String toString(String fieldValue) { - return toString(Boolean.parseBoolean(fieldValue)); - } - @Strict public static String toString(BigDecimal num, String format) { if (format.equals(DURATION_FORMAT)) { - return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue() * 1000); + return dateTimeFormatter.formatTime(FORMAT_24_HOUR, num.toBigInteger().intValue() * 1000); } else if (format.equals(DURATION_MILLIS_FORMAT)) { - return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue()); + return dateTimeFormatter.formatTime(FORMAT_24_HOUR, num.toBigInteger().intValue()); } else if (format.equals(HEX_FORMAT)) { return num.toBigInteger().toString(16); @@ -117,53 +98,17 @@ public static String toString(BigDecimal num, String format) { @Strict public static String toString(double num, String format) { - if (format.equals(DURATION_FORMAT)) { - return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num)) * 1000); - } else if (format.equals(DURATION_MILLIS_FORMAT)) { - - return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num))); - - } else if (format.equals(HEX_FORMAT)) { - return Double.toHexString(num); - } else if (format.equals(COMMAS_FORMAT)) { - NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); - return nf.format(num); - } else if (format.equals(BINARY_FORMAT)) { - return Long.toBinaryString(Double.doubleToLongBits(num)); - } - return Double.toString(num); - } - - @Strict - public static String toString(short num, String format) { - int i = (int) num; - return toString(i, format); + return toString(BigDecimal.valueOf(num), format); } @Strict public static String toString(int num, String format) { - - if (format.equals(DURATION_FORMAT)) { - return dateTimeFormatter.formatTime(format24hour, num * 1000); - } else if (format.equals(DURATION_MILLIS_FORMAT)) { - return dateTimeFormatter.formatTime(format24hour, num); - } else if (format.equals(HEX_FORMAT)) { - return Integer.toHexString(num); - } else if (format.equals(COMMAS_FORMAT)) { - NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); - return nf.format(num); - } else if (format.equals(BINARY_FORMAT)) { - return Integer.toBinaryString(num); - } - return Integer.toString(num); + return toString(BigDecimal.valueOf(num), format); } @Strict public static String toString(String str, String format) { - if (str.contains(".") || (str.length() > 10)) { - return toString(Double.parseDouble(str), format); - } else { - return toString(Integer.parseInt(str), format); - } + BigDecimal bd = new BigDecimal(str); + return toString(bd, format); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java index f0867f93327..e29d6c10d19 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -15,19 +15,6 @@ public class ToStringFunctionTest { private final ToStringFunction function = new ToStringFunction(); - @Test - void testBooleanToString() { - assertEquals("True", ToStringFunction.toString(true)); - assertEquals("False", ToStringFunction.toString(false)); - } - - @Test - void testStringBooleanToString() { - assertEquals("True", ToStringFunction.toString("true")); - assertEquals("False", ToStringFunction.toString("false")); - assertEquals("False", ToStringFunction.toString("anythingElse")); - } - @Test void testBigDecimalToStringDurationFormat() { BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second @@ -74,7 +61,7 @@ void testDoubleToStringDurationFormat() { void testDoubleToStringHexFormat() { double num = 10.5; String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertTrue(result.startsWith("0x")); + assertTrue(result.equals("a")); } @Test diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 61be5b64362..c9ee1576037 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -123,16 +123,16 @@ Example:: os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL fetched rows / total rows = 1/1 - +---------------+------------------+------------+ - | ENAME | salary_hex | SAL | - |---------------+------------------+------------+ - | SMITH | 13880 | 80000.00 | - +---------------+---------------+---------------+ + +---------------+---------------+------------+ + | ENAME | salary_hex | SAL | + |---------------+------------------+---------+ + | SMITH | 13880 | 80000.00 | + +---------------+---------------+------------+ The following example formats the column totalSales to display values with commas. Example:: - os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL fetched rows / total rows = 1/1 +---------------+------------------+------------+ | ENAME | salary_commas | SAL | @@ -143,7 +143,7 @@ Example:: The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration fetched rows / total rows = 1/1 +---------------+-------------+ | ENAME | duration | From ba3553b984529c7b68bbaba8abf7dca87c775093 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 25 Oct 2025 20:39:16 -0700 Subject: [PATCH 18/24] requested changes Signed-off-by: Asif Bashar --- .../opensearch/sql/calcite/CalciteRexNodeVisitor.java | 9 +-------- .../sql/ppl/calcite/CalcitePPLStringFunctionTest.java | 3 +-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index b1be49582ec..5c9057cc069 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -417,14 +417,7 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) { context.setInCoalesceFunction(false); } } - if (node.getFuncName().equalsIgnoreCase("tostring") && args.size() == 1) { - RelDataType targetType = - context.relBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); - // Get the source expression from arguments - RexNode sourceExpression = arguments.get(0); - // Create cast RexNode using ExtendedRexBuilder - return context.rexBuilder.makeCast(targetType, sourceExpression); - } + RexNode resolvedNode = PPLFuncImpTable.INSTANCE.resolve( context.rexBuilder, node.getFuncName(), arguments.toArray(new RexNode[0])); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index c06c536919c..14053bd7396 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -55,8 +55,7 @@ public void testToStringFormatNotSpecified() { + " fields string_value, cast_value"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(string_value=[CAST($3):VARCHAR NOT NULL], cast_value=[SAFE_CAST($3)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalProject(string_value=[CAST($3):VARCHAR], cast_value=[SAFE_CAST($3)])\n LogicalTableScan(table=[[scott, EMP]])\n"; String expectedResult = "string_value=7902; cast_value=7902\n" + "string_value=7698; cast_value=7698\n" From 08429c113b5846e22a0492e8568c2517de745d0b Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 25 Oct 2025 20:43:07 -0700 Subject: [PATCH 19/24] requested changes Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 5c9057cc069..bb6f16b2e8a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -422,7 +422,6 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) { PPLFuncImpTable.INSTANCE.resolve( context.rexBuilder, node.getFuncName(), arguments.toArray(new RexNode[0])); if (resolvedNode != null) { - return resolvedNode; } throw new IllegalArgumentException("Unsupported operator: " + node.getFuncName()); From 7e6e9ac37610b396a29bc0a52192cc4c8ef62768 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 28 Oct 2025 09:53:01 -0700 Subject: [PATCH 20/24] updated with try catch for string parsing. Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToStringFunction.java | 8 ++++++-- .../sql/ppl/calcite/CalcitePPLStringFunctionTest.java | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index 46d0ada1419..e6e8dd01df0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -108,7 +108,11 @@ public static String toString(int num, String format) { @Strict public static String toString(String str, String format) { - BigDecimal bd = new BigDecimal(str); - return toString(bd, format); + try { + BigDecimal bd = new BigDecimal(str); + return toString(bd, format); + } catch (Exception e) { + return null; + } } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index 14053bd7396..c67ba63a917 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -55,7 +55,8 @@ public void testToStringFormatNotSpecified() { + " fields string_value, cast_value"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(string_value=[CAST($3):VARCHAR], cast_value=[SAFE_CAST($3)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalProject(string_value=[CAST($3):VARCHAR], cast_value=[SAFE_CAST($3)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; String expectedResult = "string_value=7902; cast_value=7902\n" + "string_value=7698; cast_value=7698\n" From 1420e8fb127dbc50368d64f8a4394bb702a9aa04 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 28 Oct 2025 09:53:01 -0700 Subject: [PATCH 21/24] merge conflict fix Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 83 -------------------------- 1 file changed, 83 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index c9ee1576037..14f6764faf8 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -79,86 +79,3 @@ Cast function can be chained:: | True | +-------+ -TOSTRING ------------ - -Description ->>>>>>>>>>> -The following usage options are available, depending on the parameter types and the number of parameters. - -Usage with format type: tostring(ANY, [format]): Converts the value in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. -Return type: string - -Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. -Return type: string - -You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. - -Format types: - -a) "binary" Converts a number to a binary value. -b) "hex" Converts the number to a hexadecimal value. -c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. -d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -e) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. - -The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. - -Basic examples: - -You can use this function to convert a number to a string of its binary representation. -Example:: - - os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL - fetched rows / total rows = 1/1 - +---------------+------------------+------------+ - | ENAME | salary_binary | SAL | - |---------------+------------------+------------+ - | SMITH | 1001110001000000 | 80000.00 | - +---------------+------------------+------------+ - - -You can use this function to convert a number to a string of its hex representation. -Example:: - - os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL - fetched rows / total rows = 1/1 - +---------------+---------------+------------+ - | ENAME | salary_hex | SAL | - |---------------+------------------+---------+ - | SMITH | 13880 | 80000.00 | - +---------------+---------------+------------+ - -The following example formats the column totalSales to display values with commas. -Example:: - - os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL - fetched rows / total rows = 1/1 - +---------------+------------------+------------+ - | ENAME | salary_commas | SAL | - |---------------+------------------+------------+ - | SMITH | 80,000 | 80000.00 | - +---------------+------------------+------------+ - -The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. -Example:: - - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration - fetched rows / total rows = 1/1 - +---------------+-------------+ - | ENAME | duration | - |---------------+-------------+ - | SMITH | 01:48:20 | - +---------------+-------------+ - -The following example for converts boolean parameter to string. -Example:: - - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` - fetched rows / total rows = 1/1 - - +---------------------+ - | boolean_str | - |---------------------+ - | TRUE | - +---------------------+ \ No newline at end of file From f9c9a1246eb7e6062ffab4395b120e19f2db9015 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 1 Nov 2025 21:27:04 -0700 Subject: [PATCH 22/24] added back missing tostring doc Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 849d2334e41..0f2002cff54 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -117,3 +117,88 @@ Use string in comparison operator example :: | True | False | True | False | True | True | null | +------+-------+------+-------+------+------+------+ + +TOSTRING +----------- + +Description +>>>>>>>>>>> +The following usage options are available, depending on the parameter types and the number of parameters. + +Usage with format type: tostring(ANY, [format]): Converts the value in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. +Return type: string + +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. +Return type: string + +You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. + +Format types: + +a) "binary" Converts a number to a binary value. +b) "hex" Converts the number to a hexadecimal value. +c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. +d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +e) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. + +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. + +Basic examples: + +You can use this function to convert a number to a string of its binary representation. +Example:: + + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_binary | SAL | + |---------------+------------------+------------+ + | SMITH | 1001110001000000 | 80000.00 | + +---------------+------------------+------------+ + + +You can use this function to convert a number to a string of its hex representation. +Example:: + + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL + fetched rows / total rows = 1/1 + +---------------+---------------+------------+ + | ENAME | salary_hex | SAL | + |---------------+------------------+---------+ + | SMITH | 13880 | 80000.00 | + +---------------+---------------+------------+ + +The following example formats the column totalSales to display values with commas. +Example:: + + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_commas | SAL | + |---------------+------------------+------------+ + | SMITH | 80,000 | 80000.00 | + +---------------+------------------+------------+ + +The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. +Example:: + + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration + fetched rows / total rows = 1/1 + +---------------+-------------+ + | ENAME | duration | + |---------------+-------------+ + | SMITH | 01:48:20 | + +---------------+-------------+ + +The following example for converts boolean parameter to string. +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + + +---------------------+ + | boolean_str | + |---------------------+ + | TRUE | + +---------------------+ + From 123fcd5ddf06568e90f30d9febfa1784b746d756 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 4 Nov 2025 17:34:21 -0800 Subject: [PATCH 23/24] updated fix for doctest Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 67 +++++++++++++------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 0f2002cff54..c5635b8509b 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -147,58 +147,57 @@ Basic examples: You can use this function to convert a number to a string of its binary representation. Example:: - - os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL +city, city.name, city.location.latitude + os> source=accounts | where firstname = "Amber" | eval balance_binary = tostring(balance, "binary") | fields firstname, balance_binary, balance fetched rows / total rows = 1/1 - +---------------+------------------+------------+ - | ENAME | salary_binary | SAL | - |---------------+------------------+------------+ - | SMITH | 1001110001000000 | 80000.00 | - +---------------+------------------+------------+ + +-------------+------------------+-----------+ + | firstname | balance_binary | balance | + |-------------+------------------+-----------| + | Amber | 1001100100111001 | 39225 | + +-------------+------------------+-----------+ You can use this function to convert a number to a string of its hex representation. Example:: - os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL + os> source=accounts | where firstname = "Amber" | eval balance_hex = tostring(balance, "hex") | fields firstname, balance_hex, balance fetched rows / total rows = 1/1 - +---------------+---------------+------------+ - | ENAME | salary_hex | SAL | - |---------------+------------------+---------+ - | SMITH | 13880 | 80000.00 | - +---------------+---------------+------------+ + +-------------+---------------+-----------+ + | firstname | balance_hex | balance | + |-------------+---------------+-----------| + | Amber | 9939 | 39225 | + +-------------+---------------+-----------+ The following example formats the column totalSales to display values with commas. Example:: - os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL - fetched rows / total rows = 1/1 - +---------------+------------------+------------+ - | ENAME | salary_commas | SAL | - |---------------+------------------+------------+ - | SMITH | 80,000 | 80000.00 | - +---------------+------------------+------------+ + os> source=accounts | where firstname = "Amber" | eval balance_commas = tostring(balance, "commas") | fields firstname, balance_commas, balance + fetched rows / total rows = 1/1 + +-------------+------------------+-----------+ + | firstname | balance_commas | balance | + |-------------+------------------+-----------| + | Amber | 39,225 | 39225 | + +-------------+------------------+-----------+ The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration - fetched rows / total rows = 1/1 - +---------------+-------------+ - | ENAME | duration | - |---------------+-------------+ - | SMITH | 01:48:20 | - +---------------+-------------+ + os> source=accounts | where firstname = "Amber" | eval duration = tostring(6500, "duration") | fields firstname, duration + fetched rows / total rows = 1/1 + +-------------+------------+ + | firstname | duration | + |-------------+------------| + | Amber | 01:48:20 | + +-------------+------------+ The following example for converts boolean parameter to string. Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + os> source=accounts | where firstname = "Amber"| eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 - - +---------------------+ - | boolean_str | - |---------------------+ - | TRUE | - +---------------------+ + +---------------+ + | boolean_str | + |---------------| + | TRUE | + +---------------+ From e6d0c8c80772fa9138c8919d6ccb06954703ac24 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 5 Nov 2025 09:27:45 -0800 Subject: [PATCH 24/24] fix doctest failures Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 50 +++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index c5635b8509b..82d760cc3ce 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -150,11 +150,11 @@ Example:: city, city.name, city.location.latitude os> source=accounts | where firstname = "Amber" | eval balance_binary = tostring(balance, "binary") | fields firstname, balance_binary, balance fetched rows / total rows = 1/1 - +-------------+------------------+-----------+ - | firstname | balance_binary | balance | - |-------------+------------------+-----------| - | Amber | 1001100100111001 | 39225 | - +-------------+------------------+-----------+ + +-----------+------------------+---------+ + | firstname | balance_binary | balance | + |-----------+------------------+---------| + | Amber | 1001100100111001 | 39225 | + +-----------+------------------+---------+ You can use this function to convert a number to a string of its hex representation. @@ -162,42 +162,42 @@ Example:: os> source=accounts | where firstname = "Amber" | eval balance_hex = tostring(balance, "hex") | fields firstname, balance_hex, balance fetched rows / total rows = 1/1 - +-------------+---------------+-----------+ - | firstname | balance_hex | balance | - |-------------+---------------+-----------| - | Amber | 9939 | 39225 | - +-------------+---------------+-----------+ + +-----------+-------------+---------+ + | firstname | balance_hex | balance | + |-----------+-------------+---------| + | Amber | 9939 | 39225 | + +-----------+-------------+---------+ The following example formats the column totalSales to display values with commas. Example:: os> source=accounts | where firstname = "Amber" | eval balance_commas = tostring(balance, "commas") | fields firstname, balance_commas, balance fetched rows / total rows = 1/1 - +-------------+------------------+-----------+ - | firstname | balance_commas | balance | - |-------------+------------------+-----------| - | Amber | 39,225 | 39225 | - +-------------+------------------+-----------+ + +-----------+----------------+---------+ + | firstname | balance_commas | balance | + |-----------+----------------+---------| + | Amber | 39,225 | 39225 | + +-----------+----------------+---------+ The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: os> source=accounts | where firstname = "Amber" | eval duration = tostring(6500, "duration") | fields firstname, duration fetched rows / total rows = 1/1 - +-------------+------------+ - | firstname | duration | - |-------------+------------| - | Amber | 01:48:20 | - +-------------+------------+ + +-----------+----------+ + | firstname | duration | + |-----------+----------| + | Amber | 01:48:20 | + +-----------+----------+ The following example for converts boolean parameter to string. Example:: os> source=accounts | where firstname = "Amber"| eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 - +---------------+ - | boolean_str | - |---------------| - | TRUE | - +---------------+ + +-------------+ + | boolean_str | + |-------------| + | TRUE | + +-------------+