diff --git a/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java b/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java index ebec4da7ed5..041d14ab19d 100644 --- a/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/transpiler/UnifiedQueryTranspilerTest.java @@ -47,9 +47,7 @@ public void testToSqlWithCustomDialect() { normalize( "SELECT *\nFROM `catalog`.`employees`\nWHERE TRY_CAST(`name` AS DOUBLE) = 1.230E2"); assertEquals( - "Transpiled query using OpenSearchSparkSqlDialect should translate SAFE_CAST to TRY_CAST", - expectedSql, - actualSql); + "String field compared with numeric literal coerces to DOUBLE", expectedSql, actualSql); } /** Normalizes line endings to platform-specific format for cross-platform test compatibility. */ diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index c2ce4a740ec..f4ce0f09f21 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -579,26 +579,8 @@ public RexNode visitWindowFunction(WindowFunction node, CalcitePlanContext conte (arguments.isEmpty() || arguments.size() == 1) ? Collections.emptyList() : arguments.subList(1, arguments.size()); - List nodes = - PPLFuncImpTable.INSTANCE.validateAggFunctionSignature( - functionName, field, args, context.rexBuilder); - return nodes != null - ? PlanUtils.makeOver( - context, - functionName, - nodes.getFirst(), - nodes.size() <= 1 ? Collections.emptyList() : nodes.subList(1, nodes.size()), - partitions, - List.of(), - node.getWindowFrame()) - : PlanUtils.makeOver( - context, - functionName, - field, - args, - partitions, - List.of(), - node.getWindowFrame()); + return PlanUtils.makeOver( + context, functionName, field, args, partitions, List.of(), node.getWindowFrame()); }) .orElseThrow( () -> diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeUtil.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeUtil.java new file mode 100644 index 00000000000..e75bf9eced9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeUtil.java @@ -0,0 +1,225 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import lombok.experimental.UtilityClass; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.type.AbstractExprRelDataType; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; + +/** + * Utility methods for to derive types, containing special handling logics for user-defined-types. + * + * @see SqlTypeUtil utilities used during SQL validation or type derivation. + */ +@UtilityClass +public class OpenSearchTypeUtil { + /** + * Whether a given RelDataType is a user-defined type (UDT) + * + * @param type the RelDataType to check + * @return true if the type is a user-defined type, false otherwise + */ + public static boolean isUserDefinedType(RelDataType type) { + return type instanceof AbstractExprRelDataType; + } + + /** + * Checks if the RelDataType represents a numeric type. Supports standard SQL numeric types + * (INTEGER, BIGINT, SMALLINT, TINYINT, FLOAT, DOUBLE, DECIMAL, REAL), OpenSearch UDT numeric + * types, and string types (VARCHAR, CHAR). + * + * @param fieldType the RelDataType to check + * @return true if the type is numeric or string, false otherwise + */ + public static boolean isNumericOrCharacter(RelDataType fieldType) { + // Check for OpenSearch UDT numeric types + if (isUserDefinedType(fieldType)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; + ExprType udtType = exprType.getExprType(); + return ExprCoreType.numberTypes().contains(udtType); + } + + // Check standard SQL numeric types & string types (VARCHAR, CHAR) + if (SqlTypeUtil.isNumeric(fieldType) || SqlTypeUtil.isCharacter(fieldType)) { + return true; + } + + return false; + } + + /** + * Checks if the RelDataType represents a time-based field (timestamp, date, or time). Supports + * both standard SQL time types (including TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, DATE, TIME, + * and their timezone variants) and OpenSearch UDT time types. + * + * @param fieldType the RelDataType to check + * @return true if the type is time-based, false otherwise + */ + public static boolean isDatetime(RelDataType fieldType) { + // Check standard SQL time types + if (SqlTypeUtil.isDatetime(fieldType)) { + return true; + } + + // Check for OpenSearch UDT types (EXPR_TIMESTAMP mapped to VARCHAR) + if (isUserDefinedType(fieldType)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; + ExprType udtType = exprType.getExprType(); + return udtType == ExprCoreType.TIMESTAMP + || udtType == ExprCoreType.DATE + || udtType == ExprCoreType.TIME; + } + + return false; + } + + /** + * Checks whether a {@link RelDataType} represents a date type. + * + *

This method returns true for both Calcite's built-in {@link SqlTypeName#DATE} type and + * OpenSearch's user-defined date type {@link OpenSearchTypeFactory.ExprUDT#EXPR_DATE}. + * + * @param type the type to check + * @return true if the type is a date type (built-in or user-defined), false otherwise + */ + public static boolean isDate(RelDataType type) { + if (isUserDefinedType(type)) { + if (((AbstractExprRelDataType) type).getUdt() == OpenSearchTypeFactory.ExprUDT.EXPR_DATE) { + return true; + } + } + return SqlTypeName.DATE.equals(type.getSqlTypeName()); + } + + /** + * Checks whether a {@link RelDataType} represents a timestamp type. + * + *

This method returns true for both Calcite's built-in {@link SqlTypeName#TIMESTAMP} type and + * OpenSearch's user-defined timestamp type {@link OpenSearchTypeFactory.ExprUDT#EXPR_TIMESTAMP}. + * + * @param type the type to check + * @return true if the type is a timestamp type (built-in or user-defined), false otherwise + */ + public static boolean isTimestamp(RelDataType type) { + if (isUserDefinedType(type)) { + if (((AbstractExprRelDataType) type).getUdt() + == OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP) { + return true; + } + } + return SqlTypeName.TIMESTAMP.equals(type.getSqlTypeName()); + } + + /** + * Checks whether a {@link RelDataType} represents a time type. + * + *

This method returns true for both Calcite's built-in {@link SqlTypeName#TIME} type and + * OpenSearch's user-defined time type {@link OpenSearchTypeFactory.ExprUDT#EXPR_TIME}. + * + * @param type the type to check + * @return true if the type is a time type (built-in or user-defined), false otherwise + */ + public static boolean isTime(RelDataType type) { + if (isUserDefinedType(type)) { + if (((AbstractExprRelDataType) type).getUdt() == OpenSearchTypeFactory.ExprUDT.EXPR_TIME) { + return true; + } + } + return SqlTypeName.TIME.equals(type.getSqlTypeName()); + } + + /** + * This method should be used in place for {@link SqlTypeUtil#isCharacter(RelDataType)} because + * user-defined types also have VARCHAR as their SqlTypeName. + */ + public static boolean isCharacter(RelDataType type) { + return !isUserDefinedType(type) && SqlTypeUtil.isCharacter(type); + } + + /** + * Checks whether a {@link RelDataType} represents an IP address type. + * + *

This method returns true only for OpenSearch's user-defined IP type {@link + * OpenSearchTypeFactory.ExprUDT#EXPR_IP}. + * + * @param type the type to check + * @return true if the type is an IP address type, false otherwise + */ + public static boolean isIp(RelDataType type) { + return isIp(type, false); + } + + /** + * Checks whether a {@link RelDataType} represents an IP address type. If {@code acceptOther} is + * set, {@link SqlTypeName#OTHER} is also accepted as an IP type. + * + *

{@link SqlTypeName#OTHER} is "borrowed" to represent IP type during validation because + * SqlTypeName.IP does not exist + * + * @param type the type to check + * @param acceptOther whether to accept OTHER as a valid IP type + * @return true if the type is an IP address type, false otherwise + */ + public static boolean isIp(RelDataType type, boolean acceptOther) { + if (isUserDefinedType(type)) { + return ((AbstractExprRelDataType) type).getUdt() == OpenSearchTypeFactory.ExprUDT.EXPR_IP; + } + if (acceptOther) { + return type.getSqlTypeName() == SqlTypeName.OTHER; + } + return false; + } + + /** + * Checks whether a {@link RelDataType} represents a binary type. + * + *

This method returns true for both Calcite's built-in binary types (BINARY, VARBINARY) and + * OpenSearch's user-defined binary type {@link OpenSearchTypeFactory.ExprUDT#EXPR_BINARY}. + * + * @param type the type to check + * @return true if the type is a binary type (built-in or user-defined), false otherwise + */ + public static boolean isBinary(RelDataType type) { + if (isUserDefinedType(type)) { + return ((AbstractExprRelDataType) type).getUdt() + == OpenSearchTypeFactory.ExprUDT.EXPR_BINARY; + } + return SqlTypeName.BINARY_TYPES.contains(type.getSqlTypeName()); + } + + /** + * Checks whether a {@link RelDataType} represents a scalar type. + * + *

Scalar types include all primitive and atomic types such as numeric types (INTEGER, BIGINT, + * FLOAT, DOUBLE, DECIMAL), string types (VARCHAR, CHAR), boolean, temporal types (DATE, TIME, + * TIMESTAMP), and special scalar types (IP, BINARY, UUID). + * + *

This method returns false for composite types including: + * + *

+ * + * @param type the type to check; may be null + * @return true if the type is a scalar type, false if it is a composite type or null + */ + public static boolean isScalar(RelDataType type) { + if (type == null) { + return false; + } + return !type.isStruct() + && !SqlTypeUtil.isMap(type) + && !SqlTypeUtil.isCollection(type) + && !SqlTypeUtil.isRow(type); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index abf37e68392..a56473085cc 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -5,10 +5,17 @@ package org.opensearch.sql.calcite.utils; +import java.util.Locale; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** @@ -20,84 +27,32 @@ public class PPLOperandTypes { // This class is not meant to be instantiated. private PPLOperandTypes() {} - /** List of all scalar type signatures (single parameter each) */ - private static final java.util.List> - SCALAR_TYPES = - java.util.List.of( - // Numeric types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BYTE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.SHORT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.INTEGER), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.LONG), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.FLOAT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DOUBLE), - // String type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.STRING), - // Boolean type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BOOLEAN), - // Temporal types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DATE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIME), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP), - // Special scalar types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.IP), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BINARY)); - - /** Helper method to create scalar types with optional integer parameter */ - private static java.util.List> - createScalarWithOptionalInteger() { - java.util.List> result = - new java.util.ArrayList<>(SCALAR_TYPES); - - // Add scalar + integer combinations - SCALAR_TYPES.forEach( - scalarType -> - result.add( - java.util.List.of( - scalarType.get(0), org.opensearch.sql.data.type.ExprCoreType.INTEGER))); - - return result; - } - public static final UDFOperandMetadata NONE = UDFOperandMetadata.wrap(OperandTypes.family()); - public static final UDFOperandMetadata OPTIONAL_ANY = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ANY).or(OperandTypes.family())); + public static final UDFOperandMetadata OPTIONAL_INTEGER = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) OperandTypes.INTEGER.or(OperandTypes.family())); - public static final UDFOperandMetadata STRING = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.CHARACTER); - public static final UDFOperandMetadata INTEGER = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER); - public static final UDFOperandMetadata NUMERIC = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC); + UDFOperandMetadata.wrap(OperandTypes.INTEGER.or(OperandTypes.family())); + public static final UDFOperandMetadata STRING = UDFOperandMetadata.wrap(OperandTypes.CHARACTER); + public static final UDFOperandMetadata INTEGER = UDFOperandMetadata.wrap(OperandTypes.INTEGER); + public static final UDFOperandMetadata NUMERIC = UDFOperandMetadata.wrap(OperandTypes.NUMERIC); public static final UDFOperandMetadata NUMERIC_OPTIONAL_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC.or( - OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER))); + OperandTypes.NUMERIC.or( + OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata ANY_OPTIONAL_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER))); - public static final UDFOperandMetadata ANY_OPTIONAL_TIMESTAMP = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.TIMESTAMP))); + OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata INTEGER_INTEGER = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER_INTEGER); + UDFOperandMetadata.wrap(OperandTypes.INTEGER_INTEGER); public static final UDFOperandMetadata STRING_STRING = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.CHARACTER_CHARACTER); + UDFOperandMetadata.wrap(OperandTypes.CHARACTER_CHARACTER); public static final UDFOperandMetadata STRING_STRING_STRING = UDFOperandMetadata.wrap( OperandTypes.family( SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); public static final UDFOperandMetadata NUMERIC_NUMERIC = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC); + UDFOperandMetadata.wrap(OperandTypes.NUMERIC_NUMERIC); public static final UDFOperandMetadata STRING_INTEGER = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER)); public static final UDFOperandMetadata STRING_STRING_INTEGER = @@ -107,9 +62,8 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata STRING_OR_STRING_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER) - .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); + OperandTypes.family(SqlTypeFamily.CHARACTER) + .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata STRING_STRING_INTEGER_INTEGER = UDFOperandMetadata.wrap( @@ -121,16 +75,18 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata NUMERIC_STRING_OR_STRING_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - (OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) - .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + (OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER)) + .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); - public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = + public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC_SYMBOL = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_NUMERIC.or( + OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY) + .or( OperandTypes.family( - SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC))); + SqlTypeFamily.NUMERIC, + SqlTypeFamily.NUMERIC, + SqlTypeFamily.NUMERIC, + SqlTypeFamily.ANY))); public static final UDFOperandMetadata NUMERIC_NUMERIC_NUMERIC = UDFOperandMetadata.wrap( OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); @@ -144,61 +100,60 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata WIDTH_BUCKET_OPERAND = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - // 1. Numeric fields: bin age span=10 - OperandTypes.family( - SqlTypeFamily.NUMERIC, + // 1. Numeric fields: bin age span=10 + OperandTypes.family( + SqlTypeFamily.NUMERIC, + SqlTypeFamily.INTEGER, + SqlTypeFamily.NUMERIC, + SqlTypeFamily.NUMERIC) + // 2. Timestamp fields with OpenSearch type system + // Used in: Production + Integration tests (CalciteBinCommandIT) + .or( + OperandTypes.family( + SqlTypeFamily.TIMESTAMP, SqlTypeFamily.INTEGER, - SqlTypeFamily.NUMERIC, - SqlTypeFamily.NUMERIC) - // 2. Timestamp fields with OpenSearch type system - // Used in: Production + Integration tests (CalciteBinCommandIT) - .or( - OperandTypes.family( - SqlTypeFamily.TIMESTAMP, - SqlTypeFamily.INTEGER, - SqlTypeFamily.CHARACTER, // TIMESTAMP - TIMESTAMP = INTERVAL (as STRING) - SqlTypeFamily.TIMESTAMP)) - // 3. Timestamp fields with Calcite SCOTT schema - // Used in: Unit tests (CalcitePPLBinTest) - .or( - OperandTypes.family( - SqlTypeFamily.TIMESTAMP, - SqlTypeFamily.INTEGER, - SqlTypeFamily.TIMESTAMP, // TIMESTAMP - TIMESTAMP = TIMESTAMP - SqlTypeFamily.TIMESTAMP)) - // DATE field with OpenSearch type system - // Used in: Production + Integration tests (CalciteBinCommandIT) - .or( - OperandTypes.family( - SqlTypeFamily.DATE, - SqlTypeFamily.INTEGER, - SqlTypeFamily.CHARACTER, // DATE - DATE = INTERVAL (as STRING) - SqlTypeFamily.DATE)) - // DATE field with Calcite SCOTT schema - // Used in: Unit tests (CalcitePPLBinTest) - .or( - OperandTypes.family( - SqlTypeFamily.DATE, - SqlTypeFamily.INTEGER, - SqlTypeFamily.DATE, // DATE - DATE = DATE - SqlTypeFamily.DATE)) - // TIME field with OpenSearch type system - // Used in: Production + Integration tests (CalciteBinCommandIT) - .or( - OperandTypes.family( - SqlTypeFamily.TIME, - SqlTypeFamily.INTEGER, - SqlTypeFamily.CHARACTER, // TIME - TIME = INTERVAL (as STRING) - SqlTypeFamily.TIME)) - // TIME field with Calcite SCOTT schema - // Used in: Unit tests (CalcitePPLBinTest) - .or( - OperandTypes.family( - SqlTypeFamily.TIME, - SqlTypeFamily.INTEGER, - SqlTypeFamily.TIME, // TIME - TIME = TIME - SqlTypeFamily.TIME))); + SqlTypeFamily.CHARACTER, // TIMESTAMP - TIMESTAMP = INTERVAL (as STRING) + SqlTypeFamily.TIMESTAMP)) + // 3. Timestamp fields with Calcite SCOTT schema + // Used in: Unit tests (CalcitePPLBinTest) + .or( + OperandTypes.family( + SqlTypeFamily.TIMESTAMP, + SqlTypeFamily.INTEGER, + SqlTypeFamily.TIMESTAMP, // TIMESTAMP - TIMESTAMP = TIMESTAMP + SqlTypeFamily.TIMESTAMP)) + // DATE field with OpenSearch type system + // Used in: Production + Integration tests (CalciteBinCommandIT) + .or( + OperandTypes.family( + SqlTypeFamily.DATE, + SqlTypeFamily.INTEGER, + SqlTypeFamily.CHARACTER, // DATE - DATE = INTERVAL (as STRING) + SqlTypeFamily.DATE)) + // DATE field with Calcite SCOTT schema + // Used in: Unit tests (CalcitePPLBinTest) + .or( + OperandTypes.family( + SqlTypeFamily.DATE, + SqlTypeFamily.INTEGER, + SqlTypeFamily.DATE, // DATE - DATE = DATE + SqlTypeFamily.DATE)) + // TIME field with OpenSearch type system + // Used in: Production + Integration tests (CalciteBinCommandIT) + .or( + OperandTypes.family( + SqlTypeFamily.TIME, + SqlTypeFamily.INTEGER, + SqlTypeFamily.CHARACTER, // TIME - TIME = INTERVAL (as STRING) + SqlTypeFamily.TIME)) + // TIME field with Calcite SCOTT schema + // Used in: Unit tests (CalcitePPLBinTest) + .or( + OperandTypes.family( + SqlTypeFamily.TIME, + SqlTypeFamily.INTEGER, + SqlTypeFamily.TIME, // TIME - TIME = TIME + SqlTypeFamily.TIME))); public static final UDFOperandMetadata NUMERIC_NUMERIC_NUMERIC_NUMERIC_NUMERIC = UDFOperandMetadata.wrap( @@ -210,115 +165,130 @@ private PPLOperandTypes() {} SqlTypeFamily.NUMERIC)); public static final UDFOperandMetadata STRING_OR_INTEGER_INTEGER_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER))); + OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER) + .or( + OperandTypes.family( + SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata OPTIONAL_DATE_OR_TIMESTAMP_OR_NUMERIC = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.DATETIME.or(OperandTypes.NUMERIC).or(OperandTypes.family())); + OperandTypes.DATETIME.or(OperandTypes.NUMERIC).or(OperandTypes.family())); public static final UDFOperandMetadata DATETIME_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) OperandTypes.DATETIME.or(OperandTypes.CHARACTER)); + UDFOperandMetadata.wrap(OperandTypes.DATETIME.or(OperandTypes.CHARACTER)); public static final UDFOperandMetadata TIME_OR_TIMESTAMP_OR_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.CHARACTER.or(OperandTypes.TIME).or(OperandTypes.TIMESTAMP)); + OperandTypes.CHARACTER.or(OperandTypes.TIME).or(OperandTypes.TIMESTAMP)); public static final UDFOperandMetadata DATE_OR_TIMESTAMP_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) OperandTypes.DATE_OR_TIMESTAMP.or(OperandTypes.CHARACTER)); + UDFOperandMetadata.wrap(OperandTypes.DATE_OR_TIMESTAMP.or(OperandTypes.CHARACTER)); public static final UDFOperandMetadata DATETIME_OR_STRING_OR_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.DATETIME.or(OperandTypes.CHARACTER).or(OperandTypes.INTEGER)); + OperandTypes.DATETIME.or(OperandTypes.CHARACTER).or(OperandTypes.INTEGER)); public static final UDFOperandMetadata DATETIME_OPTIONAL_INTEGER = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.DATETIME.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.INTEGER))); - public static final UDFOperandMetadata ANY_DATETIME_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ANY) - .or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.DATETIME)) - .or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.STRING))); + OperandTypes.DATETIME.or( + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.INTEGER))); public static final UDFOperandMetadata DATETIME_DATETIME = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); public static final UDFOperandMetadata DATETIME_OR_STRING_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER) - .or(OperandTypes.CHARACTER_CHARACTER)); - public static final UDFOperandMetadata DATETIME_OR_STRING_DATETIME_OR_STRING = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.CHARACTER_CHARACTER - .or(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)) - .or(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER)) - .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME))); + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER) + .or(OperandTypes.CHARACTER_CHARACTER)); public static final UDFOperandMetadata STRING_TIMESTAMP = UDFOperandMetadata.wrap( OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.TIMESTAMP)); public static final UDFOperandMetadata STRING_DATETIME = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME)); public static final UDFOperandMetadata DATETIME_INTERVAL = - UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.DATETIME_INTERVAL); + UDFOperandMetadata.wrap(OperandTypes.DATETIME_INTERVAL); public static final UDFOperandMetadata TIME_TIME = UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.TIME, SqlTypeFamily.TIME)); public static final UDFOperandMetadata TIMESTAMP_OR_STRING_STRING_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.TIMESTAMP, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER))); + OperandTypes.family( + SqlTypeFamily.TIMESTAMP, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata STRING_INTEGER_DATETIME_OR_STRING = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.CHARACTER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.DATETIME))); + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.CHARACTER) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.DATETIME))); public static final UDFOperandMetadata INTERVAL_DATETIME_DATETIME = UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME)) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER)) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER, - SqlTypeFamily.CHARACTER))); + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME)) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.DATETIME, SqlTypeFamily.CHARACTER)) + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); /** * Operand type checker that accepts any scalar type. This includes numeric types, strings, * booleans, datetime types, and special scalar types like IP and BINARY. Excludes complex types * like arrays, structs, and maps. */ - public static final UDFOperandMetadata ANY_SCALAR = UDFOperandMetadata.wrapUDT(SCALAR_TYPES); + public static final UDFOperandMetadata SCALAR = + UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + return OpenSearchTypeUtil.isScalar(callBinding.getOperandType(0)); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(1); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return String.format(Locale.ROOT, "%s()", opName); + } + }); /** * Operand type checker that accepts any scalar type with an optional integer argument. This is * used for aggregation functions that take a field and an optional limit/size parameter. */ - public static final UDFOperandMetadata ANY_SCALAR_OPTIONAL_INTEGER = - UDFOperandMetadata.wrapUDT(createScalarWithOptionalInteger()); + public static final UDFOperandMetadata SCALAR_OPTIONAL_INTEGER = + UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + boolean valid = OpenSearchTypeUtil.isScalar(callBinding.getOperandType(0)); + if (callBinding.getOperandCount() == 2) { + valid = valid && SqlTypeUtil.isIntType(callBinding.getOperandType(1)); + } + return valid; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.between(1, 2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return String.format( + Locale.ROOT, "%s(), %s(, )", opName, opName); + } + }); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index f619d966cc8..db91ffa54ac 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -39,6 +39,7 @@ import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Optionality; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.udf.UserDefinedAggFunction; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; @@ -213,7 +214,7 @@ public static ImplementorUDF adaptExprMethodToUDF( String methodName, SqlReturnTypeInference returnTypeInference, NullPolicy nullPolicy, - @Nullable UDFOperandMetadata operandMetadata) { + @NonNull UDFOperandMetadata operandMetadata) { NotNullImplementor implementor = (translator, call, translatedOperands) -> { List operands = @@ -229,7 +230,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return operandMetadata; } }; @@ -261,7 +262,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return operandMetadata; } }; @@ -302,7 +303,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return operandMetadata; } }; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java index a4e924b631c..d7bfd42d57b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java @@ -8,7 +8,7 @@ import lombok.Getter; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.exception.SemanticCheckException; /** Represents a field that supports binning operations. */ @@ -33,8 +33,8 @@ public BinnableField(RexNode fieldExpr, RelDataType fieldType, String fieldName) this.fieldType = fieldType; this.fieldName = fieldName; - this.isTimeBased = OpenSearchTypeFactory.isTimeBasedType(fieldType); - this.isNumeric = OpenSearchTypeFactory.isNumericType(fieldType); + this.isTimeBased = OpenSearchTypeUtil.isDatetime(fieldType); + this.isNumeric = OpenSearchTypeUtil.isNumericOrCharacter(fieldType); // Reject truly unsupported types (e.g., BOOLEAN, ARRAY, MAP) if (!isNumeric && !isTimeBased) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java index 9a77a0d5a7c..f1c205d5d19 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ArrayFunctionImpl.java @@ -22,8 +22,10 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -59,8 +61,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.ARRAY_FUNCTION); } public static class ArrayImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java index 4b1c9586773..fb2514e1ca9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ExistsFunctionImpl.java @@ -14,8 +14,10 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -35,8 +37,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.EXISTS); } public static class ExistsImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java index 953b75303db..09d16c9d6ae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/FilterFunctionImpl.java @@ -15,8 +15,10 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -35,8 +37,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.MAP_FUNCTION); } public static class FilterImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java index 720eee841ab..2da189e27b2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForallFunctionImpl.java @@ -14,8 +14,10 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -34,8 +36,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.MAP_FUNCTION); } public static class ForallImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java index 107df5eea4e..07e4336bb5a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -18,8 +18,10 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -49,8 +51,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } private static RelDataType determineElementType( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVFindFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVFindFunctionImpl.java index 9c189bf2ff5..4c7a17865b0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVFindFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVFindFunctionImpl.java @@ -20,6 +20,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -45,7 +46,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { // Accept ARRAY and STRING for the regex pattern return UDFOperandMetadata.wrap( OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER)); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVZipFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVZipFunctionImpl.java index 7d75ae71941..a204e7f9e03 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVZipFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVZipFunctionImpl.java @@ -24,6 +24,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -62,7 +63,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { // First two arguments must be arrays, optional STRING delimiter return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java index 4cb0acae612..de90919095f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java @@ -18,8 +18,11 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -44,8 +47,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.MAP)); } public static class MapAppendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java index 1f86fcbe636..bf26e995604 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java @@ -16,7 +16,10 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -41,8 +44,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.ARRAY)); } public static class MapRemoveImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java index d60a700e816..580a6db15dc 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ReduceFunctionImpl.java @@ -5,13 +5,10 @@ package org.opensearch.sql.expression.function.CollectionUDF; -import static org.opensearch.sql.expression.function.CollectionUDF.LambdaUtils.inferReturnTypeFromLambda; import static org.opensearch.sql.expression.function.CollectionUDF.LambdaUtils.transferLambdaOutputToTargetType; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -19,14 +16,20 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCallBinding; -import org.apache.calcite.rex.RexLambda; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlLambda; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.sql.validate.SqlValidator; +import org.jspecify.annotations.NonNull; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -43,35 +46,54 @@ public ReduceFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - RexCallBinding rexCallBinding = (RexCallBinding) sqlOperatorBinding; - List rexNodes = rexCallBinding.operands(); - ArraySqlType listType = (ArraySqlType) rexNodes.get(0).getType(); - RelDataType elementType = listType.getComponentType(); - RelDataType baseType = rexNodes.get(1).getType(); - Map map = new HashMap<>(); - RexLambda mergeLambda = (RexLambda) rexNodes.get(2); - map.put(mergeLambda.getParameters().get(0).getName(), baseType); - map.put(mergeLambda.getParameters().get(1).getName(), elementType); - RelDataType mergedReturnType = - inferReturnTypeFromLambda((RexLambda) rexNodes.get(2), map, typeFactory); - if (mergedReturnType != baseType) { // For different acc, we need to recalculate - map.put(mergeLambda.getParameters().get(0).getName(), mergedReturnType); - mergedReturnType = inferReturnTypeFromLambda((RexLambda) rexNodes.get(2), map, typeFactory); - } - RelDataType finalReturnType; - if (rexNodes.size() > 3) { - finalReturnType = inferReturnTypeFromLambda((RexLambda) rexNodes.get(3), map, typeFactory); - } else { - finalReturnType = mergedReturnType; + if (sqlOperatorBinding instanceof RexCallBinding) { + return sqlOperatorBinding.getOperandType(sqlOperatorBinding.getOperandCount() - 1); + } else if (sqlOperatorBinding instanceof SqlCallBinding callBinding) { + RelDataType elementType = callBinding.getOperandType(0).getComponentType(); + RelDataType baseType = callBinding.getOperandType(1); + SqlLambda reduce1 = callBinding.getCall().operand(2); + SqlNode function1 = reduce1.getExpression(); + SqlValidator validator = callBinding.getValidator(); + // The saved types are ANY because the lambda function is defined as (ANY, ..) -> ANY + // Force it to derive types again by removing existing saved types + validator.removeValidatedNodeType(function1); + if (function1 instanceof SqlCall call) { + List operands = call.getOperandList(); + // The first argument is base (accumulator), while the second is from the array + if (!operands.isEmpty()) validator.setValidatedNodeType(operands.get(0), baseType); + if (operands.size() > 1 && elementType != null) + validator.setValidatedNodeType(operands.get(1), elementType); + } + RelDataType returnType = SqlTypeUtil.deriveType(callBinding, function1); + if (callBinding.getOperandCount() > 3) { + SqlLambda reduce2 = callBinding.getCall().operand(3); + SqlNode function2 = reduce2.getExpression(); + validator.removeValidatedNodeType(function2); + if (function2 instanceof SqlCall call) { + List operands = call.getOperandList(); + if (!operands.isEmpty()) validator.setValidatedNodeType(operands.get(0), returnType); + } + returnType = SqlTypeUtil.deriveType(callBinding, function2); + } + return returnType; } - return finalReturnType; + throw new IllegalStateException( + StringUtils.format( + "sqlOperatorBinding can only be either RexCallBinding or SqlCallBinding, but got %s", + sqlOperatorBinding.getClass())); }; } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap( + OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.ANY, SqlTypeFamily.FUNCTION) + .or( + OperandTypes.family( + SqlTypeFamily.ARRAY, + SqlTypeFamily.ANY, + SqlTypeFamily.FUNCTION, + SqlTypeFamily.FUNCTION))); } public static class ReduceImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java index d2184e12e8c..807239cff1d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/TransformFunctionImpl.java @@ -6,10 +6,12 @@ package org.opensearch.sql.expression.function.CollectionUDF; import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; +import static org.apache.calcite.util.Static.RESOURCE; import static org.opensearch.sql.expression.function.CollectionUDF.LambdaUtils.transferLambdaOutputToTargetType; import java.util.ArrayList; import java.util.List; +import java.util.stream.IntStream; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -19,12 +21,20 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexCallBinding; -import org.apache.calcite.rex.RexLambda; -import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.type.FamilyOperandTypeChecker; +import org.apache.calcite.sql.type.SqlOperandCountRanges; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlSingleOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -43,17 +53,104 @@ public TransformFunctionImpl() { public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - RexCallBinding rexCallBinding = (RexCallBinding) sqlOperatorBinding; - List operands = rexCallBinding.operands(); - RelDataType lambdaReturnType = ((RexLambda) operands.get(1)).getExpression().getType(); + RelDataType lambdaReturnType = sqlOperatorBinding.getOperandType(1); return createArrayType( typeFactory, typeFactory.createTypeWithNullability(lambdaReturnType, true), true); }; } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + // Only checks the first two arguments as it allows arbitrary number of arguments to follow them + return UDFOperandMetadata.wrap( + new SqlSingleOperandTypeChecker() { + private static final List families = + List.of(SqlTypeFamily.ARRAY, SqlTypeFamily.FUNCTION); + + /** + * Copied from {@link FamilyOperandTypeChecker#checkSingleOperandType(SqlCallBinding + * callBinding, SqlNode node, int iFormalOperand, boolean throwOnFailure)} + */ + @Override + public boolean checkSingleOperandType( + SqlCallBinding callBinding, + SqlNode operand, + int iFormalOperand, + boolean throwOnFailure) { + // Do not check types after the second operands + if (iFormalOperand > 1) { + return true; + } + SqlTypeFamily family = families.get(iFormalOperand); + switch (family) { + case ANY: + final RelDataType type = SqlTypeUtil.deriveType(callBinding, operand); + SqlTypeName typeName = type.getSqlTypeName(); + + if (typeName == SqlTypeName.CURSOR) { + // We do not allow CURSOR operands, even for ANY + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return false; + } + // fall through + case IGNORE: + // no need to check + return true; + default: + break; + } + if (SqlUtil.isNullLiteral(operand, false)) { + if (callBinding.isTypeCoercionEnabled()) { + return true; + } else if (throwOnFailure) { + throw callBinding + .getValidator() + .newValidationError(operand, RESOURCE.nullIllegal()); + } else { + return false; + } + } + RelDataType type = SqlTypeUtil.deriveType(callBinding, operand); + SqlTypeName typeName = type.getSqlTypeName(); + + // Pass type checking for operators if it's of type 'ANY'. + if (typeName.getFamily() == SqlTypeFamily.ANY) { + return true; + } + + if (!family.getTypeNames().contains(typeName)) { + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return false; + } + return true; + } + + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + return IntStream.range(0, 2) + .allMatch( + i -> + checkSingleOperandType( + callBinding, callBinding.operand(i), i, throwOnFailure)); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.from(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return ""; + } + }); } public static class TransformImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 2d769194924..7f51d0fa12e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -23,11 +23,20 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; +import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.util.BuiltInMethod; import org.opensearch.sql.calcite.udf.udaf.FirstAggFunction; import org.opensearch.sql.calcite.udf.udaf.LastAggFunction; @@ -113,66 +122,66 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { Suppliers.memoize(() -> (PPLBuiltinOperators) new PPLBuiltinOperators().init()); // Json Functions - public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); - public static final SqlOperator JSON_ARRAY_LENGTH = + public static final SqlFunction JSON = new JsonFunctionImpl().toUDF("JSON"); + public static final SqlFunction JSON_ARRAY_LENGTH = new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); - public static final SqlOperator JSON_EXTRACT = + public static final SqlFunction JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); - public static final SqlOperator JSON_EXTRACT_ALL = + public static final SqlFunction JSON_EXTRACT_ALL = new JsonExtractAllFunctionImpl().toUDF("JSON_EXTRACT_ALL"); - public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); - public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); - public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); - public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); - public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); + public static final SqlFunction JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); + public static final SqlFunction JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); + public static final SqlFunction JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); + public static final SqlFunction JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); + public static final SqlFunction JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); // Math functions - public static final SqlOperator SPAN = new SpanFunction().toUDF("SPAN"); - public static final SqlOperator E = new EulerFunction().toUDF("E"); - public static final SqlOperator CONV = new ConvFunction().toUDF("CONVERT"); - public static final SqlOperator MOD = new ModFunction().toUDF("MOD"); - public static final SqlOperator DIVIDE = new DivideFunction().toUDF("DIVIDE"); - public static final SqlOperator SHA2 = CryptographicFunction.sha2().toUDF("SHA2"); - public static final SqlOperator CIDRMATCH = new CidrMatchFunction().toUDF("CIDRMATCH"); - public static final SqlOperator SCALAR_MAX = new ScalarMaxFunction().toUDF("SCALAR_MAX"); - public static final SqlOperator SCALAR_MIN = new ScalarMinFunction().toUDF("SCALAR_MIN"); + public static final SqlFunction SPAN = new SpanFunction().toUDF("SPAN"); + public static final SqlFunction E = new EulerFunction().toUDF("E"); + public static final SqlFunction CONV = new ConvFunction().toUDF("CONVERT"); + public static final SqlFunction MOD = new ModFunction().toUDF("MOD"); + public static final SqlFunction DIVIDE = new DivideFunction().toUDF("DIVIDE"); + public static final SqlFunction SHA2 = CryptographicFunction.sha2().toUDF("SHA2"); + public static final SqlFunction CIDRMATCH = new CidrMatchFunction().toUDF("CIDRMATCH"); + public static final SqlFunction SCALAR_MAX = new ScalarMaxFunction().toUDF("SCALAR_MAX"); + public static final SqlFunction SCALAR_MIN = new ScalarMinFunction().toUDF("SCALAR_MIN"); - public static final SqlOperator COSH = + public static final SqlFunction COSH = adaptMathFunctionToUDF( "cosh", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("COSH"); - public static final SqlOperator SINH = + public static final SqlFunction SINH = adaptMathFunctionToUDF( "sinh", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("SINH"); - public static final SqlOperator RINT = + public static final SqlFunction RINT = adaptMathFunctionToUDF( "rint", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("RINT"); - public static final SqlOperator EXPM1 = + public static final SqlFunction EXPM1 = adaptMathFunctionToUDF( "expm1", ReturnTypes.DOUBLE_FORCE_NULLABLE, NullPolicy.ANY, PPLOperandTypes.NUMERIC) .toUDF("EXPM1"); // IP comparing functions - public static final SqlOperator NOT_EQUALS_IP = + public static final SqlFunction NOT_EQUALS_IP = CompareIpFunction.notEquals().toUDF("NOT_EQUALS_IP"); - public static final SqlOperator EQUALS_IP = CompareIpFunction.equals().toUDF("EQUALS_IP"); - public static final SqlOperator GREATER_IP = CompareIpFunction.greater().toUDF("GREATER_IP"); - public static final SqlOperator GTE_IP = CompareIpFunction.greaterOrEquals().toUDF("GTE_IP"); - public static final SqlOperator LESS_IP = CompareIpFunction.less().toUDF("LESS_IP"); - public static final SqlOperator LTE_IP = CompareIpFunction.lessOrEquals().toUDF("LTE_IP"); + public static final SqlFunction EQUALS_IP = CompareIpFunction.equals().toUDF("EQUALS_IP"); + public static final SqlFunction GREATER_IP = CompareIpFunction.greater().toUDF("GREATER_IP"); + public static final SqlFunction GTE_IP = CompareIpFunction.greaterOrEquals().toUDF("GTE_IP"); + public static final SqlFunction LESS_IP = CompareIpFunction.less().toUDF("LESS_IP"); + public static final SqlFunction LTE_IP = CompareIpFunction.lessOrEquals().toUDF("LTE_IP"); // Condition function - public static final SqlOperator EARLIEST = new EarliestFunction().toUDF("EARLIEST"); - public static final SqlOperator LATEST = new LatestFunction().toUDF("LATEST"); + public static final SqlFunction EARLIEST = new EarliestFunction().toUDF("EARLIEST"); + public static final SqlFunction LATEST = new LatestFunction().toUDF("LATEST"); // Datetime function - public static final SqlOperator TIMESTAMP = new TimestampFunction().toUDF("TIMESTAMP"); - public static final SqlOperator DATE = + public static final SqlFunction TIMESTAMP = new TimestampFunction().toUDF("TIMESTAMP"); + public static final SqlFunction DATE = adaptExprMethodToUDF( DateTimeFunctions.class, "exprDate", @@ -180,13 +189,13 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.DATE_OR_TIMESTAMP_OR_STRING) .toUDF("DATE"); - public static final SqlOperator YEARWEEK = new YearweekFunction().toUDF("YEARWEEK"); - public static final SqlOperator WEEKDAY = new WeekdayFunction().toUDF("WEEKDAY"); - public static final SqlOperator UNIX_TIMESTAMP = + public static final SqlFunction YEARWEEK = new YearweekFunction().toUDF("YEARWEEK"); + public static final SqlFunction WEEKDAY = new WeekdayFunction().toUDF("WEEKDAY"); + public static final SqlFunction UNIX_TIMESTAMP = new UnixTimestampFunction().toUDF("UNIX_TIMESTAMP"); - public static final SqlOperator STRFTIME = new StrftimeFunction().toUDF("STRFTIME"); - public static final SqlOperator TO_SECONDS = new ToSecondsFunction().toUDF("TO_SECONDS"); - public static final SqlOperator ADDTIME = + public static final SqlFunction STRFTIME = new StrftimeFunction().toUDF("STRFTIME"); + public static final SqlFunction TO_SECONDS = new ToSecondsFunction().toUDF("TO_SECONDS"); + public static final SqlFunction ADDTIME = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprAddTime", @@ -194,7 +203,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.DATETIME_DATETIME) .toUDF("ADDTIME"); - public static final SqlOperator SUBTIME = + public static final SqlFunction SUBTIME = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprSubTime", @@ -202,22 +211,22 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.DATETIME_DATETIME) .toUDF("SUBTIME"); - public static final SqlOperator ADDDATE = new AddSubDateFunction(true).toUDF("ADDDATE"); - public static final SqlOperator SUBDATE = new AddSubDateFunction(false).toUDF("SUBDATE"); - public static final SqlOperator DATE_ADD = new DateAddSubFunction(true).toUDF("DATE_ADD"); - public static final SqlOperator DATE_SUB = new DateAddSubFunction(false).toUDF("DATE_SUB"); - public static final SqlOperator EXTRACT = new ExtractFunction().toUDF("EXTRACT"); - public static final SqlOperator YEAR = new DatePartFunction(TimeUnit.YEAR).toUDF("YEAR"); - public static final SqlOperator QUARTER = new DatePartFunction(TimeUnit.QUARTER).toUDF("QUARTER"); - public static final SqlOperator MONTH = new DatePartFunction(TimeUnit.MONTH).toUDF("MONTH"); - public static final SqlOperator DAY = new DatePartFunction(TimeUnit.DAY).toUDF("DAY"); - public static final SqlOperator DAY_OF_WEEK = + public static final SqlFunction ADDDATE = new AddSubDateFunction(true).toUDF("ADDDATE"); + public static final SqlFunction SUBDATE = new AddSubDateFunction(false).toUDF("SUBDATE"); + public static final SqlFunction DATE_ADD = new DateAddSubFunction(true).toUDF("DATE_ADD"); + public static final SqlFunction DATE_SUB = new DateAddSubFunction(false).toUDF("DATE_SUB"); + public static final SqlFunction EXTRACT = new ExtractFunction().toUDF("EXTRACT"); + public static final SqlFunction YEAR = new DatePartFunction(TimeUnit.YEAR).toUDF("YEAR"); + public static final SqlFunction QUARTER = new DatePartFunction(TimeUnit.QUARTER).toUDF("QUARTER"); + public static final SqlFunction MONTH = new DatePartFunction(TimeUnit.MONTH).toUDF("MONTH"); + public static final SqlFunction DAY = new DatePartFunction(TimeUnit.DAY).toUDF("DAY"); + public static final SqlFunction DAY_OF_WEEK = new DatePartFunction(TimeUnit.DOW).toUDF("DAY_OF_WEEK"); - public static final SqlOperator DAY_OF_YEAR = + public static final SqlFunction DAY_OF_YEAR = new DatePartFunction(TimeUnit.DOY).toUDF("DAY_OF_YEAR"); - public static final SqlOperator HOUR = new DatePartFunction(TimeUnit.HOUR).toUDF("HOUR"); - public static final SqlOperator MINUTE = new DatePartFunction(TimeUnit.MINUTE).toUDF("MINUTE"); - public static final SqlOperator MINUTE_OF_DAY = + public static final SqlFunction HOUR = new DatePartFunction(TimeUnit.HOUR).toUDF("HOUR"); + public static final SqlFunction MINUTE = new DatePartFunction(TimeUnit.MINUTE).toUDF("MINUTE"); + public static final SqlFunction MINUTE_OF_DAY = adaptExprMethodToUDF( DateTimeFunctions.class, "exprMinuteOfDay", @@ -225,22 +234,22 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.TIME_OR_TIMESTAMP_OR_STRING) .toUDF("MINUTE_OF_DAY"); - public static final SqlOperator SECOND = new DatePartFunction(TimeUnit.SECOND).toUDF("SECOND"); - public static final SqlOperator MICROSECOND = + public static final SqlFunction SECOND = new DatePartFunction(TimeUnit.SECOND).toUDF("SECOND"); + public static final SqlFunction MICROSECOND = new DatePartFunction(TimeUnit.MICROSECOND).toUDF("MICROSECOND"); - public static final SqlOperator NOW = new CurrentFunction(ExprCoreType.TIMESTAMP).toUDF("NOW"); - public static final SqlOperator CURRENT_TIME = + public static final SqlFunction NOW = new CurrentFunction(ExprCoreType.TIMESTAMP).toUDF("NOW"); + public static final SqlFunction CURRENT_TIME = new CurrentFunction(ExprCoreType.TIME).toUDF("CURRENT_TIME"); - public static final SqlOperator CURRENT_DATE = + public static final SqlFunction CURRENT_DATE = new CurrentFunction(ExprCoreType.DATE).toUDF("CURRENT_DATE"); - public static final SqlOperator DATE_FORMAT = + public static final SqlFunction DATE_FORMAT = new FormatFunction(ExprCoreType.DATE).toUDF("DATE_FORMAT"); - public static final SqlOperator TIME_FORMAT = + public static final SqlFunction TIME_FORMAT = new FormatFunction(ExprCoreType.TIME).toUDF("TIME_FORMAT"); - public static final SqlOperator DAYNAME = new PeriodNameFunction(TimeUnit.DAY).toUDF("DAYNAME"); - public static final SqlOperator MONTHNAME = + public static final SqlFunction DAYNAME = new PeriodNameFunction(TimeUnit.DAY).toUDF("DAYNAME"); + public static final SqlFunction MONTHNAME = new PeriodNameFunction(TimeUnit.MONTH).toUDF("MONTHNAME"); - public static final SqlOperator CONVERT_TZ = + public static final SqlFunction CONVERT_TZ = adaptExprMethodToUDF( DateTimeFunctions.class, "exprConvertTZ", @@ -248,7 +257,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.TIMESTAMP_OR_STRING_STRING_STRING) .toUDF("CONVERT_TZ"); - public static final SqlOperator DATEDIFF = + public static final SqlFunction DATEDIFF = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprDateDiff", @@ -256,10 +265,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.DATETIME_DATETIME) .toUDF("DATEDIFF"); - public static final SqlOperator TIMESTAMPDIFF = + public static final SqlFunction TIMESTAMPDIFF = new TimestampDiffFunction().toUDF("TIMESTAMPDIFF"); - public static final SqlOperator LAST_DAY = new LastDayFunction().toUDF("LAST_DAY"); - public static final SqlOperator FROM_DAYS = + public static final SqlFunction LAST_DAY = new LastDayFunction().toUDF("LAST_DAY"); + public static final SqlFunction FROM_DAYS = adaptExprMethodToUDF( DateTimeFunctions.class, "exprFromDays", @@ -267,8 +276,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.INTEGER) .toUDF("FROM_DAYS"); - public static final SqlOperator FROM_UNIXTIME = new FromUnixTimeFunction().toUDF("FROM_UNIXTIME"); - public static final SqlOperator GET_FORMAT = + public static final SqlFunction FROM_UNIXTIME = new FromUnixTimeFunction().toUDF("FROM_UNIXTIME"); + public static final SqlFunction GET_FORMAT = adaptExprMethodToUDF( DateTimeFunctions.class, "exprGetFormat", @@ -276,7 +285,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.STRING_STRING) .toUDF("GET_FORMAT"); - public static final SqlOperator MAKEDATE = + public static final SqlFunction MAKEDATE = adaptExprMethodToUDF( DateTimeFunctions.class, "exprMakeDate", @@ -284,7 +293,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.NUMERIC_NUMERIC) .toUDF("MAKEDATE"); - public static final SqlOperator MAKETIME = + public static final SqlFunction MAKETIME = adaptExprMethodToUDF( DateTimeFunctions.class, "exprMakeTime", @@ -292,7 +301,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.NUMERIC_NUMERIC_NUMERIC) .toUDF("MAKETIME"); - public static final SqlOperator PERIOD_DIFF = + public static final SqlFunction PERIOD_DIFF = adaptExprMethodToUDF( DateTimeFunctions.class, "exprPeriodDiff", @@ -300,7 +309,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.INTEGER_INTEGER) .toUDF("PERIOD_DIFF"); - public static final SqlOperator PERIOD_ADD = + public static final SqlFunction PERIOD_ADD = adaptExprMethodToUDF( DateTimeFunctions.class, "exprPeriodAdd", @@ -308,7 +317,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.INTEGER_INTEGER) .toUDF("PERIOD_ADD"); - public static final SqlOperator STR_TO_DATE = + public static final SqlFunction STR_TO_DATE = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprStrToDate", @@ -316,9 +325,9 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.STRING_STRING) .toUDF("STR_TO_DATE"); - public static final SqlOperator SYSDATE = new SysdateFunction().toUDF("SYSDATE"); - public static final SqlOperator SEC_TO_TIME = new SecToTimeFunction().toUDF("SEC_TO_TIME"); - public static final SqlOperator TIME = + public static final SqlFunction SYSDATE = new SysdateFunction().toUDF("SYSDATE"); + public static final SqlFunction SEC_TO_TIME = new SecToTimeFunction().toUDF("SEC_TO_TIME"); + public static final SqlFunction TIME = adaptExprMethodToUDF( DateTimeFunctions.class, "exprTime", @@ -328,9 +337,9 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { .toUDF("TIME"); // IP cast function - public static final SqlOperator IP = + public static final SqlFunction IP = new IPFunction().toUDF(UserDefinedFunctionUtils.IP_FUNCTION_NAME); - public static final SqlOperator TIME_TO_SEC = + public static final SqlFunction TIME_TO_SEC = adaptExprMethodToUDF( DateTimeFunctions.class, "exprTimeToSec", @@ -338,7 +347,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.TIME_OR_TIMESTAMP_OR_STRING) .toUDF("TIME_TO_SEC"); - public static final SqlOperator TIMEDIFF = + public static final SqlFunction TIMEDIFF = UserDefinedFunctionUtils.adaptExprMethodToUDF( DateTimeFunctions.class, "exprTimeDiff", @@ -346,8 +355,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ANY, PPLOperandTypes.TIME_TIME) .toUDF("TIME_DIFF"); - public static final SqlOperator TIMESTAMPADD = new TimestampAddFunction().toUDF("TIMESTAMPADD"); - public static final SqlOperator TO_DAYS = + public static final SqlFunction TIMESTAMPADD = new TimestampAddFunction().toUDF("TIMESTAMPADD"); + public static final SqlFunction TO_DAYS = adaptExprMethodToUDF( DateTimeFunctions.class, "exprToDays", @@ -355,8 +364,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.ARG0, PPLOperandTypes.DATE_OR_TIMESTAMP_OR_STRING) .toUDF("TO_DAYS"); - public static final SqlOperator DATETIME = new DatetimeFunction().toUDF("DATETIME"); - public static final SqlOperator UTC_DATE = + public static final SqlFunction DATETIME = new DatetimeFunction().toUDF("DATETIME"); + public static final SqlFunction UTC_DATE = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprUtcDate", @@ -364,7 +373,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.NONE, PPLOperandTypes.NONE) .toUDF("UTC_DATE"); - public static final SqlOperator UTC_TIME = + public static final SqlFunction UTC_TIME = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprUtcTime", @@ -372,7 +381,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.NONE, PPLOperandTypes.NONE) .toUDF("UTC_TIME"); - public static final SqlOperator UTC_TIMESTAMP = + public static final SqlFunction UTC_TIMESTAMP = adaptExprMethodWithPropertiesToUDF( DateTimeFunctions.class, "exprUtcTimestamp", @@ -380,61 +389,61 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { NullPolicy.NONE, PPLOperandTypes.NONE) .toUDF("UTC_TIMESTAMP"); - public static final SqlOperator WEEK = new WeekFunction().toUDF("WEEK"); - public static final SqlOperator GROK = new ParseFunction().toUDF("GROK"); + public static final SqlFunction WEEK = new WeekFunction().toUDF("WEEK"); + public static final SqlFunction GROK = new ParseFunction().toUDF("GROK"); // TODO: Figure out if there is other option to perform multiple group match in Calcite // For now, keep V2's regexExpression logic to avoid breaking change - public static final SqlOperator PARSE = new ParseFunction().toUDF("PARSE"); - public static final SqlOperator PATTERN_PARSER = + public static final SqlFunction PARSE = new ParseFunction().toUDF("PARSE"); + public static final SqlFunction PATTERN_PARSER = new PatternParserFunctionImpl().toUDF("PATTERN_PARSER"); - public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); - public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); - public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); - public static final SqlOperator MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); - public static final SqlOperator MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); - public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); - public static final SqlOperator MVZIP = new MVZipFunctionImpl().toUDF("mvzip"); - public static final SqlOperator MVFIND = new MVFindFunctionImpl().toUDF("mvfind"); - public static final SqlOperator FILTER = new FilterFunctionImpl().toUDF("filter"); - public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform"); - public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce"); + public static final SqlFunction FORALL = new ForallFunctionImpl().toUDF("forall"); + public static final SqlFunction EXISTS = new ExistsFunctionImpl().toUDF("exists"); + public static final SqlFunction ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlFunction MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); + public static final SqlFunction MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); + public static final SqlFunction MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); + public static final SqlFunction MVZIP = new MVZipFunctionImpl().toUDF("mvzip"); + public static final SqlFunction MVFIND = new MVFindFunctionImpl().toUDF("mvfind"); + public static final SqlFunction FILTER = new FilterFunctionImpl().toUDF("filter"); + public static final SqlFunction TRANSFORM = new TransformFunctionImpl().toUDF("transform"); + public static final SqlFunction REDUCE = new ReduceFunctionImpl().toUDF("reduce"); private static final RelevanceQueryFunction RELEVANCE_QUERY_FUNCTION_INSTANCE = new RelevanceQueryFunction(); - public static final SqlOperator MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match"); - public static final SqlOperator MATCH_PHRASE = + public static final SqlFunction MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match"); + public static final SqlFunction MATCH_PHRASE = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase"); - public static final SqlOperator MATCH_BOOL_PREFIX = + public static final SqlFunction MATCH_BOOL_PREFIX = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_bool_prefix"); - public static final SqlOperator MATCH_PHRASE_PREFIX = + public static final SqlFunction MATCH_PHRASE_PREFIX = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase_prefix"); - public static final SqlOperator SIMPLE_QUERY_STRING = + public static final SqlFunction SIMPLE_QUERY_STRING = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("simple_query_string", false); - public static final SqlOperator QUERY_STRING = + public static final SqlFunction QUERY_STRING = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("query_string", false); - public static final SqlOperator MULTI_MATCH = + public static final SqlFunction MULTI_MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match", false); - public static final SqlOperator NUMBER_TO_STRING = + public static final SqlFunction NUMBER_TO_STRING = new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); - public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); - public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); - public static final SqlOperator WIDTH_BUCKET = + public static final SqlFunction TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); + public static final SqlFunction TOSTRING = new ToStringFunction().toUDF("TOSTRING"); + public static final SqlFunction WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() .toUDF("WIDTH_BUCKET"); - public static final SqlOperator SPAN_BUCKET = + public static final SqlFunction SPAN_BUCKET = new org.opensearch.sql.expression.function.udf.binning.SpanBucketFunction() .toUDF("SPAN_BUCKET"); - public static final SqlOperator MINSPAN_BUCKET = + public static final SqlFunction MINSPAN_BUCKET = new org.opensearch.sql.expression.function.udf.binning.MinspanBucketFunction() .toUDF("MINSPAN_BUCKET"); - public static final SqlOperator RANGE_BUCKET = + public static final SqlFunction RANGE_BUCKET = new org.opensearch.sql.expression.function.udf.binning.RangeBucketFunction() .toUDF("RANGE_BUCKET"); - public static final SqlOperator REX_EXTRACT = new RexExtractFunction().toUDF("REX_EXTRACT"); - public static final SqlOperator REX_EXTRACT_MULTI = + public static final SqlFunction REX_EXTRACT = new RexExtractFunction().toUDF("REX_EXTRACT"); + public static final SqlFunction REX_EXTRACT_MULTI = new RexExtractMultiFunction().toUDF("REX_EXTRACT_MULTI"); - public static final SqlOperator REX_OFFSET = new RexOffsetFunction().toUDF("REX_OFFSET"); + public static final SqlFunction REX_OFFSET = new RexOffsetFunction().toUDF("REX_OFFSET"); // Aggregation functions public static final SqlAggFunction AVG_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.AVG); @@ -463,26 +472,54 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { PercentileApproxFunction.class, "percentile_approx", ReturnTypes.ARG0_FORCE_NULLABLE, - PPLOperandTypes.NUMERIC_NUMERIC_OPTIONAL_NUMERIC); + PPLOperandTypes.NUMERIC_NUMERIC_OPTIONAL_NUMERIC_SYMBOL); public static final SqlAggFunction INTERNAL_PATTERN = createUserDefinedAggFunction( LogPatternAggFunction.class, "pattern", ReturnTypes.explicit(UserDefinedFunctionUtils.nullablePatternAggList), - null); + UDFOperandMetadata.wrap( + OperandTypes.VARIADIC)); // operand types of patterns are very flexible public static final SqlAggFunction LIST = createUserDefinedAggFunction( - ListAggFunction.class, "LIST", PPLReturnTypes.STRING_ARRAY, PPLOperandTypes.ANY_SCALAR); + ListAggFunction.class, "LIST", PPLReturnTypes.STRING_ARRAY, PPLOperandTypes.SCALAR); public static final SqlAggFunction VALUES = createUserDefinedAggFunction( ValuesAggFunction.class, "VALUES", PPLReturnTypes.STRING_ARRAY, - PPLOperandTypes.ANY_SCALAR_OPTIONAL_INTEGER); + PPLOperandTypes.SCALAR_OPTIONAL_INTEGER); public static final SqlOperator ENHANCED_COALESCE = new EnhancedCoalesceFunction().toUDF("COALESCE"); + public static final SqlFunction ATAN = + new SqlFunction( + "ATAN", + SqlKind.OTHER_FUNCTION, + ReturnTypes.DOUBLE_NULLABLE, + null, + OperandTypes.NUMERIC_OPTIONAL_NUMERIC, + SqlFunctionCategory.NUMERIC) { + @Override + public SqlNode rewriteCall(SqlValidator validator, SqlCall call) { + if (call instanceof SqlBasicCall) { + SqlOperator op = + call.getOperandList().size() == 2 + ? SqlStdOperatorTable.ATAN2 + : SqlStdOperatorTable.ATAN; + ((SqlBasicCall) call).setOperator(op); + } + return call; + } + }; + // SPARK dialect is not included in lookup table to resolve overrides issues (e.g. reverse + // function won't work if spark is included because there are multiple overrides of reverse, and + // it will choose none of them in the end.) Therefore, SPARK functions used are explicitly + // declared here for lookup. + public static final SqlFunction REGEXP = SqlLibraryOperators.REGEXP; + public static final SqlFunction MAP_CONCAT = SqlLibraryOperators.MAP_CONCAT; + /** * Returns the PPL specific operator table, creating it if necessary. * @@ -498,7 +535,7 @@ public static PPLBuiltinOperators instance() { * implementor could be substituted by a single method. */ private static Expression invokeCalciteImplementor( - RexToLixTranslator translator, RexCall call, SqlOperator operator, Expression field) + RexToLixTranslator translator, RexCall call, SqlFunction operator, Expression field) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { RexCallImplementor rexCallImplementor = RexImpTable.INSTANCE.get(operator); Method method = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 89a8f59397b..8bb5385c9ff 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -251,19 +251,17 @@ import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nullable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLambda; @@ -274,25 +272,19 @@ import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlTrimFunction.Flag; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; -import org.apache.calcite.sql.type.FamilyOperandTypeChecker; -import org.apache.calcite.sql.type.ImplicitCastOperandTypeChecker; -import org.apache.calcite.sql.type.OperandTypes; -import org.apache.calcite.sql.type.SameOperandTypeChecker; -import org.apache.calcite.sql.type.SqlOperandTypeChecker; -import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; -import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.calcite.tools.RelBuilder; -import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.sql.calcite.CalcitePlanContext; -import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; -import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.CollectionUDF.MVIndexFunctionImp; @@ -364,43 +356,39 @@ default RexNode resolve(RexBuilder builder, RexNode... args) { * implementations are independent of any specific data storage, should be registered here * internally. */ - private final ImmutableMap>> - functionRegistry; + private final ImmutableMap functionRegistry; /** * The external function registry. Functions whose implementations depend on a specific data * engine should be registered here. This reduces coupling between the core module and particular * storage backends. */ - private final Map>> - externalFunctionRegistry; + private final Map externalFunctionRegistry; /** * The registry for built-in agg functions. Agg Functions defined by the PPL specification, whose * implementations are independent of any specific data storage, should be registered here * internally. */ - private final ImmutableMap> - aggFunctionRegistry; + private final ImmutableMap aggFunctionRegistry; /** * The external agg function registry. Agg Functions whose implementations depend on a specific * data engine should be registered here. This reduces coupling between the core module and * particular storage backends. */ - private final Map> - aggExternalFunctionRegistry; + private final Map aggExternalFunctionRegistry; private PPLFuncImpTable(Builder builder, AggBuilder aggBuilder) { - final ImmutableMap.Builder>> - mapBuilder = ImmutableMap.builder(); - builder.map.forEach((k, v) -> mapBuilder.put(k, List.copyOf(v))); + final ImmutableMap.Builder mapBuilder = + ImmutableMap.builder(); + mapBuilder.putAll(builder.map); this.functionRegistry = ImmutableMap.copyOf(mapBuilder.build()); this.externalFunctionRegistry = new ConcurrentHashMap<>(); - final ImmutableMap.Builder> - aggMapBuilder = ImmutableMap.builder(); - aggBuilder.map.forEach(aggMapBuilder::put); + final ImmutableMap.Builder aggMapBuilder = + ImmutableMap.builder(); + aggMapBuilder.putAll(aggBuilder.map); this.aggFunctionRegistry = ImmutableMap.copyOf(aggMapBuilder.build()); this.aggExternalFunctionRegistry = new ConcurrentHashMap<>(); } @@ -412,20 +400,11 @@ private PPLFuncImpTable(Builder builder, AggBuilder aggBuilder) { * @param operator a SqlOperator representing an externally implemented function */ public void registerExternalOperator(BuiltinFunctionName functionName, SqlOperator operator) { - PPLTypeChecker typeChecker = - wrapSqlOperandTypeChecker( - operator.getOperandTypeChecker(), - functionName.name(), - operator instanceof SqlUserDefinedFunction); - CalciteFuncSignature signature = new CalciteFuncSignature(functionName.getName(), typeChecker); - externalFunctionRegistry.compute( - functionName, - (name, existingList) -> { - List> list = - existingList == null ? new ArrayList<>() : new ArrayList<>(existingList); - list.add(Pair.of(signature, (builder, args) -> builder.makeCall(operator, args))); - return list; - }); + if (externalFunctionRegistry.containsKey(functionName)) { + logger.warn( + String.format(Locale.ROOT, "Function %s is registered multiple times", functionName)); + } + externalFunctionRegistry.put(functionName, (builder, args) -> builder.makeCall(operator, args)); } /** @@ -437,23 +416,16 @@ public void registerExternalOperator(BuiltinFunctionName functionName, SqlOperat */ public void registerExternalAggOperator( BuiltinFunctionName functionName, SqlUserDefinedAggFunction aggFunction) { - PPLTypeChecker typeChecker = - wrapSqlOperandTypeChecker(aggFunction.getOperandTypeChecker(), functionName.name(), true); - CalciteFuncSignature signature = new CalciteFuncSignature(functionName.getName(), typeChecker); + if (aggExternalFunctionRegistry.containsKey(functionName)) { + logger.warn( + String.format( + Locale.ROOT, "Aggregate function %s is registered multiple times", functionName)); + } AggHandler handler = (distinct, field, argList, ctx) -> UserDefinedFunctionUtils.makeAggregateCall( aggFunction, List.of(field), argList, ctx.relBuilder); - aggExternalFunctionRegistry.put(functionName, Pair.of(signature, handler)); - } - - public List validateAggFunctionSignature( - BuiltinFunctionName functionName, - RexNode field, - List argList, - RexBuilder rexBuilder) { - var implementation = getImplementation(functionName); - return validateFunctionArgs(implementation, functionName, field, argList, rexBuilder); + aggExternalFunctionRegistry.put(functionName, handler); } public RelBuilder.AggCall resolveAgg( @@ -463,62 +435,110 @@ public RelBuilder.AggCall resolveAgg( List argList, CalcitePlanContext context) { var implementation = getImplementation(functionName); - - // Validation is done based on original argument types to generate error from user perspective. - List nodes = - validateFunctionArgs(implementation, functionName, field, argList, context.rexBuilder); - - var handler = implementation.getValue(); - return nodes != null - ? handler.apply(distinct, nodes.getFirst(), nodes.subList(1, nodes.size()), context) - : handler.apply(distinct, field, argList, context); + // For numeric aggregation functions (SUM, AVG, etc.), coerce VARCHAR fields to DOUBLE. + // This handles cases like stats sum(strnum) where strnum is a keyword (VARCHAR) field. + if (field != null && NUMERIC_AGG_FUNCTIONS.contains(functionName)) { + ExprType fieldType = OpenSearchTypeFactory.convertRelDataTypeToExprType(field.getType()); + if (fieldType == ExprCoreType.STRING) { + RelDataType doubleType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.DOUBLE); + field = context.rexBuilder.makeCast(doubleType, field, true, true); + } + } + return implementation.apply(distinct, field, argList, context); } - static List validateFunctionArgs( - Pair implementation, - BuiltinFunctionName functionName, - RexNode field, - List argList, - RexBuilder rexBuilder) { - CalciteFuncSignature signature = implementation.getKey(); + private static final java.util.Set NUMERIC_AGG_FUNCTIONS = + java.util.Set.of( + BuiltinFunctionName.SUM, + BuiltinFunctionName.AVG, + BuiltinFunctionName.VARSAMP, + BuiltinFunctionName.VARPOP, + BuiltinFunctionName.STDDEV_SAMP, + BuiltinFunctionName.STDDEV_POP); - List argTypes = new ArrayList<>(); - if (field != null) { - argTypes.add(field.getType()); - } + /** + * Arithmetic operators where ALL args (including numeric) should be widened to DOUBLE when any + * string arg is present. This ensures type consistency for binary arithmetic (e.g., string + int + * -> double + double). + */ + private static final java.util.Set ARITHMETIC_COERCE_FUNCTIONS = + java.util.Set.of( + ADD, + ADDFUNCTION, + SUBTRACT, + BuiltinFunctionName.SUBTRACTFUNCTION, + MULTIPLY, + BuiltinFunctionName.MULTIPLYFUNCTION, + DIVIDE, + DIVIDEFUNCTION, + BuiltinFunctionName.MOD, + BuiltinFunctionName.MODULUS, + BuiltinFunctionName.MODULUSFUNCTION); - // Currently only PERCENTILE_APPROX, TAKE, EARLIEST, and LATEST have additional arguments. - // Their additional arguments will always come as a map of - List additionalArgTypes = - argList.stream().map(PlanUtils::derefMapCall).map(RexNode::getType).toList(); - argTypes.addAll(additionalArgTypes); - List coercionNodes = null; - if (!signature.match(functionName.getName(), argTypes)) { - List fields = new ArrayList<>(); - fields.add(field); - fields.addAll(argList); - if (CoercionUtils.hasString(fields)) { - coercionNodes = CoercionUtils.castArguments(rexBuilder, signature.typeChecker(), fields); - } - if (coercionNodes == null) { - String errorMessagePattern = - argTypes.size() <= 1 - ? "Aggregation function %s expects field type {%s}, but got %s" - : "Aggregation function %s expects field type and additional arguments {%s}, but" - + " got %s"; - throw new ExpressionEvaluationException( - String.format( - errorMessagePattern, - functionName, - signature.typeChecker().getAllowedSignatures(), - PlanUtils.getActualSignature(argTypes))); + /** + * Non-arithmetic functions that expect numeric operands and should coerce only string args to + * DOUBLE (leaving integer args like precision parameters unchanged). + */ + private static final java.util.Set NUMERIC_COERCE_FUNCTIONS = + java.util.Set.of( + // Bin functions + BuiltinFunctionName.SPAN_BUCKET, + BuiltinFunctionName.WIDTH_BUCKET, + BuiltinFunctionName.MINSPAN_BUCKET, + BuiltinFunctionName.RANGE_BUCKET, + // Math functions + ABS, + CEIL, + CEILING, + FLOOR, + BuiltinFunctionName.ROUND, + BuiltinFunctionName.TRUNCATE, + SQRT, + CBRT, + EXP, + EXPM1, + BuiltinFunctionName.LN, + BuiltinFunctionName.LOG, + BuiltinFunctionName.LOG2, + BuiltinFunctionName.LOG10, + BuiltinFunctionName.POW, + BuiltinFunctionName.POWER, + BuiltinFunctionName.SIGN, + SIN, + COS, + BuiltinFunctionName.TAN, + COT, + ASIN, + ACOS, + ATAN, + ATAN2, + DEGREES, + BuiltinFunctionName.RADIANS, + BuiltinFunctionName.COSH); + + /** + * Determines if string args should be coerced to numeric for the given function. ADD is special: + * string+string is concatenation, so only coerce ADD when mixed with numeric args. + */ + private static boolean shouldCoerceStringsToNumeric( + BuiltinFunctionName functionName, RexNode[] args) { + if (functionName == ADD || functionName == ADDFUNCTION) { + // For ADD: only coerce if there's a mix of string and numeric args + boolean hasString = false; + boolean hasNumeric = false; + for (RexNode arg : args) { + ExprType type = OpenSearchTypeFactory.convertRelDataTypeToExprType(arg.getType()); + if (type == ExprCoreType.STRING) hasString = true; + else if (ExprCoreType.numberTypes().contains(type)) hasNumeric = true; } + return hasString && hasNumeric; } - return coercionNodes; + return ARITHMETIC_COERCE_FUNCTIONS.contains(functionName) + || NUMERIC_COERCE_FUNCTIONS.contains(functionName); } - private Pair getImplementation( - BuiltinFunctionName functionName) { + private AggHandler getImplementation(BuiltinFunctionName functionName) { var implementation = aggExternalFunctionRegistry.get(functionName); if (implementation == null) { implementation = aggFunctionRegistry.get(functionName); @@ -541,13 +561,12 @@ public RexNode resolve( final RexBuilder builder, final BuiltinFunctionName functionName, RexNode... args) { // Check the external function registry first. This allows the data-storage-dependent // function implementations to override the internal ones with the same name. - List> implementList = - externalFunctionRegistry.get(functionName); - // If the function is not part of the external registry, check the internal registry. - if (implementList == null) { - implementList = functionRegistry.get(functionName); - } - if (implementList == null || implementList.isEmpty()) { + // If the function is not part of the external registry, check the internal registry. + FunctionImp implementation = + externalFunctionRegistry.get(functionName) != null + ? externalFunctionRegistry.get(functionName) + : functionRegistry.get(functionName); + if (implementation == null) { throw new IllegalStateException(String.format("Cannot resolve function: %s", functionName)); } @@ -556,38 +575,29 @@ public RexNode resolve( // return type of the lambda function. compulsoryCast(builder, functionName, args); - List argTypes = Arrays.stream(args).map(RexNode::getType).toList(); - try { - for (Map.Entry implement : implementList) { - if (implement.getKey().match(functionName.getName(), argTypes)) { - return implement.getValue().resolve(builder, args); - } - } - - // If no implementation found with exact match, try to cast arguments to match the - // signatures. - RexNode coerced = resolveWithCoercion(builder, functionName, implementList, args); + // For comparators, apply type coercion before resolving: + // 1. If one arg is datetime and the other is string, cast string to the datetime type + // 2. If both are datetime but different types (DATE vs TIME), widen both to TIMESTAMP + // 3. If one arg is string and the other is numeric, widen both to a common type + if (BuiltinFunctionName.COMPARATORS.contains(functionName) && args.length == 2) { + RexNode[] coerced = coerceComparatorArgs(builder, args); if (coerced != null) { - return coerced; + return implementation.resolve(builder, coerced); } - } catch (Exception e) { - throw new ExpressionEvaluationException( - String.format( - "Cannot resolve function: %s, arguments: %s, caused by: %s", - functionName, PlanUtils.getActualSignature(argTypes), e.getMessage()), - e); } - StringJoiner allowedSignatures = new StringJoiner(","); - for (var implement : implementList) { - String signature = implement.getKey().typeChecker().getAllowedSignatures(); - if (!signature.isEmpty()) { - allowedSignatures.add(signature); + + // For functions that expect numeric args, coerce string args to DOUBLE. + // This handles rex-extracted fields (VARCHAR) in arithmetic, math, or bin functions. + // ADD is special: string+string is concatenation, so only coerce if mixed with numeric. + if (shouldCoerceStringsToNumeric(functionName, args)) { + boolean widenAllToDouble = ARITHMETIC_COERCE_FUNCTIONS.contains(functionName); + RexNode[] coerced = coerceArgsToNumeric(builder, args, widenAllToDouble); + if (coerced != null) { + return implementation.resolve(builder, coerced); } } - throw new ExpressionEvaluationException( - String.format( - "%s function expects {%s}, but got %s", - functionName, allowedSignatures, PlanUtils.getActualSignature(argTypes))); + + return implementation.resolve(builder, args); } /** @@ -610,91 +620,155 @@ private void compulsoryCast( } } - private @Nullable RexNode resolveWithCoercion( - final RexBuilder builder, - final BuiltinFunctionName functionName, - List> implementList, - RexNode... args) { - if (BuiltinFunctionName.COMPARATORS.contains(functionName)) { - for (Map.Entry implement : implementList) { - var widenedArgs = CoercionUtils.widenArguments(builder, List.of(args)); - if (widenedArgs != null) { - boolean matchSignature = - implement - .getKey() - .typeChecker() - .checkOperandTypes(widenedArgs.stream().map(RexNode::getType).toList()); - if (matchSignature) { - return implement.getValue().resolve(builder, widenedArgs.toArray(new RexNode[0])); - } - } + /** + * Coerce comparator arguments when they have mismatched types. Returns a new array of coerced + * arguments, or null if no coercion is needed. + * + *

Rules: + * + *

    + *
  • If one arg is a datetime type and the other is a string, cast the string to the datetime + * type (e.g., DATE field compared with VARCHAR literal). + *
  • If both args are different datetime types (e.g., DATE vs TIME), widen both to TIMESTAMP. + *
+ */ + private static RexNode[] coerceComparatorArgs(RexBuilder builder, RexNode[] args) { + ExprType type0 = OpenSearchTypeFactory.convertRelDataTypeToExprType(args[0].getType()); + ExprType type1 = OpenSearchTypeFactory.convertRelDataTypeToExprType(args[1].getType()); + + if (type0 == type1) { + return null; // No coercion needed + } + + boolean isDatetime0 = isDatetimeExprType(type0); + boolean isDatetime1 = isDatetimeExprType(type1); + boolean isString0 = type0 == ExprCoreType.STRING; + boolean isString1 = type1 == ExprCoreType.STRING; + + // Case 1: One is datetime, the other is string -> cast string to datetime type + if (isDatetime0 && isString1) { + RelDataType targetType = args[0].getType(); + return new RexNode[] {args[0], builder.makeCast(targetType, args[1], true, true)}; + } + if (isDatetime1 && isString0) { + RelDataType targetType = args[1].getType(); + return new RexNode[] {builder.makeCast(targetType, args[0], true, true), args[1]}; + } + + // Case 2: Both are datetime but different types -> widen to TIMESTAMP + if (isDatetime0 && isDatetime1) { + var widenedArgs = CoercionUtils.widenArguments(builder, List.of(args)); + if (widenedArgs != null) { + return widenedArgs.toArray(new RexNode[0]); } - } else { - for (Map.Entry implement : implementList) { - var signature = implement.getKey(); - var castedArgs = - CoercionUtils.castArguments(builder, signature.typeChecker(), List.of(args)); - if (castedArgs != null) { - // If compatible function is found, replace the original RexNode with cast node - // TODO: check - this is a return-once-found implementation, rest possible combinations - // will be skipped. - // Maybe can be improved to return the best match? E.g. convert to timestamp when date, - // time, and timestamp are all possible. - return implement.getValue().resolve(builder, castedArgs.toArray(new RexNode[0])); - } + } + + // Case 3: One is string and the other is numeric -> widen to common type (DOUBLE) + boolean isNumeric0 = ExprCoreType.numberTypes().contains(type0); + boolean isNumeric1 = ExprCoreType.numberTypes().contains(type1); + if ((isString0 && isNumeric1) || (isNumeric0 && isString1)) { + var widenedArgs = CoercionUtils.widenArguments(builder, List.of(args)); + if (widenedArgs != null) { + return widenedArgs.toArray(new RexNode[0]); } } + + // Case 4: One is string and the other is boolean -> cast string to boolean + boolean isBoolean0 = type0 == ExprCoreType.BOOLEAN; + boolean isBoolean1 = type1 == ExprCoreType.BOOLEAN; + if ((isString0 && isBoolean1) || (isBoolean0 && isString1)) { + RelDataType boolType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.BOOLEAN); + RexNode arg0 = isString0 ? builder.makeCast(boolType, args[0], true, true) : args[0]; + RexNode arg1 = isString1 ? builder.makeCast(boolType, args[1], true, true) : args[1]; + return new RexNode[] {arg0, arg1}; + } + return null; } + /** + * Coerce string arguments to DOUBLE for numeric functions. If any arg is VARCHAR/STRING, cast it + * to DOUBLE. When {@code widenAllToDouble} is true (for arithmetic operators), also cast + * non-DOUBLE numeric args to DOUBLE for type consistency. When false (for math functions like + * ROUND), only STRING args are cast, preserving integer args (e.g., precision parameter). + */ + private static RexNode[] coerceArgsToNumeric( + RexBuilder builder, RexNode[] args, boolean widenAllToDouble) { + boolean hasString = false; + boolean hasDatetime = false; + for (RexNode arg : args) { + ExprType type = OpenSearchTypeFactory.convertRelDataTypeToExprType(arg.getType()); + if (type == ExprCoreType.STRING) { + hasString = true; + } else if (isDatetimeExprType(type)) { + hasDatetime = true; + } + } + // Skip coercion if no string args, or if datetime args are present (the "string" args may + // actually be datetime-derived values like TIMESTAMP - TIMESTAMP which produces VARCHAR). + if (!hasString || hasDatetime) { + return null; + } + RelDataType doubleType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.DOUBLE); + RexNode[] coerced = new RexNode[args.length]; + for (int i = 0; i < args.length; i++) { + ExprType type = OpenSearchTypeFactory.convertRelDataTypeToExprType(args[i].getType()); + if (type == ExprCoreType.STRING) { + coerced[i] = builder.makeCast(doubleType, args[i], true, true); + } else if (widenAllToDouble + && ExprCoreType.numberTypes().contains(type) + && args[i].getType().getSqlTypeName() != SqlTypeName.DOUBLE) { + coerced[i] = builder.makeCast(doubleType, args[i], true, true); + } else { + coerced[i] = args[i]; + } + } + return coerced; + } + + private static boolean isDatetimeExprType(ExprType type) { + return type == ExprCoreType.DATE || type == ExprCoreType.TIME || type == ExprCoreType.TIMESTAMP; + } + @SuppressWarnings({"UnusedReturnValue", "SameParameterValue"}) private abstract static class AbstractBuilder { /** Maps an operator to an implementation. */ - abstract void register( - BuiltinFunctionName functionName, FunctionImp functionImp, PPLTypeChecker typeChecker); + abstract void register(BuiltinFunctionName functionName, FunctionImp functionImp); /** - * Register one or multiple operators under a single function name. This allows function - * overloading based on operand types. + * Registers an operator for a built-in function name. * - *

When a function is called, the system will try each registered operator in sequence, - * checking if the provided arguments match the operator's type requirements. The first operator - * whose type checker accepts the arguments will be used to execute the function. + *

Each function name can only be registered to one operator. Use {@code + * register(BuiltinFunctionName, FunctionImp)} to dynamically register a built-in function name + * to different operators based on argument count or types if override is desired. * * @param functionName the built-in function name under which to register the operators - * @param operators the operators to associate with this function name, tried in sequence until - * one matches the argument types during resolution + * @param operator the operator to associate with this function name */ - protected void registerOperator(BuiltinFunctionName functionName, SqlOperator... operators) { - for (SqlOperator operator : operators) { - SqlOperandTypeChecker typeChecker; - if (operator instanceof SqlUserDefinedFunction udfOperator) { - typeChecker = extractTypeCheckerFromUDF(udfOperator); - } else { - typeChecker = operator.getOperandTypeChecker(); - } - PPLTypeChecker pplTypeChecker = - wrapSqlOperandTypeChecker( - typeChecker, operator.getName(), operator instanceof SqlUserDefinedFunction); - registerOperator(functionName, operator, pplTypeChecker); - } + protected void registerOperator(BuiltinFunctionName functionName, SqlOperator operator) { + register( + functionName, (RexBuilder builder, RexNode... args) -> builder.makeCall(operator, args)); } /** - * Registers an operator for a built-in function name with a specified {@link PPLTypeChecker}. - * This allows custom type checking logic to be associated with the operator. - * - * @param functionName the built-in function name - * @param operator the SQL operator to register - * @param typeChecker the type checker to use for validating argument types + * Registers a comparison operator that dispatches to IP comparison functions when any operand + * is an IP type, and falls back to the standard operator otherwise. */ - protected void registerOperator( - BuiltinFunctionName functionName, SqlOperator operator, PPLTypeChecker typeChecker) { + protected void registerComparisonWithIp( + BuiltinFunctionName functionName, SqlOperator stdOperator, SqlOperator ipOperator) { register( functionName, - (RexBuilder builder, RexNode... args) -> builder.makeCall(operator, args), - typeChecker); + (RexBuilder builder, RexNode... args) -> { + if (args.length == 2 + && (OpenSearchTypeUtil.isIp(args[0].getType()) + || OpenSearchTypeUtil.isIp(args[1].getType()))) { + return builder.makeCall(ipOperator, args); + } + return builder.makeCall(stdOperator, args); + }); } protected void registerDivideFunction(BuiltinFunctionName functionName) { @@ -707,39 +781,28 @@ protected void registerDivideFunction(BuiltinFunctionName functionName) { ? PPLBuiltinOperators.DIVIDE : SqlLibraryOperators.SAFE_DIVIDE; return builder.makeCall(operator, left, right); - }, - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); + }); } void populate() { - // register operators for comparison - registerOperator(NOTEQUAL, PPLBuiltinOperators.NOT_EQUALS_IP, SqlStdOperatorTable.NOT_EQUALS); - registerOperator(EQUAL, PPLBuiltinOperators.EQUALS_IP, SqlStdOperatorTable.EQUALS); - registerOperator(GREATER, PPLBuiltinOperators.GREATER_IP, SqlStdOperatorTable.GREATER_THAN); - registerOperator(GTE, PPLBuiltinOperators.GTE_IP, SqlStdOperatorTable.GREATER_THAN_OR_EQUAL); - registerOperator(LESS, PPLBuiltinOperators.LESS_IP, SqlStdOperatorTable.LESS_THAN); - registerOperator(LTE, PPLBuiltinOperators.LTE_IP, SqlStdOperatorTable.LESS_THAN_OR_EQUAL); + // register operators for comparison, with IP type dispatch + registerComparisonWithIp( + NOTEQUAL, SqlStdOperatorTable.NOT_EQUALS, PPLBuiltinOperators.NOT_EQUALS_IP); + registerComparisonWithIp(EQUAL, SqlStdOperatorTable.EQUALS, PPLBuiltinOperators.EQUALS_IP); + registerComparisonWithIp( + GREATER, SqlStdOperatorTable.GREATER_THAN, PPLBuiltinOperators.GREATER_IP); + registerComparisonWithIp( + GTE, SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, PPLBuiltinOperators.GTE_IP); + registerComparisonWithIp(LESS, SqlStdOperatorTable.LESS_THAN, PPLBuiltinOperators.LESS_IP); + registerComparisonWithIp( + LTE, SqlStdOperatorTable.LESS_THAN_OR_EQUAL, PPLBuiltinOperators.LTE_IP); // Register std operator registerOperator(AND, SqlStdOperatorTable.AND); registerOperator(OR, SqlStdOperatorTable.OR); registerOperator(NOT, SqlStdOperatorTable.NOT); - - // Register ADDFUNCTION for numeric addition only - registerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); - registerOperator( - SUBTRACTFUNCTION, - SqlStdOperatorTable.MINUS, - PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - registerOperator( - SUBTRACT, - SqlStdOperatorTable.MINUS, - PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - // Add DATETIME-DATETIME variant for timestamp binning support - registerOperator( - SUBTRACT, - SqlStdOperatorTable.MINUS, - PPLTypeChecker.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); + registerOperator(SUBTRACTFUNCTION, SqlStdOperatorTable.MINUS); + registerOperator(SUBTRACT, SqlStdOperatorTable.MINUS); registerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); registerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); registerOperator(TRUNCATE, SqlStdOperatorTable.TRUNCATE); @@ -788,46 +851,26 @@ void populate() { } } return builder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, args); - }, - wrapSqlOperandTypeChecker( - SqlLibraryOperators.REGEXP_REPLACE_3.getOperandTypeChecker(), REPLACE.name(), false)); + }); registerOperator(UPPER, SqlStdOperatorTable.UPPER); registerOperator(ABS, SqlStdOperatorTable.ABS); registerOperator(ACOS, SqlStdOperatorTable.ACOS); registerOperator(ASIN, SqlStdOperatorTable.ASIN); - registerOperator(ATAN, SqlStdOperatorTable.ATAN); + // ATAN with 1 arg -> SqlStdOperatorTable.ATAN, with 2 args -> ATAN2 + register( + ATAN, + (RexBuilder b, RexNode... a) -> + a.length == 2 + ? b.makeCall(SqlStdOperatorTable.ATAN2, a) + : b.makeCall(SqlStdOperatorTable.ATAN, a)); registerOperator(ATAN2, SqlStdOperatorTable.ATAN2); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - CEIL, - SqlStdOperatorTable.CEIL, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_OR_INTERVAL.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), - false)); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - CEILING, - SqlStdOperatorTable.CEIL, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_OR_INTERVAL.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), - false)); + registerOperator(CEIL, SqlStdOperatorTable.CEIL); + registerOperator(CEILING, SqlStdOperatorTable.CEIL); registerOperator(COS, SqlStdOperatorTable.COS); registerOperator(COT, SqlStdOperatorTable.COT); registerOperator(DEGREES, SqlStdOperatorTable.DEGREES); registerOperator(EXP, SqlStdOperatorTable.EXP); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - FLOOR, - SqlStdOperatorTable.FLOOR, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC_OR_INTERVAL.or( - OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), - false)); + registerOperator(FLOOR, SqlStdOperatorTable.FLOOR); registerOperator(LN, SqlStdOperatorTable.LN); registerOperator(LOG10, SqlStdOperatorTable.LOG10); registerOperator(PI, SqlStdOperatorTable.PI); @@ -835,15 +878,7 @@ void populate() { registerOperator(POWER, SqlStdOperatorTable.POWER); registerOperator(RADIANS, SqlStdOperatorTable.RADIANS); registerOperator(RAND, SqlStdOperatorTable.RAND); - // TODO, workaround to support sequence CompositeOperandTypeChecker. - registerOperator( - ROUND, - SqlStdOperatorTable.ROUND, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.NUMERIC.or( - OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.INTEGER)), - false)); + registerOperator(ROUND, SqlStdOperatorTable.ROUND); registerOperator(SIGN, SqlStdOperatorTable.SIGN); registerOperator(SIGNUM, SqlStdOperatorTable.SIGN); registerOperator(SIN, SqlStdOperatorTable.SIN); @@ -855,14 +890,14 @@ void populate() { registerOperator(COALESCE, PPLBuiltinOperators.ENHANCED_COALESCE); // Register library operator - registerOperator(REGEXP, SqlLibraryOperators.REGEXP); + registerOperator(REGEXP, PPLBuiltinOperators.REGEXP); registerOperator(REGEXP_MATCH, SqlLibraryOperators.REGEXP_CONTAINS); registerOperator(CONCAT, SqlLibraryOperators.CONCAT_FUNCTION); registerOperator(CONCAT_WS, SqlLibraryOperators.CONCAT_WS); - registerOperator(CONCAT_WS, SqlLibraryOperators.CONCAT_WS); registerOperator(REVERSE, SqlLibraryOperators.REVERSE); registerOperator(RIGHT, SqlLibraryOperators.RIGHT); registerOperator(LEFT, SqlLibraryOperators.LEFT); + registerOperator(LOG, SqlLibraryOperators.LOG_MYSQL); registerOperator(LOG2, SqlLibraryOperators.LOG2); registerOperator(MD5, SqlLibraryOperators.MD5); registerOperator(SHA1, SqlLibraryOperators.SHA1); @@ -982,13 +1017,15 @@ void populate() { registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER); registerOperator(TONUMBER, PPLBuiltinOperators.TONUMBER); - registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING); register( TOSTRING, - (FunctionImp1) - (builder, source) -> - builder.makeCast(TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true), source), - PPLTypeChecker.family(SqlTypeFamily.ANY)); + (builder, args) -> { + if (args.length == 1) { + return builder.makeCast( + TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true), args[0]); + } + return builder.makeCall(PPLBuiltinOperators.TOSTRING, args); + }); // Register MVJOIN to use Calcite's ARRAY_JOIN registerOperator(MVJOIN, SqlLibraryOperators.ARRAY_JOIN); @@ -1018,20 +1055,10 @@ void populate() { // CASE WHEN isEmptyDelimiter THEN splitChars ELSE normalSplit END return builder.makeCall( SqlStdOperatorTable.CASE, isEmptyDelimiter, splitChars, normalSplit); - }, - PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); + }); // Register MVINDEX to use Calcite's ITEM/ARRAY_SLICE with index normalization - register( - MVINDEX, - new MVIndexFunctionImp(), - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), - false)); + register(MVINDEX, new MVIndexFunctionImp()); registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); registerOperator(MVAPPEND, PPLBuiltinOperators.MVAPPEND); @@ -1040,7 +1067,7 @@ void populate() { registerOperator(MVZIP, PPLBuiltinOperators.MVZIP); registerOperator(MVMAP, PPLBuiltinOperators.TRANSFORM); registerOperator(MAP_APPEND, PPLBuiltinOperators.MAP_APPEND); - registerOperator(MAP_CONCAT, SqlLibraryOperators.MAP_CONCAT); + registerOperator(MAP_CONCAT, PPLBuiltinOperators.MAP_CONCAT); registerOperator(MAP_REMOVE, PPLBuiltinOperators.MAP_REMOVE); registerOperator(ARRAY_LENGTH, SqlLibraryOperators.ARRAY_LENGTH); registerOperator(ARRAY_SLICE, SqlLibraryOperators.ARRAY_SLICE); @@ -1058,16 +1085,14 @@ void populate() { builder.makeCall( SqlStdOperatorTable.JSON_ARRAY, Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) - .toArray(RexNode[]::new))), - null); + .toArray(RexNode[]::new)))); register( JSON_OBJECT, ((builder, args) -> builder.makeCall( SqlStdOperatorTable.JSON_OBJECT, Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) - .toArray(RexNode[]::new))), - null); + .toArray(RexNode[]::new)))); registerOperator(JSON, PPLBuiltinOperators.JSON); registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH); registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); @@ -1079,57 +1104,44 @@ void populate() { registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); registerOperator(JSON_EXTRACT_ALL, PPLBuiltinOperators.JSON_EXTRACT_ALL); // internal - // Register operators with a different type checker - - // Register ADD (+ symbol) for string concatenation - // Replaced type checker since CONCAT also supports array concatenation - registerOperator( - ADD, - SqlStdOperatorTable.CONCAT, - PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); - // Register ADD (+ symbol) for numeric addition - // Replace type checker since PLUS also supports binary addition - registerOperator( - ADD, - SqlStdOperatorTable.PLUS, - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); - // Replace with a custom CompositeOperandTypeChecker to check both operands as - // SqlStdOperatorTable.ITEM.getOperandTypeChecker() checks only the first - // operand instead - // of all operands. - registerOperator( - INTERNAL_ITEM, - SqlStdOperatorTable.ITEM, - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER) - .or(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.ANY)), - false)); - registerOperator( - INTERNAL_ITEM, - SqlStdOperatorTable.ITEM, - PPLTypeChecker.family(SqlTypeFamily.IGNORE, SqlTypeFamily.CHARACTER)); - registerOperator( - XOR, - SqlStdOperatorTable.NOT_EQUALS, - PPLTypeChecker.family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.BOOLEAN)); - // SqlStdOperatorTable.CASE.getOperandTypeChecker is null. We manually create a type checker - // for it. The second and third operands are required to be of the same type. If not, it will - // throw an IllegalArgumentException with information Can't find leastRestrictive type - registerOperator( - IF, - SqlStdOperatorTable.CASE, - PPLTypeChecker.family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.ANY)); - // Re-define the type checker for is not null, is present, and is null since - // their original type checker ANY isn't compatible with struct types. - registerOperator( - IS_NOT_NULL, - SqlStdOperatorTable.IS_NOT_NULL, - PPLTypeChecker.family(SqlTypeFamily.IGNORE)); - registerOperator( - IS_PRESENT, SqlStdOperatorTable.IS_NOT_NULL, PPLTypeChecker.family(SqlTypeFamily.IGNORE)); - registerOperator( - IS_NULL, SqlStdOperatorTable.IS_NULL, PPLTypeChecker.family(SqlTypeFamily.IGNORE)); + // Register ADD (+ symbol) for string concatenation and numeric addition + // Not creating PPL builtin operator as it will cause confusion during function resolution + FunctionImp add = + (builder, args) -> { + if (Stream.of(args).map(RexNode::getType).allMatch(OpenSearchTypeUtil::isCharacter)) { + return builder.makeCall(SqlStdOperatorTable.CONCAT, args); + } + // If all args are numeric, use PLUS directly + if (Stream.of(args).map(RexNode::getType).allMatch(SqlTypeUtil::isNumeric)) { + return builder.makeCall(SqlStdOperatorTable.PLUS, args); + } + // Mixed types: widen all operands to DOUBLE for addition + // (e.g. VARCHAR from JSON extract + INTEGER) + RexNode[] plusArgs = new RexNode[args.length]; + for (int i = 0; i < args.length; i++) { + RelDataType t = args[i].getType(); + RelDataType doubleType = + builder + .getTypeFactory() + .createTypeWithNullability( + builder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE), + t.isNullable()); + if (SqlTypeUtil.isNumeric(t)) { + plusArgs[i] = builder.makeCast(doubleType, args[i]); + } else { + plusArgs[i] = builder.makeCast(doubleType, args[i], true, true); + } + } + return builder.makeCall(SqlStdOperatorTable.PLUS, plusArgs); + }; + register(ADD, add); + register(ADDFUNCTION, add); + registerOperator(INTERNAL_ITEM, SqlStdOperatorTable.ITEM); + registerOperator(XOR, SqlStdOperatorTable.NOT_EQUALS); + registerOperator(IF, SqlStdOperatorTable.CASE); + registerOperator(IS_NOT_NULL, SqlStdOperatorTable.IS_NOT_NULL); + registerOperator(IS_PRESENT, SqlStdOperatorTable.IS_NOT_NULL); + registerOperator(IS_NULL, SqlStdOperatorTable.IS_NULL); // Register implementation. // Note, make the implementation an individual class if too complex. @@ -1141,8 +1153,7 @@ void populate() { SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.BOTH), builder.makeLiteral(" "), - arg), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER)); + arg)); register( LTRIM, @@ -1152,8 +1163,7 @@ void populate() { SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.LEADING), builder.makeLiteral(" "), - arg), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER)); + arg)); register( RTRIM, (FunctionImp1) @@ -1162,60 +1172,19 @@ void populate() { SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.TRAILING), builder.makeLiteral(" "), - arg), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER)); - registerOperator( - ATAN, - SqlStdOperatorTable.ATAN2, - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); + arg)); register( STRCMP, (FunctionImp2) - (builder, arg1, arg2) -> builder.makeCall(SqlLibraryOperators.STRCMP, arg2, arg1), - PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); - // SqlStdOperatorTable.SUBSTRING.getOperandTypeChecker is null. We manually - // create a type - // checker for it. + (builder, arg1, arg2) -> builder.makeCall(SqlLibraryOperators.STRCMP, arg2, arg1)); register( SUBSTRING, (RexBuilder builder, RexNode... args) -> - builder.makeCall(SqlStdOperatorTable.SUBSTRING, args), - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.INTEGER, - SqlTypeFamily.INTEGER)), - false)); + builder.makeCall(SqlStdOperatorTable.SUBSTRING, args)); register( SUBSTR, (RexBuilder builder, RexNode... args) -> - builder.makeCall(SqlStdOperatorTable.SUBSTRING, args), - PPLTypeChecker.wrapComposite( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER) - .or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, - SqlTypeFamily.INTEGER, - SqlTypeFamily.INTEGER)), - false)); - register( - LOG, - (FunctionImp2) - (builder, arg1, arg2) -> builder.makeCall(SqlLibraryOperators.LOG, arg2, arg1), - PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); - register( - LOG, - (FunctionImp1) - (builder, arg) -> - builder.makeCall( - SqlLibraryOperators.LOG, - arg, - builder.makeApproxLiteral(BigDecimal.valueOf(Math.E))), - PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + builder.makeCall(SqlStdOperatorTable.SUBSTRING, args)); // SqlStdOperatorTable.SQRT is declared but not implemented. The call to SQRT in Calcite is // converted to POWER(x, 0.5). register( @@ -1225,14 +1194,12 @@ void populate() { builder.makeCall( SqlStdOperatorTable.POWER, arg, - builder.makeApproxLiteral(BigDecimal.valueOf(0.5))), - PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + builder.makeApproxLiteral(BigDecimal.valueOf(0.5)))); register( TYPEOF, (FunctionImp1) (builder, arg) -> - builder.makeLiteral(getLegacyTypeName(arg.getType(), QueryType.PPL)), - null); + builder.makeLiteral(getLegacyTypeName(arg.getType(), QueryType.PPL))); register( NULLIF, (FunctionImp2) @@ -1241,8 +1208,7 @@ void populate() { SqlStdOperatorTable.CASE, builder.makeCall(SqlStdOperatorTable.EQUALS, arg1, arg2), builder.makeNullLiteral(arg1.getType()), - arg1), - PPLTypeChecker.wrapComparable((SameOperandTypeChecker) OperandTypes.SAME_SAME)); + arg1)); register( IS_EMPTY, (FunctionImp1) @@ -1250,8 +1216,7 @@ void populate() { builder.makeCall( SqlStdOperatorTable.OR, builder.makeCall(SqlStdOperatorTable.IS_NULL, arg), - builder.makeCall(SqlStdOperatorTable.IS_EMPTY, arg)), - PPLTypeChecker.family(SqlTypeFamily.ANY)); + builder.makeCall(SqlStdOperatorTable.EQUALS, arg, builder.makeLiteral("")))); register( IS_BLANK, (FunctionImp1) @@ -1260,20 +1225,19 @@ void populate() { SqlStdOperatorTable.OR, builder.makeCall(SqlStdOperatorTable.IS_NULL, arg), builder.makeCall( - SqlStdOperatorTable.IS_EMPTY, + SqlStdOperatorTable.EQUALS, builder.makeCall( SqlStdOperatorTable.TRIM, builder.makeFlag(Flag.BOTH), builder.makeLiteral(" "), - arg))), - PPLTypeChecker.family(SqlTypeFamily.ANY)); + arg), + builder.makeLiteral("")))); register( ILIKE, (FunctionImp2) (builder, arg1, arg2) -> builder.makeCall( - SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")), - PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)); + SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\"))); register( LIKE, (FunctionImp3) @@ -1282,44 +1246,35 @@ void populate() { ? builder.makeCall( SqlStdOperatorTable.LIKE, arg1, arg2, builder.makeLiteral("\\")) : builder.makeCall( - SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")), - PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.BOOLEAN)); + SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\"))); } } private static class Builder extends AbstractBuilder { - private final Map>> map = - new HashMap<>(); + private final Map map = new HashMap<>(); @Override - void register( - BuiltinFunctionName functionName, FunctionImp implement, PPLTypeChecker typeChecker) { - CalciteFuncSignature signature = - new CalciteFuncSignature(functionName.getName(), typeChecker); + void register(BuiltinFunctionName functionName, FunctionImp implement) { if (map.containsKey(functionName)) { - map.get(functionName).add(Pair.of(signature, implement)); - } else { - map.put(functionName, new ArrayList<>(List.of(Pair.of(signature, implement)))); + throw new IllegalStateException( + String.format( + Locale.ROOT, + "Each function can only be registered with one operator: %s", + functionName)); } + map.put(functionName, implement); } } private static class AggBuilder { private static final double MEDIAN_PERCENTILE = 50.0; - private final Map> map = - new HashMap<>(); - - void register( - BuiltinFunctionName functionName, AggHandler aggHandler, PPLTypeChecker typeChecker) { - CalciteFuncSignature signature = - new CalciteFuncSignature(functionName.getName(), typeChecker); - map.put(functionName, Pair.of(signature, aggHandler)); + private final Map map = new HashMap<>(); + + void register(BuiltinFunctionName functionName, AggHandler aggHandler) { + map.put(functionName, aggHandler); } void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFunction) { - SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(aggFunction); - PPLTypeChecker typeChecker = - wrapSqlOperandTypeChecker(innerTypeChecker, functionName.name(), true); AggHandler handler = (distinct, field, argList, ctx) -> { List newArgList = @@ -1327,7 +1282,7 @@ void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFuncti return UserDefinedFunctionUtils.makeAggregateCall( aggFunction, List.of(field), newArgList, ctx.relBuilder); }; - register(functionName, handler, typeChecker); + register(functionName, handler); } void populate() { @@ -1343,11 +1298,7 @@ void populate() { registerOperator(LIST, PPLBuiltinOperators.LIST); registerOperator(VALUES, PPLBuiltinOperators.VALUES); - register( - AVG, - (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field), - wrapSqlOperandTypeChecker( - SqlStdOperatorTable.AVG.getOperandTypeChecker(), AVG.name(), false)); + register(AVG, (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field)); register( COUNT, @@ -1359,8 +1310,7 @@ void populate() { // count(field) should count non-null values of the field return ctx.relBuilder.count(distinct, null, field); } - }, - wrapSqlOperandTypeChecker(PPLOperandTypes.OPTIONAL_ANY, COUNT.name(), false)); + }); register( PERCENTILE_APPROX, @@ -1373,11 +1323,7 @@ void populate() { newArgList.add(ctx.rexBuilder.makeFlag(field.getType().getSqlTypeName())); return UserDefinedFunctionUtils.makeAggregateCall( PPLBuiltinOperators.PERCENTILE_APPROX, List.of(field), newArgList, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - extractTypeCheckerFromUDF(PPLBuiltinOperators.PERCENTILE_APPROX), - PERCENTILE_APPROX.name(), - false)); + }); register( MEDIAN, @@ -1400,9 +1346,7 @@ void populate() { List.of(field), medianArgList, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - PPLOperandTypes.NUMERIC.getInnerTypeChecker(), MEDIAN.name(), false)); + }); register( EARLIEST, @@ -1410,9 +1354,7 @@ void populate() { List args = resolveTimeField(argList, ctx); return UserDefinedFunctionUtils.makeAggregateCall( SqlStdOperatorTable.ARG_MIN, List.of(field), args, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - PPLOperandTypes.ANY_OPTIONAL_TIMESTAMP, EARLIEST.name(), false)); + }); register( LATEST, @@ -1420,9 +1362,7 @@ void populate() { List args = resolveTimeField(argList, ctx); return UserDefinedFunctionUtils.makeAggregateCall( SqlStdOperatorTable.ARG_MAX, List.of(field), args, ctx.relBuilder); - }, - wrapSqlOperandTypeChecker( - PPLOperandTypes.ANY_OPTIONAL_TIMESTAMP, EARLIEST.name(), false)); + }); // Register FIRST function - uses document order register( @@ -1430,9 +1370,7 @@ void populate() { (distinct, field, argList, ctx) -> { // Use our custom FirstAggFunction for document order aggregation return ctx.relBuilder.aggregateCall(PPLBuiltinOperators.FIRST, field); - }, - wrapSqlOperandTypeChecker( - PPLBuiltinOperators.FIRST.getOperandTypeChecker(), FIRST.name(), false)); + }); // Register LAST function - uses document order register( @@ -1440,9 +1378,7 @@ void populate() { (distinct, field, argList, ctx) -> { // Use our custom LastAggFunction for document order aggregation return ctx.relBuilder.aggregateCall(PPLBuiltinOperators.LAST, field); - }, - wrapSqlOperandTypeChecker( - PPLBuiltinOperators.LAST.getOperandTypeChecker(), LAST.name(), false)); + }); } } @@ -1460,79 +1396,4 @@ static List resolveTimeField(List argList, CalcitePlanContext return argList.stream().map(PlanUtils::derefMapCall).collect(Collectors.toList()); } } - - /** - * Wraps a {@link SqlOperandTypeChecker} into a {@link PPLTypeChecker} for use in function - * signature validation. - * - * @param typeChecker the original SQL operand type checker - * @param functionName the name of the function for error reporting - * @param isUserDefinedFunction true if the function is user-defined, false otherwise - * @return a {@link PPLTypeChecker} that delegates to the provided {@code typeChecker} - */ - private static PPLTypeChecker wrapSqlOperandTypeChecker( - SqlOperandTypeChecker typeChecker, String functionName, boolean isUserDefinedFunction) { - PPLTypeChecker pplTypeChecker; - if (typeChecker instanceof ImplicitCastOperandTypeChecker implicitCastTypeChecker) { - pplTypeChecker = PPLTypeChecker.wrapFamily(implicitCastTypeChecker); - } else if (typeChecker instanceof CompositeOperandTypeChecker compositeTypeChecker) { - // UDFs implement their own composite type checkers, which always use OR logic for - // argument - // types. Verifying the composition type would require accessing a protected field in - // CompositeOperandTypeChecker. If access to this field is not allowed, type checking will - // be skipped, so we avoid checking the composition type here. - - // If compositeTypeChecker contains operand checkers other than family type checkers or - // other than OR compositions, the function with be registered with a null type checker, - // which means the function will not be type checked. - try { - pplTypeChecker = PPLTypeChecker.wrapComposite(compositeTypeChecker, !isUserDefinedFunction); - } catch (IllegalArgumentException | UnsupportedOperationException e) { - logger.debug( - String.format( - "Failed to create composite type checker for operator: %s. Will skip its type" - + " checking", - functionName), - e); - pplTypeChecker = null; - } - } else if (typeChecker instanceof SameOperandTypeChecker comparableTypeChecker) { - // Comparison operators like EQUAL, GREATER_THAN, LESS_THAN, etc. - // SameOperandTypeCheckers like COALESCE, IFNULL, etc. - pplTypeChecker = PPLTypeChecker.wrapComparable(comparableTypeChecker); - } else if (typeChecker instanceof UDFOperandMetadata.UDTOperandMetadata udtOperandMetadata) { - pplTypeChecker = PPLTypeChecker.wrapUDT(udtOperandMetadata.allowedParamTypes()); - } else if (typeChecker != null) { - pplTypeChecker = PPLTypeChecker.wrapDefault(typeChecker); - } else { - logger.info( - "Cannot create type checker for function: {}. Will skip its type checking", functionName); - pplTypeChecker = null; - } - return pplTypeChecker; - } - - /** - * Extracts the underlying {@link SqlOperandTypeChecker} from a {@link SqlOperator}. - * - *

For user-defined functions (UDFs) and user-defined aggregate functions (UDAFs), the {@link - * SqlOperandTypeChecker} is typically wrapped in a {@link UDFOperandMetadata}, which contains the - * actual type checker used for operand validation. Most of these wrapped type checkers are - * defined in {@link org.opensearch.sql.calcite.utils.PPLOperandTypes}. This method retrieves the - * inner type checker from {@link UDFOperandMetadata} if present. - * - *

For Calcite's built-in operators, its type checker is returned directly. - * - * @param operator the {@link SqlOperator}, which may be a Calcite built-in operator, a - * user-defined function, or a user-defined aggregation function - * @return the underlying {@link SqlOperandTypeChecker} instance, or {@code null} if not available - */ - private static SqlOperandTypeChecker extractTypeCheckerFromUDF(SqlOperator operator) { - SqlOperandTypeChecker typeChecker = operator.getOperandTypeChecker(); - if (typeChecker instanceof UDFOperandMetadata) { - UDFOperandMetadata udfOperandMetadata = (UDFOperandMetadata) typeChecker; - return udfOperandMetadata.getInnerTypeChecker(); - } - return typeChecker; - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PatternParserFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/PatternParserFunctionImpl.java index e4f7f1f9d1c..70cc923541a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PatternParserFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PatternParserFunctionImpl.java @@ -31,6 +31,7 @@ import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.logging.log4j.util.Strings; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.common.patterns.BrainLogParser; import org.opensearch.sql.common.patterns.PatternUtils; @@ -50,7 +51,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.ARRAY, SqlTypeFamily.BOOLEAN) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java index dc4761b26e7..7b131383b95 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java @@ -5,19 +5,15 @@ package org.opensearch.sql.expression.function; -import java.util.Collections; import java.util.List; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; -import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandMetadata; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; -import org.opensearch.sql.data.type.ExprType; /** * This class is created for the compatibility with {@link SqlUserDefinedFunction} constructors when @@ -27,7 +23,7 @@ public interface UDFOperandMetadata extends SqlOperandMetadata { SqlOperandTypeChecker getInnerTypeChecker(); - static UDFOperandMetadata wrap(FamilyOperandTypeChecker typeChecker) { + static UDFOperandMetadata wrap(SqlOperandTypeChecker typeChecker) { return new UDFOperandMetadata() { @Override public SqlOperandTypeChecker getInnerTypeChecker() { @@ -36,57 +32,16 @@ public SqlOperandTypeChecker getInnerTypeChecker() { @Override public List paramTypes(RelDataTypeFactory typeFactory) { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); + // This function is not used in the current context + throw new UnsupportedOperationException( + "paramTypes of UDFOperandMetadata is not implemented and should not be called"); } @Override public List paramNames() { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); - } - - @Override - public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { - return typeChecker.checkOperandTypesWithoutTypeCoercion(callBinding, throwOnFailure); - } - - @Override - public SqlOperandCountRange getOperandCountRange() { - return typeChecker.getOperandCountRange(); - } - - @Override - public String getAllowedSignatures(SqlOperator op, String opName) { - return typeChecker.getAllowedSignatures(op, opName); - } - }; - } - - static UDFOperandMetadata wrap(CompositeOperandTypeChecker typeChecker) { - for (SqlOperandTypeChecker rule : typeChecker.getRules()) { - if (!(rule instanceof FamilyOperandTypeChecker)) { - throw new IllegalArgumentException( - "Currently only compositions of ImplicitCastOperandTypeChecker are supported"); - } - } - - return new UDFOperandMetadata() { - @Override - public SqlOperandTypeChecker getInnerTypeChecker() { - return typeChecker; - } - - @Override - public List paramTypes(RelDataTypeFactory typeFactory) { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); - } - - @Override - public List paramNames() { - // This function is not used in the current context, so we return an empty list. - return Collections.emptyList(); + // This function is not used in the current context + throw new UnsupportedOperationException( + "paramNames of UDFOperandMetadata is not implemented and should not be called"); } @Override @@ -105,40 +60,4 @@ public String getAllowedSignatures(SqlOperator op, String opName) { } }; } - - static UDFOperandMetadata wrapUDT(List> allowSignatures) { - return new UDTOperandMetadata(allowSignatures); - } - - record UDTOperandMetadata(List> allowedParamTypes) implements UDFOperandMetadata { - @Override - public SqlOperandTypeChecker getInnerTypeChecker() { - return this; - } - - @Override - public List paramTypes(RelDataTypeFactory typeFactory) { - return List.of(); - } - - @Override - public List paramNames() { - return List.of(); - } - - @Override - public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { - return false; - } - - @Override - public SqlOperandCountRange getOperandCountRange() { - return null; - } - - @Override - public String getAllowedSignatures(SqlOperator op, String opName) { - return ""; - } - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java index f52c6b69f0e..64e04a5c936 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java @@ -9,10 +9,13 @@ import org.apache.calcite.schema.ImplementableFunction; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.InferTypes; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; +import org.jspecify.annotations.NonNull; /** * The interface helps to construct a SqlUserDefinedFunction @@ -22,7 +25,8 @@ *

2. getReturnTypeInference - returns the return type of the UDF * *

3. getOperandMetadata - returns the operand metadata of the UDF. This is for checking the - * operand when validation, default null without checking. + * operand when validation, default to VARIADIC (accepts any number of operands without type + * checking). */ public interface UserDefinedFunctionBuilder { @@ -30,7 +34,21 @@ public interface UserDefinedFunctionBuilder { SqlReturnTypeInference getReturnTypeInference(); - UDFOperandMetadata getOperandMetadata(); + /** + * Define the operand metadata of the UDF. This is for checking the operand when validation, + * default to VARIADIC (accepts any number of operands without type checking). + * + * @return a {@link UDFOperandMetadata} that defines operand types, typically wrapping a {@link + * org.apache.calcite.sql.type.SqlOperandTypeChecker} to reuse Calcite's type checkers. + */ + default @NonNull UDFOperandMetadata getOperandMetadata() { + // Use the most permissive type checker by default, effectively skipping type checking + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); + } + + default SqlKind getKind() { + return SqlKind.OTHER_FUNCTION; + } default SqlUserDefinedFunction toUDF(String functionName) { return toUDF(functionName, true); @@ -50,7 +68,7 @@ default SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministi new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); return new SqlUserDefinedFunction( udfLtrimIdentifier, - SqlKind.OTHER_FUNCTION, + getKind(), getReturnTypeInference(), InferTypes.ANY_NULLABLE, getOperandMetadata(), @@ -66,6 +84,11 @@ public SqlIdentifier getSqlIdentifier() { // check the code SqlUtil.unparseFunctionSyntax() return null; } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return getOperandMetadata().getOperandCountRange(); + } }; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index dd76a002e06..de56ee4c6b7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -22,7 +22,9 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -37,8 +39,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.ONE_OR_MORE); } public static class JsonAppendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java index cc12d8de7c5..f10aa62415e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java @@ -19,6 +19,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -34,7 +35,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index b3a884a4f17..d8f24cbaad9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -7,7 +7,6 @@ import static org.apache.calcite.runtime.JsonFunctions.jsonRemove; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; import java.util.Arrays; @@ -20,7 +19,9 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -35,8 +36,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.ONE_OR_MORE); } public static class JsonDeleteImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index dd91f1d95bd..647ee799aae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -22,7 +22,9 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -37,8 +39,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.ONE_OR_MORE); } public static class JsonExtendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java index 8168700b6da..17a4e14b8a7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java @@ -29,8 +29,8 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -57,8 +57,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.STRING)); + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.CHARACTER.or(OperandTypes.ARRAY)); } public static class JsonExtractAllImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index 6d80bf70aba..d6e2d3c6f52 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -8,7 +8,6 @@ import static org.apache.calcite.sql.SqlJsonQueryEmptyOrErrorBehavior.NULL; import static org.apache.calcite.sql.SqlJsonQueryWrapperBehavior.WITHOUT_ARRAY; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import java.util.ArrayList; import java.util.Arrays; @@ -25,7 +24,9 @@ import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -40,8 +41,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } public static class JsonExtractImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java index 0379aeecb72..c263e611a83 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -15,8 +15,10 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -35,8 +37,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.ANY); } public static class JsonImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java index 40214ca7556..820d4ae0b8f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -17,7 +17,9 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -32,8 +34,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.ANY); } public static class JsonKeysImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 27346b478e4..981c95ccf2b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.apache.calcite.util.Static.RESOURCE; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; @@ -18,10 +19,19 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlJsonModifyFunction; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,8 +46,39 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + /** + * Copied from {@link SqlJsonModifyFunction#checkOperandTypes(SqlCallBinding, boolean)} + * (Calcite 1.41) + */ + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + final int count = callBinding.getOperandCount(); + for (int i = 1; i < count; i += 2) { + RelDataType nameType = callBinding.getOperandType(i); + if (!OpenSearchTypeUtil.isCharacter(nameType)) { + if (throwOnFailure) { + throw callBinding.newError(RESOURCE.expectedCharacter()); + } + return false; + } + } + return true; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.from(3); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "(json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY" + + " ...)"; + } + }); } public static class JsonSetImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java index 0f46e5cbb1c..3d63d1ca5ce 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java @@ -18,6 +18,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.MessageDigestAlgorithms; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -30,7 +31,7 @@ private CryptographicFunction(NotNullImplementor implementor, NullPolicy nullPol public static CryptographicFunction sha2() { return new CryptographicFunction(new Sha2Implementor(), NullPolicy.ANY) { @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_INTEGER; } }; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ParseFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ParseFunction.java index 404b2075ee4..92ada9ac549 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ParseFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ParseFunction.java @@ -23,6 +23,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.commons.lang3.tuple.Pair; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.ast.expression.ParseMethod; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprValue; @@ -50,7 +51,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_STRING_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java index d8e53704804..4ef9d6ab48f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java @@ -5,18 +5,17 @@ package org.opensearch.sql.expression.function.udf; -import com.google.common.collect.ImmutableList; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeFamily; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -39,55 +38,11 @@ public SqlReturnTypeInference getReturnTypeInference() { * Query parameter is always required and cannot be null. */ @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - ImmutableList.of( - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP), - i -> i > 0 && i < 14) // Parameters 3-14 are optional - .or( - OperandTypes.family( - ImmutableList.of( - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP, - SqlTypeFamily.MAP), - i -> i > 0 && i < 25))); // Parameters 3-25 are optional + OperandTypes.repeat( + SqlOperandCountRanges.between(1, 25), + OperandTypes.MAP)); // Parameters 2-25 are optional } public static class RelevanceQueryImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java index 569e03f84c1..6aabd2178cc 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java @@ -18,6 +18,7 @@ import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,7 +37,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { // Support both (field, pattern, groupIndex) and (field, pattern, groupName) return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java index 2e0cb0cd06c..2e5e7318cfb 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java @@ -21,6 +21,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -43,7 +44,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { // Support both (field, pattern, groupIndex, maxMatch) and (field, pattern, groupName, maxMatch) return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexOffsetFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexOffsetFunction.java index c52b27537e5..261276d1099 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexOffsetFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexOffsetFunction.java @@ -17,6 +17,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -34,7 +35,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java index f28f12e30b9..edda8532a41 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java @@ -18,12 +18,13 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.BuiltInMethod; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.type.ExprSqlType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.data.model.ExprValue; @@ -45,29 +46,41 @@ public SpanFunction() { public SqlReturnTypeInference getReturnTypeInference() { // Return arg0 type if it has a unit (i.e. time related span) return callBinding -> { - if (SqlTypeUtil.isString(callBinding.getOperandType(2))) { + RelDataType unitType = callBinding.getOperandType(2); + if (SqlTypeUtil.isString(unitType)) { return callBinding.getOperandType(0); } + // When unit is NULL/ANY and field is a datetime type, return the field type. + // This handles cases like timechart with millisecond spans where the unit may be NULL. + RelDataType fieldType = callBinding.getOperandType(0); + if ((SqlTypeUtil.isNull(unitType) || SqlTypeName.ANY.equals(unitType.getSqlTypeName())) + && (SqlTypeUtil.isDatetime(fieldType) || fieldType instanceof ExprSqlType)) { + return fieldType; + } // Use the least restrictive type between the field type and the interval type if it's a // numeric span. E.g. span(int_field, double_literal) -> double - return callBinding - .getTypeFactory() - .leastRestrictive(List.of(callBinding.getOperandType(0), callBinding.getOperandType(1))); + RelDataType leastRestrictive = + callBinding + .getTypeFactory() + .leastRestrictive( + List.of(callBinding.getOperandType(0), callBinding.getOperandType(1))); + return leastRestrictive != null ? leastRestrictive : callBinding.getOperandType(0); }; } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER) - .or( - OperandTypes.family( - SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER)) - .or( - OperandTypes.family( - SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY))); + OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER) + .or( + OperandTypes.family( + SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER)) + .or( + OperandTypes.family( + SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY)) + .or( + OperandTypes.family( + SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.ANY))); } public static class SpanImplementor implements NotNullImplementor { @@ -86,7 +99,7 @@ public Expression implement( if (SqlTypeUtil.isDecimal(intervalType)) { interval = Expressions.call(interval, "doubleValue"); } - if (SqlTypeUtil.isNull(unitType)) { + if (SqlTypeUtil.isNull(unitType) || SqlTypeName.ANY.equals(unitType.getSqlTypeName())) { return switch (call.getType().getSqlTypeName()) { case BIGINT, INTEGER, SMALLINT, TINYINT -> Expressions.multiply(Expressions.divide(field, interval), interval); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index bc0a6dffa8d..6e5acede002 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -16,6 +16,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -52,7 +53,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_OR_STRING_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index e6e8dd01df0..ea20f3ec26e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -18,8 +18,8 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.runtime.SqlFunctions; -import org.apache.calcite.sql.*; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -56,7 +56,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_STRING_OR_STRING_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java index 11e1a33afbd..ecb78727ff3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -47,7 +48,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_NUMERIC_NUMERIC_NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java index a8f2625b20f..02a43a22a5e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -51,7 +52,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_NUMERIC_NUMERIC_NUMERIC_NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java index 6970e485525..db057b78127 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -45,7 +46,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java index 08daf9c314b..23fcf02e707 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java @@ -16,8 +16,8 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.sql.calcite.type.ExprSqlType; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.jspecify.annotations.NonNull; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.binning.BinConstants; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -51,21 +51,15 @@ public SqlReturnTypeInference getReturnTypeInference() { return (opBinding) -> { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); RelDataType arg0Type = opBinding.getOperandType(0); - return dateRelatedType(arg0Type) + return OpenSearchTypeUtil.isDatetime(arg0Type) ? arg0Type : typeFactory.createTypeWithNullability( typeFactory.createSqlType(SqlTypeName.VARCHAR, 2000), true); }; } - public static boolean dateRelatedType(RelDataType type) { - return type instanceof ExprSqlType exprSqlType - && List.of(ExprUDT.EXPR_DATE, ExprUDT.EXPR_TIME, ExprUDT.EXPR_TIMESTAMP) - .contains(exprSqlType.getUdt()); - } - @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.WIDTH_BUCKET_OPERAND; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EarliestFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EarliestFunction.java index 24e37ab83ca..71232859eef 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EarliestFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EarliestFunction.java @@ -22,6 +22,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.model.ExprValue; @@ -40,7 +41,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_TIMESTAMP; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/LatestFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/LatestFunction.java index 95d67ac2b40..f42b075c06a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/LatestFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/LatestFunction.java @@ -22,6 +22,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.model.ExprValue; @@ -40,7 +41,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_TIMESTAMP; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java index 9456e6b857a..a55ad342ca1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java @@ -25,6 +25,7 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.calcite.utils.datetime.DateTimeConversionUtils; @@ -73,7 +74,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.DATETIME_INTERVAL.or( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java index 49e06afa3d6..94ff496f283 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java @@ -15,6 +15,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.model.ExprDateValue; @@ -58,7 +59,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NONE; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java index dc1a4bb1d16..dab00fa9928 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java @@ -16,6 +16,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -47,7 +48,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_INTERVAL; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatePartFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatePartFunction.java index 35ce04f6858..8654b00a74d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatePartFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatePartFunction.java @@ -19,6 +19,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; @@ -71,7 +72,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OR_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatetimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatetimeFunction.java index d7a876cfeaa..c9be918c706 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatetimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatetimeFunction.java @@ -15,6 +15,7 @@ import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.model.ExprStringValue; @@ -46,7 +47,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.TIMESTAMP_STRING diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ExtractFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ExtractFunction.java index 90b7c8a783f..0daddd28c6f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ExtractFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ExtractFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; @@ -48,7 +49,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_DATETIME; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FormatFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FormatFunction.java index 0a7f214b939..0599468f6c7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FormatFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FormatFunction.java @@ -18,6 +18,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -57,7 +58,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OR_STRING_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java index 00fa9f690da..9cbb661ea31 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java @@ -19,6 +19,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprStringValue; @@ -53,7 +54,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_OPTIONAL_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/LastDayFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/LastDayFunction.java index c1eb8aa776d..c7c4aae5db9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/LastDayFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/LastDayFunction.java @@ -13,6 +13,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -46,7 +47,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OR_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java index 109bad16bf1..28219a06b54 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java @@ -17,6 +17,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -50,7 +51,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATE_OR_TIMESTAMP_OR_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java index 319a22ed372..b7bdf03b9ff 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java @@ -16,6 +16,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.MathUtils; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -45,7 +46,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/StrftimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/StrftimeFunction.java index 653455609d4..150faf9ef27 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/StrftimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/StrftimeFunction.java @@ -18,6 +18,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.*; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -53,14 +54,15 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - // Accepts (NUMERIC|TIMESTAMP, STRING) -> STRING - // Note: STRING is NOT accepted for first parameter - use unix_timestamp() to convert + public @NonNull UDFOperandMetadata getOperandMetadata() { + // Accepts (NUMERIC|TIMESTAMP|CHARACTER, STRING) -> STRING + // CHARACTER is accepted for first parameter to allow string numeric values like '1521467703' // Calcite will auto-cast DATE and TIME to TIMESTAMP return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER) - .or(OperandTypes.family(SqlTypeFamily.TIMESTAMP, SqlTypeFamily.CHARACTER))); + .or(OperandTypes.family(SqlTypeFamily.TIMESTAMP, SqlTypeFamily.CHARACTER)) + .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); } public static class StrftimeImplementor implements NotNullImplementor { @@ -144,8 +146,13 @@ private static Double extractNumericValue(ExprValue value) { return (double) value.floatValue(); } - // Not a numeric type - return null; + // Try to parse string values as numeric (handles string literals like '1521467703') + try { + return Double.parseDouble(value.stringValue()); + } catch (Exception e) { + // Not a parseable numeric string + return null; + } } /** diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SysdateFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SysdateFunction.java index 672d8f75e8c..958adb9dd82 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SysdateFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SysdateFunction.java @@ -15,6 +15,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprTimestampValue; @@ -43,7 +44,7 @@ public SysdateFunction() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.OPTIONAL_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampAddFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampAddFunction.java index 29de7881fa6..56069274219 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampAddFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampAddFunction.java @@ -17,6 +17,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -55,7 +56,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_INTEGER_DATETIME_OR_STRING; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampDiffFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampDiffFunction.java index 5adf7bf8bf7..cecced28b61 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampDiffFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampDiffFunction.java @@ -17,6 +17,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; @@ -52,7 +53,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.INTERVAL_DATETIME_DATETIME; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampFunction.java index a13a1ce1894..47c9235d062 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/TimestampFunction.java @@ -20,6 +20,7 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.function.FunctionProperties; @@ -48,7 +49,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.CHARACTER diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ToSecondsFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ToSecondsFunction.java index 7a89d5e918b..b0e349dad1d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ToSecondsFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/ToSecondsFunction.java @@ -20,6 +20,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprValue; @@ -50,7 +51,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OR_STRING_OR_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/UnixTimestampFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/UnixTimestampFunction.java index 5103173b46f..2b0ee5304fe 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/UnixTimestampFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/UnixTimestampFunction.java @@ -17,6 +17,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.function.FunctionProperties; @@ -48,7 +49,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.OPTIONAL_DATE_OR_TIMESTAMP_OR_NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekFunction.java index 11564ce47f0..f4fe2211035 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekFunction.java @@ -15,6 +15,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprIntegerValue; @@ -49,7 +50,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OPTIONAL_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekdayFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekdayFunction.java index 472203f3079..d01dc6331b8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekdayFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/WeekdayFunction.java @@ -5,7 +5,6 @@ package org.opensearch.sql.expression.function.udf.datetime; -import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.*; import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprWeekday; import static org.opensearch.sql.expression.datetime.DateTimeFunctions.formatNow; @@ -17,6 +16,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -50,7 +50,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OPTIONAL_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/YearweekFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/YearweekFunction.java index 9bfe30df39c..71c5f95c842 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/YearweekFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/YearweekFunction.java @@ -17,6 +17,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprValue; @@ -49,7 +50,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.DATETIME_OPTIONAL_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java index c3a4fe4efe6..d18aa825ac9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java @@ -11,13 +11,21 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.sql.expression.ip.IPFunctions; @@ -43,14 +51,33 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { // EXPR_IP is mapped to SqlTypeFamily.OTHER in // UserDefinedFunctionUtils.convertRelDataTypeToSqlTypeName // We use a specific type checker to serve - return UDFOperandMetadata.wrapUDT( - List.of( - List.of(ExprCoreType.IP, ExprCoreType.STRING), - List.of(ExprCoreType.STRING, ExprCoreType.STRING))); + return UDFOperandMetadata.wrap( + OperandTypes.CHARACTER_CHARACTER.or( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + List types = callBinding.collectOperandTypes(); + return OpenSearchTypeUtil.isIp(types.get(0), true) + && OpenSearchTypeUtil.isCharacter(types.get(1)); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "CIDRMATCH(, )"; + } + })); } public static class CidrMatchImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index ce200323f60..2f3cdeb5457 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -15,18 +15,23 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.data.model.ExprIpValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -119,8 +124,28 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT(List.of(List.of(ExprCoreType.IP, ExprCoreType.IP))); + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + return OpenSearchTypeUtil.isIp(callBinding.getOperandType(0), true) + && OpenSearchTypeUtil.isIp(callBinding.getOperandType(1), true); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return String.format(Locale.ROOT, "%s(, )", opName); + } + }); } public static class CompareImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java index baf6b8a37e1..e7858f36970 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java @@ -11,9 +11,17 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -39,9 +47,29 @@ public IPFunction() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT( - List.of(List.of(ExprCoreType.IP), List.of(ExprCoreType.STRING))); + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + RelDataType type = callBinding.getOperandType(0); + boolean valid = OpenSearchTypeUtil.isIp(type) || OpenSearchTypeUtil.isCharacter(type); + if (!valid && throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return valid; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(1); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "IP(), IP()"; + } + }); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ConvFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ConvFunction.java index e29c17dba49..125f0957326 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ConvFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ConvFunction.java @@ -16,6 +16,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -38,7 +39,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.STRING_OR_INTEGER_INTEGER_INTEGER; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java index 1767a2fc69b..89606c6b115 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java @@ -15,9 +15,11 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.MathUtils; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -41,13 +43,18 @@ public DivideFunction() { super(new DivideImplementor(), NullPolicy.ANY); } + @Override + public SqlKind getKind() { + return SqlKind.DIVIDE; + } + @Override public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.QUOTIENT_FORCE_NULLABLE; } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/EulerFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/EulerFunction.java index f33d21c887f..e0a5f033499 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/EulerFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/EulerFunction.java @@ -14,6 +14,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -39,7 +40,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NONE; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java index 7a8f8e75f92..6e72e96ea3b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java @@ -14,10 +14,12 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeTransforms; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.MathUtils; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -37,7 +39,12 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public SqlKind getKind() { + return SqlKind.MOD; + } + + @Override + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC_NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/NumberToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/NumberToStringFunction.java index 24488ea5746..da6c08a0869 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/NumberToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/NumberToStringFunction.java @@ -13,6 +13,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -35,7 +36,7 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { + public @NonNull UDFOperandMetadata getOperandMetadata() { return PPLOperandTypes.NUMERIC; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java index 9b4b0b48c73..c54a340b30a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMaxFunction.java @@ -14,8 +14,10 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.data.utils.MixedTypeComparator; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,8 +38,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } public static class MaxImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java index 441257a422e..583cd00cc0b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ScalarMinFunction.java @@ -14,8 +14,10 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.sql.data.utils.MixedTypeComparator; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,8 +38,8 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return null; + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.VARIADIC); } public static class MinImplementor implements NotNullImplementor { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java index 102e3d8784c..ab6ad5fe6d7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java @@ -197,11 +197,11 @@ public void testStrftimeWithPercentLiteral() { @Test public void testStrftimeWithStringTimestamp() { - // String inputs are not supported - should return null + // String numeric timestamps are parsed and formatted String result = StrftimeFunction.strftime( new ExprStringValue("1521467703"), new ExprStringValue("%Y-%m-%d")); - assertNull(result); + assertEquals("2018-03-19", result); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java b/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java index d20841a2cee..8215c99b695 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/AggFunctionTestBase.java @@ -16,22 +16,19 @@ public abstract class AggFunctionTestBase { @SuppressWarnings("unchecked") - protected Map> - getAggFunctionRegistry() { + protected Map getAggFunctionRegistry() { try { PPLFuncImpTable funcTable = PPLFuncImpTable.INSTANCE; Field field = PPLFuncImpTable.class.getDeclaredField("aggFunctionRegistry"); field.setAccessible(true); - return (Map>) - field.get(funcTable); + return (Map) field.get(funcTable); } catch (Exception e) { throw new RuntimeException("Failed to access aggFunctionRegistry", e); } } protected void assertFunctionIsRegistered(BuiltinFunctionName functionName) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); assertTrue( registry.containsKey(functionName), functionName.getName().getFunctionName() @@ -48,36 +45,23 @@ protected void assertFunctionsAreRegistered(BuiltinFunctionName... functionNames } protected void assertFunctionHandlerTypes(BuiltinFunctionName... functionNames) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); for (BuiltinFunctionName functionName : functionNames) { - org.apache.commons.lang3.tuple.Pair registryEntry = registry.get(functionName); - assertNotNull( - registryEntry, functionName.getName().getFunctionName() + " should be registered"); - - // Extract the AggHandler from the pair - AggHandler handler = registryEntry.getRight(); - + AggHandler handler = registry.get(functionName); assertNotNull( handler, functionName.getName().getFunctionName() + " handler should not be null"); - assertTrue( - handler instanceof AggHandler, - functionName.getName().getFunctionName() - + " handler should implement AggHandler interface"); } } protected void assertRegistryMinimumSize(int expectedMinimumSize) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); assertTrue( registry.size() >= expectedMinimumSize, "Registry should contain at least " + expectedMinimumSize + " aggregate functions"); } protected void assertKnownFunctionsPresent(Set knownFunctions) { - Map> registry = - getAggFunctionRegistry(); + Map registry = getAggFunctionRegistry(); long foundFunctions = registry.keySet().stream().filter(knownFunctions::contains).count(); assertTrue( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java index ef0c0599b57..b869f9d6030 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java @@ -120,12 +120,12 @@ public void testStrftimeWithExpressions() throws IOException { verifyDataRows(result2, rows(currentYear)); // Should return current year - // Test 3: Using unix_timestamp to convert string first + // Test 3: strftime with a different format JSONObject result3 = executeQuery( String.format( - "source=%s | eval ts = unix_timestamp('2018-03-19 13:55:03') | " - + "eval result = strftime(ts, '%s') | fields result | head 1", + "source=%s | eval result = strftime(1521467703, '%s') | " + + "fields result | head 1", TEST_INDEX_DATE, "%m/%d/%Y")); verifyDataRows(result3, rows("03/19/2018")); } @@ -148,12 +148,11 @@ public void testStrftimeStringHandling() throws IOException { TEST_INDEX_DATE, "%Y-%m-%d")); verifyDataRows(result1, rows("2018-03-19")); - // Test 3: For date strings, users must use unix_timestamp() first - // This is the recommended approach for converting date strings + // Test 3: strftime with a numeric variable from eval JSONObject result2 = executeQuery( String.format( - "source=%s | eval ts = unix_timestamp('2018-03-19 13:55:03') | " + "source=%s | eval ts = 1521467703 | " + "eval result = strftime(ts, '%s') | fields result | head 1", TEST_INDEX_DATE, "%Y-%m-%d")); verifyDataRows(result2, rows("2018-03-19")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteNoMvCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteNoMvCommandIT.java index 3ad50cdb4b0..9c9601daf72 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteNoMvCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteNoMvCommandIT.java @@ -225,7 +225,8 @@ public void testNoMvMissingFieldShouldReturn4xx() throws IOException { || msg.contains("field") || msg.contains("Field") || msg.contains("ARRAY_COMPACT") - || msg.contains("ARRAY"), + || msg.contains("ARRAY") + || msg.contains("inferred array element type"), msg); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index c17bd10e18a..4acf1581f45 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiRElWSURFIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json index bb62be7b990..30022247b90 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->GREATER_IP($0, IP('1.1.1.1':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, '1.1.1.1':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->GREATER_IP($0, '1.1.1.1':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json index 8ebbbbe885d..efe8ba75157 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP(IP('::ffff:1234':VARCHAR), $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->LTE_IP(IP('::ffff:1234':VARCHAR), $0), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"::ffff:1234\",\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP('::ffff:1234':VARCHAR, $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->LTE_IP('::ffff:1234':VARCHAR, $0), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"::ffff:1234\",\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_query_string_with_boolean_string_literal.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_query_string_with_boolean_string_literal.yaml new file mode 100644 index 00000000000..4718cdf4d26 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_query_string_with_boolean_string_literal.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1]) + LogicalFilter(condition=[=($12, 'TRUE':VARCHAR)]) + LogicalFilter(condition=[query_string(MAP('query', 'firstname:Amber':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[firstname, male], FILTER->AND(query_string(MAP('query', 'firstname:Amber':VARCHAR)), =($1, 'TRUE':VARCHAR)), PROJECT->[firstname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"query_string":{"query":"firstname:Amber","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},{"term":{"male":{"value":"TRUE","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml index d262a4af76d..957d5e8305b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) + LogicalFilter(condition=[OR(IS NULL($1), =(TRIM(FLAG(BOTH), ' ', $1), ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQFHHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiVFJJTSIsCiAgICAgICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogIkJPVEgiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiU1lNQk9MIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0],"DIGESTS":["firstname.keyword"," ","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =(TRIM(FLAG(BOTH), ' ', $1), '')), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDXnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlRSSU0iLAogICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6ICJCT1RIIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJTWU1CT0wiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,2],"DIGESTS":[" ","firstname.keyword",""]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml index df97332ae98..dca3555434f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[SEARCH($1, Sarg[''; NULL AS TRUE]:CHAR(0))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC13sKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["firstname.keyword","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->SEARCH($1, Sarg[''; NULL AS TRUE]:CHAR(0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"terms":{"firstname.keyword":[""],"boost":1.0}},{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml index 06215911134..313458ba42e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[OR(=($4, 'M'), SEARCH($1, Sarg[''; NULL AS TRUE]:CHAR(0)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =($4, 'M'), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEtnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIEVNUFRZIiwKICAgICAgICAia2luZCI6ICJPVEhFUiIsCiAgICAgICAgInN5bnRheCI6ICJQT1NURklYIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,0],"DIGESTS":["firstname.keyword","gender.keyword","M","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->OR(=($4, 'M'), SEARCH($1, Sarg[''; NULL AS TRUE]:CHAR(0))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"term":{"gender.keyword":{"value":"M","boost":1.0}}},{"bool":{"should":[{"terms":{"firstname.keyword":[""],"boost":1.0}},{"bool":{"must_not":[{"exists":{"field":"firstname","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml index c32f7d1a2d6..d271a07a5f9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject(balance2=[CEIL(DIVIDE($3, 10000.0:DECIMAL(6, 1)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDUnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDUnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDFXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiRElWSURFIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyNywKICAgICAgICAic2NhbGUiOiA3CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDFXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiRElWSURFIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyNywKICAgICAgICAic2NhbGUiOiA3CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",10000.0]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json index 25891f0e629..5c1195ac999 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_week_argument_coercion.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(w=[WEEK(TIMESTAMP('2024-12-10':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10':VARCHAR], expr#18=[TIMESTAMP($t17)], expr#19=[WEEK($t18)], w=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(w=[WEEK('2024-12-10':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10':VARCHAR], expr#18=[WEEK($t17)], w=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json index 9d963dd5747..36086533e76 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['1.1.1.1':VARCHAR], expr#13=[IP($t12)], expr#14=[GREATER_IP($t0, $t13)], host=[$t0], $condition=[$t14])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, '1.1.1.1':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['1.1.1.1':VARCHAR], expr#13=[GREATER_IP($t0, $t12)], host=[$t0], $condition=[$t13])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json index bf130cd5789..527bc3435a3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP(IP('::ffff:1234':VARCHAR), $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['::ffff:1234':VARCHAR], expr#13=[IP($t12)], expr#14=[LTE_IP($t13, $t0)], host=[$t0], $condition=[$t14])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP('::ffff:1234':VARCHAR, $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['::ffff:1234':VARCHAR], expr#13=[LTE_IP($t12, $t0)], host=[$t0], $condition=[$t13])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml index 887fd96408b..f0dbbc36d56 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) + LogicalFilter(condition=[OR(IS NULL($1), =(TRIM(FLAG(BOTH), ' ', $1), ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[FLAG(BOTH)], expr#19=[' '], expr#20=[TRIM($t18, $t19, $t1)], expr#21=[IS EMPTY($t20)], expr#22=[OR($t17, $t21)], proj#0..10=[{exprs}], $condition=[$t22]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[FLAG(BOTH)], expr#19=[' '], expr#20=[TRIM($t18, $t19, $t1)], expr#21=[''], expr#22=[=($t20, $t21)], expr#23=[OR($t17, $t22)], proj#0..10=[{exprs}], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml index 6115f98e23f..45e93bd3771 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[SEARCH($1, Sarg[''; NULL AS TRUE]:CHAR(0))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[IS EMPTY($t1)], expr#19=[OR($t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[''; NULL AS TRUE]:CHAR(0)], expr#18=[SEARCH($t1, $t17)], proj#0..10=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml index 7f43f48dc57..d2ad8d60037 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) + LogicalFilter(condition=[OR(=($4, 'M'), SEARCH($1, Sarg[''; NULL AS TRUE]:CHAR(0)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=['M'], expr#19=[=($t4, $t18)], expr#20=[IS EMPTY($t1)], expr#21=[OR($t17, $t19, $t20)], proj#0..10=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['M'], expr#18=[=($t4, $t17)], expr#19=[Sarg[''; NULL AS TRUE]:CHAR(0)], expr#20=[SEARCH($t1, $t19)], expr#21=[OR($t18, $t20)], proj#0..10=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json index c475102597f..6b32b1989d9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_week_argument_coercion.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(w=[WEEK(TIMESTAMP('2024-12-10':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10':VARCHAR], expr#18=[TIMESTAMP($t17)], expr#19=[WEEK($t18)], w=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(w=[WEEK('2024-12-10':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['2024-12-10':VARCHAR], expr#18=[WEEK($t17)], w=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json index 7afd6497b9a..e472cb6e168 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ip.json @@ -8,10 +8,10 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" }, "children": [] } ] } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json index ff004cfeb4c..efe8ba75157 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json @@ -1,17 +1,6 @@ { - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[host]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IANG9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMJDI9znTUIQE9bAIABUwADXZhbCRhcmd1bWVudHNxAH4AAUwADHZhbCRmdW5jdGlvbnQAP0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0wAEHZhbCRmdW5jdGlvbk5hbWVxAH4AA0wAFnZhbCRmdW5jdGlvblByb3BlcnRpZXNxAH4ABEwADnZhbCRyZXR1cm5UeXBlcQB+AAV4cQB+AAZzcQB+AAgAAAABdwQAAAABc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJTdHJpbmdWYWx1ZQBBMiVziQ4TAgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL1N0cmluZzt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAALOjpmZmZmOjEyMzR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+ABJ4cHQACmNhc3RfdG9faXBxAH4ADXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzdAATW0xqYXZhL2xhbmcvT2JqZWN0O0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AEkwAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+ABJMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+ABJMAAlpbXBsQ2xhc3NxAH4AEkwADmltcGxNZXRob2ROYW1lcQB+ABJMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+ABJMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+ABJ4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAZAAAABnVxAH4AHQAAAAFzcQB+ABkAAAAGdXEAfgAdAAAAAHZyAEBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5jb252ZXJ0LlR5cGVDYXN0T3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAO29yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9udAAFYXBwbHl0ACYoTGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QAQG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL2NvbnZlcnQvVHlwZUNhc3RPcGVyYXRvcnN0ABpsYW1iZGEkY2FzdFRvSXAkMTJjN2RjNDgkMXQAVChMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAqdnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAlcQB+ACZxAH4AJ3QAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckODc4MDY5YzgkMXQAkShMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AKnEAfgAsdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnEAfgAmdAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ALXQAFmxhbWJkYSRpbXBsJDhkNTg2Y2RjJDF0AMwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0AI8oTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAXc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AOncNAgAAAABoifJmKvyT4Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAA1BQTH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHEAfgA+dAACSVBzcgAxb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uUmVmZXJlbmNlRXhwcmVzc2lvbqtE71wSB4XWAgAETAAEYXR0cnEAfgASTAAFcGF0aHNxAH4AAUwAB3Jhd1BhdGhxAH4AEkwABHR5cGVxAH4ABXhwdAAEaG9zdHNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXEAfgAaeHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AEZxAH4ARnEAfgBCeHNxAH4AFnQAAjw9cQB+AAlzcQB+ABkAAAAGdXEAfgAdAAAAAXNxAH4AGQAAAAZ1cQB+AB0AAAABc3EAfgAZAAAABnVxAH4AHQAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHBxAH4AMHEAfgAmcQB+ADF0AElvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9wcmVkaWNhdGUvQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzdAAVbGFtYmRhJGx0ZSQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AV3EAfgAscQB+ADBxAH4AJnEAfgAxcQB+AC10ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AV3EAfgAsdAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AJnQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+AC10ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AS3EAfgA5fnEAfgBBdAAHQk9PTEVBTg==\\\"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP('::ffff:1234':VARCHAR, $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->LTE_IP('::ffff:1234':VARCHAR, $0), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"::ffff:1234\",\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml index 766e6eb5f22..05d51fcd274 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml @@ -11,6 +11,6 @@ root: :{\"wildcard\":{\"firstname.keyword\":{\"wildcard\":\"*mbe*\",\"boost\"\ :1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\"\ ,\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\"\ - ,\"lastname\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + ,\"lastname\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - children: [] \ No newline at end of file + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml index ff157e036d0..190da3d1488 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml @@ -13,4 +13,4 @@ root: firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"\ state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - children: [] \ No newline at end of file + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json index bc88f5eedfa..fb098898264 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_week_argument_coercion.json @@ -16,7 +16,7 @@ { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" }, "children": [] } @@ -24,4 +24,4 @@ } ] } -} \ No newline at end of file +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java index 83b1915f6b5..bede63bb30f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java @@ -16,15 +16,21 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.jspecify.annotations.NonNull; import org.opensearch.geospatial.action.IpEnrichmentActionClient; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.transport.client.node.NodeClient; @@ -57,11 +63,33 @@ public SqlReturnTypeInference getReturnTypeInference() { } @Override - public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT( - List.of( - List.of(ExprCoreType.STRING, ExprCoreType.IP), - List.of(ExprCoreType.STRING, ExprCoreType.IP, ExprCoreType.STRING))); + public @NonNull UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap( + new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!getOperandCountRange().isValidCount(callBinding.getOperandCount())) { + return false; + } + boolean valid = + OpenSearchTypeUtil.isCharacter(callBinding.getOperandType(0)) + && OpenSearchTypeUtil.isIp(callBinding.getOperandType(1), true); + if (callBinding.getOperandCount() == 3) { + valid = valid && OpenSearchTypeUtil.isCharacter(callBinding.getOperandType(2)); + } + return valid; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.between(2, 3); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return "GEOIP(, ), GEOIP(, , )"; + } + }); } public static class GeoIPImplementor implements NotNullImplementor { @@ -101,10 +129,15 @@ public Expression implement( ExprIpValue ipAddress, String commaSeparatedOptions, NodeClient nodeClient) { + return fetchIpEnrichment(dataSource, ipAddress.toString(), commaSeparatedOptions, nodeClient); + } + + public static Map fetchIpEnrichment( + String dataSource, String ipAddress, String commaSeparatedOptions, NodeClient nodeClient) { String unquotedOptions = StringUtils.unquoteText(commaSeparatedOptions); final Set options = Arrays.stream(unquotedOptions.split(",")).map(String::trim).collect(Collectors.toSet()); - return fetchIpEnrichment(dataSource, ipAddress.toString(), options, nodeClient); + return fetchIpEnrichment(dataSource, ipAddress, options, nodeClient); } private static Map fetchIpEnrichment( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java index bd9f17abd2a..a860211607e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java @@ -24,8 +24,8 @@ import org.apache.calcite.tools.RelBuilder; import org.immutables.value.Value; import org.opensearch.sql.calcite.plan.rule.OpenSearchRuleConfig; +import org.opensearch.sql.calcite.utils.OpenSearchTypeUtil; import org.opensearch.sql.calcite.utils.PlanUtils; -import org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction; import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; @@ -243,8 +243,7 @@ static boolean containsWidthBucketFuncOnDate(LogicalProject project) { expr -> expr instanceof RexCall rexCall && rexCall.getOperator().equals(WIDTH_BUCKET) - && WidthBucketFunction.dateRelatedType( - rexCall.getOperands().getFirst().getType())); + && OpenSearchTypeUtil.isDatetime(rexCall.getOperands().getFirst().getType())); } } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 5c31a36cfc8..f758c666f9a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -834,7 +834,8 @@ void isNullOr_ScriptPushDown() throws ExpressionNotAnalyzableException { .add("a", builder.getTypeFactory().createSqlType(SqlTypeName.BIGINT)) .add("b", builder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)) .build(); - // PPL IS_EMPTY is translated to OR(IS_NULL(arg), IS_EMPTY(arg)) + // PPL IS_EMPTY is translated to OR(IS_NULL(arg), EQUALS("")) (not IS_EMPTY because IS_EMPTY is + // only for collections) RexNode call = PPLFuncImpTable.INSTANCE.resolve(builder, BuiltinFunctionName.IS_EMPTY, field2); Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); QueryExpression expression = @@ -842,7 +843,33 @@ void isNullOr_ScriptPushDown() throws ExpressionNotAnalyzableException { assert (expression .builder() .toString() - .contains("\"lang\" : \"opensearch_compounded_script\"")); + .contains( + """ + "should" : [ + { + "bool" : { + "must_not" : [ + { + "exists" : { + "field" : "b", + "boost" : 1.0 + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + { + "term" : { + "b.keyword" : { + "value" : "", + "boost" : 1.0 + } + } + } + ]\ + """)); } @Test diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java index b363be9bee1..ef3bbf70e96 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl.calcite; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; @@ -16,166 +17,273 @@ public CalcitePPLAggregateFunctionTypeTest() { @Test public void testAvgWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats avg(HIREDATE) as avg_name", - "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + String ppl = "source=EMP | stats avg(HIREDATE) as avg_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], avg_name=[AVG($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "avg_name=1982-05-02\n"); + verifyPPLToSparkSQL(root, "SELECT AVG(`HIREDATE`) `avg_name`\nFROM `scott`.`EMP`"); } @Test public void testVarsampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats var_samp(HIREDATE) as varsamp_name", - "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + String ppl = "source=EMP | stats var_samp(HIREDATE) as varsamp_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], varsamp_name=[VAR_SAMP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "varsamp_name=8277-10-13\n"); + verifyPPLToSparkSQL(root, "SELECT VAR_SAMP(`HIREDATE`) `varsamp_name`\nFROM `scott`.`EMP`"); } @Test public void testVarpopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats var_pop(HIREDATE) as varpop_name", - "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + String ppl = "source=EMP | stats var_pop(HIREDATE) as varpop_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], varpop_name=[VAR_POP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "varpop_name=3541-07-07\n"); + verifyPPLToSparkSQL(root, "SELECT VAR_POP(`HIREDATE`) `varpop_name`\nFROM `scott`.`EMP`"); } @Test public void testStddevSampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats stddev_samp(HIREDATE) as stddev_name", - "Aggregation function STDDEV_SAMP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + String ppl = "source=EMP | stats stddev_samp(HIREDATE) as stddev_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_name=[STDDEV_SAMP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "stddev_name=1983-06-23\n"); + verifyPPLToSparkSQL(root, "SELECT STDDEV_SAMP(`HIREDATE`) `stddev_name`\nFROM `scott`.`EMP`"); } @Test public void testStddevPopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | stats stddev_pop(HIREDATE) as stddev_name", - "Aggregation function STDDEV_POP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + String ppl = "source=EMP | stats stddev_pop(HIREDATE) as stddev_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_name=[STDDEV_POP($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "stddev_name=1982-12-26\n"); + verifyPPLToSparkSQL(root, "SELECT STDDEV_POP(`HIREDATE`) `stddev_name`\nFROM `scott`.`EMP`"); } @Test public void testPercentileApproxWithWrongArgType() { // First argument should be numeric - verifyQueryThrowsException( - "source=EMP | stats percentile_approx(HIREDATE, 50) as percentile", - "Aggregation function PERCENTILE_APPROX expects field type and additional arguments" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [DATE,INTEGER]"); + // This test verifies logical plan generation, but execution fails with ClassCastException + String ppl = "source=EMP | stats percentile_approx(HIREDATE, 50) as percentile"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], percentile=[percentile_approx($0, $1, $2)])\n" + + " LogicalProject(HIREDATE=[$4], $f1=[50], $f2=[FLAG(DATE)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: ClassCastException: Double cannot be cast to Integer } @Test public void testListFunctionWithArrayArgType() { // Test LIST function with array expression (which is not a supported scalar type) - verifyQueryThrowsException( - "source=EMP | stats list(array(ENAME, JOB)) as name_list", - "Aggregation function LIST expects field type" - + " {[BYTE]|[SHORT]|[INTEGER]|[LONG]|[FLOAT]|[DOUBLE]|[STRING]|[BOOLEAN]|[DATE]|[TIME]|[TIMESTAMP]|[IP]|[BINARY]}," - + " but got [ARRAY]"); + String ppl = "source=EMP | stats list(array(ENAME, JOB)) as name_list"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], name_list=[LIST($0)])\n" + + " LogicalProject($f2=[array($1, $2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "name_list=[[SMITH, CLERK], [ALLEN, SALESMAN], [WARD, SALESMAN], [JONES, MANAGER], [MARTIN," + + " SALESMAN], [BLAKE, MANAGER], [CLARK, MANAGER], [SCOTT, ANALYST], [KING, PRESIDENT]," + + " [TURNER, SALESMAN], [ADAMS, CLERK], [JAMES, CLERK], [FORD, ANALYST], [MILLER," + + " CLERK]]\n"); + verifyPPLToSparkSQL( + root, "SELECT `LIST`(ARRAY(`ENAME`, `JOB`)) `name_list`\nFROM `scott`.`EMP`"); } @Test public void testCountWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats count(EMPNO, DEPTNO)", - "Aggregation function COUNT expects field type and additional arguments {[]|[ANY]}," - + " but got [SHORT,BYTE]"); + // COUNT with extra parameters throws IllegalArgumentException + Exception e = + org.junit.Assert.assertThrows( + IllegalArgumentException.class, + () -> getRelNode("source=EMP | stats count(EMPNO, DEPTNO)")); + verifyErrorMessageContains(e, "Field [DEPTNO] not found"); } @Test public void testAvgWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats avg(EMPNO, DEPTNO)", - "Aggregation function AVG expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + // AVG with extra parameters throws IllegalArgumentException + Exception e = + org.junit.Assert.assertThrows( + IllegalArgumentException.class, + () -> getRelNode("source=EMP | stats avg(EMPNO, DEPTNO)")); + verifyErrorMessageContains(e, "Field [DEPTNO] not found"); } @Test public void testSumWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats sum(EMPNO, DEPTNO)", - "Aggregation function SUM expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + String ppl = "source=EMP | stats sum(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], sum(EMPNO, DEPTNO)=[SUM($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "sum(EMPNO, DEPTNO)=108172\n"); + verifyPPLToSparkSQL( + root, "SELECT SUM(`EMPNO`, `DEPTNO`) `sum(EMPNO, DEPTNO)`\nFROM `scott`.`EMP`"); } @Test public void testMinWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats min(EMPNO, DEPTNO)", - "Aggregation function MIN expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + String ppl = "source=EMP | stats min(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], min(EMPNO, DEPTNO)=[MIN($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "min(EMPNO, DEPTNO)=7369\n"); + verifyPPLToSparkSQL( + root, "SELECT MIN(`EMPNO`, `DEPTNO`) `min(EMPNO, DEPTNO)`\nFROM `scott`.`EMP`"); } @Test public void testMaxWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats max(EMPNO, DEPTNO)", - "Aggregation function MAX expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + String ppl = "source=EMP | stats max(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], max(EMPNO, DEPTNO)=[MAX($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "max(EMPNO, DEPTNO)=7934\n"); + verifyPPLToSparkSQL( + root, "SELECT MAX(`EMPNO`, `DEPTNO`) `max(EMPNO, DEPTNO)`\nFROM `scott`.`EMP`"); } @Test public void testVarSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats var_samp(EMPNO, DEPTNO)", - "Aggregation function VARSAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats var_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], var_samp(EMPNO, DEPTNO)=[VAR_SAMP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testVarPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats var_pop(EMPNO, DEPTNO)", - "Aggregation function VARPOP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats var_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], var_pop(EMPNO, DEPTNO)=[VAR_POP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testStddevSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats stddev_samp(EMPNO, DEPTNO)", - "Aggregation function STDDEV_SAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats stddev_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_samp(EMPNO, DEPTNO)=[STDDEV_SAMP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testStddevPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats stddev_pop(EMPNO, DEPTNO)", - "Aggregation function STDDEV_POP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with AssertionError + String ppl = "source=EMP | stats stddev_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], stddev_pop(EMPNO, DEPTNO)=[STDDEV_POP($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: AssertionError [0, 1] in AggregateReduceFunctionsRule } @Test public void testPercentileWithMissingParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats percentile(EMPNO)", - "Aggregation function PERCENTILE_APPROX expects field type" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [SHORT]"); + // This test verifies logical plan generation, but execution fails with ClassCastException + String ppl = "source=EMP | stats percentile(EMPNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], percentile(EMPNO)=[percentile_approx($0, $1)])\n" + + " LogicalProject(EMPNO=[$0], $f1=[FLAG(SMALLINT)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: ClassCastException: SqlTypeName cannot be cast to Number } @Test public void testPercentileWithInvalidParameterTypesThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats percentile(EMPNO, 50, HIREDATE)", - "Aggregation function PERCENTILE_APPROX expects field type and additional arguments" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [SHORT,INTEGER,DATE]"); + // This test verifies logical plan generation, but execution fails with ClassCastException + String ppl = "source=EMP | stats percentile(EMPNO, 50, HIREDATE)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], percentile(EMPNO, 50, HIREDATE)=[percentile_approx($0, $2," + + " $1, $3)])\n" + + " LogicalProject(EMPNO=[$0], HIREDATE=[$4], $f2=[50], $f3=[FLAG(SMALLINT)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + // Execution fails with: ClassCastException: Double cannot be cast to Short } @Test public void testEarliestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats earliest(ENAME, HIREDATE, JOB)", - "Aggregation function EARLIEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + String ppl = "source=EMP | stats earliest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], earliest(ENAME, HIREDATE, JOB)=[ARG_MIN($0, $1, $2)])\n" + + " LogicalProject(ENAME=[$1], HIREDATE=[$4], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "earliest(ENAME, HIREDATE, JOB)=SMITH\n"); + verifyPPLToSparkSQL( + root, + "SELECT MIN_BY(`ENAME`, `HIREDATE`, `JOB`) `earliest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testLatestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | stats latest(ENAME, HIREDATE, JOB)", - "Aggregation function LATEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + String ppl = "source=EMP | stats latest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], latest(ENAME, HIREDATE, JOB)=[ARG_MAX($0, $1, $2)])\n" + + " LogicalProject(ENAME=[$1], HIREDATE=[$4], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult(root, "latest(ENAME, HIREDATE, JOB)=ADAMS\n"); + verifyPPLToSparkSQL( + root, + "SELECT MAX_BY(`ENAME`, `HIREDATE`, `JOB`) `latest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java index 24bd9ac18d0..2444a6c4ba9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl.calcite; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; @@ -16,128 +17,507 @@ public CalcitePPLEventstatsTypeTest() { @Test public void testCountWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats count(EMPNO, DEPTNO)", - "Aggregation function COUNT expects field type and additional arguments {[]|[ANY]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + // Note: COUNT ignores the second parameter + String ppl = "source=EMP | eventstats count(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], count(EMPNO, DEPTNO)=[COUNT($0) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; count(EMPNO, DEPTNO)=14\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; count(EMPNO, DEPTNO)=14\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, COUNT(`EMPNO`)" + + " OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) `count(EMPNO," + + " DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testAvgWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats avg(EMPNO, DEPTNO)", - "Aggregation function AVG expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + // Note: AVG ignores the second parameter + String ppl = "source=EMP | eventstats avg(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], avg(EMPNO, DEPTNO)=[/(SUM($0) OVER (), CAST(COUNT($0) OVER" + + " ()):DOUBLE NOT NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; avg(EMPNO, DEPTNO)=7726.571428571428\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; avg(EMPNO, DEPTNO)=7726.571428571428\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, (SUM(`EMPNO`)" + + " OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) /" + + " CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE) `avg(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testSumWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats sum(EMPNO, DEPTNO)", - "Aggregation function SUM expects field type and additional arguments {[INTEGER]|[DOUBLE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats sum(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], sum(EMPNO, DEPTNO)=[SUM($0, $7) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; sum(EMPNO, DEPTNO)=108172\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; sum(EMPNO, DEPTNO)=108172\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, SUM(`EMPNO`," + + " `DEPTNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + " `sum(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testMinWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats min(EMPNO, DEPTNO)", - "Aggregation function MIN expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats min(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], min(EMPNO, DEPTNO)=[MIN($0, $7) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; min(EMPNO, DEPTNO)=7369\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; min(EMPNO, DEPTNO)=7369\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MIN(`EMPNO`," + + " `DEPTNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + " `min(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testMaxWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats max(EMPNO, DEPTNO)", - "Aggregation function MAX expects field type and additional arguments {[COMPARABLE_TYPE]}," - + " but got [SHORT,BYTE]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats max(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(EMPNO, DEPTNO)=[MAX($0, $7) OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; max(EMPNO, DEPTNO)=7934\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; max(EMPNO, DEPTNO)=7934\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`EMPNO`," + + " `DEPTNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + " `max(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testVarSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats var_samp(EMPNO, DEPTNO)", - "Aggregation function VARSAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats var_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], var_samp(EMPNO, DEPTNO)=[/(-(SUM(*($0, $0)) OVER ()," + + " /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT NULL))," + + " -(CAST(COUNT($0) OVER ()):DOUBLE NOT NULL, 1))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, ((SUM(`EMPNO`" + + " * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) *" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) /" + + " CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE)) / (CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) - 1) `var_samp(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testVarPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats var_pop(EMPNO, DEPTNO)", - "Aggregation function VARPOP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats var_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], var_pop(EMPNO, DEPTNO)=[/(-(SUM(*($0, $0)) OVER ()," + + " /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT NULL))," + + " CAST(COUNT($0) OVER ()):DOUBLE NOT NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, ((SUM(`EMPNO`" + + " * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) *" + + " (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) /" + + " CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) `var_pop(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testStddevSampWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_samp(EMPNO, DEPTNO)", - "Aggregation function STDDEV_SAMP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats stddev_samp(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], stddev_samp(EMPNO, DEPTNO)=[POWER(/(-(SUM(*($0, $0)) OVER" + + " (), /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT" + + " NULL)), -(CAST(COUNT($0) OVER ()):DOUBLE NOT NULL, 1)), 0.5E0:DOUBLE)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " POWER(((SUM(`EMPNO` * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) - (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) * (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING" + + " AND UNBOUNDED FOLLOWING) AS DOUBLE)) / (CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN" + + " UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE) - 1), 5E-1)" + + " `stddev_samp(EMPNO, DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testStddevPopWithExtraParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_pop(EMPNO, DEPTNO)", - "Aggregation function STDDEV_POP expects field type and additional arguments" - + " {[INTEGER]|[DOUBLE]}, but got [SHORT,BYTE]"); + // This test verifies logical plan generation, but execution fails with ArithmeticException + String ppl = "source=EMP | eventstats stddev_pop(EMPNO, DEPTNO)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], stddev_pop(EMPNO, DEPTNO)=[POWER(/(-(SUM(*($0, $0)) OVER" + + " (), /(*(SUM($0) OVER (), SUM($0) OVER ()), CAST(COUNT($0) OVER ()):DOUBLE NOT" + + " NULL)), CAST(COUNT($0) OVER ()):DOUBLE NOT NULL), 0.5E0:DOUBLE)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " POWER(((SUM(`EMPNO` * `EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) - (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) * (SUM(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING" + + " AND UNBOUNDED FOLLOWING) AS DOUBLE)) / CAST(COUNT(`EMPNO`) OVER (RANGE BETWEEN" + + " UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 5E-1) `stddev_pop(EMPNO," + + " DEPTNO)`\n" + + "FROM `scott`.`EMP`"); + // Execution fails with: ArithmeticException: Value out of range + Exception e = + org.junit.Assert.assertThrows(RuntimeException.class, () -> verifyResultCount(root, 0)); + verifyErrorMessageContains(e, "out of range"); } @Test public void testEarliestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats earliest(ENAME, HIREDATE, JOB)", - "Aggregation function EARLIEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats earliest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], earliest(ENAME, HIREDATE, JOB)=[ARG_MIN($1, $4, $2) OVER" + + " ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; earliest(ENAME, HIREDATE, JOB)=SMITH\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; earliest(ENAME, HIREDATE, JOB)=SMITH\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " MIN_BY(`ENAME`, `HIREDATE`, `JOB`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING) `earliest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testLatestWithTooManyParametersThrowsException() { - verifyQueryThrowsException( - "source=EMP | eventstats latest(ENAME, HIREDATE, JOB)", - "Aggregation function LATEST expects field type and additional arguments" - + " {[ANY]|[ANY,ANY]}, but got" - + " [STRING,DATE,STRING]"); + // This test verifies logical plan generation for window functions + String ppl = "source=EMP | eventstats latest(ENAME, HIREDATE, JOB)"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], latest(ENAME, HIREDATE, JOB)=[ARG_MAX($1, $4, $2) OVER" + + " ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + verifyResult( + root, + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20; latest(ENAME, HIREDATE, JOB)=ADAMS\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10; latest(ENAME, HIREDATE, JOB)=ADAMS\n"); + verifyPPLToSparkSQL( + root, + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " MAX_BY(`ENAME`, `HIREDATE`, `JOB`) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND" + + " UNBOUNDED FOLLOWING) `latest(ENAME, HIREDATE, JOB)`\n" + + "FROM `scott`.`EMP`"); } @Test public void testAvgWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats avg(HIREDATE) as avg_name", - "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // AVG with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats avg(HIREDATE) as avg_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testVarsampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats var_samp(HIREDATE) as varsamp_name", - "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // VAR_SAMP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats var_samp(HIREDATE) as varsamp_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testVarpopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats var_pop(HIREDATE) as varpop_name", - "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // VAR_POP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats var_pop(HIREDATE) as varpop_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testStddevSampWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_samp(HIREDATE) as stddev_name", - "Aggregation function STDDEV_SAMP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + // STDDEV_SAMP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats stddev_samp(HIREDATE) as stddev_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } @Test public void testStddevPopWithWrongArgType() { - verifyQueryThrowsException( - "source=EMP | eventstats stddev_pop(HIREDATE) as stddev_name", - "Aggregation function STDDEV_POP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [DATE]"); + // STDDEV_POP with DATE argument throws CalciteContextException during plan generation + String ppl = "source=EMP | eventstats stddev_pop(HIREDATE) as stddev_name"; + Exception e = + org.junit.Assert.assertThrows( + org.apache.calcite.runtime.CalciteContextException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Cannot infer return type for /; operand types: [DATE, DOUBLE]"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java index 4383acf40e0..c0f2f593136 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java @@ -6,13 +6,11 @@ package org.opensearch.sql.ppl.calcite; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Assert; import org.junit.Test; -import org.opensearch.sql.exception.ExpressionEvaluationException; public class CalcitePPLFunctionTypeTest extends CalcitePPLAbstractTest { @@ -22,23 +20,23 @@ public CalcitePPLFunctionTypeTest() { @Test public void testLowerWithIntegerType() { - verifyQueryThrowsException( - "source=EMP | eval lower_name = lower(EMPNO) | fields lower_name", - "LOWER function expects {[STRING]}, but got [SHORT]"); + // Lower with IntegerType no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval lower_name = lower(EMPNO) | fields lower_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(lower_name=[LOWER($0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testTimeDiffWithUdtInputType() { - String strPpl = - "source=EMP | eval time_diff = timediff('12:00:00', '12:00:06') | fields time_diff"; - String timePpl = - "source=EMP | eval time_diff = timediff(time('13:00:00'), time('12:00:06')) | fields" - + " time_diff"; - getRelNode(strPpl); - getRelNode(timePpl); - verifyQueryThrowsException( - "source=EMP | eval time_diff = timediff(12, '2009-12-10') | fields time_diff", - "TIMEDIFF function expects {[TIME,TIME]}, but got [INTEGER,STRING]"); + // TimeDiff with UdtInputType no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval time_diff = timediff(12, '2009-12-10') | fields time_diff"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(time_diff=[TIME_DIFF(12, '2009-12-10':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -49,6 +47,31 @@ public void testComparisonWithDifferentType() { getRelNode("source=EMP | where ENAME < 6 | fields ENAME"); } + /** + * Test that safe numeric widening (e.g., SMALLINT → INTEGER) uses regular CAST instead of + * SAFE_CAST. This avoids generating scripts in OpenSearch queries, improving performance. + * + *

EMPNO is SMALLINT in the SCOTT schema, and literal 6 is INTEGER. The comparison requires + * widening EMPNO to INTEGER. Since SMALLINT → INTEGER is a safe widening (no data loss), regular + * CAST is used instead of SAFE_CAST. + * + *

With regular CAST, Calcite can optimize it away entirely because numeric comparisons between + * compatible integer types are semantically equivalent. With SAFE_CAST, the cast would be + * preserved because SAFE_CAST has different semantics (returns NULL on failure). + */ + @Test + public void testSafeNumericWideningUsesCastInsteadOfSafeCast() { + // EMPNO is SMALLINT, 6 is INTEGER + // Safe widening SMALLINT → INTEGER uses CAST, which Calcite optimizes away + // The result is a direct comparison >($0, 6) without any cast + RelNode root = getRelNode("source=EMP | where EMPNO > 6 | fields ENAME"); + verifyLogical( + root, + "LogicalProject(ENAME=[$1])\n" + + " LogicalFilter(condition=[>($0, 6)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); + } + @Test public void testCoalesceWithSameType() { String ppl = "source=EMP | eval coalesce_name = coalesce(ENAME, 'Jack') | fields coalesce_name"; @@ -65,116 +88,148 @@ public void testCoalesceWithDifferentType() { @Test public void testSubstringWithWrongType() { - getRelNode("source=EMP | eval sub_name = substring(ENAME, 1, 3) | fields sub_name"); - getRelNode("source=EMP | eval sub_name = substring(ENAME, 1) | fields sub_name"); - verifyQueryThrowsException( - "source=EMP | eval sub_name = substring(ENAME, 1, '3') | fields sub_name", - "SUBSTRING function expects {[STRING,INTEGER]|[STRING,INTEGER,INTEGER]}, but got" - + " [STRING,INTEGER,STRING]"); + // Substring with wrong type no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval sub_name = substring(ENAME, 1, '3') | fields sub_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(sub_name=[SUBSTRING($1, 1, '3')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testIfWithWrongType() { - getRelNode("source=EMP | eval if_name = if(EMPNO > 6, 'Jack', ENAME) | fields if_name"); - getRelNode("source=EMP | eval if_name = if(EMPNO > 6, EMPNO, DEPTNO) | fields if_name"); - verifyQueryThrowsException( - "source=EMP | eval if_name = if(EMPNO, 1, DEPTNO) | fields if_name", - "IF function expects {[BOOLEAN,ANY,ANY]}, but got [SHORT,INTEGER,BYTE]"); - verifyQueryThrowsException( - "source=EMP | eval if_name = if(EMPNO > 6, 'Jack', 1) | fields if_name", - "Cannot resolve function: IF, arguments: [BOOLEAN,STRING,INTEGER], caused by: Can't find" - + " leastRestrictive type for [VARCHAR, INTEGER]"); + // If with wrong type no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval if_name = if(EMPNO, 1, DEPTNO) | fields if_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(if_name=[CASE($0, 1, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testTimestampWithWrongArg() { - verifyQueryThrowsException( + // Timestamp with wrong argument no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval timestamp = timestamp('2020-08-26 13:49:00', 2009) | fields timestamp |" - + " head 1", - "TIMESTAMP function expects" - + " {[STRING]|[TIMESTAMP]|[DATE]|[TIME]|[STRING,STRING]|[TIMESTAMP,TIMESTAMP]|[TIMESTAMP,DATE]|[TIMESTAMP,TIME]|[DATE,TIMESTAMP]|[DATE,DATE]|[DATE,TIME]|[TIME,TIMESTAMP]|[TIME,DATE]|[TIME,TIME]|[STRING,TIMESTAMP]|[STRING,DATE]|[STRING,TIME]|[TIMESTAMP,STRING]|[DATE,STRING]|[TIME,STRING]}," - + " but got [STRING,INTEGER]"); + + " head 1"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalSort(fetch=[1])\n" + + " LogicalProject(timestamp=[TIMESTAMP('2020-08-26 13:49:00':VARCHAR, 2009)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testCurDateWithArg() { - verifyQueryThrowsException( - "source=EMP | eval curdate = CURDATE(1) | fields curdate | head 1", - "CURDATE function expects {[]}, but got [INTEGER]"); + // CurDate with Arg no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval curdate = CURDATE(1) | fields curdate | head 1"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalSort(fetch=[1])\n" + + " LogicalProject(curdate=[CURRENT_DATE(1)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } + // Test directly registered UDF: register(funcname, FuncImp) // Test directly registered UDF: register(funcname, FuncImp) @Test public void testLtrimWrongArg() { - verifyQueryThrowsException( - "source=EMP | where ltrim(EMPNO, DEPTNO) = 'Jim' | fields name, age", - "LTRIM function expects {[STRING]}, but got [SHORT,BYTE]"); + String ppl = "source=EMP | where ltrim(EMPNO, DEPTNO) = 'Jim' | fields name, age"; + Exception e = Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "This function requires exactly 1 arguments"); } + // Test udf registered via sql library operator: registerOperator(REVERSE, + // SqlLibraryOperators.REVERSE); // Test udf registered via sql library operator: registerOperator(REVERSE, // SqlLibraryOperators.REVERSE); @Test public void testReverseWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | where reverse(EMPNO) = '3202' | fields year", - "REVERSE function expects {[STRING]}, but got [SHORT]"); + String ppl = "source=EMP | where reverse(EMPNO) = '3202' | fields year"; + Throwable e = Assert.assertThrows(AssertionError.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Was not expecting value 'SMALLINT'"); } + // test type checking on UDF with direct registration: register(funcname, FuncImp) // test type checking on UDF with direct registration: register(funcname, FuncImp) @Test public void testStrCmpWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | where strcmp(10, 'Jane') = 0 | fields name, age", - "STRCMP function expects {[STRING,STRING]}, but got [INTEGER,STRING]"); + String ppl = "source=EMP | where strcmp(10, 'Jane') = 0 | fields name, age"; + Exception e = Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Field [name] not found"); } + // Test registered Sql Std Operator: registerOperator(funcName, SqlStdOperatorTable.OPERATOR) // Test registered Sql Std Operator: registerOperator(funcName, SqlStdOperatorTable.OPERATOR) @Test public void testLowerWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | where lower(EMPNO) = 'hello' | fields name, age", - "LOWER function expects {[STRING]}, but got [SHORT]"); + String ppl = "source=EMP | where lower(EMPNO) = 'hello' | fields name, age"; + Exception e = Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains(e, "Field [name] not found"); } @Test public void testSha2WrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | head 1 | eval sha256 = SHA2('hello', '256') | fields sha256", - "SHA2 function expects {[STRING,INTEGER]}, but got [STRING,STRING]"); + // Sha2WrongArg should throw no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | head 1 | eval sha256 = SHA2('hello', '256') | fields sha256"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(sha256=[SHA2('hello':VARCHAR, '256':VARCHAR)])\n" + + " LogicalSort(fetch=[1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test type checking on udf with direct registration: register(SQRT, funcImp) @Test public void testSqrtWithWrongArg() { - verifyQueryThrowsException( - "source=EMP | head 1 | eval sqrt_name = sqrt(HIREDATE) | fields sqrt_name", - "SQRT function expects {[INTEGER]|[DOUBLE]}, but got [DATE]"); + // Sqrt with wrong argument no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | head 1 | eval sqrt_name = sqrt(HIREDATE) | fields sqrt_name"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(sqrt_name=[POWER($4, 0.5E0:DOUBLE)])\n" + + " LogicalSort(fetch=[1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test UDF registered with PPL builtin operators: registerOperator(MOD, PPLBuiltinOperators.MOD); @Test public void testModWithWrongArg() { - verifyQueryThrowsException( - "source=EMP | eval z = mod(0.5, 1, 2) | fields z", - "MOD function expects" - + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]}, but got" - + " [DOUBLE,INTEGER,INTEGER]"); + // Mod with wrong argument no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval z = mod(0.5, 1, 2) | fields z"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(z=[MOD(0.5:DECIMAL(2, 1), 1, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test UDF registered with sql std operators: registerOperator(PI, SqlStdOperatorTable.PI) @Test public void testPiWithArg() { - verifyQueryThrowsException( - "source=EMP | eval pi = pi(1) | fields pi", "PI function expects {[]}, but got [INTEGER]"); + // Pi with Arg no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval pi = pi(1) | fields pi"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, "LogicalProject(pi=[PI(1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test UDF registered with sql library operators: registerOperator(LOG2, // SqlLibraryOperators.LOG2) @Test public void testLog2WithWrongArgShouldThrow() { - verifyQueryThrowsException( - "source=EMP | eval log2 = log2(ENAME, JOB) | fields log2", - "LOG2 function expects {[INTEGER]|[DOUBLE]}, but got [STRING,STRING]"); + // Log2 with string args: string args are coerced to DOUBLE via SAFE_CAST + String ppl = "source=EMP | eval log2 = log2(ENAME, JOB) | fields log2"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(log2=[LOG2(SAFE_CAST($1), SAFE_CAST($2))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -198,13 +253,13 @@ public void testStrftimeWithCorrectTypes() { @Test public void testStrftimeWithWrongFirstArgType() { - // First argument should be numeric/timestamp, not boolean + // First argument should be numeric/timestamp, but Calcite handles type coercion String ppl = "source=EMP | eval formatted = strftime(EMPNO > 5, '%Y-%m-%d') | fields formatted"; - Throwable t = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl)); - verifyErrorMessageContains( - t, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [BOOLEAN,STRING]"); + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(formatted=[STRFTIME(>($0, 5), '%Y-%m-%d':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -215,9 +270,10 @@ public void testStrftimeAcceptsDateInput() { "source=EMP | eval formatted = strftime(date('2020-09-16'), '%Y-%m-%d') | fields formatted"; RelNode relNode = getRelNode(ppl); assertNotNull(relNode); - // The plan should show TIMESTAMP(DATE(...)) indicating auto-conversion - String planString = relNode.explain(); - assertTrue(planString.contains("STRFTIME") && planString.contains("TIMESTAMP")); + verifyLogical( + relNode, + "LogicalProject(formatted=[STRFTIME(DATE('2020-09-16':VARCHAR), '%Y-%m-%d':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test @@ -254,50 +310,57 @@ public void testStrftimeWithDateReturningFunctions() { @Test public void testStrftimeWithWrongSecondArgType() { - // Second argument should be string, not numeric + // Second argument should be string, but Calcite handles type coercion String ppl = "source=EMP | eval formatted = strftime(1521467703, 123) | fields formatted"; - Throwable t = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl)); - verifyErrorMessageContains( - t, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [INTEGER,INTEGER]"); + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalProject(formatted=[STRFTIME(1521467703, 123)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } @Test public void testStrftimeWithWrongNumberOfArgs() { - // strftime requires exactly 2 arguments + // strftime now accepts variable arguments, so these no longer throw exceptions String ppl1 = "source=EMP | eval formatted = strftime(1521467703) | fields formatted"; - Throwable t1 = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl1)); - verifyErrorMessageContains( - t1, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [INTEGER]"); + RelNode root1 = getRelNode(ppl1); + verifyLogical( + root1, + "LogicalProject(formatted=[STRFTIME(1521467703)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); String ppl2 = "source=EMP | eval formatted = strftime(1521467703, '%Y', 'extra') | fields formatted"; - Throwable t2 = Assert.assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl2)); - verifyErrorMessageContains( - t2, - "STRFTIME function expects {[INTEGER,STRING]|[DOUBLE,STRING]|[TIMESTAMP,STRING]}, but got" - + " [INTEGER,STRING,STRING]"); + RelNode root2 = getRelNode(ppl2); + verifyLogical( + root2, + "LogicalProject(formatted=[STRFTIME(1521467703, '%Y':VARCHAR, 'extra':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // Test VALUES function with array expression (which is not a supported scalar type) @Test public void testValuesFunctionWithArrayArgType() { - verifyQueryThrowsException( - "source=EMP | stats values(array(ENAME, JOB)) as unique_values", - "Aggregation function VALUES expects field type" - + " {[BYTE]|[SHORT]|[INTEGER]|[LONG]|[FLOAT]|[DOUBLE]|[STRING]|[BOOLEAN]|[DATE]|[TIME]|[TIMESTAMP]|[IP]|[BINARY]|[BYTE,INTEGER]" - + "|[SHORT,INTEGER]|[INTEGER,INTEGER]|[LONG,INTEGER]|[FLOAT,INTEGER]|[DOUBLE,INTEGER]|[STRING,INTEGER]|[BOOLEAN,INTEGER]|[DATE,INTEGER]|[TIME,INTEGER]|[TIMESTAMP,INTEGER]|[IP,INTEGER]|[BINARY,INTEGER]}," - + " but got [ARRAY]"); + // ValuesFunction with ArrayArgType no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | stats values(array(ENAME, JOB)) as unique_values"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalAggregate(group=[{}], unique_values=[VALUES($0)])\n" + + " LogicalProject($f2=[array($1, $2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } // mvjoin should reject non-string single values @Test public void testMvjoinRejectsNonStringValues() { - verifyQueryThrowsException( - "source=EMP | eval result = mvjoin(42, ',') | fields result | head 1", - "MVJOIN function expects {[ARRAY,STRING]|[ARRAY,STRING,STRING]}, but got [INTEGER,STRING]"); + // Mvjoin rejects non-stringValues no longer throws exception, Calcite handles type coercion + String ppl = "source=EMP | eval result = mvjoin(42, ',') | fields result | head 1"; + RelNode root = getRelNode(ppl); + verifyLogical( + root, + "LogicalSort(fetch=[1])\n" + + " LogicalProject(result=[ARRAY_JOIN(42, ',')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java index 58e29c49a28..3b57aaa656f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java @@ -200,10 +200,9 @@ public void testLn() { public void testLog() { RelNode root = getRelNode("source=EMP | eval LOG = log(2) | fields LOG"); String expectedLogical = - "LogicalProject(LOG=[LOG(2, 2.718281828459045E0:DOUBLE)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalProject(LOG=[LOG(2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedSparkSql = "SELECT LOG(2, 2.718281828459045E0) `LOG`\nFROM `scott`.`EMP`"; + String expectedSparkSql = "SELECT LOG(2) `LOG`\nFROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java index 7185c85aa1d..0177193a257 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java @@ -290,9 +290,9 @@ public void testMultisearchWithTimestampFiltering() { "LogicalSort(sort0=[$3], dir0=[DESC-nulls-last])\n" + " LogicalSort(sort0=[$3], dir0=[DESC])\n" + " LogicalUnion(all=[true])\n" - + " LogicalFilter(condition=[>($3, TIMESTAMP('2025-07-31 23:00:00':VARCHAR))])\n" + + " LogicalFilter(condition=[>($3, 2025-07-31 23:00:00)])\n" + " LogicalTableScan(table=[[scott, TIME_DATA1]])\n" - + " LogicalFilter(condition=[>($3, TIMESTAMP('2025-07-31 23:00:00':VARCHAR))])\n" + + " LogicalFilter(condition=[>($3, 2025-07-31 23:00:00)])\n" + " LogicalTableScan(table=[[scott, TIME_DATA2]])\n"; verifyLogical(root, expectedLogical); @@ -301,11 +301,11 @@ public void testMultisearchWithTimestampFiltering() { + "FROM (SELECT `timestamp`, `value`, `category`, `@timestamp`\n" + "FROM (SELECT *\n" + "FROM `scott`.`TIME_DATA1`\n" - + "WHERE `@timestamp` > TIMESTAMP('2025-07-31 23:00:00')\n" + + "WHERE `@timestamp` > TIMESTAMP '2025-07-31 23:00:00'\n" + "UNION ALL\n" + "SELECT *\n" + "FROM `scott`.`TIME_DATA2`\n" - + "WHERE `@timestamp` > TIMESTAMP('2025-07-31 23:00:00'))\n" + + "WHERE `@timestamp` > TIMESTAMP '2025-07-31 23:00:00')\n" + "ORDER BY `@timestamp` DESC NULLS FIRST) `t2`\n" + "ORDER BY `@timestamp` DESC"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLNoMvTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLNoMvTest.java index 5d7669d20a1..c92ca4dc8e9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLNoMvTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLNoMvTest.java @@ -200,7 +200,8 @@ public void testNoMvNonExistentField() { msg.toLowerCase().contains("does_not_exist") || msg.toLowerCase().contains("field") || msg.contains("ARRAY_COMPACT") - || msg.contains("ARRAY")); + || msg.contains("ARRAY") + || msg.contains("inferred array element type")); } @Test diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java index 48c0e5cfa62..261e208ebda 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java @@ -33,8 +33,8 @@ public void testStreamstatsBy() { String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" - + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" - + " `max(SAL)`\n" + + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN UNBOUNDED" + + " PRECEDING AND CURRENT ROW) `max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" + "FROM `scott`.`EMP`) `t`\n" @@ -52,8 +52,8 @@ public void testStreamstatsByNullBucket() { + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[CASE(IS NOT" - + " NULL($7), MAX($5) OVER (PARTITION BY $7 ROWS UNBOUNDED PRECEDING), null:DECIMAL(7," - + " 2))])\n" + + " NULL($7), MAX($5) OVER (PARTITION BY $7 ROWS UNBOUNDED" + + " PRECEDING), null:DECIMAL(7, 2))])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -76,14 +76,15 @@ public void testStreamstatsCurrent() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7], max(SAL)=[MAX($5) OVER (ROWS BETWEEN UNBOUNDED PRECEDING" - + " AND 1 PRECEDING)])\n" + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[MAX($5) OVER (ROWS BETWEEN" + + " UNBOUNDED PRECEDING AND 1 PRECEDING)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" - + " OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) `max(SAL)`\n" + + " OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)" + + " `max(SAL)`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -147,7 +148,8 @@ public void testStreamstatsGlobal() { String expectedSparkSql = "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" - + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) `max(SAL)`\n" + + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN 4 PRECEDING" + + " AND CURRENT ROW) `max(SAL)`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + " ROW_NUMBER() OVER () `__stream_seq__`\n" + "FROM `scott`.`EMP`) `t`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java index b6b60c530e7..70e411d3c0f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java @@ -105,8 +105,8 @@ public void testMultipleAggregatesWithAliasesTranspose() { + " `t1`.`_row_number_transpose_`, `t2`.`column`, CASE WHEN `t2`.`column` = 'avg_sal'" + " THEN NUMBER_TO_STRING(`t1`.`avg_sal`) WHEN `t2`.`column` = 'max_sal' THEN" + " NUMBER_TO_STRING(`t1`.`max_sal`) WHEN `t2`.`column` = 'min_sal' THEN" - + " NUMBER_TO_STRING(`t1`.`min_sal`) WHEN `t2`.`column` = 'cnt' THEN CAST(`t1`.`cnt` AS" - + " STRING) ELSE NULL END `value`\n" + + " NUMBER_TO_STRING(`t1`.`min_sal`) WHEN `t2`.`column` = 'cnt' THEN" + + " CAST(`t1`.`cnt` AS STRING) ELSE NULL END `value`\n" + "FROM (SELECT AVG(`SAL`) `avg_sal`, MAX(`SAL`) `max_sal`, MIN(`SAL`) `min_sal`," + " COUNT(*) `cnt`, ROW_NUMBER() OVER () `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t1`\n" @@ -238,9 +238,9 @@ public void testTransposeWithLimitColumnName() { + "FROM (SELECT `t`.`ENAME`, `t`.`COMM`, `t`.`JOB`, `t`.`SAL`," + " `t`.`_row_number_transpose_`, `t0`.`column_names`, CASE WHEN `t0`.`column_names` =" + " 'ENAME' THEN CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column_names` = 'COMM' THEN" - + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column_names` = 'JOB' THEN CAST(`t`.`JOB`" - + " AS STRING) WHEN `t0`.`column_names` = 'SAL' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE" - + " NULL END `value`\n" + + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column_names` = 'JOB' THEN" + + " CAST(`t`.`JOB` AS STRING) WHEN `t0`.`column_names` = 'SAL' THEN" + + " NUMBER_TO_STRING(`t`.`SAL`) ELSE NULL END `value`\n" + "FROM (SELECT `ENAME`, `COMM`, `JOB`, `SAL`, ROW_NUMBER() OVER ()" + " `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t`\n"