diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 18a5bdce8f6..4b0d3a5a2eb 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -32,13 +32,13 @@ Examples Here is an example for different type of literals:: - os> SELECT 123, 'hello', false, -4.567, DATE '2020-07-07', TIME '01:01:01', TIMESTAMP '2020-07-07 01:01:01'; + os> SELECT 123, 'hello', false, -4.567, 9.876E-1, DATE '2020-07-07', TIME '01:01:01', TIMESTAMP '2020-07-07 01:01:01'; fetched rows / total rows = 1/1 - +-----+---------+-------+--------+-------------------+-----------------+---------------------------------+ - | 123 | 'hello' | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | - |-----+---------+-------+--------+-------------------+-----------------+---------------------------------| - | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | - +-----+---------+-------+--------+-------------------+-----------------+---------------------------------+ + +-----+---------+-------+--------+----------+-------------------+-----------------+---------------------------------+ + | 123 | 'hello' | false | -4.567 | 9.876E-1 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | + |-----+---------+-------+--------+----------+-------------------+-----------------+---------------------------------| + | 123 | hello | False | -4.567 | 0.9876 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | + +-----+---------+-------+--------+----------+-------------------+-----------------+---------------------------------+ os> SELECT "Hello", 'Hello', "It""s", 'It''s', "It's", '"Its"', 'It\'s', 'It\\\'s', "\I\t\s" diff --git a/docs/user/ppl/functions/expressions.rst b/docs/user/ppl/functions/expressions.rst index 3c4b2a5f1a2..80cd8fb3136 100644 --- a/docs/user/ppl/functions/expressions.rst +++ b/docs/user/ppl/functions/expressions.rst @@ -14,6 +14,52 @@ Introduction Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in ``Filter``, ``Stats`` command. +Literal Values +============== + +Description +----------- + +A literal is a symbol that represents a value. The most common literal values include: + +1. Numeric literals: specify numeric values such as integer and floating-point numbers. +2. String literals: specify a string enclosed by single or double quotes. +3. Boolean literals: ``true`` or ``false``. +4. Date and Time literals: DATE 'YYYY-MM-DD' represent the date, TIME 'hh:mm:ss' represent the time, TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the timestamp. + +Examples +-------- + +Here is an example for different type of literals:: + + os> source=accounts | eval `123`=123, `'hello'`='hello', `false`=false, `-4.567`=-4.567, `9.876E-1`=9.876E-1, `DATE '2020-07-07'`=DATE '2020-07-07', `TIME '01:01:01'`=TIME '01:01:01', `TIMESTAMP '2020-07-07 01:01:01'`=TIMESTAMP '2020-07-07 01:01:01' | fields `123`, `'hello'`, `false`, `-4.567`, `9.876E-1`, `DATE '2020-07-07'`, `TIME '01:01:01'`, `TIMESTAMP '2020-07-07 01:01:01'` | head 1; + fetched rows / total rows = 1/1 + +-----+---------+-------+--------+----------+-------------------+-----------------+---------------------------------+ + | 123 | 'hello' | false | -4.567 | 9.876E-1 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | + |-----+---------+-------+--------+----------+-------------------+-----------------+---------------------------------| + | 123 | hello | False | -4.567 | 0.9876 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | + +-----+---------+-------+--------+----------+-------------------+-----------------+---------------------------------+ + + + os> source=accounts | eval `"Hello"`="Hello", `'Hello'`='Hello', `"It""s"`="It""s", `'It''s'`='It''s', `"It's"`="It's", `'"Its"'`='"Its"', `'It\'s'`='It\'s', `'It\\\'s'`='It\\\'s', `"\I\t\s"`="\I\t\s" | fields `"Hello"`, `'Hello'`, `"It""s"`, `'It''s'`, `"It's"`, `'"Its"'`, `'It\'s'`, `'It\\\'s'`, `"\I\t\s"` | head 1; + fetched rows / total rows = 1/1 + +---------+---------+---------+---------+--------+---------+---------+-----------+----------+ + | "Hello" | 'Hello' | "It""s" | 'It''s' | "It's" | '"Its"' | 'It\'s' | 'It\\\'s' | "\I\t\s" | + |---------+---------+---------+---------+--------+---------+---------+-----------+----------| + | Hello | Hello | It"s | It's | It's | "Its" | It's | It\'s | \I\t\s | + +---------+---------+---------+---------+--------+---------+---------+-----------+----------+ + + + os> source=accounts | eval `{DATE '2020-07-07'}`={DATE '2020-07-07'}, `{TIME '01:01:01'}`={TIME '01:01:01'}, `{TIMESTAMP '2020-07-07 01:01:01'}`={TIMESTAMP '2020-07-07 01:01:01'} | fields `{DATE '2020-07-07'}`, `{TIME '01:01:01'}`, `{TIMESTAMP '2020-07-07 01:01:01'}` | head 1; + fetched rows / total rows = 1/1 + +---------------------+-------------------+-----------------------------------+ + | {DATE '2020-07-07'} | {TIME '01:01:01'} | {TIMESTAMP '2020-07-07 01:01:01'} | + |---------------------+-------------------+-----------------------------------| + | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | + +---------------------+-------------------+-----------------------------------+ + + + Arithmetic Operators ==================== diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 5a1c82e8da7..c10cc3d4d04 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -110,4 +110,30 @@ public void test_alias_data_type() throws IOException { verifySchema(result, schema("original_col", "int"), schema("alias_col", "int")); verifyDataRows(result, rows(2, 2), rows(3, 3)); } + + @Test + public void test_exponent_literal_converting_to_double_type() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval `9e1` = 9e1, `+9e+1` = +9e+1, `900e-1` = 900e-1, `9.0e1` =" + + " 9.0e1, `9.0e+1` = 9.0e+1, `9.0E1` = 9.0E1, `.9e+2` = .9e+2, `0.09e+3` =" + + " 0.09e+3, `900.0e-1` = 900.0e-1, `-900.0E-1` = -900.0E-1 | fields `9e1`," + + " `+9e+1`, `900e-1`, `9.0e1`, `9.0e+1`, `9.0E1`, `.9e+2`, `0.09e+3`," + + " `900.0e-1`, `-900.0E-1`", + TEST_INDEX_DATATYPE_NUMERIC)); + verifySchema( + result, + schema("9e1", "double"), + schema("+9e+1", "double"), + schema("900e-1", "double"), + schema("9.0e1", "double"), + schema("9.0e+1", "double"), + schema("9.0E1", "double"), + schema(".9e+2", "double"), + schema("0.09e+3", "double"), + schema("900.0e-1", "double"), + schema("-900.0E-1", "double")); + verifyDataRows(result, rows(90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, -90.0)); + } } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2826.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2826.yml new file mode 100644 index 00000000000..0b1b3af26f3 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2826.yml @@ -0,0 +1,49 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + 90: + type: double + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + plugins.calcite.fallback.allowed : false + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + plugins.calcite.fallback.allowed : true + +--- +"Exponent literal converts to double type": + - skip: + features: + - headers + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"90": 90.0}' + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=test | eval `9e1` = 9e1, `+9e+1` = +9e+1, `900e-1` = 900e-1, `9.0e1` = 9.0e1, `9.0e+1` = 9.0e+1, `9.0E1` = 9.0E1, `.9e+2` = .9e+2, `0.09e+3` = 0.09e+3, `900.0e-1` = 900.0e-1, `-900.0E-1` = -900.0E-1' + - match: {"total": 1} + - match: {"schema": [{"name": "90", "type": "double"}, {"name": "9e1", "type": "double"}, {"name": "+9e+1", "type": "double"}, {"name": "900e-1", "type": "double"}, {"name": "9.0e1", "type": "double"}, {"name": "9.0e+1", "type": "double"}, {"name": "9.0E1", "type": "double"}, {"name": ".9e+2", "type": "double"}, {"name": "0.09e+3", "type": "double"}, {"name": "900.0e-1", "type": "double"}, {"name": "-900.0E-1", "type": "double"}]} + - match: {"datarows": [[90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, 90.0, -90.0]]} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 521e7d09a13..448766aba87 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -439,8 +439,9 @@ Y: 'Y'; //STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; ID: ID_LITERAL; CLUSTER: CLUSTER_PREFIX_LITERAL; -INTEGER_LITERAL: DEC_DIGIT+; -DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; +INTEGER_LITERAL: INTEGER_NUM; +DECIMAL_LITERAL: DECIMAL_NUM; +EXPONENT_LITERAL: INTEGER_NUM EXPONENT_NUM | DECIMAL_NUM EXPONENT_NUM; fragment DATE_SUFFIX: ([\-.][*0-9]+)+; fragment CLUSTER_PREFIX_LITERAL: [*A-Z]+?[*A-Z_\-0-9]* COLON; @@ -449,6 +450,9 @@ DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; fragment DEC_DIGIT: [0-9]; +fragment INTEGER_NUM: DEC_DIGIT+; +fragment DECIMAL_NUM: (DEC_DIGIT+)? '.' DEC_DIGIT+; +fragment EXPONENT_NUM: 'E' [-+]? DEC_DIGIT+; // Identifiers cannot start with a single '_' since this an OpenSearch reserved // metadata field. Two underscores (or more) is acceptable, such as '__field'. diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 08c9e78160b..9bff97ac3ef 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -419,6 +419,7 @@ percentileApproxFunction numericLiteral : integerLiteral | decimalLiteral + | exponentLiteral ; // expressions @@ -909,6 +910,7 @@ literalValue | stringLiteral | integerLiteral | decimalLiteral + | exponentLiteral | booleanLiteral | datetimeLiteral //#datetime ; @@ -930,6 +932,10 @@ decimalLiteral : (PLUS | MINUS)? DECIMAL_LITERAL ; +exponentLiteral + : (PLUS | MINUS)? EXPONENT_LITERAL + ; + booleanLiteral : TRUE | FALSE diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 679565e22f6..4c81559a122 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -20,6 +20,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalFunctionCallContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ExponentLiteralContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldExpressionContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsQualifiedNameContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsTableQualifiedNameContext; @@ -410,6 +411,11 @@ public UnresolvedExpression visitDecimalLiteral(DecimalLiteralContext ctx) { return new Literal(Double.valueOf(ctx.getText()), DataType.DOUBLE); } + @Override + public UnresolvedExpression visitExponentLiteral(ExponentLiteralContext ctx) { + return new Literal(Double.valueOf(ctx.getText()), DataType.DOUBLE); + } + @Override public UnresolvedExpression visitBooleanLiteral(BooleanLiteralContext ctx) { return new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index b2fd8693d3d..5e6aa4fbd2c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -436,6 +436,28 @@ public void testCanParseTimestampdiffFunction() { .parse("SOURCE=test | eval k = TIMESTAMPDIFF(WEEK,'2003-01-02','2003-01-02')")); } + @Test + public void testExponentLiteralShouldPass() { + List scientificNotationList = + List.of( + "9e1", + "+9e+1", + "9e-1", + "-9e1", + "9.0e1", + "9.0e+1", + "9.0E1", + ".9e+2", + "0.9e+2", + "900e-1", + "900.0E-1"); + for (String exponentLiteral : scientificNotationList) { + ParseTree tree = + new PPLSyntaxParser().parse("search source=t | eval scientific = " + exponentLiteral); + assertNotEquals(null, tree); + } + } + @Test public void testCanParseFillNullSameValue() { assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a")); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 0efb006846b..87348acb547 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -664,6 +664,44 @@ public void testDoubleLiteralExpr() { "source=t b=0.1", filter(relation("t"), compare("=", field("b"), doubleLiteral(0.1)))); } + @Test + public void testExponentLiteralExpr() { + List scientificNotationList = + List.of( + "9e1", + "+9e+1", + "900e-1", + "9.0e1", + "9.0e+1", + "9.0E1", + ".9e+2", + "0.09e+3", + "900.0e-1", + "+900.0E-1"); + for (String scientificNotation : scientificNotationList) { + assertEqual( + "source=t b=" + scientificNotation, + filter(relation("t"), compare("=", field("b"), doubleLiteral(90.0)))); + } + List negativeScientificNotationList = + List.of( + "-9e1", + "-9e+1", + "-900e-1", + "-9.0e1", + "-9.0e+1", + "-9.0E1", + "-.9e+2", + "-0.09e+3", + "-900.0e-1", + "-900.0E-1"); + for (String negativeScientificNotation : negativeScientificNotationList) { + assertEqual( + "source=t b=" + negativeScientificNotation, + filter(relation("t"), compare("=", field("b"), doubleLiteral(-90.0)))); + } + } + @Test public void testBooleanLiteralExpr() { assertEqual(