Skip to content

ESQL: Begin documenting MV behaviors on non-aggs #123917

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,22 @@ ROW end_23=TO_DATETIME("2023-12-31T23:59:59.999Z"),
// end::evalDateDiffYearForDocs-result[]
;

dateDiffMv
// tag::date-diff-mv[]
ROW lhs=TO_DATETIME(["2024-01-01", "2024-02-01"]),
rhs=TO_DATETIME("2024-03-01")
| EVAL diff=DATE_DIFF("year", lhs, rhs)
// end::date-diff-mv[]
;
warning:Line 3:13: evaluation of [DATE_DIFF(\"year\", lhs, rhs)] failed, treating result as null. Only first 20 failures recorded.
warning:Line 3:13: java.lang.IllegalArgumentException: single-value function encountered multi-value

// tag::date-diff-mv-result[]
lhs:date | rhs:date | diff:integer
["2024-01-01", "2024-02-01"]| 2024-03-01T00:00:00.000Z| null
// end::date-diff-mv-result[]
;

evalDateParseWithSimpleDate
row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b;

Expand Down Expand Up @@ -619,6 +635,34 @@ emp_no:integer | new_date:datetime | birth_date:datetime | bool:
10050 | 1958-05-21T00:00:00.000Z | 1958-05-21T00:00:00.000Z | true
;

dateParseSimple
// tag::date-parse[]
ROW v = "2022-05-06"
| EVAL date = DATE_PARSE("yyyy-MM-dd", v)
// end::date-parse[]
;

//tag::date-parse-result[]
v:keyword | date:date
2022-05-06 | 2022-05-06T00:00:00.000Z
// end::date-parse-result[]
;

dateParseMv
// tag::date-parse-mv[]
ROW v = ["2022-05-06", "2022-06-06"]
| EVAL date = DATE_PARSE("yyyy-MM-dd", v)
// end::date-parse-mv[]
;
warning:Line 2:15: evaluation of [DATE_PARSE(\"yyyy-MM-dd\", v)] failed, treating result as null. Only first 20 failures recorded.
warning:Line 2:15: java.lang.IllegalArgumentException: single-value function encountered multi-value

//tag::date-parse-mv-result[]
v:keyword | date:date
["2022-05-06", "2022-06-06"]| null
// end::date-parse-mv-result[]
;

dateFields
from employees | where emp_no == 10049 or emp_no == 10050
| eval year = date_extract("year", birth_date), month = date_extract("month_of_year", birth_date), day = date_extract("day_of_month", birth_date)
Expand Down Expand Up @@ -1061,6 +1105,21 @@ a:integer | df:keyword
1 | 1989-06-02
;

dateFormatMv
// tag::date-format-mv[]
ROW v = TO_DATETIME(["2024-01-01", "2024-02-01"])
| EVAL fmt = DATE_FORMAT("yyyy-MM-dd", v)
// end::date-format-mv[]
;
warning:Line 2:14: evaluation of [DATE_FORMAT(\"yyyy-MM-dd\", v)] failed, treating result as null. Only first 20 failures recorded.
warning:Line 2:14: java.lang.IllegalArgumentException: single-value function encountered multi-value

// tag::date-format-mv-result[]
v:date | fmt:keyword
["2024-01-01", "2024-02-01"] | null
// end::date-format-mv-result[]
;

docsDateTrunc
// tag::docsDateTrunc[]
FROM employees
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,19 +310,6 @@ Saniya |Kalloufi |2.1 |6.9
// end::round-result[]
;

dateParse
// tag::dateParse[]
ROW date_string = "2022-05-06"
| EVAL date = DATE_PARSE("yyyy-MM-dd", date_string)
// end::dateParse[]
;

//tag::dateParse-result[]
date_string:keyword | date:date
2022-05-06 | 2022-05-06T00:00:00.000Z
// end::dateParse-result[]
;

docsReplace
//tag::replaceString[]
ROW str = "Hello World"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1492,6 +1492,19 @@ l:integer
-10000
;

leastMv
// tag::least-mv[]
ROW a = 10, b = [20, 1]
| EVAL g = LEAST(a, b)
// end::least-mv[]
;

// tag::least-mv-result[]
a:integer | b:integer | g:integer
10 | [20, 1] | 1
// end::least-mv-result[]
;


greatest
// tag::greatest[]
Expand Down Expand Up @@ -1521,10 +1534,16 @@ g:integer
;

greatestMv
ROW g=GREATEST([10, 4], 1);
// tag::greatest-mv[]
ROW a = 10, b = [20, 1]
| EVAL g = GREATEST(a, b)
// end::greatest-mv[]
;

g:integer
10
// tag::greatest-mv-result[]
a:integer | b:integer | g:integer
10 | [20, 1] | 20
// end::greatest-mv-result[]
;

leastGreatestMany
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,19 @@ a:null | b:keyword | COALESCE(a, b):keyword
// end::coalesce-result[]
;

coalesceMv#[skip:-8.12.99,reason:expression spaces are maintained since 8.13]
// tag::coalesce-mv[]
ROW a=null, b=["1", "2"]
| EVAL COALESCE(a, b)
// end::coalesce-mv[]
;

// tag::coalesce-mv-result[]
a:null | b:keyword | COALESCE(a, b):keyword
null |["1", "2"] | ["1", "2"]
// end::coalesce-mv-result[]
;

coalesce
FROM employees
| EVAL first_name = COALESCE(first_name, "X")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;

/**
* Returns the maximum value of multiple columns.
* Returns the maximum value of multiple columns. This will correctly return the
* greatest value across multivalue fields.
*/
public class Greatest extends EsqlScalarFunction implements OptionalArgument {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Greatest", Greatest::new);
Expand All @@ -48,7 +49,9 @@ public class Greatest extends EsqlScalarFunction implements OptionalArgument {
+ "except it is intended to run on multiple columns at once.",
note = "When run on `keyword` or `text` fields, this returns the last string in alphabetical order. "
+ "When run on `boolean` columns this will return `true` if any values are `true`.",
examples = @Example(file = "math", tag = "greatest")
examples = { @Example(file = "math", tag = "greatest"), @Example(description = """
Returns the maximum value from multivalued fields.
""", file = "math", tag = "greatest-mv"), }
)
public Greatest(
Source source,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;

/**
* Returns the minimum value of multiple columns.
* Returns the minimum value of multiple columns. This will correctly return the
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this doc change be reflected in the FunctionInfo.description instead? Same for Greatest

* least value across multivalue fields.
*/
public class Least extends EsqlScalarFunction implements OptionalArgument {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Least", Least::new);
Expand All @@ -46,7 +47,9 @@ public class Least extends EsqlScalarFunction implements OptionalArgument {
returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "version" },
description = "Returns the minimum value from multiple columns. "
+ "This is similar to <<esql-mv_min>> except it is intended to run on multiple columns at once.",
examples = @Example(file = "math", tag = "least")
examples = { @Example(file = "math", tag = "least"), @Example(description = """
Returns the minimum value from multivalued fields.
""", file = "math", tag = "least-mv"), }
)
public Least(
Source source,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,9 @@ public static Part resolve(String dateTimeUnit) {
examples = { @Example(file = "date", tag = "docsDateDiff"), @Example(description = """
When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted.
To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly.
""", file = "date", tag = "evalDateDiffYearForDocs") }
""", file = "date", tag = "evalDateDiffYearForDocs"), @Example(description = """
If any column is multivalued, this will return a `null` result.
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'd been considering this behavior to be "undefined" but the null-and-warning behavior as something we may change in the future. I'm not sure that that's really true though - folks may be relying on it even though it's a warning.

""", file = "date", tag = "date-diff-mv") }
)
public DateDiff(
Source source,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public class DateFormat extends EsqlConfigurationFunction implements OptionalArg
@FunctionInfo(
returnType = "keyword",
description = "Returns a string representation of a date, in the provided format.",
examples = @Example(file = "date", tag = "docsDateFormat")
examples = { @Example(file = "date", tag = "docsDateFormat"), @Example(file = "date", tag = "docsDateFormat"), }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both examples are identical. Wrong tag?

)
public DateFormat(
Source source,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ public class DateParse extends EsqlScalarFunction implements OptionalArgument {
@FunctionInfo(
returnType = "date",
description = "Returns a date by parsing the second argument using the format specified in the first argument.",
examples = @Example(file = "docs", tag = "dateParse")
examples = { @Example(file = "date", tag = "date-parse"), @Example(description = """
If any column is multivalued, this will return a `null` result.
""", file = "date", tag = "date-parse-mv"), }
)
public DateParse(
Source source,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ public class Coalesce extends EsqlScalarFunction implements OptionalArgument {
"long",
"version" },
description = "Returns the first of its arguments that is not null. If all arguments are null, it returns `null`.",
examples = { @Example(file = "null", tag = "coalesce") }
examples = { @Example(file = "null", tag = "coalesce"), @Example(description = """
COALESCE keeps multivalued fields.
""", file = "null", tag = "coalesce-mv") }
)
public Coalesce(
Source source,
Expand Down