diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java index 30821e68cd2d7..5bd003fe4271f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java @@ -48,9 +48,9 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; diff --git a/docs/changelog/128393.yaml b/docs/changelog/128393.yaml new file mode 100644 index 0000000000000..1f4a2bf8697f3 --- /dev/null +++ b/docs/changelog/128393.yaml @@ -0,0 +1,6 @@ +pr: 128393 +summary: Pushdown constructs doing case-insensitive regexes +area: ES|QL +type: enhancement +issues: + - 127479 diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index bfe32ca591eeb..57ba6de0b973c 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -272,6 +272,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_VERTEXAI_CHATCOMPLETION_ADDED = def(9_083_0_00); public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00); public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00); + public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java index 8d681977b5b42..1de62f750346d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java @@ -16,11 +16,11 @@ public abstract class AbstractStringPattern implements StringPattern { private Automaton automaton; - public abstract Automaton createAutomaton(); + public abstract Automaton createAutomaton(boolean ignoreCase); private Automaton automaton() { if (automaton == null) { - automaton = createAutomaton(); + automaton = createAutomaton(false); } return automaton; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLike.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLike.java deleted file mode 100644 index b4bccf162d9e4..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLike.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.regex; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; - -public abstract class RLike extends RegexMatch { - - public RLike(Source source, Expression value, RLikePattern pattern) { - super(source, value, pattern, false); - } - - public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) { - super(source, field, rLikePattern, caseInsensitive); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java index 4e559f564acb1..0744977170911 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java @@ -21,9 +21,10 @@ public RLikePattern(String regexpPattern) { } @Override - public Automaton createAutomaton() { + public Automaton createAutomaton(boolean ignoreCase) { + int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0; return Operations.determinize( - new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton(), + new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT ); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardLike.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardLike.java deleted file mode 100644 index 05027707326bd..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardLike.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.regex; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; - -public abstract class WildcardLike extends RegexMatch { - - public WildcardLike(Source source, Expression left, WildcardPattern pattern) { - this(source, left, pattern, false); - } - - public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) { - super(source, left, pattern, caseInsensitive); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java index 3e9cbf92727c2..d2e1bcff2c204 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java @@ -10,10 +10,13 @@ import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.xpack.esql.core.util.StringUtils; import java.util.Objects; +import static org.elasticsearch.xpack.esql.core.util.StringUtils.luceneWildcardToRegExp; + /** * Similar to basic regex, supporting '?' wildcard for single character (same as regex ".") * and '*' wildcard for multiple characters (same as regex ".*") @@ -37,8 +40,14 @@ public String pattern() { } @Override - public Automaton createAutomaton() { - return WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + public Automaton createAutomaton(boolean ignoreCase) { + return ignoreCase + ? Operations.determinize( + new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE) + .toAutomaton(), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT + ) + : WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java index 288612c9a593d..2a5349309d9ee 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.util; import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.spell.LevenshteinDistance; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CollectionUtil; @@ -178,6 +179,44 @@ public static String wildcardToJavaPattern(String pattern, char escape) { return regex.toString(); } + /** + * Translates a Lucene wildcard pattern to a Lucene RegExp one. + * @param wildcard Lucene wildcard pattern + * @return Lucene RegExp pattern + */ + public static String luceneWildcardToRegExp(String wildcard) { + StringBuilder regex = new StringBuilder(); + + for (int i = 0, wcLen = wildcard.length(); i < wcLen; i++) { + char c = wildcard.charAt(i); // this will work chunking through Unicode as long as all values matched are ASCII + switch (c) { + case WildcardQuery.WILDCARD_STRING -> regex.append(".*"); + case WildcardQuery.WILDCARD_CHAR -> regex.append("."); + case WildcardQuery.WILDCARD_ESCAPE -> { + if (i + 1 < wcLen) { + // consume the wildcard escaping, consider the next char + char next = wildcard.charAt(i + 1); + i++; + switch (next) { + case WildcardQuery.WILDCARD_STRING, WildcardQuery.WILDCARD_CHAR, WildcardQuery.WILDCARD_ESCAPE -> + // escape `*`, `.`, `\`, since these are special chars in RegExp as well + regex.append("\\"); + // default: unnecessary escaping -- just ignore the escaping + } + regex.append(next); + } else { + // "else fallthru, lenient parsing with a trailing \" -- according to WildcardQuery#toAutomaton + regex.append("\\\\"); + } + } + case '$', '(', ')', '+', '.', '[', ']', '^', '{', '|', '}' -> regex.append("\\").append(c); + default -> regex.append(c); + } + } + + return regex.toString(); + } + /** * Translates a like pattern to a Lucene wildcard. * This methods pays attention to the custom escape char which gets converted into \ (used by Lucene). diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/StringUtilsTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/StringUtilsTests.java index e584357b25b09..fbb956177b0ad 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/StringUtilsTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/StringUtilsTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.test.ESTestCase; +import static org.elasticsearch.xpack.esql.core.util.StringUtils.luceneWildcardToRegExp; import static org.elasticsearch.xpack.esql.core.util.StringUtils.wildcardToJavaPattern; +import static org.hamcrest.Matchers.is; public class StringUtilsTests extends ESTestCase { @@ -55,4 +57,21 @@ public void testWildcard() { public void testEscapedEscape() { assertEquals("^\\\\\\\\$", wildcardToJavaPattern("\\\\\\\\", '\\')); } + + public void testLuceneWildcardToRegExp() { + assertThat(luceneWildcardToRegExp(""), is("")); + assertThat(luceneWildcardToRegExp("*"), is(".*")); + assertThat(luceneWildcardToRegExp("?"), is(".")); + assertThat(luceneWildcardToRegExp("\\\\"), is("\\\\")); + assertThat(luceneWildcardToRegExp("foo?bar"), is("foo.bar")); + assertThat(luceneWildcardToRegExp("foo*bar"), is("foo.*bar")); + assertThat(luceneWildcardToRegExp("foo\\\\bar"), is("foo\\\\bar")); + assertThat(luceneWildcardToRegExp("foo*bar?baz"), is("foo.*bar.baz")); + assertThat(luceneWildcardToRegExp("foo\\*bar"), is("foo\\*bar")); + assertThat(luceneWildcardToRegExp("foo\\?bar\\?"), is("foo\\?bar\\?")); + assertThat(luceneWildcardToRegExp("foo\\?bar\\"), is("foo\\?bar\\\\")); + assertThat(luceneWildcardToRegExp("[](){}^$.|+"), is("\\[\\]\\(\\)\\{\\}\\^\\$\\.\\|\\+")); + assertThat(luceneWildcardToRegExp("foo\\\uD83D\uDC14bar"), is("foo\uD83D\uDC14bar")); + assertThat(luceneWildcardToRegExp("foo\uD83D\uDC14bar"), is("foo\uD83D\uDC14bar")); + } } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java index 34a477c70c504..08426d84c898a 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import java.util.Locale; import java.util.regex.Pattern; import static java.util.Collections.emptyMap; @@ -61,4 +62,15 @@ public static FieldAttribute getFieldAttribute(String name, DataType dataType) { public static String stripThrough(String input) { return WS_PATTERN.matcher(input).replaceAll(StringUtils.EMPTY); } + + /** Returns the input string, but with parts of it having the letter casing changed. */ + public static String randomCasing(String input) { + StringBuilder sb = new StringBuilder(input.length()); + for (int i = 0, inputLen = input.length(), step = (int) Math.sqrt(inputLen), chunkEnd; i < inputLen; i += step) { + chunkEnd = Math.min(i + step, inputLen); + var chunk = input.substring(i, chunkEnd); + sb.append(randomBoolean() ? chunk.toLowerCase(Locale.ROOT) : chunk.toUpperCase(Locale.ROOT)); + } + return sb.toString(); + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index dc5608efe67b1..ebada01e72b41 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -63,8 +63,8 @@ import org.elasticsearch.xpack.esql.core.util.DateUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.Range; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec index 2a62117be8169..aa04878ea12d0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec @@ -319,3 +319,107 @@ warningRegex:java.lang.IllegalArgumentException: single-value function encounter emp_no:integer | job_positions:keyword 10025 | Accountant ; + +likeWithUpperTurnedInsensitive +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_UPPER(first_name) LIKE "GEOR*" +; + +emp_no:integer |first_name:keyword +10001 |Georgi +10055 |Georgy +; + +likeWithLowerTurnedInsensitive +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_LOWER(TO_UPPER(first_name)) LIKE "geor*" +; + +emp_no:integer |first_name:keyword +10001 |Georgi +10055 |Georgy +; + +likeWithLowerConflictingFolded +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_UPPER(first_name) LIKE "geor*" +; + +emp_no:integer |first_name:keyword +; + +likeWithLowerTurnedInsensitiveNotPushedDown +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_LOWER(first_name) LIKE "geor*" OR emp_no + 1 IN (10002, 10056) +; + +emp_no:integer |first_name:keyword +10001 |Georgi +10055 |Georgy +; + +rlikeWithUpperTurnedInsensitive +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_UPPER(first_name) RLIKE "GEOR.*" +; + +emp_no:integer |first_name:keyword +10001 |Georgi +10055 |Georgy +; + +rlikeWithLowerTurnedInsensitive +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE "geor.*" +; + +emp_no:integer |first_name:keyword +10001 |Georgi +10055 |Georgy +; + +rlikeWithLowerConflictingFolded +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_UPPER(first_name) RLIKE "geor.*" +; + +emp_no:integer |first_name:keyword +; + +negatedRLikeWithLowerTurnedInsensitive +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_LOWER(TO_UPPER(first_name)) NOT RLIKE "geor.*" +| STATS c = COUNT() +; + +c:long +88 +; + +rlikeWithLowerTurnedInsensitiveNotPushedDown +FROM employees +| KEEP emp_no, first_name +| SORT emp_no +| WHERE TO_LOWER(first_name) RLIKE "geor.*" OR emp_no + 1 IN (10002, 10056) +; + +emp_no:integer |first_name:keyword +10001 |Georgi +10055 |Georgy +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java index 7e17d25f67e72..c617f66fb2533 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java @@ -68,11 +68,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLike.java similarity index 70% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLike.java index 43c4705d1c750..dea36bba2c4fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLike.java @@ -5,21 +5,17 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.function.scalar.string; +package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; @@ -29,14 +25,9 @@ import java.io.IOException; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; - -public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike - implements - EvaluatorMapper, - TranslationAware.SingleValueTranslationAware { +public class RLike extends RegexMatch { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "RLike", RLike::new); + public static final String NAME = "RLIKE"; @FunctionInfo(returnType = "boolean", description = """ Use `RLIKE` to filter data based on string patterns using using @@ -52,13 +43,13 @@ Matching special characters (eg. `.`, `*`, `(`...) will require escaping. To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"` <> - """, operator = "RLIKE", examples = @Example(file = "docs", tag = "rlike")) + """, operator = NAME, examples = @Example(file = "docs", tag = "rlike")) public RLike( Source source, @Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value, @Param(name = "pattern", type = { "keyword", "text" }, description = "A regular expression.") RLikePattern pattern ) { - super(source, value, pattern); + this(source, value, pattern, false); } public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) { @@ -66,7 +57,12 @@ public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean } private RLike(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), new RLikePattern(in.readString())); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + new RLikePattern(in.readString()), + deserializeCaseInsensitivity(in) + ); } @Override @@ -74,6 +70,12 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(field()); out.writeString(pattern().asJavaRegex()); + serializeCaseInsensitivity(out); + } + + @Override + public String name() { + return NAME; } @Override @@ -91,35 +93,10 @@ protected RLike replaceChild(Expression newChild) { return new RLike(source(), newChild, pattern(), caseInsensitive()); } - @Override - protected TypeResolution resolveType() { - return isString(field(), sourceText(), DEFAULT); - } - - @Override - public Boolean fold(FoldContext ctx) { - return (Boolean) EvaluatorMapper.super.fold(source(), ctx); - } - - @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - return AutomataMatch.toEvaluator(source(), toEvaluator.apply(field()), pattern().createAutomaton()); - } - - @Override - public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO; - } - @Override public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var fa = LucenePushdownPredicates.checkIsFieldAttribute(field()); // TODO: see whether escaping is needed return new RegexQuery(source(), handler.nameOf(fa.exactAttribute()), pattern().asJavaRegex(), caseInsensitive()); } - - @Override - public Expression singleValueField() { - return field(); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java new file mode 100644 index 0000000000000..eb5e06a686320 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex; + +import org.apache.lucene.util.automaton.Automata; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.AbstractStringPattern; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.AutomataMatch; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; + +import java.io.IOException; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +abstract class RegexMatch

extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch< + P> implements EvaluatorMapper, TranslationAware.SingleValueTranslationAware { + + abstract String name(); + + RegexMatch(Source source, Expression field, P pattern, boolean caseInsensitive) { + super(source, field, pattern, caseInsensitive); + } + + @Override + protected TypeResolution resolveType() { + return isString(field(), sourceText(), DEFAULT); + } + + @Override + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + return AutomataMatch.toEvaluator( + source(), + toEvaluator.apply(field()), + // The empty pattern will accept the empty string + pattern().pattern().isEmpty() ? Automata.makeEmptyString() : pattern().createAutomaton(caseInsensitive()) + ); + } + + @Override + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO; + } + + @Override + public Expression singleValueField() { + return field(); + } + + @Override + public String nodeString() { + return name() + "(" + field().nodeString() + ", \"" + pattern().pattern() + "\", " + caseInsensitive() + ")"; + } + + void serializeCaseInsensitivity(StreamOutput out) throws IOException { + if (out.getTransportVersion().before(TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY)) { + if (caseInsensitive()) { + // The plan has been optimized to run a case-insensitive match, which the remote peer cannot be notified of. Simply avoiding + // the serialization of the boolean would result in wrong results. + throw new EsqlIllegalArgumentException( + name() + " with case insensitivity is not supported in peer node's version [{}]. Upgrade to version [{}] or newer.", + out.getTransportVersion(), + TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY + ); + } // else: write nothing, the remote peer can execute the case-sensitive query + } else { + out.writeBoolean(caseInsensitive()); + } + } + + static boolean deserializeCaseInsensitivity(StreamInput in) throws IOException { + return in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY) && in.readBoolean(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java similarity index 69% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java index 3f90f9b70766f..73ac9e1c50969 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java @@ -5,23 +5,18 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.function.scalar.string; +package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex; -import org.apache.lucene.util.automaton.Automata; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; @@ -31,18 +26,13 @@ import java.io.IOException; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; - -public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike - implements - EvaluatorMapper, - TranslationAware.SingleValueTranslationAware { +public class WildcardLike extends RegexMatch { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "WildcardLike", WildcardLike::new ); + public static final String NAME = "LIKE"; @FunctionInfo(returnType = "boolean", description = """ Use `LIKE` to filter data based on string patterns using wildcards. `LIKE` @@ -63,17 +53,26 @@ also act on a constant (literal) expression. The right-hand side of the operator To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"` <> - """, operator = "LIKE", examples = @Example(file = "docs", tag = "like")) + """, operator = NAME, examples = @Example(file = "docs", tag = "like")) public WildcardLike( Source source, @Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left, @Param(name = "pattern", type = { "keyword", "text" }, description = "Pattern.") WildcardPattern pattern ) { - super(source, left, pattern, false); + this(source, left, pattern, false); + } + + public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) { + super(source, left, pattern, caseInsensitive); } private WildcardLike(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), new WildcardPattern(in.readString())); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + new WildcardPattern(in.readString()), + deserializeCaseInsensitivity(in) + ); } @Override @@ -81,41 +80,27 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(field()); out.writeString(pattern().pattern()); + serializeCaseInsensitivity(out); } @Override - public String getWriteableName() { - return ENTRY.name; - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, WildcardLike::new, field(), pattern()); - } - - @Override - protected WildcardLike replaceChild(Expression newLeft) { - return new WildcardLike(source(), newLeft, pattern()); + public String name() { + return NAME; } @Override - protected TypeResolution resolveType() { - return isString(field(), sourceText(), DEFAULT); + public String getWriteableName() { + return ENTRY.name; } @Override - public Boolean fold(FoldContext ctx) { - return (Boolean) EvaluatorMapper.super.fold(source(), ctx); + protected NodeInfo info() { + return NodeInfo.create(this, WildcardLike::new, field(), pattern(), caseInsensitive()); } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - return AutomataMatch.toEvaluator( - source(), - toEvaluator.apply(field()), - // The empty pattern will accept the empty string - pattern().pattern().length() == 0 ? Automata.makeEmptyString() : pattern().createAutomaton() - ); + protected WildcardLike replaceChild(Expression newLeft) { + return new WildcardLike(source(), newLeft, pattern(), caseInsensitive()); } @Override @@ -134,9 +119,4 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand private Query translateField(String targetFieldName) { return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive()); } - - @Override - public Expression singleValueField() { - return field(); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 0c2ecdcb71fe7..5f675adbcd507 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEmptyRelation; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStatsFilteredAggWithEval; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.LocalPropagateEmptyRelation; @@ -33,19 +34,17 @@ */ public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor { - private static final List> RULES = replaceRules( - arrayAsArrayList( - new Batch<>( - "Local rewrite", - Limiter.ONCE, - new ReplaceTopNWithLimitAndSort(), - new ReplaceFieldWithConstantOrNull(), - new InferIsNotNull(), - new InferNonNullAggConstraint() - ), - operators(), - cleanup() - ) + private static final List> RULES = arrayAsArrayList( + new Batch<>( + "Local rewrite", + Limiter.ONCE, + new ReplaceTopNWithLimitAndSort(), + new ReplaceFieldWithConstantOrNull(), + new InferIsNotNull(), + new InferNonNullAggConstraint() + ), + localOperators(), + cleanup() ); public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimizerContext) { @@ -58,27 +57,26 @@ protected List> batches() { } @SuppressWarnings("unchecked") - private static List> replaceRules(List> listOfRules) { - List> newBatches = new ArrayList<>(listOfRules.size()); - for (var batch : listOfRules) { - var rules = batch.rules(); - List> newRules = new ArrayList<>(rules.length); - boolean updated = false; - for (var r : rules) { - if (r instanceof PropagateEmptyRelation) { - newRules.add(new LocalPropagateEmptyRelation()); - updated = true; - } else if (r instanceof ReplaceStatsFilteredAggWithEval) { - // skip it: once a fragment contains an Agg, this can no longer be pruned, which the rule can do - updated = true; - } else { - newRules.add(r); + private static Batch localOperators() { + var operators = operators(); + var rules = operators().rules(); + List> newRules = new ArrayList<>(rules.length); + + // apply updates to existing rules that have different applicability locally + for (var r : rules) { + switch (r) { + case PropagateEmptyRelation ignoredPropagate -> newRules.add(new LocalPropagateEmptyRelation()); + // skip it: once a fragment contains an Agg, this can no longer be pruned, which the rule can do + case ReplaceStatsFilteredAggWithEval ignoredReplace -> { } + default -> newRules.add(r); } - batch = updated ? batch.with(newRules.toArray(Rule[]::new)) : batch; - newBatches.add(batch); } - return newBatches; + + // add rule that should only apply locally + newRules.add(new ReplaceStringCasingWithInsensitiveRegexMatch()); + + return operators.with(newRules.toArray(Rule[]::new)); } public LogicalPlan localOptimize(LogicalPlan plan) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java index 921db7f7ad23e..fbf3dcf8470a5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java @@ -66,7 +66,7 @@ private static Expression replaceChangeCase(LogicalOptimizerContext ctx, BinaryC return e; } - private static Expression unwrapCase(Expression e) { + static Expression unwrapCase(Expression e) { for (; e instanceof ChangeCase cc; e = cc.field()) { } return e; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveRegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveRegexMatch.java new file mode 100644 index 0000000000000..f26af58195e81 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveRegexMatch.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.ChangeCase; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; + +import static org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveEquals.unwrapCase; + +public class ReplaceStringCasingWithInsensitiveRegexMatch extends OptimizerRules.OptimizerExpressionRule< + RegexMatch> { + + public ReplaceStringCasingWithInsensitiveRegexMatch() { + super(OptimizerRules.TransformDirection.DOWN); + } + + @Override + protected Expression rule(RegexMatch regexMatch, LogicalOptimizerContext unused) { + Expression e = regexMatch; + if (regexMatch.field() instanceof ChangeCase changeCase) { + var pattern = regexMatch.pattern().pattern(); + e = changeCase.caseType().matchesCase(pattern) ? insensitiveRegexMatch(regexMatch) : Literal.of(regexMatch, Boolean.FALSE); + } + return e; + } + + private static Expression insensitiveRegexMatch(RegexMatch regexMatch) { + return switch (regexMatch) { + case RLike rlike -> new RLike(rlike.source(), unwrapCase(rlike.field()), rlike.pattern(), true); + case WildcardLike wildcardLike -> new WildcardLike( + wildcardLike.source(), + unwrapCase(wildcardLike.field()), + wildcardLike.pattern(), + true + ); + default -> regexMatch; + }; + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 8e14f6275eb0c..0530bb8728046 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -42,8 +42,8 @@ import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 958ac84ac82bb..2f49d68992db1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -20,8 +20,8 @@ import org.elasticsearch.xpack.esql.CsvTestsDataLoader; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeSerializationTests.java index 655d1a75470a3..0316e86ffd1bd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeSerializationTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index c2397f0340e67..d42b9c9b48cc4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -20,13 +20,17 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; import org.junit.AfterClass; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; +import static org.elasticsearch.xpack.esql.core.util.TestUtils.randomCasing; import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -39,12 +43,13 @@ public RLikeTests(@Name("TestCase") Supplier testCase @ParametersFactory public static Iterable parameters() { - return parameters(str -> { + final Function escapeString = str -> { for (String syntax : new String[] { "\\", ".", "?", "+", "*", "|", "{", "}", "[", "]", "(", ")", "\"", "<", ">", "#", "&" }) { str = str.replace(syntax, "\\" + syntax); } return str; - }, () -> randomAlphaOfLength(1) + "?"); + }; + return parameters(escapeString, () -> randomAlphaOfLength(1) + "?"); } static Iterable parameters(Function escapeString, Supplier optionalPattern) { @@ -88,24 +93,52 @@ private static void casesForString( String text = textSupplier.get(); return new TextAndPattern(text, escapeString.apply(text)); }, true); + cases(cases, title + " matches self case insensitive", () -> { + String text = textSupplier.get(); + return new TextAndPattern(randomCasing(text), escapeString.apply(text)); + }, true, true); cases(cases, title + " doesn't match self with trailing", () -> { String text = textSupplier.get(); return new TextAndPattern(text, escapeString.apply(text) + randomAlphaOfLength(1)); }, false); + cases(cases, title + " doesn't match self with trailing case insensitive", () -> { + String text = textSupplier.get(); + return new TextAndPattern(randomCasing(text), escapeString.apply(text) + randomAlphaOfLength(1)); + }, true, false); cases(cases, title + " matches self with optional trailing", () -> { String text = randomAlphaOfLength(1); return new TextAndPattern(text, escapeString.apply(text) + optionalPattern.get()); }, true); + cases(cases, title + " matches self with optional trailing case insensitive", () -> { + String text = randomAlphaOfLength(1); + return new TextAndPattern(randomCasing(text), escapeString.apply(text) + optionalPattern.get()); + }, true, true); if (canGenerateDifferent) { cases(cases, title + " doesn't match different", () -> { String text = textSupplier.get(); String different = escapeString.apply(randomValueOtherThan(text, textSupplier)); return new TextAndPattern(text, different); }, false); + cases(cases, title + " doesn't match different case insensitive", () -> { + String text = textSupplier.get(); + Predicate predicate = t -> t.toLowerCase(Locale.ROOT).equals(text.toLowerCase(Locale.ROOT)); + String different = escapeString.apply(randomValueOtherThanMany(predicate, textSupplier)); + return new TextAndPattern(text, different); + }, true, false); } } private static void cases(List cases, String title, Supplier textAndPattern, boolean expected) { + cases(cases, title, textAndPattern, false, expected); + } + + private static void cases( + List cases, + String title, + Supplier textAndPattern, + boolean caseInsensitive, + boolean expected + ) { for (DataType type : DataType.stringTypes()) { cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD, DataType.BOOLEAN), () -> { TextAndPattern v = textAndPattern.get(); @@ -113,25 +146,27 @@ private static void cases(List cases, String title, Supplier { - TextAndPattern v = textAndPattern.get(); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"), - new TestCaseSupplier.TypedData(new BytesRef(v.pattern), DataType.KEYWORD, "pattern").forceLiteral() + new TestCaseSupplier.TypedData(caseInsensitive, DataType.BOOLEAN, "caseInsensitive").forceLiteral() ), startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"), DataType.BOOLEAN, equalTo(expected) ); })); + if (caseInsensitive == false) { + cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD), () -> { + TextAndPattern v = textAndPattern.get(); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"), + new TestCaseSupplier.TypedData(new BytesRef(v.pattern), DataType.KEYWORD, "pattern").forceLiteral() + ), + startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"), + DataType.BOOLEAN, + equalTo(expected) + ); + })); + } } } @@ -150,7 +185,9 @@ static Expression buildRLike(Logger logger, Source source, List args return caseInsensitiveBool ? new RLike(source, expression, new RLikePattern(patternString), true) - : new RLike(source, expression, new RLikePattern(patternString)); + : (randomBoolean() + ? new RLike(source, expression, new RLikePattern(patternString)) + : new RLike(source, expression, new RLikePattern(patternString), false)); } @AfterClass diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeSerializationTests.java index 1bbf124864682..d1399d5e635c6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeSerializationTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index bb59f9e501efb..a9e9e5f917785 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -20,10 +20,12 @@ import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.junit.AfterClass; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator; @@ -38,12 +40,13 @@ public WildcardLikeTests(@Name("TestCase") Supplier t @ParametersFactory public static Iterable parameters() { - List cases = (List) RLikeTests.parameters(str -> { - for (String syntax : new String[] { "\\", "*" }) { + final Function escapeString = str -> { + for (String syntax : new String[] { "\\", "*", "?" }) { str = str.replace(syntax, "\\" + syntax); } return str; - }, () -> "*"); + }; + List cases = (List) RLikeTests.parameters(escapeString, () -> "*"); List suppliers = new ArrayList<>(); addCases(suppliers); @@ -83,11 +86,15 @@ protected Expression build(Source source, List args) { static Expression buildWildcardLike(Source source, List args) { Expression expression = args.get(0); Literal pattern = (Literal) args.get(1); - if (args.size() > 2) { - Literal caseInsensitive = (Literal) args.get(2); - assertThat(caseInsensitive.fold(FoldContext.small()), equalTo(false)); - } - return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString())); + Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null; + boolean caseInsesitiveBool = caseInsensitive != null && (boolean) caseInsensitive.fold(FoldContext.small()); + + WildcardPattern wildcardPattern = new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString()); + return caseInsesitiveBool + ? new WildcardLike(source, expression, wildcardPattern, true) + : (randomBoolean() + ? new WildcardLike(source, expression, wildcardPattern) + : new WildcardLike(source, expression, wildcardPattern, false)); } @AfterClass diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 0d806771787dd..0901ff3d15a91 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -32,6 +32,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -79,6 +81,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -635,6 +638,88 @@ public void testIsNotNullOnCase_With_IS_NULL() { var source = as(filter.child(), EsRelation.class); } + /* + * Limit[1000[INTEGER],false] + * \_Filter[RLIKE(first_name{f}#4, "VALÜ*", true)] + * \_EsRelation[test][_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..] + */ + public void testReplaceUpperStringCasinqgWithInsensitiveRLike() { + var plan = localPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) RLIKE \"VALÜ*\""); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var rlike = as(filter.condition(), RLike.class); + var field = as(rlike.field(), FieldAttribute.class); + assertThat(field.fieldName(), is("first_name")); + assertThat(rlike.pattern().pattern(), is("VALÜ*")); + assertThat(rlike.caseInsensitive(), is(true)); + var source = as(filter.child(), EsRelation.class); + } + + // same plan as above, but lower case pattern + public void testReplaceLowerStringCasingWithInsensitiveRLike() { + var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE \"valü*\""); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var rlike = as(filter.condition(), RLike.class); + var field = as(rlike.field(), FieldAttribute.class); + assertThat(field.fieldName(), is("first_name")); + assertThat(rlike.pattern().pattern(), is("valü*")); + assertThat(rlike.caseInsensitive(), is(true)); + var source = as(filter.child(), EsRelation.class); + } + + /** + * LocalRelation[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu + * ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],EMPTY] + */ + public void testReplaceStringCasingAndRLikeWithLocalRelation() { + var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE \"VALÜ*\""); + + var local = as(plan, LocalRelation.class); + assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY)); + } + + // same plan as in testReplaceUpperStringCasingWithInsensitiveRLike, but with LIKE instead of RLIKE + public void testReplaceUpperStringCasingWithInsensitiveLike() { + var plan = localPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) LIKE \"VALÜ*\""); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var wlike = as(filter.condition(), WildcardLike.class); + var field = as(wlike.field(), FieldAttribute.class); + assertThat(field.fieldName(), is("first_name")); + assertThat(wlike.pattern().pattern(), is("VALÜ*")); + assertThat(wlike.caseInsensitive(), is(true)); + var source = as(filter.child(), EsRelation.class); + } + + // same plan as above, but lower case pattern + public void testReplaceLowerStringCasingWithInsensitiveLike() { + var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) LIKE \"valü*\""); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var wlike = as(filter.condition(), WildcardLike.class); + var field = as(wlike.field(), FieldAttribute.class); + assertThat(field.fieldName(), is("first_name")); + assertThat(wlike.pattern().pattern(), is("valü*")); + assertThat(wlike.caseInsensitive(), is(true)); + var source = as(filter.child(), EsRelation.class); + } + + /** + * LocalRelation[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu + * ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],EMPTY] + */ + public void testReplaceStringCasingAndLikeWithLocalRelation() { + var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) LIKE \"VALÜ*\""); + + var local = as(plan, LocalRelation.class); + assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY)); + } + private IsNotNull isNotNull(Expression field) { return new IsNotNull(EMPTY, field); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 8ce7124b501f2..8aeef36e8d9c1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -83,6 +83,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; @@ -205,7 +206,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; -// @TestLogging(value = "org.elasticsearch.xpack.esql:DEBUG", reason = "debug") +// @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class PhysicalPlanOptimizerTests extends ESTestCase { private static final String PARAM_FORMATTING = "%1$s"; @@ -2201,6 +2202,163 @@ public void testNoPushDownChangeCase() { assertThat(source.query(), nullValue()); } + /* + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu + * ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],false] + * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu + * ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8]] + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]<[],[]> + * \_EsQueryExec[test], indexMode[standard], query[{"esql_single_value":{"field":"first_name","next":{"regexp":{"first_name": + * {"value":"foo*","flags_value":65791,"case_insensitive":true,"max_determinized_states":10000,"boost":0.0}}}, + * "source":"TO_LOWER(first_name) RLIKE \"foo*\"@2:9"}}][_doc{f}#25], limit[1000], sort[] estimatedRowSize[332] + */ + private void doTestPushDownCaseChangeRegexMatch(String query, String expected) { + var plan = physicalPlan(query); + var optimized = optimizedPlan(plan); + + var topLimit = as(optimized, LimitExec.class); + var exchange = asRemoteExchange(topLimit.child()); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var source = as(fieldExtract.child(), EsQueryExec.class); + + var singleValue = as(source.query(), SingleValueQuery.Builder.class); + assertThat(stripThrough(singleValue.toString()), is(stripThrough(expected))); + } + + public void testPushDownLowerCaseChangeRLike() { + doTestPushDownCaseChangeRegexMatch(""" + FROM test + | WHERE TO_LOWER(first_name) RLIKE "foo*" + """, """ + { + "esql_single_value": { + "field": "first_name", + "next": { + "regexp": { + "first_name": { + "value": "foo*", + "flags_value": 65791, + "case_insensitive": true, + "max_determinized_states": 10000, + "boost": 0.0 + } + } + }, + "source": "TO_LOWER(first_name) RLIKE \\"foo*\\"@2:9" + } + } + """); + } + + public void testPushDownUpperCaseChangeRLike() { + doTestPushDownCaseChangeRegexMatch(""" + FROM test + | WHERE TO_UPPER(first_name) RLIKE "FOO*" + """, """ + { + "esql_single_value": { + "field": "first_name", + "next": { + "regexp": { + "first_name": { + "value": "FOO*", + "flags_value": 65791, + "case_insensitive": true, + "max_determinized_states": 10000, + "boost": 0.0 + } + } + }, + "source": "TO_UPPER(first_name) RLIKE \\"FOO*\\"@2:9" + } + } + """); + } + + public void testPushDownLowerCaseChangeLike() { + doTestPushDownCaseChangeRegexMatch(""" + FROM test + | WHERE TO_LOWER(first_name) LIKE "foo*" + """, """ + { + "esql_single_value": { + "field": "first_name", + "next": { + "wildcard": { + "first_name": { + "wildcard": "foo*", + "case_insensitive": true, + "boost": 0.0 + } + } + }, + "source": "TO_LOWER(first_name) LIKE \\"foo*\\"@2:9" + } + } + """); + } + + public void testPushDownUpperCaseChangeLike() { + doTestPushDownCaseChangeRegexMatch(""" + FROM test + | WHERE TO_UPPER(first_name) LIKE "FOO*" + """, """ + { + "esql_single_value": { + "field": "first_name", + "next": { + "wildcard": { + "first_name": { + "wildcard": "FOO*", + "case_insensitive": true, + "boost": 0.0 + } + } + }, + "source": "TO_UPPER(first_name) LIKE \\"FOO*\\"@2:9" + } + } + """); + } + + /* + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang + * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9],false] + * \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang + * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9]] + * \_FieldExtractExec[_meta_field{f}#10, gender{f}#6, hire_date{f}#11, jo..]<[],[]> + * \_LimitExec[1000[INTEGER]] + * \_FilterExec[LIKE(first_name{f}#5, "FOO*", true) OR IN(1[INTEGER],2[INTEGER],3[INTEGER],emp_no{f}#4 + 1[INTEGER])] + * \_FieldExtractExec[first_name{f}#5, emp_no{f}#4]<[],[]> + * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#26], limit[], sort[] estimatedRowSize[332] + */ + public void testChangeCaseAsInsensitiveWildcardLikeNotPushedDown() { + var esql = """ + FROM test + | WHERE TO_UPPER(first_name) LIKE "FOO*" OR emp_no + 1 IN (1, 2, 3) + """; + var plan = physicalPlan(esql); + var optimized = optimizedPlan(plan); + + var topLimit = as(optimized, LimitExec.class); + var exchange = asRemoteExchange(topLimit.child()); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var limit = as(fieldExtract.child(), LimitExec.class); + var filter = as(limit.child(), FilterExec.class); + fieldExtract = as(filter.child(), FieldExtractExec.class); + var source = as(fieldExtract.child(), EsQueryExec.class); + + var or = as(filter.condition(), Or.class); + var wildcard = as(or.left(), WildcardLike.class); + assertThat(Expressions.name(wildcard.field()), is("first_name")); + assertThat(wildcard.pattern().pattern(), is("FOO*")); + assertThat(wildcard.caseInsensitive(), is(true)); + } + public void testPushDownNotRLike() { var plan = physicalPlan(""" from test @@ -2432,6 +2590,17 @@ public void testDissect() { assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES + KEYWORD_EST)); } + /* + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang + * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2],false] + * \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang + * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2]] + * \_FieldExtractExec[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, ge..]<[],[]> + * \_EsQueryExec[test], indexMode[standard], query[{"wildcard":{"_index":{"wildcard":"test*","boost":0.0}}}][_doc{f}#27], + * limit[1000], sort[] estimatedRowSize[382] + * + */ public void testPushDownMetadataIndexInWildcard() { var plan = physicalPlan(""" from test metadata _index @@ -8097,7 +8266,7 @@ private PhysicalPlan physicalPlan(String query, TestDataSource dataSource, boole var physical = mapper.map(logical); // System.out.println("Physical\n" + physical); if (assertSerialization) { - assertSerialization(physical); + assertSerialization(physical, config); } return physical; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java index 864a59338c6a9..f7ea5d17f485b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java @@ -18,8 +18,8 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.Range; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java index 254b9197204a0..96e26fbd37a4c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java @@ -47,8 +47,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java index d8cae21257201..ef3cacfb0e471 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java @@ -18,8 +18,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java index 449728f43a3a2..39926c63e7210 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java @@ -15,8 +15,8 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.util.StringUtils; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 4f73b2b99d628..22e99c0cf2a64 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -28,8 +28,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;