From 7629d8246983ffdfc51634813fffbae7ea909aab Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Tue, 30 Jun 2026 17:53:26 +0800 Subject: [PATCH] [feature](be) Support expression zonemap pruning (#63389) Problem Summary: Support expression-based ZoneMap pruning for internal and Parquet scan paths. Supported expressions include comparisons, IN/NOT IN, IS NULL/IS NOT NULL, and starts_with. The implementation adds segment/page/row-group pruning, conservative fallback semantics, and profile counters, with BE unit and SQL regression coverage. CHAR string range statistics are supported through the trimmed logical bounds produced by ZoneMap deserialization, so CHAR predicates can participate in expression ZoneMap pruning. This follow-up also removes duplicated capability checks between `can_evaluate_zonemap_filter` and `evaluate_zonemap_filter`. Capability checks now focus on expression shape and NULL-literal cases, while storage/runtime datatype invariants are asserted with `DORIS_CHECK` in the evaluation path. The latest follow-up centralizes HybridSet-to-Field materialization for scalar and direct IN predicates, removes identity-cast unwrapping from expression ZoneMap matching, and lets null-aware runtime filters use ZoneMap pruning when the target zone has no NULL values while conservatively keeping zones that may contain NULL. The latest crash-fix follow-up makes comparison expression ZoneMap capability checks conservative when the scan slot type and literal type are not directly compatible, such as external table predicates comparing `Nullable(DateTimeV2(6))` with a `DateTimeV2(0)` literal. In that case Doris now skips expression ZoneMap pruning instead of triggering a BE `DORIS_CHECK`, leaving the normal expression evaluation path to filter rows. Support expression-based ZoneMap pruning for scan predicates to skip data using ZoneMap statistics. - Test: Unit Test / Regression test / Format check - DORIS_HOME= ninja -C be/ut_build_ASAN Exprs Storage Exec Format doris_be_test - DORIS_HOME= ninja -C be/ut_build_ASAN Exprs doris_be_test - ./run-be-ut.sh --run --filter=SegmentFilterHelpersTest.* - ./run-be-ut.sh --run --filter=ScanNormalizePredicateTest.*:RuntimeFilterConsumerHelperTest.*:VRuntimeFilterWrapperSamplingTest.*:ScannerLateArrivalRFTest.* - ./run-be-ut.sh --run --filter=ParquetExprTest.test_expr_zonemap_* - ./run-be-ut.sh --run --filter=ExprZonemapFilterTest.* - ./run-be-ut.sh --run --filter='ExprZonemapFilterTest.FunctionStringStartsWithZonemapUsesPrefixRange:function_string_test.function_starts_with_test' - ./run-be-ut.sh --run --filter='ExprZonemapFilterTest.*:ParquetExprTest.test_expr_zonemap_*' - ninja -C be/ut_build_ASAN src/storage/CMakeFiles/Storage.dir/segment/segment.cpp.o src/storage/CMakeFiles/Storage.dir/segment/segment_iterator.cpp.o - ./run-be-ut.sh --run --filter=ParquetExprTest.test_expr_zonemap_page_filter_keeps_unsupported_results_and_counts_stats - doris-local-regression.sh all -d inverted_index_p0 -s test_index_range_in_select - ./run-regression-test.sh --conf --run -d query_p0/expr_zonemap -s test_expr_zonemap_pruning - ./run-regression-test.sh --conf --run -d query_p1/expr_zonemap -s test_expr_zonemap_pruning_p1 - doris-local-regression --network 10.26.20.3/24 all -d query_p0/expr_zonemap -s test_expr_zonemap_pruning - doris-local-regression --network 10.26.20.3/24 all -d query_p1/expr_zonemap -s test_expr_zonemap_pruning_p1 - doris-local-regression --network 10.26.20.3/24 all -d mtmv_p0 -s test_iceberg_mtmv -conf enableIcebergTest=true -conf iceberg_rest_uri_port=18283 -conf iceberg_minio_port=19001 -conf externalEnvIp=127.0.0.1 - build-support/clang-format.sh --files be/src/exprs/function/functions_comparison.h be/test/exprs/expr_zonemap_filter_test.cpp - build-support/check-format.sh --files be/src/exprs/function/functions_comparison.h be/test/exprs/expr_zonemap_filter_test.cpp - build-support/clang-format.sh - build-support/check-format.sh - doris-local-regression --network 10.26.20.3/24 run -d query_p0/expr_zonemap -s test_expr_zonemap_pruning - doris-local-regression --network 10.26.20.3/24 run -d query_p1/expr_zonemap -s test_expr_zonemap_pruning_p1 - git diff --check - Behavior changed: Yes. Supported scan predicates, including CHAR string predicates, can now prune data with expression ZoneMap evaluation. - Does this need documentation: No (cherry picked from commit 257417b029595580f6632b2e0fcf5e5ac3ccdb7d) --- be/src/exec/operator/olap_scan_operator.cpp | 10 + be/src/exec/operator/olap_scan_operator.h | 11 + .../runtime_filter_consumer.cpp | 2 +- be/src/exec/scan/olap_scanner.cpp | 8 + be/src/exprs/expr_zonemap_filter.cpp | 267 ++++++ be/src/exprs/expr_zonemap_filter.h | 124 +++ be/src/exprs/function/function.cpp | 6 + be/src/exprs/function/function.h | 19 + be/src/exprs/function/function_string.cpp | 90 +- be/src/exprs/function/functions_comparison.h | 144 +++ be/src/exprs/function/is_not_null.h | 13 +- be/src/exprs/function/is_null.h | 11 + be/src/exprs/vcompound_pred.h | 48 + be/src/exprs/vdirect_in_predicate.h | 69 +- be/src/exprs/vectorized_fn_call.cpp | 10 + be/src/exprs/vectorized_fn_call.h | 2 + be/src/exprs/vexpr.cpp | 6 + be/src/exprs/vexpr.h | 5 + be/src/exprs/vexpr_context.cpp | 17 + be/src/exprs/vexpr_context.h | 5 + be/src/exprs/vin_predicate.cpp | 53 +- be/src/exprs/vin_predicate.h | 13 +- be/src/exprs/virtual_slot_ref.h | 2 +- be/src/exprs/vruntimefilter_wrapper.cpp | 38 +- be/src/exprs/vruntimefilter_wrapper.h | 5 + be/src/exprs/vslot_ref.h | 2 +- be/src/format/orc/vorc_reader.cpp | 2 +- .../format/parquet/vparquet_group_reader.cpp | 2 +- be/src/format/parquet/vparquet_reader.cpp | 221 ++++- be/src/format/parquet/vparquet_reader.h | 17 + .../index/zone_map/zonemap_eval_context.cpp | 49 ++ .../index/zone_map/zonemap_eval_context.h | 70 ++ .../index/zone_map/zonemap_filter_result.h | 30 + be/src/storage/olap_common.h | 5 + be/src/storage/segment/column_reader.cpp | 29 + be/src/storage/segment/column_reader.h | 7 + be/src/storage/segment/segment.cpp | 79 ++ be/src/storage/segment/segment_iterator.cpp | 125 ++- be/src/storage/segment/segment_iterator.h | 3 + be/test/exprs/expr_zonemap_filter_test.cpp | 819 ++++++++++++++++++ be/test/format/parquet/parquet_expr_test.cpp | 415 ++++++++- .../segment_iterator_expr_zonemap_test.cpp | 244 ++++++ .../org/apache/doris/qe/SessionVariable.java | 20 + gensrc/thrift/PaloInternalService.thrift | 4 + .../test_expr_zonemap_pruning.groovy | 227 +++++ .../test_expr_zonemap_pruning_p1.groovy | 157 ++++ 46 files changed, 3481 insertions(+), 24 deletions(-) create mode 100644 be/src/exprs/expr_zonemap_filter.cpp create mode 100644 be/src/exprs/expr_zonemap_filter.h create mode 100644 be/src/storage/index/zone_map/zonemap_eval_context.cpp create mode 100644 be/src/storage/index/zone_map/zonemap_eval_context.h create mode 100644 be/src/storage/index/zone_map/zonemap_filter_result.h create mode 100644 be/test/exprs/expr_zonemap_filter_test.cpp create mode 100644 be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp create mode 100644 regression-test/suites/query_p0/expr_zonemap/test_expr_zonemap_pruning.groovy create mode 100644 regression-test/suites/query_p1/expr_zonemap/test_expr_zonemap_pruning_p1.groovy diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index da855aff32d4d8..911a321f21f5f9 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -222,6 +222,16 @@ Status OlapScanLocalState::_init_profile() { _stats_filtered_counter = ADD_COUNTER(_segment_profile, "RowsStatsFiltered", TUnit::UNIT); _stats_rp_filtered_counter = ADD_COUNTER(_segment_profile, "RowsZoneMapRuntimePredicateFiltered", TUnit::UNIT); + _expr_zonemap_filtered_segment_counter = + ADD_COUNTER(_segment_profile, "ExprZoneMapFilteredSegments", TUnit::UNIT); + _expr_zonemap_filtered_page_counter = + ADD_COUNTER(_segment_profile, "ExprZoneMapFilteredPages", TUnit::UNIT); + _expr_zonemap_unusable_counter = + ADD_COUNTER(_segment_profile, "ExprZoneMapUnusableEvals", TUnit::UNIT); + _in_zonemap_point_check_counter = + ADD_COUNTER(_segment_profile, "InZoneMapPointCheckCount", TUnit::UNIT); + _in_zonemap_range_only_counter = + ADD_COUNTER(_segment_profile, "InZoneMapRangeOnlyCount", TUnit::UNIT); _bf_filtered_counter = ADD_COUNTER(_segment_profile, "RowsBloomFilterFiltered", TUnit::UNIT); _dict_filtered_counter = ADD_COUNTER(_segment_profile, "SegmentDictFiltered", TUnit::UNIT); _del_filtered_counter = ADD_COUNTER(_scanner_profile, "RowsDelFiltered", TUnit::UNIT); diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index e8664257714f16..57c4912b777f6b 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -156,6 +156,17 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _stats_filtered_counter = nullptr; RuntimeProfile::Counter* _stats_rp_filtered_counter = nullptr; + // Number of whole segments skipped by expression ZoneMap evaluation. + RuntimeProfile::Counter* _expr_zonemap_filtered_segment_counter = nullptr; + // Number of pages skipped by expression ZoneMap evaluation after page index ranges are built. + RuntimeProfile::Counter* _expr_zonemap_filtered_page_counter = nullptr; + // Number of expression ZoneMap evaluations that reached the evaluator but could not use the + // current ZoneMap context, such as missing slot/page ZoneMap or unusable range statistics. + RuntimeProfile::Counter* _expr_zonemap_unusable_counter = nullptr; + // Number of IN-predicate ZoneMap evaluations that used per-value point checks. + RuntimeProfile::Counter* _in_zonemap_point_check_counter = nullptr; + // Number of IN-predicate ZoneMap evaluations that fell back to min/max range overlap only. + RuntimeProfile::Counter* _in_zonemap_range_only_counter = nullptr; RuntimeProfile::Counter* _bf_filtered_counter = nullptr; RuntimeProfile::Counter* _dict_filtered_counter = nullptr; RuntimeProfile::Counter* _del_filtered_counter = nullptr; diff --git a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp index e33bb8e41c39d8..d82900d2540c22 100644 --- a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp @@ -101,7 +101,7 @@ Status RuntimeFilterConsumer::_get_push_exprs(std::vector& co node.in_predicate.__set_is_not_in(false); node.__set_opcode(TExprOpcode::FILTER_IN); node.__set_is_nullable(false); - auto in_pred = VDirectInPredicate::create_shared(node, _wrapper->hybrid_set()); + auto in_pred = VDirectInPredicate::create_shared(node, _wrapper->hybrid_set(), true); in_pred->add_child(probe_ctx->root()); auto wrapper = VRuntimeFilterWrapper::create_shared( node, in_pred, get_in_list_ignore_thredhold(_wrapper->hybrid_set()->size()), diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 5dec25aca0c4b5..56684d5ffff802 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -739,6 +739,14 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_rows_expr_cond_input_counter, stats.expr_cond_input_rows); COUNTER_UPDATE(local_state->_stats_filtered_counter, stats.rows_stats_filtered); COUNTER_UPDATE(local_state->_stats_rp_filtered_counter, stats.rows_stats_rp_filtered); + COUNTER_UPDATE(local_state->_expr_zonemap_filtered_segment_counter, + stats.expr_zonemap_filtered_segments); + COUNTER_UPDATE(local_state->_expr_zonemap_filtered_page_counter, + stats.expr_zonemap_filtered_pages); + COUNTER_UPDATE(local_state->_expr_zonemap_unusable_counter, stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(local_state->_in_zonemap_point_check_counter, + stats.in_zonemap_point_check_count); + COUNTER_UPDATE(local_state->_in_zonemap_range_only_counter, stats.in_zonemap_range_only_count); COUNTER_UPDATE(local_state->_dict_filtered_counter, stats.segment_dict_filtered); COUNTER_UPDATE(local_state->_bf_filtered_counter, stats.rows_bf_filtered); COUNTER_UPDATE(local_state->_del_filtered_counter, stats.rows_del_filtered); diff --git a/be/src/exprs/expr_zonemap_filter.cpp b/be/src/exprs/expr_zonemap_filter.cpp new file mode 100644 index 00000000000000..19457beb37416b --- /dev/null +++ b/be/src/exprs/expr_zonemap_filter.cpp @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/expr_zonemap_filter.h" + +#include +#include +#include + +#include "common/logging.h" +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" +#include "exprs/hybrid_set.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "runtime/runtime_state.h" + +namespace doris::expr_zonemap { +namespace { + +constexpr size_t kInZoneMapPointCheckThreshold = 64; + +std::optional> field_from_literal_expr(const VExprSPtr& expr) { + auto literal = std::dynamic_pointer_cast(expr); + if (literal == nullptr) { + return std::nullopt; + } + Field field; + literal->get_column_ptr()->get(0, field); + return std::make_pair(std::move(field), literal->get_data_type()); +} + +bool value_in_range(const Field& value, const Field& min_value, const Field& max_value) { + return value >= min_value && value <= max_value; +} + +} // namespace + +TExprNode create_texpr_node_from_hybrid_set_value(const void* data, const PrimitiveType& type, + int precision, int scale) { + if (is_string_type(type)) { + const auto* value = reinterpret_cast(data); + auto field = Field::create_field(String(value->data, value->size)); + return create_texpr_node_from(field, type, precision, scale); + } + return create_texpr_node_from(data, type, precision, scale); +} + +Status materialize_hybrid_set_for_zonemap_filter(HybridSetBase& set, const DataTypePtr& data_type, + InZonemapMaterializedSet* result) { + DORIS_CHECK(result != nullptr); + DORIS_CHECK(data_type != nullptr); + const auto value_type = remove_nullable(data_type); + DORIS_CHECK(value_type != nullptr); + + result->contains_null = set.contain_null(); + result->values.clear(); + result->min_value = Field(); + result->max_value = Field(); + + auto* iterator = set.begin(); + while (iterator->has_next()) { + const void* value = iterator->get_value(); + if (value != nullptr) { + TExprNode literal_node = create_texpr_node_from_hybrid_set_value( + value, value_type->get_primitive_type(), value_type->get_precision(), + value_type->get_scale()); + auto literal = VLiteral::create_shared(literal_node); + Field field; + literal->get_column_ptr()->get(0, field); + result->values.emplace_back(std::move(field)); + } + iterator->next(); + } + + if (!result->values.empty()) { + auto minmax = std::ranges::minmax_element( + result->values, [](const Field& lhs, const Field& rhs) { return lhs < rhs; }); + result->min_value = *minmax.min; + result->max_value = *minmax.max; + } + return Status::OK(); +} + +std::optional extract_slot_and_literal(const VExprSPtrs& args) { + if (args.size() != 2) { + return std::nullopt; + } + + if (auto slot = std::dynamic_pointer_cast(args[0]); slot) { + auto literal = field_from_literal_expr(args[1]); + if (!literal.has_value()) { + return std::nullopt; + } + auto [literal_value, literal_type] = std::move(*literal); + return SlotLiteral {.slot_index = slot->column_id(), + .slot_type = slot->data_type(), + .literal = std::move(literal_value), + .literal_type = std::move(literal_type), + .literal_on_left = false}; + } + + if (auto slot = std::dynamic_pointer_cast(args[1]); slot) { + auto literal = field_from_literal_expr(args[0]); + if (!literal.has_value()) { + return std::nullopt; + } + auto [literal_value, literal_type] = std::move(*literal); + return SlotLiteral {.slot_index = slot->column_id(), + .slot_type = slot->data_type(), + .literal = std::move(literal_value), + .literal_type = std::move(literal_type), + .literal_on_left = true}; + } + + return std::nullopt; +} + +bool range_stats_usable_for_zonemap(const segment_v2::ZoneMap& zone_map, + const DataTypePtr& data_type) { + if (zone_map.pass_all || zone_map.has_nan || zone_map.has_positive_inf || + zone_map.has_negative_inf) { + return false; + } + DORIS_CHECK(data_type != nullptr); + auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + DORIS_CHECK(field_types_compatible(zone_map.min_value.get_type(), primitive_type)); + DORIS_CHECK(field_types_compatible(zone_map.max_value.get_type(), primitive_type)); + return true; +} + +ZoneMapFilterResult eval_null_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtrs& arguments, + bool is_null) { + DORIS_CHECK(arguments.size() == 1); + auto slot = std::dynamic_pointer_cast(arguments[0]); + DORIS_CHECK(slot != nullptr); + auto zone_map_ptr = ctx.zone_map(slot->column_id()); + if (zone_map_ptr == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ptr; + if (is_null) { + return zone_map.has_null ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch; + } + return zone_map.has_not_null ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch; +} + +ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtr& slot_expr, + bool is_not_in, const std::vector& values, + const Field& min_value, const Field& max_value) { + auto slot = std::dynamic_pointer_cast(slot_expr); + DORIS_CHECK(slot != nullptr); + // Empty IN has no candidate values, while NOT IN with an empty set cannot filter anything. + if (values.empty()) { + return is_not_in ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch; + } + + // The caller has materialized the IN set and precomputed its non-null min/max. They must match + // the expression slot type before being compared with storage zone-map statistics. + DORIS_CHECK(!min_value.is_null()); + DORIS_CHECK(!max_value.is_null()); + auto data_type = remove_nullable(slot->data_type()); + DORIS_CHECK(data_type != nullptr); + DORIS_CHECK(field_types_compatible(min_value.get_type(), data_type->get_primitive_type())); + DORIS_CHECK(field_types_compatible(max_value.get_type(), data_type->get_primitive_type())); + + // Re-check against the reader-schema type and the available zone map. Missing or unsupported + // metadata must conservatively fall back to may-match. + auto slot_type = fetch_compatible_slot_type(ctx, slot->column_id(), slot->data_type()); + if (slot_type == nullptr) { + return unsupported_zonemap_filter(ctx); + } + auto zone_map_ptr = ctx.zone_map(slot->column_id()); + if (zone_map_ptr == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ptr; + // IN values are all non-null here, so an all-null zone cannot match. + if (!zone_map.has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + + if (!range_stats_usable_for_zonemap(zone_map, slot_type)) { + return unsupported_zonemap_filter(ctx); + } + + if (is_not_in) { + // NOT IN can only prune when the whole zone contains exactly one non-null value and that + // value is excluded by the set. Wider ranges may contain values that are not filtered. + if (zone_map.min_value == zone_map.max_value) { + const bool only_value_is_filtered = std::ranges::any_of( + values, [&](const Field& value) { return value == zone_map.min_value; }); + return only_value_is_filtered ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + return ZoneMapFilterResult::kMayMatch; + } + + // First use the materialized IN-set min/max to rule out disjoint zone-map ranges. + if (zone_map.max_value < min_value || zone_map.min_value > max_value) { + return ZoneMapFilterResult::kNoMatch; + } + + // For large IN sets, avoid checking every point on the scan hot path. The range overlap above + // is only a coarse may-match signal. + if (values.size() > kInZoneMapPointCheckThreshold) { + ++ctx.stats.in_zonemap_range_only_count; + return ZoneMapFilterResult::kMayMatch; + } + + // For small IN sets, verify whether any candidate value can fall into the zone-map range. + ++ctx.stats.in_zonemap_point_check_count; + for (const auto& value : values) { + if (value_in_range(value, zone_map.min_value, zone_map.max_value)) { + return ZoneMapFilterResult::kMayMatch; + } + } + return ZoneMapFilterResult::kNoMatch; +} + +// Return the only slot ordinal referenced by a zonemap-evaluable expression. A negative result is +// the conservative fallback marker for unsupported expressions, multi-slot expressions, or invalid +// slot ordinals, so callers can skip schema-indexed zonemap pruning safely. +int single_slot_zonemap_index(const VExprContextSPtr& ctx) { + DORIS_CHECK(ctx != nullptr); + const auto& root = ctx->root(); + DORIS_CHECK(root != nullptr); + if (!root->can_evaluate_zonemap_filter()) { + return -1; + } + + std::set slot_indexes; + root->collect_slot_column_ids(slot_indexes); + if (slot_indexes.size() != 1) { + return -1; + } + + return *slot_indexes.begin(); +} + +bool is_expr_zonemap_filter_enabled(const RuntimeState* state) { + if (state == nullptr) { + return true; + } + const auto& query_options = state->query_options(); + return !query_options.__isset.enable_expr_zonemap_filter || + query_options.enable_expr_zonemap_filter; +} + +} // namespace doris::expr_zonemap diff --git a/be/src/exprs/expr_zonemap_filter.h b/be/src/exprs/expr_zonemap_filter.h new file mode 100644 index 00000000000000..2d8ff4d0c72207 --- /dev/null +++ b/be/src/exprs/expr_zonemap_filter.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/define_primitive_type.h" +#include "core/field.h" +#include "exprs/vexpr_fwd.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" + +namespace doris { +class HybridSetBase; +class RuntimeState; +class TExprNode; +} // namespace doris + +namespace doris::expr_zonemap { + +struct InZonemapMaterializedSet { + bool contains_null = false; + std::vector values; + Field min_value; + Field max_value; +}; + +struct SlotLiteral { + // Slot ordinal in the current expression binding. It is also the key used to look up the + // corresponding reader-schema type and zone map from ZoneMapEvalContext. + int slot_index; + // Type carried by the slot expression. It is kept separately because evaluation first has to + // verify that the expression binding is compatible with the reader-schema type stored in the + // ZoneMapEvalContext before using schema-indexed zone-map statistics. + DataTypePtr slot_type; + // Constant literal value paired with the slot, already materialized as a Field for zone-map + // comparisons. + Field literal; + // Type carried by the literal expression. It is needed with slot_type so capability checks can + // reject incompatible slot/literal comparisons before runtime evaluation instead of evaluating + // zone-map ranges with mismatched Field types. + DataTypePtr literal_type; + // Whether the original expression shape is literal slot instead of slot literal. The + // comparison operator is directional, so evaluators need this flag to normalize cases like + // `5 < slot` into the equivalent slot-side comparison before checking the zone map. Non-symmetric + // function evaluators also use it to reject unsupported shapes such as `starts_with(literal, + // slot)`, because only `starts_with(slot, literal)` can be mapped to a safe zone-map range. + bool literal_on_left; +}; + +std::optional extract_slot_and_literal(const VExprSPtrs& args); + +TExprNode create_texpr_node_from_hybrid_set_value(const void* data, const PrimitiveType& type, + int precision, int scale); + +Status materialize_hybrid_set_for_zonemap_filter(HybridSetBase& set, const DataTypePtr& data_type, + InZonemapMaterializedSet* result); + +inline bool field_types_compatible(PrimitiveType lhs, PrimitiveType rhs) { + return lhs == rhs || (is_string_type(lhs) && is_string_type(rhs)); +} + +inline bool data_types_compatible(const DataTypePtr& lhs, const DataTypePtr& rhs) { + if (lhs == nullptr || rhs == nullptr) { + return false; + } + const auto lhs_type = remove_nullable(lhs); + const auto rhs_type = remove_nullable(rhs); + const auto lhs_primitive_type = lhs_type->get_primitive_type(); + const auto rhs_primitive_type = rhs_type->get_primitive_type(); + if (is_string_type(lhs_primitive_type) && is_string_type(rhs_primitive_type)) { + return true; + } + return lhs_type->equals(*rhs_type); +} + +inline DataTypePtr fetch_compatible_slot_type(const ZoneMapEvalContext& ctx, int slot_index, + const DataTypePtr& expr_slot_type) { + auto slot_type = ctx.data_type(slot_index); + if (slot_type == nullptr) { + return nullptr; + } + DORIS_CHECK(data_types_compatible(slot_type, expr_slot_type)); + return slot_type; +} + +bool range_stats_usable_for_zonemap(const segment_v2::ZoneMap& zone_map, + const DataTypePtr& data_type); + +ZoneMapFilterResult eval_null_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtrs& arguments, + bool is_null); + +ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtr& slot_expr, + bool is_not_in, const std::vector& values, + const Field& min_value, const Field& max_value); + +// Return the only slot ordinal referenced by a zonemap-evaluable expression in its current +// binding. Expressions that are unsupported by zonemap pruning, reference multiple slots, or use an +// invalid negative slot ordinal return a negative value. +int single_slot_zonemap_index(const VExprContextSPtr& ctx); + +bool is_expr_zonemap_filter_enabled(const RuntimeState* state); + +} // namespace doris::expr_zonemap diff --git a/be/src/exprs/function/function.cpp b/be/src/exprs/function/function.cpp index fa5b12fea2ec6d..6cec038864e8b7 100644 --- a/be/src/exprs/function/function.cpp +++ b/be/src/exprs/function/function.cpp @@ -39,6 +39,7 @@ #include "exec/common/util.hpp" #include "exprs/aggregate/aggregate_function.h" #include "exprs/function/function_helpers.h" +#include "storage/index/zone_map/zonemap_eval_context.h" namespace doris { #include "common/compile_check_begin.h" @@ -407,5 +408,10 @@ bool FunctionBuilderImpl::is_nested_type_date_or_datetime_or_decimal( } } +ZoneMapFilterResult IFunctionBase::evaluate_zonemap_filter( + const ZoneMapEvalContext& ctx, const VExprSPtrs& function_arguments) const { + return unsupported_zonemap_filter(ctx); +} + #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 9e0aebdabaf032..171e889f79b21d 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -43,8 +43,10 @@ #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "exprs/function_context.h" +#include "exprs/vexpr_fwd.h" #include "storage/index/inverted/inverted_index_iterator.h" // IWYU pragma: keep #include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/zone_map/zonemap_filter_result.h" namespace doris { struct InvertedIndexAnalyzerCtx; @@ -76,6 +78,7 @@ struct FunctionAttr { class Field; class VExpr; +class ZoneMapEvalContext; // Only use dispose the variadic argument template @@ -226,6 +229,13 @@ class IFunctionBase { virtual bool can_push_down_to_index() const { return false; } virtual bool is_blockable() const { return false; } + + virtual ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx, + const VExprSPtrs& function_arguments) const; + + virtual bool can_evaluate_zonemap_filter(const VExprSPtrs& /*function_arguments*/) const { + return false; + } }; using FunctionBasePtr = std::shared_ptr; @@ -492,6 +502,15 @@ class DefaultFunction final : public IFunctionBase { bool is_blockable() const override { return function->is_blockable(); } + ZoneMapFilterResult evaluate_zonemap_filter( + const ZoneMapEvalContext& ctx, const VExprSPtrs& function_arguments) const override { + return function->evaluate_zonemap_filter(ctx, function_arguments); + } + + bool can_evaluate_zonemap_filter(const VExprSPtrs& function_arguments) const override { + return function->can_evaluate_zonemap_filter(function_arguments); + } + private: std::shared_ptr function; DataTypes arguments; diff --git a/be/src/exprs/function/function_string.cpp b/be/src/exprs/function/function_string.cpp index 4ce5e484b8609d..4eb0b1462cb3d5 100644 --- a/be/src/exprs/function/function_string.cpp +++ b/be/src/exprs/function/function_string.cpp @@ -24,16 +24,23 @@ #include #include +#include #include #include +#include +#include +#include #include #include "common/cast_set.h" +#include "common/logging.h" #include "common/status.h" #include "core/column/column.h" #include "core/column/column_string.h" +#include "core/data_type/data_type_nullable.h" #include "core/pod_array_fwd.h" #include "core/string_ref.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function/function_reverse.h" #include "exprs/function/function_string_concat.h" #include "exprs/function/function_string_format.h" @@ -1330,8 +1337,87 @@ using FunctionCrc32 = FunctionUnaryToType; using FunctionStringUTF8Length = FunctionUnaryToType; using FunctionStringSpace = FunctionUnaryToType; using FunctionIsValidUTF8 = FunctionUnaryToType; -using FunctionStringStartsWith = - FunctionBinaryToType; + +class FunctionStringStartsWith : public FunctionBinaryToType { +public: + static FunctionPtr create() { return std::make_shared(); } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx, + const VExprSPtrs& arguments) const override { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + auto slot_type = expr_zonemap::fetch_compatible_slot_type(ctx, slot_literal->slot_index, + slot_literal->slot_type); + if (slot_type == nullptr) { + return unsupported_zonemap_filter(ctx); + } + auto zone_map_ref = ctx.zone_map(slot_literal->slot_index); + if (zone_map_ref == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ref; + if (!zone_map.has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + if (!expr_zonemap::range_stats_usable_for_zonemap(zone_map, slot_type)) { + return unsupported_zonemap_filter(ctx); + } + + const auto prefix = slot_literal->literal.as_string_view(); + auto lower = Field::create_field(std::string(prefix)); + if (zone_map.max_value < lower) { + return ZoneMapFilterResult::kNoMatch; + } + auto upper_prefix = _next_prefix_for_starts_with_zonemap(prefix); + if (upper_prefix.has_value() && + !(zone_map.min_value < Field::create_field(*upper_prefix))) { + return ZoneMapFilterResult::kNoMatch; + } + return ZoneMapFilterResult::kMayMatch; + } + + bool can_evaluate_zonemap_filter(const VExprSPtrs& arguments) const override { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + if (!slot_literal.has_value() || slot_literal->literal_on_left) { + return false; + } + + // A NULL prefix makes starts_with(slot, NULL) evaluate to NULL. An empty prefix matches + // every non-NULL string and cannot prune by range. Reject both shapes here before + // evaluate_zonemap_filter is called. + if (slot_literal->literal.is_null()) { + return false; + } + + DORIS_CHECK(slot_literal->slot_type != nullptr); + DORIS_CHECK(slot_literal->literal_type != nullptr); + DORIS_CHECK(is_string_type(remove_nullable(slot_literal->slot_type)->get_primitive_type())); + DORIS_CHECK( + is_string_type(remove_nullable(slot_literal->literal_type)->get_primitive_type())); + + const auto prefix = slot_literal->literal.as_string_view(); + return !prefix.empty(); + } + +private: + static std::optional _next_prefix_for_starts_with_zonemap( + std::string_view prefix) { + // ZoneMap string bounds are compared by bytewise Field ordering. For starts_with(s, p), + // the safe upper bound is the next byte string after p: p <= s < next_prefix(p). + // For example, starts_with(s, "ab") can use the range "ab" <= s < "ac". + std::string upper(prefix); + for (auto i = static_cast(upper.size()) - 1; i >= 0; --i) { + auto byte = static_cast(upper[i]); + if (byte != std::numeric_limits::max()) { + upper[i] = static_cast(byte + 1); + upper.resize(i + 1); + return upper; + } + } + return std::nullopt; + } +}; + using FunctionStringEndsWith = FunctionBinaryToType; using FunctionStringInstr = diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index d51e6add28517b..062e5d857e390b 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -20,16 +20,21 @@ #pragma once +#include #include +#include +#include #include #include "common/logging.h" +#include "common/status.h" #include "core/accurate_comparison.h" #include "core/assert_cast.h" #include "core/column/column_const.h" #include "core/column/column_decimal.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/define_primitive_type.h" @@ -37,9 +42,11 @@ #include "core/field.h" #include "core/memcmp_small.h" #include "core/value/vdatetime_value.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function/function.h" #include "exprs/function/function_helpers.h" #include "exprs/function/functions_logical.h" +#include "exprs/vexpr.h" #include "storage/index/index_reader_helper.h" namespace doris { @@ -265,6 +272,131 @@ struct NameGreaterOrEquals { static constexpr auto name = "ge"; }; +namespace comparison_zonemap_detail { +enum class Op { + EQ, + NE, + LT, + LE, + GT, + GE, +}; + +inline Op symmetric_op(Op op) { + switch (op) { + case Op::EQ: + case Op::NE: + return op; + case Op::LT: + return Op::GT; + case Op::LE: + return Op::GE; + case Op::GT: + return Op::LT; + case Op::GE: + return Op::LE; + } + __builtin_unreachable(); +} + +inline ZoneMapFilterResult evaluate(const ZoneMapEvalContext& ctx, const VExprSPtrs& arguments, + Op op) { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + + auto slot_type = expr_zonemap::fetch_compatible_slot_type(ctx, slot_literal->slot_index, + slot_literal->slot_type); + if (slot_type == nullptr) { + return unsupported_zonemap_filter(ctx); + } + auto zone_map_ptr = ctx.zone_map(slot_literal->slot_index); + if (zone_map_ptr == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ptr; + if (!zone_map.has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + if (!expr_zonemap::range_stats_usable_for_zonemap(zone_map, slot_type)) { + return unsupported_zonemap_filter(ctx); + } + + const auto effective_op = slot_literal->literal_on_left ? symmetric_op(op) : op; + const auto& literal = slot_literal->literal; + switch (effective_op) { + case Op::EQ: + return literal < zone_map.min_value || zone_map.max_value < literal + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::NE: + return zone_map.min_value == literal && zone_map.max_value == literal + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::LT: + return zone_map.min_value >= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::LE: + return zone_map.min_value > literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::GT: + return zone_map.max_value <= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::GE: + return zone_map.max_value < literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + + // keep this to avoid compile failure with g++. + __builtin_unreachable(); +} + +inline bool can_evaluate(const VExprSPtrs& arguments) { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + if (!slot_literal.has_value()) { + return false; + } + + // A NULL literal makes the comparison evaluate to NULL instead of a byte range predicate on + // the slot. This zonemap evaluator only derives bounds from non-NULL literals, so reject this + // shape here before evaluate_zonemap_filter is called. + if (slot_literal->literal.is_null()) { + return false; + } + + DORIS_CHECK(slot_literal->slot_type != nullptr); + DORIS_CHECK(slot_literal->literal_type != nullptr); + if (!expr_zonemap::data_types_compatible(slot_literal->slot_type, slot_literal->literal_type)) { + // The optimizer may generate a bare slot/literal comparison whose logical types differ + // only by attributes such as DATETIMEV2 scale. Expr zonemap evaluates stored Field + // values without running expression casts, so conservatively skip this optimization. + return false; + } + + return true; +} + +inline std::optional op_from_name(std::string_view name) { + if (name == NameEquals::name) { + return Op::EQ; + } + if (name == NameNotEquals::name) { + return Op::NE; + } + if (name == NameLess::name) { + return Op::LT; + } + if (name == NameLessOrEquals::name) { + return Op::LE; + } + if (name == NameGreater::name) { + return Op::GT; + } + if (name == NameGreaterOrEquals::name) { + return Op::GE; + } + return std::nullopt; +} +} // namespace comparison_zonemap_detail + template