diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index da855aff32d4d8..911a321f21f5f9 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -222,6 +222,16 @@ Status OlapScanLocalState::_init_profile() { _stats_filtered_counter = ADD_COUNTER(_segment_profile, "RowsStatsFiltered", TUnit::UNIT); _stats_rp_filtered_counter = ADD_COUNTER(_segment_profile, "RowsZoneMapRuntimePredicateFiltered", TUnit::UNIT); + _expr_zonemap_filtered_segment_counter = + ADD_COUNTER(_segment_profile, "ExprZoneMapFilteredSegments", TUnit::UNIT); + _expr_zonemap_filtered_page_counter = + ADD_COUNTER(_segment_profile, "ExprZoneMapFilteredPages", TUnit::UNIT); + _expr_zonemap_unusable_counter = + ADD_COUNTER(_segment_profile, "ExprZoneMapUnusableEvals", TUnit::UNIT); + _in_zonemap_point_check_counter = + ADD_COUNTER(_segment_profile, "InZoneMapPointCheckCount", TUnit::UNIT); + _in_zonemap_range_only_counter = + ADD_COUNTER(_segment_profile, "InZoneMapRangeOnlyCount", TUnit::UNIT); _bf_filtered_counter = ADD_COUNTER(_segment_profile, "RowsBloomFilterFiltered", TUnit::UNIT); _dict_filtered_counter = ADD_COUNTER(_segment_profile, "SegmentDictFiltered", TUnit::UNIT); _del_filtered_counter = ADD_COUNTER(_scanner_profile, "RowsDelFiltered", TUnit::UNIT); diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index e8664257714f16..57c4912b777f6b 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -156,6 +156,17 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _stats_filtered_counter = nullptr; RuntimeProfile::Counter* _stats_rp_filtered_counter = nullptr; + // Number of whole segments skipped by expression ZoneMap evaluation. + RuntimeProfile::Counter* _expr_zonemap_filtered_segment_counter = nullptr; + // Number of pages skipped by expression ZoneMap evaluation after page index ranges are built. + RuntimeProfile::Counter* _expr_zonemap_filtered_page_counter = nullptr; + // Number of expression ZoneMap evaluations that reached the evaluator but could not use the + // current ZoneMap context, such as missing slot/page ZoneMap or unusable range statistics. + RuntimeProfile::Counter* _expr_zonemap_unusable_counter = nullptr; + // Number of IN-predicate ZoneMap evaluations that used per-value point checks. + RuntimeProfile::Counter* _in_zonemap_point_check_counter = nullptr; + // Number of IN-predicate ZoneMap evaluations that fell back to min/max range overlap only. + RuntimeProfile::Counter* _in_zonemap_range_only_counter = nullptr; RuntimeProfile::Counter* _bf_filtered_counter = nullptr; RuntimeProfile::Counter* _dict_filtered_counter = nullptr; RuntimeProfile::Counter* _del_filtered_counter = nullptr; diff --git a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp index e33bb8e41c39d8..d82900d2540c22 100644 --- a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp @@ -101,7 +101,7 @@ Status RuntimeFilterConsumer::_get_push_exprs(std::vector& co node.in_predicate.__set_is_not_in(false); node.__set_opcode(TExprOpcode::FILTER_IN); node.__set_is_nullable(false); - auto in_pred = VDirectInPredicate::create_shared(node, _wrapper->hybrid_set()); + auto in_pred = VDirectInPredicate::create_shared(node, _wrapper->hybrid_set(), true); in_pred->add_child(probe_ctx->root()); auto wrapper = VRuntimeFilterWrapper::create_shared( node, in_pred, get_in_list_ignore_thredhold(_wrapper->hybrid_set()->size()), diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 5dec25aca0c4b5..56684d5ffff802 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -739,6 +739,14 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_rows_expr_cond_input_counter, stats.expr_cond_input_rows); COUNTER_UPDATE(local_state->_stats_filtered_counter, stats.rows_stats_filtered); COUNTER_UPDATE(local_state->_stats_rp_filtered_counter, stats.rows_stats_rp_filtered); + COUNTER_UPDATE(local_state->_expr_zonemap_filtered_segment_counter, + stats.expr_zonemap_filtered_segments); + COUNTER_UPDATE(local_state->_expr_zonemap_filtered_page_counter, + stats.expr_zonemap_filtered_pages); + COUNTER_UPDATE(local_state->_expr_zonemap_unusable_counter, stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(local_state->_in_zonemap_point_check_counter, + stats.in_zonemap_point_check_count); + COUNTER_UPDATE(local_state->_in_zonemap_range_only_counter, stats.in_zonemap_range_only_count); COUNTER_UPDATE(local_state->_dict_filtered_counter, stats.segment_dict_filtered); COUNTER_UPDATE(local_state->_bf_filtered_counter, stats.rows_bf_filtered); COUNTER_UPDATE(local_state->_del_filtered_counter, stats.rows_del_filtered); diff --git a/be/src/exprs/expr_zonemap_filter.cpp b/be/src/exprs/expr_zonemap_filter.cpp new file mode 100644 index 00000000000000..19457beb37416b --- /dev/null +++ b/be/src/exprs/expr_zonemap_filter.cpp @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/expr_zonemap_filter.h" + +#include +#include +#include + +#include "common/logging.h" +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" +#include "exprs/hybrid_set.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "runtime/runtime_state.h" + +namespace doris::expr_zonemap { +namespace { + +constexpr size_t kInZoneMapPointCheckThreshold = 64; + +std::optional> field_from_literal_expr(const VExprSPtr& expr) { + auto literal = std::dynamic_pointer_cast(expr); + if (literal == nullptr) { + return std::nullopt; + } + Field field; + literal->get_column_ptr()->get(0, field); + return std::make_pair(std::move(field), literal->get_data_type()); +} + +bool value_in_range(const Field& value, const Field& min_value, const Field& max_value) { + return value >= min_value && value <= max_value; +} + +} // namespace + +TExprNode create_texpr_node_from_hybrid_set_value(const void* data, const PrimitiveType& type, + int precision, int scale) { + if (is_string_type(type)) { + const auto* value = reinterpret_cast(data); + auto field = Field::create_field(String(value->data, value->size)); + return create_texpr_node_from(field, type, precision, scale); + } + return create_texpr_node_from(data, type, precision, scale); +} + +Status materialize_hybrid_set_for_zonemap_filter(HybridSetBase& set, const DataTypePtr& data_type, + InZonemapMaterializedSet* result) { + DORIS_CHECK(result != nullptr); + DORIS_CHECK(data_type != nullptr); + const auto value_type = remove_nullable(data_type); + DORIS_CHECK(value_type != nullptr); + + result->contains_null = set.contain_null(); + result->values.clear(); + result->min_value = Field(); + result->max_value = Field(); + + auto* iterator = set.begin(); + while (iterator->has_next()) { + const void* value = iterator->get_value(); + if (value != nullptr) { + TExprNode literal_node = create_texpr_node_from_hybrid_set_value( + value, value_type->get_primitive_type(), value_type->get_precision(), + value_type->get_scale()); + auto literal = VLiteral::create_shared(literal_node); + Field field; + literal->get_column_ptr()->get(0, field); + result->values.emplace_back(std::move(field)); + } + iterator->next(); + } + + if (!result->values.empty()) { + auto minmax = std::ranges::minmax_element( + result->values, [](const Field& lhs, const Field& rhs) { return lhs < rhs; }); + result->min_value = *minmax.min; + result->max_value = *minmax.max; + } + return Status::OK(); +} + +std::optional extract_slot_and_literal(const VExprSPtrs& args) { + if (args.size() != 2) { + return std::nullopt; + } + + if (auto slot = std::dynamic_pointer_cast(args[0]); slot) { + auto literal = field_from_literal_expr(args[1]); + if (!literal.has_value()) { + return std::nullopt; + } + auto [literal_value, literal_type] = std::move(*literal); + return SlotLiteral {.slot_index = slot->column_id(), + .slot_type = slot->data_type(), + .literal = std::move(literal_value), + .literal_type = std::move(literal_type), + .literal_on_left = false}; + } + + if (auto slot = std::dynamic_pointer_cast(args[1]); slot) { + auto literal = field_from_literal_expr(args[0]); + if (!literal.has_value()) { + return std::nullopt; + } + auto [literal_value, literal_type] = std::move(*literal); + return SlotLiteral {.slot_index = slot->column_id(), + .slot_type = slot->data_type(), + .literal = std::move(literal_value), + .literal_type = std::move(literal_type), + .literal_on_left = true}; + } + + return std::nullopt; +} + +bool range_stats_usable_for_zonemap(const segment_v2::ZoneMap& zone_map, + const DataTypePtr& data_type) { + if (zone_map.pass_all || zone_map.has_nan || zone_map.has_positive_inf || + zone_map.has_negative_inf) { + return false; + } + DORIS_CHECK(data_type != nullptr); + auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + DORIS_CHECK(field_types_compatible(zone_map.min_value.get_type(), primitive_type)); + DORIS_CHECK(field_types_compatible(zone_map.max_value.get_type(), primitive_type)); + return true; +} + +ZoneMapFilterResult eval_null_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtrs& arguments, + bool is_null) { + DORIS_CHECK(arguments.size() == 1); + auto slot = std::dynamic_pointer_cast(arguments[0]); + DORIS_CHECK(slot != nullptr); + auto zone_map_ptr = ctx.zone_map(slot->column_id()); + if (zone_map_ptr == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ptr; + if (is_null) { + return zone_map.has_null ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch; + } + return zone_map.has_not_null ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch; +} + +ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtr& slot_expr, + bool is_not_in, const std::vector& values, + const Field& min_value, const Field& max_value) { + auto slot = std::dynamic_pointer_cast(slot_expr); + DORIS_CHECK(slot != nullptr); + // Empty IN has no candidate values, while NOT IN with an empty set cannot filter anything. + if (values.empty()) { + return is_not_in ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch; + } + + // The caller has materialized the IN set and precomputed its non-null min/max. They must match + // the expression slot type before being compared with storage zone-map statistics. + DORIS_CHECK(!min_value.is_null()); + DORIS_CHECK(!max_value.is_null()); + auto data_type = remove_nullable(slot->data_type()); + DORIS_CHECK(data_type != nullptr); + DORIS_CHECK(field_types_compatible(min_value.get_type(), data_type->get_primitive_type())); + DORIS_CHECK(field_types_compatible(max_value.get_type(), data_type->get_primitive_type())); + + // Re-check against the reader-schema type and the available zone map. Missing or unsupported + // metadata must conservatively fall back to may-match. + auto slot_type = fetch_compatible_slot_type(ctx, slot->column_id(), slot->data_type()); + if (slot_type == nullptr) { + return unsupported_zonemap_filter(ctx); + } + auto zone_map_ptr = ctx.zone_map(slot->column_id()); + if (zone_map_ptr == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ptr; + // IN values are all non-null here, so an all-null zone cannot match. + if (!zone_map.has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + + if (!range_stats_usable_for_zonemap(zone_map, slot_type)) { + return unsupported_zonemap_filter(ctx); + } + + if (is_not_in) { + // NOT IN can only prune when the whole zone contains exactly one non-null value and that + // value is excluded by the set. Wider ranges may contain values that are not filtered. + if (zone_map.min_value == zone_map.max_value) { + const bool only_value_is_filtered = std::ranges::any_of( + values, [&](const Field& value) { return value == zone_map.min_value; }); + return only_value_is_filtered ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + return ZoneMapFilterResult::kMayMatch; + } + + // First use the materialized IN-set min/max to rule out disjoint zone-map ranges. + if (zone_map.max_value < min_value || zone_map.min_value > max_value) { + return ZoneMapFilterResult::kNoMatch; + } + + // For large IN sets, avoid checking every point on the scan hot path. The range overlap above + // is only a coarse may-match signal. + if (values.size() > kInZoneMapPointCheckThreshold) { + ++ctx.stats.in_zonemap_range_only_count; + return ZoneMapFilterResult::kMayMatch; + } + + // For small IN sets, verify whether any candidate value can fall into the zone-map range. + ++ctx.stats.in_zonemap_point_check_count; + for (const auto& value : values) { + if (value_in_range(value, zone_map.min_value, zone_map.max_value)) { + return ZoneMapFilterResult::kMayMatch; + } + } + return ZoneMapFilterResult::kNoMatch; +} + +// Return the only slot ordinal referenced by a zonemap-evaluable expression. A negative result is +// the conservative fallback marker for unsupported expressions, multi-slot expressions, or invalid +// slot ordinals, so callers can skip schema-indexed zonemap pruning safely. +int single_slot_zonemap_index(const VExprContextSPtr& ctx) { + DORIS_CHECK(ctx != nullptr); + const auto& root = ctx->root(); + DORIS_CHECK(root != nullptr); + if (!root->can_evaluate_zonemap_filter()) { + return -1; + } + + std::set slot_indexes; + root->collect_slot_column_ids(slot_indexes); + if (slot_indexes.size() != 1) { + return -1; + } + + return *slot_indexes.begin(); +} + +bool is_expr_zonemap_filter_enabled(const RuntimeState* state) { + if (state == nullptr) { + return true; + } + const auto& query_options = state->query_options(); + return !query_options.__isset.enable_expr_zonemap_filter || + query_options.enable_expr_zonemap_filter; +} + +} // namespace doris::expr_zonemap diff --git a/be/src/exprs/expr_zonemap_filter.h b/be/src/exprs/expr_zonemap_filter.h new file mode 100644 index 00000000000000..2d8ff4d0c72207 --- /dev/null +++ b/be/src/exprs/expr_zonemap_filter.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/define_primitive_type.h" +#include "core/field.h" +#include "exprs/vexpr_fwd.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" + +namespace doris { +class HybridSetBase; +class RuntimeState; +class TExprNode; +} // namespace doris + +namespace doris::expr_zonemap { + +struct InZonemapMaterializedSet { + bool contains_null = false; + std::vector values; + Field min_value; + Field max_value; +}; + +struct SlotLiteral { + // Slot ordinal in the current expression binding. It is also the key used to look up the + // corresponding reader-schema type and zone map from ZoneMapEvalContext. + int slot_index; + // Type carried by the slot expression. It is kept separately because evaluation first has to + // verify that the expression binding is compatible with the reader-schema type stored in the + // ZoneMapEvalContext before using schema-indexed zone-map statistics. + DataTypePtr slot_type; + // Constant literal value paired with the slot, already materialized as a Field for zone-map + // comparisons. + Field literal; + // Type carried by the literal expression. It is needed with slot_type so capability checks can + // reject incompatible slot/literal comparisons before runtime evaluation instead of evaluating + // zone-map ranges with mismatched Field types. + DataTypePtr literal_type; + // Whether the original expression shape is literal slot instead of slot literal. The + // comparison operator is directional, so evaluators need this flag to normalize cases like + // `5 < slot` into the equivalent slot-side comparison before checking the zone map. Non-symmetric + // function evaluators also use it to reject unsupported shapes such as `starts_with(literal, + // slot)`, because only `starts_with(slot, literal)` can be mapped to a safe zone-map range. + bool literal_on_left; +}; + +std::optional extract_slot_and_literal(const VExprSPtrs& args); + +TExprNode create_texpr_node_from_hybrid_set_value(const void* data, const PrimitiveType& type, + int precision, int scale); + +Status materialize_hybrid_set_for_zonemap_filter(HybridSetBase& set, const DataTypePtr& data_type, + InZonemapMaterializedSet* result); + +inline bool field_types_compatible(PrimitiveType lhs, PrimitiveType rhs) { + return lhs == rhs || (is_string_type(lhs) && is_string_type(rhs)); +} + +inline bool data_types_compatible(const DataTypePtr& lhs, const DataTypePtr& rhs) { + if (lhs == nullptr || rhs == nullptr) { + return false; + } + const auto lhs_type = remove_nullable(lhs); + const auto rhs_type = remove_nullable(rhs); + const auto lhs_primitive_type = lhs_type->get_primitive_type(); + const auto rhs_primitive_type = rhs_type->get_primitive_type(); + if (is_string_type(lhs_primitive_type) && is_string_type(rhs_primitive_type)) { + return true; + } + return lhs_type->equals(*rhs_type); +} + +inline DataTypePtr fetch_compatible_slot_type(const ZoneMapEvalContext& ctx, int slot_index, + const DataTypePtr& expr_slot_type) { + auto slot_type = ctx.data_type(slot_index); + if (slot_type == nullptr) { + return nullptr; + } + DORIS_CHECK(data_types_compatible(slot_type, expr_slot_type)); + return slot_type; +} + +bool range_stats_usable_for_zonemap(const segment_v2::ZoneMap& zone_map, + const DataTypePtr& data_type); + +ZoneMapFilterResult eval_null_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtrs& arguments, + bool is_null); + +ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const VExprSPtr& slot_expr, + bool is_not_in, const std::vector& values, + const Field& min_value, const Field& max_value); + +// Return the only slot ordinal referenced by a zonemap-evaluable expression in its current +// binding. Expressions that are unsupported by zonemap pruning, reference multiple slots, or use an +// invalid negative slot ordinal return a negative value. +int single_slot_zonemap_index(const VExprContextSPtr& ctx); + +bool is_expr_zonemap_filter_enabled(const RuntimeState* state); + +} // namespace doris::expr_zonemap diff --git a/be/src/exprs/function/function.cpp b/be/src/exprs/function/function.cpp index fa5b12fea2ec6d..6cec038864e8b7 100644 --- a/be/src/exprs/function/function.cpp +++ b/be/src/exprs/function/function.cpp @@ -39,6 +39,7 @@ #include "exec/common/util.hpp" #include "exprs/aggregate/aggregate_function.h" #include "exprs/function/function_helpers.h" +#include "storage/index/zone_map/zonemap_eval_context.h" namespace doris { #include "common/compile_check_begin.h" @@ -407,5 +408,10 @@ bool FunctionBuilderImpl::is_nested_type_date_or_datetime_or_decimal( } } +ZoneMapFilterResult IFunctionBase::evaluate_zonemap_filter( + const ZoneMapEvalContext& ctx, const VExprSPtrs& function_arguments) const { + return unsupported_zonemap_filter(ctx); +} + #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 9e0aebdabaf032..171e889f79b21d 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -43,8 +43,10 @@ #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "exprs/function_context.h" +#include "exprs/vexpr_fwd.h" #include "storage/index/inverted/inverted_index_iterator.h" // IWYU pragma: keep #include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/zone_map/zonemap_filter_result.h" namespace doris { struct InvertedIndexAnalyzerCtx; @@ -76,6 +78,7 @@ struct FunctionAttr { class Field; class VExpr; +class ZoneMapEvalContext; // Only use dispose the variadic argument template @@ -226,6 +229,13 @@ class IFunctionBase { virtual bool can_push_down_to_index() const { return false; } virtual bool is_blockable() const { return false; } + + virtual ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx, + const VExprSPtrs& function_arguments) const; + + virtual bool can_evaluate_zonemap_filter(const VExprSPtrs& /*function_arguments*/) const { + return false; + } }; using FunctionBasePtr = std::shared_ptr; @@ -492,6 +502,15 @@ class DefaultFunction final : public IFunctionBase { bool is_blockable() const override { return function->is_blockable(); } + ZoneMapFilterResult evaluate_zonemap_filter( + const ZoneMapEvalContext& ctx, const VExprSPtrs& function_arguments) const override { + return function->evaluate_zonemap_filter(ctx, function_arguments); + } + + bool can_evaluate_zonemap_filter(const VExprSPtrs& function_arguments) const override { + return function->can_evaluate_zonemap_filter(function_arguments); + } + private: std::shared_ptr function; DataTypes arguments; diff --git a/be/src/exprs/function/function_string.cpp b/be/src/exprs/function/function_string.cpp index 4ce5e484b8609d..4eb0b1462cb3d5 100644 --- a/be/src/exprs/function/function_string.cpp +++ b/be/src/exprs/function/function_string.cpp @@ -24,16 +24,23 @@ #include #include +#include #include #include +#include +#include +#include #include #include "common/cast_set.h" +#include "common/logging.h" #include "common/status.h" #include "core/column/column.h" #include "core/column/column_string.h" +#include "core/data_type/data_type_nullable.h" #include "core/pod_array_fwd.h" #include "core/string_ref.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function/function_reverse.h" #include "exprs/function/function_string_concat.h" #include "exprs/function/function_string_format.h" @@ -1330,8 +1337,87 @@ using FunctionCrc32 = FunctionUnaryToType; using FunctionStringUTF8Length = FunctionUnaryToType; using FunctionStringSpace = FunctionUnaryToType; using FunctionIsValidUTF8 = FunctionUnaryToType; -using FunctionStringStartsWith = - FunctionBinaryToType; + +class FunctionStringStartsWith : public FunctionBinaryToType { +public: + static FunctionPtr create() { return std::make_shared(); } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx, + const VExprSPtrs& arguments) const override { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + auto slot_type = expr_zonemap::fetch_compatible_slot_type(ctx, slot_literal->slot_index, + slot_literal->slot_type); + if (slot_type == nullptr) { + return unsupported_zonemap_filter(ctx); + } + auto zone_map_ref = ctx.zone_map(slot_literal->slot_index); + if (zone_map_ref == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ref; + if (!zone_map.has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + if (!expr_zonemap::range_stats_usable_for_zonemap(zone_map, slot_type)) { + return unsupported_zonemap_filter(ctx); + } + + const auto prefix = slot_literal->literal.as_string_view(); + auto lower = Field::create_field(std::string(prefix)); + if (zone_map.max_value < lower) { + return ZoneMapFilterResult::kNoMatch; + } + auto upper_prefix = _next_prefix_for_starts_with_zonemap(prefix); + if (upper_prefix.has_value() && + !(zone_map.min_value < Field::create_field(*upper_prefix))) { + return ZoneMapFilterResult::kNoMatch; + } + return ZoneMapFilterResult::kMayMatch; + } + + bool can_evaluate_zonemap_filter(const VExprSPtrs& arguments) const override { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + if (!slot_literal.has_value() || slot_literal->literal_on_left) { + return false; + } + + // A NULL prefix makes starts_with(slot, NULL) evaluate to NULL. An empty prefix matches + // every non-NULL string and cannot prune by range. Reject both shapes here before + // evaluate_zonemap_filter is called. + if (slot_literal->literal.is_null()) { + return false; + } + + DORIS_CHECK(slot_literal->slot_type != nullptr); + DORIS_CHECK(slot_literal->literal_type != nullptr); + DORIS_CHECK(is_string_type(remove_nullable(slot_literal->slot_type)->get_primitive_type())); + DORIS_CHECK( + is_string_type(remove_nullable(slot_literal->literal_type)->get_primitive_type())); + + const auto prefix = slot_literal->literal.as_string_view(); + return !prefix.empty(); + } + +private: + static std::optional _next_prefix_for_starts_with_zonemap( + std::string_view prefix) { + // ZoneMap string bounds are compared by bytewise Field ordering. For starts_with(s, p), + // the safe upper bound is the next byte string after p: p <= s < next_prefix(p). + // For example, starts_with(s, "ab") can use the range "ab" <= s < "ac". + std::string upper(prefix); + for (auto i = static_cast(upper.size()) - 1; i >= 0; --i) { + auto byte = static_cast(upper[i]); + if (byte != std::numeric_limits::max()) { + upper[i] = static_cast(byte + 1); + upper.resize(i + 1); + return upper; + } + } + return std::nullopt; + } +}; + using FunctionStringEndsWith = FunctionBinaryToType; using FunctionStringInstr = diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index d51e6add28517b..062e5d857e390b 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -20,16 +20,21 @@ #pragma once +#include #include +#include +#include #include #include "common/logging.h" +#include "common/status.h" #include "core/accurate_comparison.h" #include "core/assert_cast.h" #include "core/column/column_const.h" #include "core/column/column_decimal.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/define_primitive_type.h" @@ -37,9 +42,11 @@ #include "core/field.h" #include "core/memcmp_small.h" #include "core/value/vdatetime_value.h" +#include "exprs/expr_zonemap_filter.h" #include "exprs/function/function.h" #include "exprs/function/function_helpers.h" #include "exprs/function/functions_logical.h" +#include "exprs/vexpr.h" #include "storage/index/index_reader_helper.h" namespace doris { @@ -265,6 +272,131 @@ struct NameGreaterOrEquals { static constexpr auto name = "ge"; }; +namespace comparison_zonemap_detail { +enum class Op { + EQ, + NE, + LT, + LE, + GT, + GE, +}; + +inline Op symmetric_op(Op op) { + switch (op) { + case Op::EQ: + case Op::NE: + return op; + case Op::LT: + return Op::GT; + case Op::LE: + return Op::GE; + case Op::GT: + return Op::LT; + case Op::GE: + return Op::LE; + } + __builtin_unreachable(); +} + +inline ZoneMapFilterResult evaluate(const ZoneMapEvalContext& ctx, const VExprSPtrs& arguments, + Op op) { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + + auto slot_type = expr_zonemap::fetch_compatible_slot_type(ctx, slot_literal->slot_index, + slot_literal->slot_type); + if (slot_type == nullptr) { + return unsupported_zonemap_filter(ctx); + } + auto zone_map_ptr = ctx.zone_map(slot_literal->slot_index); + if (zone_map_ptr == nullptr) { + return unsupported_zonemap_filter(ctx); + } + const auto& zone_map = *zone_map_ptr; + if (!zone_map.has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + if (!expr_zonemap::range_stats_usable_for_zonemap(zone_map, slot_type)) { + return unsupported_zonemap_filter(ctx); + } + + const auto effective_op = slot_literal->literal_on_left ? symmetric_op(op) : op; + const auto& literal = slot_literal->literal; + switch (effective_op) { + case Op::EQ: + return literal < zone_map.min_value || zone_map.max_value < literal + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::NE: + return zone_map.min_value == literal && zone_map.max_value == literal + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::LT: + return zone_map.min_value >= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::LE: + return zone_map.min_value > literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::GT: + return zone_map.max_value <= literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + case Op::GE: + return zone_map.max_value < literal ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + + // keep this to avoid compile failure with g++. + __builtin_unreachable(); +} + +inline bool can_evaluate(const VExprSPtrs& arguments) { + auto slot_literal = expr_zonemap::extract_slot_and_literal(arguments); + if (!slot_literal.has_value()) { + return false; + } + + // A NULL literal makes the comparison evaluate to NULL instead of a byte range predicate on + // the slot. This zonemap evaluator only derives bounds from non-NULL literals, so reject this + // shape here before evaluate_zonemap_filter is called. + if (slot_literal->literal.is_null()) { + return false; + } + + DORIS_CHECK(slot_literal->slot_type != nullptr); + DORIS_CHECK(slot_literal->literal_type != nullptr); + if (!expr_zonemap::data_types_compatible(slot_literal->slot_type, slot_literal->literal_type)) { + // The optimizer may generate a bare slot/literal comparison whose logical types differ + // only by attributes such as DATETIMEV2 scale. Expr zonemap evaluates stored Field + // values without running expression casts, so conservatively skip this optimization. + return false; + } + + return true; +} + +inline std::optional op_from_name(std::string_view name) { + if (name == NameEquals::name) { + return Op::EQ; + } + if (name == NameNotEquals::name) { + return Op::NE; + } + if (name == NameLess::name) { + return Op::LT; + } + if (name == NameLessOrEquals::name) { + return Op::LE; + } + if (name == NameGreater::name) { + return Op::GT; + } + if (name == NameGreaterOrEquals::name) { + return Op::GE; + } + return std::nullopt; +} +} // namespace comparison_zonemap_detail + template