From 674fb7af84ea6773e2cdc3850399fc239b974644 Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Tue, 16 Jun 2026 13:15:39 +0800 Subject: [PATCH] [refactor](storage) Drop PredicateColumnType (#64128) Drop PredicateColumnType Issue Number: close #xxx Related PR: #xxx Problem Summary: None - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [x] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --------- Co-authored-by: Claude Opus 4.7 (1M context) (cherry picked from commit ca4af95e2c7bfbb875d4e9d0ee01c7a1b0c43c61) --- be/src/core/column/column_decimal.h | 12 + be/src/core/column/column_dictionary.h | 17 +- be/src/core/column/column_execute_util.h | 48 +- be/src/core/column/column_vector.h | 12 + be/src/core/column/predicate_column.h | 434 ------- be/src/exprs/bloom_filter_func.h | 9 +- be/src/exprs/bloom_filter_func_adaptor.h | 26 +- be/src/exprs/bloom_filter_func_impl.h | 26 +- be/src/exprs/create_predicate_function.h | 11 +- be/src/exprs/function/like.h | 1 - .../iceberg_delete_file_reader_helper.cpp | 6 +- .../predicate/bloom_filter_predicate.h | 9 +- .../storage/predicate/comparison_predicate.h | 37 +- be/src/storage/predicate/in_list_predicate.h | 36 +- .../predicate/like_column_predicate.cpp | 33 +- .../storage/predicate/like_column_predicate.h | 7 +- be/src/storage/predicate/predicate_creator.h | 2 +- be/src/storage/schema.cpp | 102 +- be/src/storage/schema.h | 2 +- be/src/storage/segment/column_reader.cpp | 3 +- be/src/storage/segment/segment_iterator.cpp | 6 +- be/src/storage/tablet/tablet_reader.cpp | 3 +- .../core/column/column_dictionary_test.cpp | 23 +- be/test/core/column/column_nullable_test.cpp | 5 +- be/test/core/column/common_column_test.h | 1 + be/test/core/column/predicate_column_test.cpp | 1082 ----------------- be/test/exprs/bloom_filter_func_test.cpp | 42 +- .../parquet/byte_array_dict_decoder_test.cpp | 6 +- .../parquet/fix_length_dict_decoder_test.cpp | 6 +- .../predicate/block_column_predicate_test.cpp | 22 +- 30 files changed, 217 insertions(+), 1812 deletions(-) delete mode 100644 be/src/core/column/predicate_column.h delete mode 100644 be/test/core/column/predicate_column_test.cpp diff --git a/be/src/core/column/column_decimal.h b/be/src/core/column/column_decimal.h index 4d6344fdfcd52a..55e429f3183749 100644 --- a/be/src/core/column/column_decimal.h +++ b/be/src/core/column/column_decimal.h @@ -149,6 +149,18 @@ class ColumnDecimal final : public COWHelper> { memset(data.data() + old_size, 0, length * sizeof(data[0])); } + Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) override { + Self* output = assert_cast(col_ptr); + auto& res_data = output->get_data(); + DCHECK(res_data.empty()) + << "filter_by_selector requires the destination column to be empty"; + res_data.resize(sel_size); + for (size_t i = 0; i < sel_size; i++) { + res_data[i] = data[sel[i]]; + } + return Status::OK(); + } + void insert_many_from(const IColumn& src, size_t position, size_t length) override; void pop_back(size_t n) override { data.resize_assume_reserved(data.size() - n); } diff --git a/be/src/core/column/column_dictionary.h b/be/src/core/column/column_dictionary.h index b70433f338c293..3f6987d6baa370 100644 --- a/be/src/core/column/column_dictionary.h +++ b/be/src/core/column/column_dictionary.h @@ -23,7 +23,6 @@ #include "core/column/column.h" #include "core/column/column_string.h" -#include "core/column/predicate_column.h" #include "core/pod_array.h" #include "core/string_ref.h" #include "core/types.h" @@ -50,7 +49,7 @@ class ColumnDictI32 final : public COWHelper { ColumnDictI32(const ColumnDictI32& src) { throw doris::Exception(ErrorCode::INTERNAL_ERROR, "copy not supported in ColumnDictionary"); } - ColumnDictI32(FieldType type) : _type(type) {} + ColumnDictI32() = default; public: using Self = ColumnDictI32; @@ -108,7 +107,7 @@ class ColumnDictI32 final : public COWHelper { MutableColumnPtr clone_resized(size_t size) const override { DCHECK(size == 0); - return create(_type); + return create(); } void insert(const Field& x) override { @@ -257,16 +256,7 @@ class ColumnDictI32 final : public COWHelper { if (is_dict_sorted() && !is_dict_code_converted()) { convert_dict_codes_if_necessary(); } - // if type is OLAP_FIELD_TYPE_CHAR, we need to construct TYPE_CHAR PredicateColumnType, - // because the string length will different from varchar and string which needed to be processed after. - auto create_column = [this]() -> MutableColumnPtr { - if (_type == FieldType::OLAP_FIELD_TYPE_CHAR) { - return PredicateColumnType::create(); - } - return PredicateColumnType::create(); - }; - - auto res = create_column(); + auto res = ColumnString::create(); res->reserve(_codes.capacity()); for (int code : _codes) { auto value = _dict.get_value(code); @@ -464,7 +454,6 @@ class ColumnDictI32 final : public COWHelper { bool _dict_code_converted = false; Dictionary _dict; Container _codes; - FieldType _type; std::pair _rowset_segment_id; std::vector _strings; }; diff --git a/be/src/core/column/column_execute_util.h b/be/src/core/column/column_execute_util.h index 0b6513ba5b779b..599c7da9f748a5 100644 --- a/be/src/core/column/column_execute_util.h +++ b/be/src/core/column/column_execute_util.h @@ -16,13 +16,17 @@ // under the License. #pragma once +#include #include +#include #include +#include #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" @@ -32,27 +36,55 @@ namespace doris { // Utility tools for convenient column execution -// ColumnElementView is used to distinguish between scalar columns and string columns +// Per-row read view over a column. The pointer returned by ptr_at on the +// string specialization is valid only until the next ptr_at call. +namespace detail { + template -struct ColumnElementView { +struct NumericElementView { using ColumnType = typename PrimitiveTypeTraits::ColumnType; using ElementType = typename ColumnType::value_type; const typename ColumnType::Container& data; - ElementType get_element(size_t idx) const { return data[idx]; } - const ElementType* get_data() const { return data.data(); } - ColumnElementView(const IColumn& column) + NumericElementView(const IColumn& column) : data(assert_cast(column).get_data()) {} + + ElementType get_element(size_t idx) const { return data[idx]; } + const ElementType* get_data() const { return data.data(); } + ElementType operator[](size_t idx) const { return data[idx]; } + const ElementType* ptr_at(size_t idx) const { return data.data() + idx; } + size_t size() const { return data.size(); } }; -template <> -struct ColumnElementView { +struct StringElementView { using ColumnType = ColumnString; using ElementType = StringRef; const ColumnString& string_column; - ColumnElementView(const IColumn& column) + mutable StringRef _cell {}; // staging for ptr_at + + StringElementView(const IColumn& column) : string_column(assert_cast(column)) {} + StringRef get_element(size_t idx) const { return string_column.get_data_at(idx); } + StringRef operator[](size_t idx) const { return string_column.get_data_at(idx); } + const StringRef* ptr_at(size_t idx) const { + _cell = string_column.get_data_at(idx); + return &_cell; + } + size_t size() const { return string_column.size(); } +}; + +} // namespace detail + +template +using ColumnElementView = std::conditional_t>; + +template +concept ColumnElementSubscriptable = requires(const T& v, size_t i) { + typename T::ElementType; + { v[i] } -> std::convertible_to; + { v.size() } -> std::convertible_to; }; // ColumnView is used to handle the nullable and const properties of a column. diff --git a/be/src/core/column/column_vector.h b/be/src/core/column/column_vector.h index 699f0115eb36f8..22a807f6929124 100644 --- a/be/src/core/column/column_vector.h +++ b/be/src/core/column/column_vector.h @@ -271,6 +271,18 @@ class ColumnVector final : public COWHelper> { void insert_value(const value_type value) { data.push_back(value); } + Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) override { + Self* output = assert_cast(col_ptr); + auto& res_data = output->get_data(); + DCHECK(res_data.empty()) + << "filter_by_selector requires the destination column to be empty"; + res_data.resize(sel_size); + for (size_t i = 0; i < sel_size; i++) { + res_data[i] = data[sel[i]]; + } + return Status::OK(); + } + /// This method implemented in header because it could be possibly devirtualized. int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override { return Compare::compare( diff --git a/be/src/core/column/predicate_column.h b/be/src/core/column/predicate_column.h deleted file mode 100644 index 538f1838d3db9d..00000000000000 --- a/be/src/core/column/predicate_column.h +++ /dev/null @@ -1,434 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "core/arena.h" -#include "core/column/column.h" -#include "core/column/column_decimal.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/define_primitive_type.h" -#include "core/data_type/primitive_type.h" -#include "core/decimal12.h" -#include "core/string_ref.h" -#include "core/types.h" -#include "core/uint24.h" - -namespace doris { -#include "common/compile_check_begin.h" - -/** - * used to keep predicate column in storage layer - * - * T = predicate column type - */ -template -class PredicateColumnType final : public COWHelper> { -private: - PredicateColumnType() = default; - PredicateColumnType(const size_t n) : data(n) {} - PredicateColumnType(const PredicateColumnType& src) : data(src.data.begin(), src.data.end()) {} - friend class COWHelper>; - using T = std::conditional_t::CppType>; - using ColumnType = typename PrimitiveTypeTraits::ColumnType; - - void insert_string_to_res_column(const uint16_t* sel, size_t sel_size, ColumnString* res_ptr) { - _refs.resize(sel_size); - size_t length = 0; - for (size_t i = 0; i < sel_size; i++) { - uint16_t n = sel[i]; - auto& sv = reinterpret_cast(data[n]); - _refs[i].data = sv.data; - _refs[i].size = sv.size; - length += sv.size; - } - res_ptr->get_offsets().reserve(sel_size + res_ptr->get_offsets().size()); - res_ptr->get_chars().reserve(length + res_ptr->get_chars().size()); - res_ptr->insert_many_strings_without_reserve(_refs.data(), sel_size); - } - - template typename ColumnContainer> - void insert_default_value_res_column(const uint16_t* sel, size_t sel_size, - ColumnContainer* res_ptr) { - static_assert(std::is_same_v, ColumnType>); - auto& res_data = res_ptr->get_data(); - DCHECK(res_data.empty()); - // Has to reserve first, could not call resize or reserve after get_end_ptr - // because reserve or resize may change memory block. - size_t org_num = res_data.size(); - res_data.reserve(sel_size); - auto* y = (typename PrimitiveTypeTraits::CppType*)res_data.get_end_ptr(); - for (size_t i = 0; i < sel_size; i++) { - y[i] = data[sel[i]]; - } - res_data.resize(org_num + sel_size); - } - - void insert_byte_to_res_column(const uint16_t* sel, size_t sel_size, IColumn* res_ptr) { - for (size_t i = 0; i < sel_size; i++) { - uint16_t n = sel[i]; - char* ch_val = reinterpret_cast(&data[n]); - res_ptr->insert_data(ch_val, 0); - } - } - - void insert_many_default_type(const char* data_ptr, size_t num) { - auto old_size = data.size(); - data.resize(old_size + num); - memcpy(reinterpret_cast(data.data() + old_size), data_ptr, num * sizeof(T)); - } - -public: - using Self = PredicateColumnType; - using value_type = T; - using Container = PaddedPODArray; - - size_t size() const override { return data.size(); } - - StringRef get_data_at(size_t n) const override { - if constexpr (std::is_same_v) { - return reinterpret_cast(data[n]); - } else { - throw doris::Exception( - ErrorCode::INTERNAL_ERROR, - "should not call get_data_at in predicate column except for string type"); - } - } - - void insert_from(const IColumn& src, size_t n) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "should not call insert_from in predicate column"); - } - - void insert_range_from(const IColumn& src, size_t start, size_t length) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "should not call insert_range_from in predicate column"); - } - - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "should not call insert_indices_from in predicate column"); - } - - void pop_back(size_t n) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "should not call pop_back in predicate column"); - } - - void update_hash_with_value(size_t n, SipHash& hash) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "should not call update_hash_with_value in predicate column"); - } - - void insert_string_value(const char* data_ptr, size_t length) { - StringRef sv((char*)data_ptr, length); - data.push_back_without_reserve(sv); - } - - // used for int128 - void insert_in_copy_way(const char* data_ptr, size_t length) { - T val {}; - memcpy(&val, data_ptr, sizeof(val)); - data.push_back_without_reserve(val); - } - - void insert_default_type(const char* data_ptr, size_t length) { - T* val = (T*)data_ptr; - data.push_back_without_reserve(*val); - } - - void insert_data(const char* data_ptr, size_t length) override { - if constexpr (std::is_same_v) { - insert_string_value(data_ptr, length); - } else if constexpr (std::is_same_v) { - insert_in_copy_way(data_ptr, length); - } else { - insert_default_type(data_ptr, length); - } - } - - void insert_many_date(const char* data_ptr, size_t num) { - constexpr size_t input_type_size = sizeof(PrimitiveTypeTraits::StorageFieldType); - static_assert(input_type_size == sizeof(uint24_t)); - const auto* input_data_ptr = reinterpret_cast(data_ptr); - auto* res_ptr = reinterpret_cast(data.get_end_ptr()); - size_t old_size = data.size(); - for (int i = 0; i < num; i++) { - res_ptr[i].set_olap_date(unaligned_load(&input_data_ptr[i])); - } - data.resize(old_size + num); - } - - void insert_many_datetime(const char* data_ptr, size_t num) { - constexpr size_t input_type_size = - sizeof(PrimitiveTypeTraits::StorageFieldType); - static_assert(input_type_size == sizeof(uint64_t)); - const auto* input_data_ptr = reinterpret_cast(data_ptr); - auto* res_ptr = reinterpret_cast(data.get_end_ptr()); - size_t old_size = data.size(); - for (int i = 0; i < num; i++) { - res_ptr[i].from_olap_datetime(input_data_ptr[i]); - } - data.resize(old_size + num); - } - - // The logic is same to ColumnDecimal::insert_many_fix_len_data - void insert_many_decimalv2(const char* data_ptr, size_t num) { - size_t old_size = data.size(); - data.resize(old_size + num); - - auto* target = (DecimalV2Value*)(data.data() + old_size); - for (int i = 0; i < num; i++) { - const char* cur_ptr = data_ptr + sizeof(decimal12_t) * i; - auto int_value = unaligned_load(cur_ptr); - int32_t frac_value = *(int32_t*)(cur_ptr + sizeof(int64_t)); - target[i].from_olap_decimal(int_value, frac_value); - } - } - - void insert_many_fix_len_data(const char* data_ptr, size_t num) override { - if constexpr (Type == TYPE_DECIMALV2) { - // DecimalV2 is special, its storage is , but its compute type is - // should convert here, but it may have some performance lost - insert_many_decimalv2(data_ptr, num); - } else if constexpr (std::is_same_v) { - // here is unreachable, just for compilation to be able to pass - } else if constexpr (Type == TYPE_DATE) { - // Datev1 is special, its storage is uint24, but its compute type is actual int64. - insert_many_date(data_ptr, num); - } else if constexpr (Type == TYPE_DATETIME) { - insert_many_datetime(data_ptr, num); - } else { - insert_many_default_type(data_ptr, num); - } - } - - void insert_many_dict_data(const int32_t* data_array, size_t start_index, const StringRef* dict, - size_t num, uint32_t /*dict_num*/) override { - if constexpr (std::is_same_v) { - for (size_t end_index = start_index + num; start_index < end_index; ++start_index) { - int32_t codeword = data_array[start_index]; - insert_string_value(dict[codeword].data, dict[codeword].size); - } - } - } - - void insert_many_continuous_binary_data(const char* data_, const uint32_t* offsets, - const size_t num) override { - if (UNLIKELY(num == 0)) { - return; - } - if constexpr (std::is_same_v) { - const auto total_mem_size = offsets[num] - offsets[0]; - char* destination = _arena.alloc(total_mem_size); - memcpy(destination, data_ + offsets[0], total_mem_size); - size_t org_elem_num = data.size(); - data.resize(org_elem_num + num); - - auto* data_ptr = &data[org_elem_num]; - for (size_t i = 0; i != num; ++i) { - data_ptr[i].data = destination + offsets[i] - offsets[0]; - data_ptr[i].size = offsets[i + 1] - offsets[i]; - } - DCHECK(data_ptr[num - 1].data + data_ptr[num - 1].size == destination + total_mem_size); - } - } - - void insert_many_strings(const StringRef* strings, size_t num) override { - if (num == 0) { - return; - } - if constexpr (std::is_same_v) { - size_t total_mem_size = 0; - for (size_t i = 0; i < num; i++) { - total_mem_size += strings[i].size; - } - - char* destination = _arena.alloc(total_mem_size); - char* org_dst = destination; - size_t org_elem_num = data.size(); - data.resize(org_elem_num + num); - uint64_t fragment_start_offset = 0; - size_t fragment_len = 0; - for (size_t i = 0; i < num; i++) { - data[org_elem_num + i].data = destination + fragment_len; - data[org_elem_num + i].size = strings[i].size; - fragment_len += strings[i].size; - // Compute the largest continuous memcpy block and copy them. - // If this is the last element in data array, then should copy the current memory block. - if (i == num - 1 || strings[i + 1].data != strings[i].data + strings[i].size) { - memcpy(destination, strings[fragment_start_offset].data, fragment_len); - destination += fragment_len; - fragment_start_offset = i == num - 1 ? 0 : i + 1; - fragment_len = 0; - } - } - CHECK(destination - org_dst == total_mem_size) - << "Copied size not equal to expected size"; - } else { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "Method insert_many_binary_data is not supported"); - } - } - - void insert_default() override { data.push_back(T()); } - - void clear() override { - data.clear(); - _arena.clear(); - } - - size_t byte_size() const override { return data.size() * sizeof(T); } - - size_t allocated_bytes() const override { return byte_size(); } - - bool has_enough_capacity(const IColumn& src) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "has_enough_capacity not supported in PredicateColumnType"); - } - - void reserve(size_t n) override { data.reserve(n); } - - std::string get_name() const override { return type_to_string(Type); } - - MutableColumnPtr clone_resized(size_t size) const override { - DCHECK(size == 0); - return this->create(); - } - - void insert_duplicate_fields(const Field& x, const size_t n) override { - if constexpr (is_string_type(Type)) { - const auto& str = x.get(); - auto* dst = _arena.alloc(str.size() * n); - for (size_t i = 0; i < n; i++) { - memcpy(dst, str.data(), str.size()); - insert_string_value(dst, str.size()); - dst += str.size(); - } - } else if constexpr (Type == TYPE_LARGEINT) { - const auto& v = x.get(); - for (size_t i = 0; i < n; i++) { - insert_in_copy_way(reinterpret_cast(&v), sizeof(v)); - } - } else { - const auto& v = x.get(); - for (size_t i = 0; i < n; i++) { - insert_default_type(reinterpret_cast(&v), sizeof(v)); - } - } - } - - void insert(const Field& x) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "insert not supported in PredicateColumnType"); - } - - [[noreturn]] Field operator[](size_t n) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "operator[] not supported in PredicateColumnType"); - } - - void get(size_t n, Field& res) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "get field not supported in PredicateColumnType"); - } - - [[noreturn]] bool get_bool(size_t n) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "get field not supported in PredicateColumnType"); - } - - [[noreturn]] Int64 get_int(size_t n) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "get field not supported in PredicateColumnType"); - } - - // it's impossible to use ComplexType as key , so we don't have to implement them - [[noreturn]] StringRef serialize_value_into_arena(size_t n, Arena& arena, - char const*& begin) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "serialize_value_into_arena not supported in PredicateColumnType"); - } - - [[noreturn]] const char* deserialize_and_insert_from_arena(const char* pos) override { - throw doris::Exception( - ErrorCode::INTERNAL_ERROR, - "deserialize_and_insert_from_arena not supported in PredicateColumnType"); - } - - [[noreturn]] StringRef get_raw_data() const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "get_raw_data not supported in PredicateColumnType"); - } - - [[noreturn]] bool structure_equals(const IColumn& rhs) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "structure_equals not supported in PredicateColumnType"); - } - - [[noreturn]] ColumnPtr filter(const IColumn::Filter& filt, - ssize_t result_size_hint) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "filter not supported in PredicateColumnType"); - } - - [[noreturn]] size_t filter(const IColumn::Filter&) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "filter not supported in PredicateColumnType"); - } - - [[noreturn]] MutableColumnPtr permute(const IColumn::Permutation& perm, - size_t limit) const override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "permute not supported in PredicateColumnType"); - } - - Container& get_data() { return data; } - - const Container& get_data() const { return data; } - - Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) override { - ColumnType* column = assert_cast(col_ptr); - if constexpr (is_string_type(Type)) { - insert_string_to_res_column(sel, sel_size, column); - } else if constexpr (Type == TYPE_BOOLEAN) { - insert_byte_to_res_column(sel, sel_size, col_ptr); - } else { - insert_default_value_res_column(sel, sel_size, column); - } - return Status::OK(); - } - - void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) override { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "should not call replace_column_data in predicate column"); - } - -private: - Container data; - // manages the memory for slice's data(For string type) - Arena _arena; - std::vector _refs; -}; - -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index 4a38d4c91b13b7..d9e5ccab577afe 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -137,7 +137,7 @@ class BloomFilterFuncBase : public FilterBase { virtual void find_fixed_len(const ColumnPtr& column, uint8_t* results, const uint8_t* __restrict filter = nullptr) = 0; - virtual uint16_t find_fixed_len_olap_engine(const char* data, const uint8_t* nullmap, + virtual uint16_t find_fixed_len_olap_engine(const IColumn& column, const uint8_t* nullmap, uint16_t* offsets, int number, bool is_parse_column) = 0; @@ -199,9 +199,10 @@ class BloomFilterFunc final : public BloomFilterFuncBase { return new_size; } - uint16_t find_fixed_len_olap_engine(const char* data, const uint8_t* nullmap, uint16_t* offsets, - int number, bool is_parse_column) override { - return OpV2::find_batch_olap_engine(*_bloom_filter, data, nullmap, offsets, number, + uint16_t find_fixed_len_olap_engine(const IColumn& column, const uint8_t* nullmap, + uint16_t* offsets, int number, + bool is_parse_column) override { + return OpV2::find_batch_olap_engine(*_bloom_filter, column, nullmap, offsets, number, is_parse_column); } diff --git a/be/src/exprs/bloom_filter_func_adaptor.h b/be/src/exprs/bloom_filter_func_adaptor.h index 5cc5d33215d359..62778bbf549fc2 100644 --- a/be/src/exprs/bloom_filter_func_adaptor.h +++ b/be/src/exprs/bloom_filter_func_adaptor.h @@ -80,11 +80,19 @@ class BloomFilterAdaptor : public FilterBase { template struct CommonFindOp { - static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, const char* data, - const uint8_t* nullmap, uint16_t* offsets, int number, + static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, + const doris::IColumn& column, const uint8_t* nullmap, + uint16_t* offsets, int number, const bool is_parse_column) { - return find_batch_olap(bloom_filter, data, nullmap, offsets, - number, is_parse_column); + // This path reinterpret-casts the column's raw data as a contiguous T[], which is only + // valid for fixed-length types. String types (StringRef/String) are not laid out this way + // and must go through StringFindOp instead. + static_assert(!std::is_same_v && !std::is_same_v, + "find_batch_olap_engine does not support string types; use StringFindOp"); + const T* __restrict data = reinterpret_cast(column.get_raw_data().data); + return find_batch_olap_impl( + bloom_filter, [data](int i) { return data[i]; }, nullmap, offsets, number, + is_parse_column); } template @@ -171,6 +179,16 @@ template struct StringFindOp : CommonFindOp { using CommonFindOp::for_each_with_filter; + static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, + const doris::IColumn& column, const uint8_t* nullmap, + uint16_t* offsets, int number, + const bool is_parse_column) { + const auto& col = assert_cast(column); + return find_batch_olap_impl( + bloom_filter, [&col](int i) { return col.get_data_at(i); }, nullmap, offsets, + number, is_parse_column); + } + static void insert_batch(BloomFilterAdaptor& bloom_filter, const ColumnPtr& column, size_t start) { auto _insert_batch_col_str = [&](const auto& col, const uint8_t* __restrict nullmap, diff --git a/be/src/exprs/bloom_filter_func_impl.h b/be/src/exprs/bloom_filter_func_impl.h index 0ee5da80f881fa..05b5993cecbdbb 100644 --- a/be/src/exprs/bloom_filter_func_impl.h +++ b/be/src/exprs/bloom_filter_func_impl.h @@ -19,6 +19,7 @@ #include +#include "common/compiler_util.h" #include "core/data_type/primitive_type.h" #include "core/string_ref.h" #include "exec/common/hash_table/hash.h" @@ -55,21 +56,18 @@ struct fixed_len_to_uint32_v2 { } }; -template -uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* data, - const uint8_t* nullmap, uint16_t* offsets, int number, - const bool is_parse_column) { - auto get_element = [](const char* input_data, int idx) { - return ((const T*)(input_data))[idx]; - }; - +// Per-row probe driver. `read(idx)` yields the row value (T or StringRef). +template +inline ALWAYS_INLINE uint16_t find_batch_olap_impl(const BloomFilterAdaptor& bloom_filter, + Accessor read, const uint8_t* nullmap, + uint16_t* offsets, int number, + const bool is_parse_column) { uint16_t new_size = 0; if (is_parse_column) { if (nullmap == nullptr) { for (uint16_t i = 0; i < number; i++) { uint16_t idx = offsets[i]; - if (!bloom_filter.test_element( - get_element(data, idx))) { + if (!bloom_filter.test_element(read(idx))) { continue; } offsets[new_size++] = idx; @@ -82,8 +80,7 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat continue; } } else { - if (!bloom_filter.test_element( - get_element(data, idx))) { + if (!bloom_filter.test_element(read(idx))) { continue; } } @@ -93,7 +90,7 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat } else { if (nullmap == nullptr) { for (uint16_t i = 0; i < number; i++) { - if (!bloom_filter.test_element(get_element(data, i))) { + if (!bloom_filter.test_element(read(i))) { continue; } offsets[new_size++] = i; @@ -105,8 +102,7 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat continue; } } else { - if (!bloom_filter.test_element( - get_element(data, i))) { + if (!bloom_filter.test_element(read(i))) { continue; } } diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index 9edcd39703bc28..e91a55df90deb2 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -193,14 +193,9 @@ std::shared_ptr create_olap_column_predicate( uint32_t column_id, const std::shared_ptr& filter, const TabletColumn* column, bool) { // currently only support like predicate - if constexpr (PT == TYPE_CHAR) { - return LikeColumnPredicate::create_shared(filter->_opposite, column_id, - column->name(), filter->_fn_ctx, - filter->_string_param); - } else if constexpr (PT == TYPE_VARCHAR || PT == TYPE_STRING) { - return LikeColumnPredicate::create_shared(filter->_opposite, column_id, - column->name(), filter->_fn_ctx, - filter->_string_param); + if constexpr (PT == TYPE_CHAR || PT == TYPE_VARCHAR || PT == TYPE_STRING) { + return LikeColumnPredicate::create_shared(filter->_opposite, column_id, column->name(), + filter->_fn_ctx, filter->_string_param); } throw Exception(ErrorCode::INTERNAL_ERROR, "function filter do not support type {}", PT); } diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h index 4c2631d10c355a..461c97956bcc7f 100644 --- a/be/src/exprs/function/like.h +++ b/be/src/exprs/function/like.h @@ -33,7 +33,6 @@ #include "common/status.h" #include "core/block/column_numbers.h" #include "core/column/column_string.h" -#include "core/column/predicate_column.h" #include "core/data_type/data_type_number.h" #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index 301e1b5337a360..521c2c07400c44 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -255,9 +255,9 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi while (!eof) { Block block; if (dictionary_coded) { - block.insert(ColumnWithTypeAndName( - ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR), - std::make_shared(), ICEBERG_FILE_PATH)); + block.insert(ColumnWithTypeAndName(ColumnDictI32::create(), + std::make_shared(), + ICEBERG_FILE_PATH)); } else { block.insert(ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), diff --git a/be/src/storage/predicate/bloom_filter_predicate.h b/be/src/storage/predicate/bloom_filter_predicate.h index 2878175644c191..cdc5ad50d3da13 100644 --- a/be/src/storage/predicate/bloom_filter_predicate.h +++ b/be/src/storage/predicate/bloom_filter_predicate.h @@ -20,8 +20,8 @@ #include "core/assert_cast.h" #include "core/column/column_dictionary.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" #include "core/column/column_vector.h" -#include "core/column/predicate_column.h" #include "core/data_type/primitive_type.h" #include "exprs/bloom_filter_func.h" #include "exprs/vruntimefilter_wrapper.h" @@ -80,11 +80,8 @@ class BloomFilterColumnPredicate final : public ColumnPredicate { new_size = _specific_filter->template find_dict_olap_engine( dict_col, null_map, sel, size); } else { - const auto& data = - assert_cast>*>(&column) - ->get_data(); - new_size = _specific_filter->find_fixed_len_olap_engine((char*)data.data(), null_map, - sel, size, data.size() != size); + new_size = _specific_filter->find_fixed_len_olap_engine( + column, null_map, sel, size, /*is_parse_column=*/column.size() != size); } return new_size; } diff --git a/be/src/storage/predicate/comparison_predicate.h b/be/src/storage/predicate/comparison_predicate.h index eb33d042e78feb..657fae0ec6d7a7 100644 --- a/be/src/storage/predicate/comparison_predicate.h +++ b/be/src/storage/predicate/comparison_predicate.h @@ -22,6 +22,8 @@ #include "common/compare.h" #include "core/column/column_dictionary.h" +#include "core/column/column_execute_util.h" +#include "core/column/column_string.h" #include "core/field.h" #include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/inverted/inverted_index_cache.h" // IWYU pragma: keep @@ -426,12 +428,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { __builtin_unreachable(); } } else { - auto* data_array = check_and_get_column< - const PredicateColumnType>>( - nested_column) - ->get_data() - .data(); - + ColumnElementView data_array {nested_column}; _base_loop_vec(size, flags, null_map.data(), data_array, _value); } } else { @@ -455,12 +452,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { __builtin_unreachable(); } } else { - auto* data_array = - check_and_get_column>>( - column) - ->get_data() - .data(); - + ColumnElementView data_array {column}; _base_loop_vec(size, flags, nullptr, data_array, _value); } } @@ -556,10 +548,12 @@ class ComparisonPredicateBase final : public ColumnPredicate { } } + // `data_array` is raw pointer or ColumnElementView wrapper; passed by value + // (no __restrict on struct), SIMD preserved via loop versioning. template void __attribute__((flatten)) _base_loop_vec(uint16_t size, bool* __restrict bflags, const uint8_t* __restrict null_map, - const TArray* __restrict data_array, const TValue& value) const { + TArray data_array, const TValue& value) const { //uint8_t helps compiler to generate vectorized code auto* flags = reinterpret_cast(bflags); if constexpr (is_and) { @@ -583,7 +577,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { template void _base_loop_bit(const uint16_t* sel, uint16_t size, bool* flags, - const uint8_t* __restrict null_map, const TArray* __restrict data_array, + const uint8_t* __restrict null_map, TArray data_array, const TValue& value) const { for (uint16_t i = 0; i < size; i++) { if (is_and ^ flags[i]) { @@ -617,11 +611,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { __builtin_unreachable(); } } else { - auto* data_array = - check_and_get_column>>(column) - ->get_data() - .data(); - + ColumnElementView data_array {*column}; _base_loop_bit(sel, size, flags, null_map, data_array, _value); } } @@ -655,14 +645,11 @@ class ComparisonPredicateBase final : public ColumnPredicate { return 0; } } else { - auto& pred_col = - check_and_get_column>>(column) - ->get_data(); - auto pred_col_data = pred_col.data(); uint16_t new_size = 0; + ColumnElementView pred_col {*column}; #define EVALUATE_WITH_NULL_IMPL(IDX) \ - _opposite ^ (!null_map[IDX] && _operator(pred_col_data[IDX], _value)) -#define EVALUATE_WITHOUT_NULL_IMPL(IDX) _opposite ^ _operator(pred_col_data[IDX], _value) + _opposite ^ (!null_map[IDX] && _operator(pred_col[IDX], _value)) +#define EVALUATE_WITHOUT_NULL_IMPL(IDX) _opposite ^ _operator(pred_col[IDX], _value) EVALUATE_BY_SELECTOR(EVALUATE_WITH_NULL_IMPL, EVALUATE_WITHOUT_NULL_IMPL) #undef EVALUATE_WITH_NULL_IMPL #undef EVALUATE_WITHOUT_NULL_IMPL diff --git a/be/src/storage/predicate/in_list_predicate.h b/be/src/storage/predicate/in_list_predicate.h index eef1796399d990..c5267acd7f2881 100644 --- a/be/src/storage/predicate/in_list_predicate.h +++ b/be/src/storage/predicate/in_list_predicate.h @@ -22,6 +22,8 @@ #include "common/exception.h" #include "core/column/column_dictionary.h" +#include "core/column/column_execute_util.h" +#include "core/column/column_string.h" #include "core/data_type/data_type.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" @@ -535,17 +537,12 @@ class InListPredicateBase final : public ColumnPredicate { __builtin_unreachable(); } } else { - auto& pred_col = - check_and_get_column>>(column) - ->get_data(); - auto pred_col_data = pred_col.data(); - + // ptr_at safe: HybridSet::find consumes the pointer synchronously. + ColumnElementView pred_col {*column}; #define EVALUATE_WITH_NULL_IMPL(IDX) \ - is_opposite ^ \ - (!(*null_map)[IDX] && \ - _operator(_values->find(reinterpret_cast(&pred_col_data[IDX])), false)) + is_opposite ^ (!(*null_map)[IDX] && _operator(_values->find(pred_col.ptr_at(IDX)), false)) #define EVALUATE_WITHOUT_NULL_IMPL(IDX) \ - is_opposite ^ _operator(_values->find(reinterpret_cast(&pred_col_data[IDX])), false) + is_opposite ^ _operator(_values->find(pred_col.ptr_at(IDX)), false) EVALUATE_BY_SELECTOR(EVALUATE_WITH_NULL_IMPL, EVALUATE_WITHOUT_NULL_IMPL) #undef EVALUATE_WITH_NULL_IMPL #undef EVALUATE_WITHOUT_NULL_IMPL @@ -596,15 +593,8 @@ class InListPredicateBase final : public ColumnPredicate { __builtin_unreachable(); } } else { - auto* nested_col_ptr = - check_and_get_column>>(column); - if (nested_col_ptr == nullptr) { - throw Exception(ErrorCode::INTERNAL_ERROR, - "InListPredicateBase: _base_evaluate_bit get invalid column type"); - } - - auto& data_array = nested_col_ptr->get_data(); - + // ptr_at safe: HybridSet::find consumes the pointer synchronously. + ColumnElementView view {*column}; for (uint16_t i = 0; i < size; i++) { if (is_and ^ flags[i]) { continue; @@ -618,17 +608,13 @@ class InListPredicateBase final : public ColumnPredicate { continue; } } - + bool hit = _operator(_values->find(view.ptr_at(idx)), false); if constexpr (!is_opposite) { - if (is_and ^ - _operator(_values->find(reinterpret_cast(&data_array[idx])), - false)) { + if (is_and ^ hit) { flags[i] = !is_and; } } else { - if (is_and ^ - !_operator(_values->find(reinterpret_cast(&data_array[idx])), - false)) { + if (is_and ^ !hit) { flags[i] = !is_and; } } diff --git a/be/src/storage/predicate/like_column_predicate.cpp b/be/src/storage/predicate/like_column_predicate.cpp index 6b60cdc3e78ab4..b23cf938a33ee6 100644 --- a/be/src/storage/predicate/like_column_predicate.cpp +++ b/be/src/storage/predicate/like_column_predicate.cpp @@ -17,7 +17,7 @@ #include "storage/predicate/like_column_predicate.h" -#include "core/column/predicate_column.h" +#include "core/column/column_string.h" #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" #include "exprs/function/like.h" @@ -25,32 +25,25 @@ namespace doris { -template -LikeColumnPredicate::LikeColumnPredicate(bool opposite, uint32_t column_id, std::string col_name, - doris::FunctionContext* fn_ctx, doris::StringRef val) - : ColumnPredicate(column_id, col_name, T, opposite), pattern(val) { - static_assert(T == TYPE_VARCHAR || T == TYPE_CHAR || T == TYPE_STRING, - "LikeColumnPredicate only supports the following types: TYPE_VARCHAR, TYPE_CHAR, " - "TYPE_STRING"); +LikeColumnPredicate::LikeColumnPredicate(bool opposite, uint32_t column_id, std::string col_name, + doris::FunctionContext* fn_ctx, doris::StringRef val) + : ColumnPredicate(column_id, col_name, TYPE_STRING, opposite), pattern(val) { _state = reinterpret_cast( fn_ctx->get_function_state(doris::FunctionContext::THREAD_LOCAL)); THROW_IF_ERROR(_state->search_state.clone(_like_state)); } -template -void LikeColumnPredicate::evaluate_vec(const IColumn& column, uint16_t size, bool* flags) const { +void LikeColumnPredicate::evaluate_vec(const IColumn& column, uint16_t size, bool* flags) const { _evaluate_vec(column, size, flags); } -template -void LikeColumnPredicate::evaluate_and_vec(const IColumn& column, uint16_t size, - bool* flags) const { +void LikeColumnPredicate::evaluate_and_vec(const IColumn& column, uint16_t size, + bool* flags) const { _evaluate_vec(column, size, flags); } -template -uint16_t LikeColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t* sel, - uint16_t size) const { +uint16_t LikeColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t* sel, + uint16_t size) const { uint16_t new_size = 0; if (column.is_nullable()) { auto* nullable_col = check_and_get_column(column); @@ -80,7 +73,7 @@ uint16_t LikeColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t } } } else { - auto* str_col = check_and_get_column>(nested_col); + auto* str_col = assert_cast(&nested_col); if (!nullable_col->has_null()) { ColumnUInt8::Container res(size, 0); for (uint16_t i = 0; i != size; i++) { @@ -121,8 +114,7 @@ uint16_t LikeColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t new_size += _opposite ^ flag; } } else { - const PredicateColumnType* str_col = - check_and_get_column>(column); + const auto* str_col = assert_cast(&column); ColumnUInt8::Container res(size, 0); for (uint16_t i = 0; i != size; i++) { @@ -138,7 +130,4 @@ uint16_t LikeColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t return new_size; } -template class LikeColumnPredicate; -template class LikeColumnPredicate; - } //namespace doris diff --git a/be/src/storage/predicate/like_column_predicate.h b/be/src/storage/predicate/like_column_predicate.h index 578b98ac6090b7..4c0ffd25f0c4fd 100644 --- a/be/src/storage/predicate/like_column_predicate.h +++ b/be/src/storage/predicate/like_column_predicate.h @@ -43,23 +43,22 @@ class Roaring; namespace doris { class FunctionContext; -template class LikeColumnPredicate final : public ColumnPredicate { public: ENABLE_FACTORY_CREATOR(LikeColumnPredicate); LikeColumnPredicate(bool opposite, uint32_t column_id, std::string col_name, doris::FunctionContext* fn_ctx, doris::StringRef val); ~LikeColumnPredicate() override = default; - LikeColumnPredicate(const LikeColumnPredicate& other, uint32_t col_id) + LikeColumnPredicate(const LikeColumnPredicate& other, uint32_t col_id) : ColumnPredicate(other, col_id) { _origin = other._origin; pattern = other.pattern; _state = other._state; _opposite = other._opposite; } - LikeColumnPredicate(const LikeColumnPredicate& other) = delete; + LikeColumnPredicate(const LikeColumnPredicate& other) = delete; std::shared_ptr clone(uint32_t col_id) const override { - return LikeColumnPredicate::create_shared(*this, col_id); + return LikeColumnPredicate::create_shared(*this, col_id); } std::string debug_string() const override { fmt::memory_buffer debug_string_buffer; diff --git a/be/src/storage/predicate/predicate_creator.h b/be/src/storage/predicate/predicate_creator.h index bff564761228f5..d4cd676cb4f668 100644 --- a/be/src/storage/predicate/predicate_creator.h +++ b/be/src/storage/predicate/predicate_creator.h @@ -51,7 +51,7 @@ std::shared_ptr build_set() { return std::make_shared>, HybridSet::CppType>, - PredicateColumnType>>>>(false); + typename PrimitiveTypeTraits::ColumnType>>>(false); } std::shared_ptr create_bloom_filter_predicate( diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index df10a7e3b55a81..8af8b3292b21a0 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -26,11 +26,13 @@ #include "common/config.h" #include "core/column/column_array.h" +#include "core/column/column_decimal.h" #include "core/column/column_dictionary.h" #include "core/column/column_map.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" #include "core/column/column_struct.h" -#include "core/column/predicate_column.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" @@ -87,96 +89,20 @@ DataTypePtr Schema::get_data_type_ptr(const TabletColumn& column) { return DataTypeFactory::instance().create_data_type(column); } -IColumn::MutablePtr Schema::get_predicate_column_ptr(const FieldType& type, bool is_nullable, +IColumn::MutablePtr Schema::get_predicate_column_ptr(const DataTypePtr& data_type, const ReaderType reader_type) { - IColumn::MutablePtr ptr = nullptr; - switch (type) { - case FieldType::OLAP_FIELD_TYPE_BOOL: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_TINYINT: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_SMALLINT: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_INT: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_FLOAT: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DOUBLE: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_BIGINT: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_LARGEINT: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DATE: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DATEV2: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DATETIME: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_CHAR: - if (config::enable_low_cardinality_optimize && reader_type == ReaderType::READER_QUERY) { - ptr = doris::ColumnDictI32::create(type); - } else { - ptr = doris::PredicateColumnType::create(); + // Low-cardinality dictionary optimization substitutes a ColumnDictI32 for the + // canonical string column during query reads. Every other case just materializes + // the data type's own canonical column (which already wraps nullable for us). + if (config::enable_low_cardinality_optimize && reader_type == ReaderType::READER_QUERY && + is_string_type(data_type->get_primitive_type())) { + IColumn::MutablePtr ptr = doris::ColumnDictI32::create(); + if (data_type->is_nullable()) { + return doris::ColumnNullable::create(std::move(ptr), doris::ColumnUInt8::create()); } - break; - case FieldType::OLAP_FIELD_TYPE_VARCHAR: - case FieldType::OLAP_FIELD_TYPE_STRING: - case FieldType::OLAP_FIELD_TYPE_JSONB: - if (config::enable_low_cardinality_optimize && reader_type == ReaderType::READER_QUERY) { - ptr = doris::ColumnDictI32::create(type); - } else { - ptr = doris::PredicateColumnType::create(); - } - break; - case FieldType::OLAP_FIELD_TYPE_DECIMAL: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DECIMAL32: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DECIMAL64: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_DECIMAL256: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_IPV4: - ptr = doris::PredicateColumnType::create(); - break; - case FieldType::OLAP_FIELD_TYPE_IPV6: - ptr = doris::PredicateColumnType::create(); - break; - default: - throw Exception( - ErrorCode::SCHEMA_SCHEMA_FIELD_INVALID, - fmt::format("Unexpected type when choosing predicate column, type={}", int(type))); - } - - if (is_nullable) { - return doris::ColumnNullable::create(std::move(ptr), doris::ColumnUInt8::create()); + return ptr; } - return ptr; + return data_type->create_column(); } } // namespace doris diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index 831f85312216a5..80d2ce9e9cb797 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -78,7 +78,7 @@ class Schema { static DataTypePtr get_data_type_ptr(const TabletColumn& column); - static IColumn::MutablePtr get_predicate_column_ptr(const FieldType& type, bool is_nullable, + static IColumn::MutablePtr get_predicate_column_ptr(const DataTypePtr& data_type, const ReaderType reader_type); const std::vector& columns() const { return _cols; } diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 9ca528cc122b81..36d4ad2a2e4b3f 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -323,8 +323,7 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) ? assert_cast(dst.get())->get_nested_column_ptr().get() : dst.get(); - /// `PredicateColumnType` does not support `void get(size_t n, Field& res)`, - /// So here only check `CoumnVector` + /// Only verify when the destination column carries Field-accessible TYPE_INT data. if (check_and_get_column>(non_nullable_column) == nullptr) { return; } diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 75becf85d60ed6..24c078814f05aa 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -547,8 +547,7 @@ Status SegmentIterator::_lazy_init(Block* block) { // Here, cid will not go out of bounds // because the size of _current_return_columns equals _schema->tablet_columns().size() _current_return_columns[cid] = Schema::get_predicate_column_ptr( - storage_column_type->get_storage_field_type(), - storage_column_type->is_nullable(), _opts.io_ctx.reader_type)); + storage_column_type, _opts.io_ctx.reader_type)); _current_return_columns[cid]->set_rowset_segment_id( {_segment->rowset_id(), _segment->id()}); _current_return_columns[cid]->reserve(nrows_reserve_limit); @@ -1185,8 +1184,7 @@ bool SegmentIterator::_check_apply_by_inverted_index(std::shared_ptr*>(pred.get()) != nullptr || - dynamic_cast*>(pred.get()) != nullptr) { + if (dynamic_cast(pred.get()) != nullptr) { return false; } diff --git a/be/src/storage/tablet/tablet_reader.cpp b/be/src/storage/tablet/tablet_reader.cpp index 62fca98ac7956d..7e3f3ffd701ab3 100644 --- a/be/src/storage/tablet/tablet_reader.cpp +++ b/be/src/storage/tablet/tablet_reader.cpp @@ -438,8 +438,7 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { std::inserter(predicates, predicates.begin())); // Function filter push down to storage engine auto is_like_predicate = [](std::shared_ptr _pred) { - return dynamic_cast*>(_pred.get()) != nullptr || - dynamic_cast*>(_pred.get()) != nullptr; + return dynamic_cast(_pred.get()) != nullptr; }; for (const auto& filter : read_params.function_filters) { diff --git a/be/test/core/column/column_dictionary_test.cpp b/be/test/core/column/column_dictionary_test.cpp index 929da3a9d89ec9..e802eae7541c9f 100644 --- a/be/test/core/column/column_dictionary_test.cpp +++ b/be/test/core/column/column_dictionary_test.cpp @@ -54,10 +54,10 @@ class ColumnDictionaryTest : public CommonColumnTest { column_dict_data = ColumnString::create(); column_dict_indices = ColumnInt32::create(); - column_dict_char = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_CHAR); + column_dict_char = ColumnDictI32::create(); EXPECT_TRUE(column_dict_char->is_dict_empty()); - column_dict_varchar = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_STRING); - column_dict_str = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_STRING); + column_dict_varchar = ColumnDictI32::create(); + column_dict_str = ColumnDictI32::create(); load_columns_data(); } @@ -184,7 +184,7 @@ TEST_F(ColumnDictionaryTest, insert_default) { } */ TEST_F(ColumnDictionaryTest, clear) { - auto target_column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_CHAR); + auto target_column = ColumnDictI32::create(); target_column->insert_many_dict_data(dict_array.data(), dict_array.size()); target_column->clear(); @@ -197,9 +197,7 @@ TEST_F(ColumnDictionaryTest, allocated_bytes) { EXPECT_EQ(column_dict_char->allocated_bytes(), column_dict_char->size() * 4); } TEST_F(ColumnDictionaryTest, has_enough_capacity) { - EXPECT_THROW(column_dict_char->has_enough_capacity( - ColumnDictI32(FieldType::OLAP_FIELD_TYPE_VARCHAR)), - Exception); + EXPECT_THROW(column_dict_char->has_enough_capacity(*ColumnDictI32::create()), Exception); } TEST_F(ColumnDictionaryTest, pop_back) { EXPECT_THROW(column_dict_char->pop_back(9), Exception); @@ -281,8 +279,7 @@ TEST_F(ColumnDictionaryTest, filter_by_selector) { test_func(column_dict_char); } TEST_F(ColumnDictionaryTest, insert_many_dict_data) { - ColumnDictI32::MutablePtr tmp_column_dict = - ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_CHAR); + ColumnDictI32::MutablePtr tmp_column_dict = ColumnDictI32::create(); tmp_column_dict->insert_many_dict_data(dict_array.data(), dict_array.size()); for (size_t i = 0; i != dict_array.size(); ++i) { EXPECT_EQ(tmp_column_dict->get_value(i), dict_array[i]); @@ -290,8 +287,7 @@ TEST_F(ColumnDictionaryTest, insert_many_dict_data) { } TEST_F(ColumnDictionaryTest, convert_dict_codes_if_necessary) { { - ColumnDictI32::MutablePtr tmp_column_dict = - ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_CHAR); + ColumnDictI32::MutablePtr tmp_column_dict = ColumnDictI32::create(); tmp_column_dict->convert_dict_codes_if_necessary(); EXPECT_FALSE(tmp_column_dict->is_dict_sorted()); EXPECT_FALSE(tmp_column_dict->is_dict_code_converted()); @@ -316,7 +312,7 @@ TEST_F(ColumnDictionaryTest, find_code) { /* TEST_F(ColumnDictionaryTest, initialize_hash_values_for_runtime_filter) { ColumnDictI32::MutablePtr tmp_column_dict = - ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_CHAR); + ColumnDictI32::create(); auto dict_data_row_count = column_dict_data->size(); auto dict_indices_row_count = column_dict_indices->size(); tmp_column_dict->reserve(dict_indices_row_count); @@ -344,8 +340,7 @@ TEST_F(ColumnDictionaryTest, rowset_segment_id) { EXPECT_EQ(ids.second, segment_id); } TEST_F(ColumnDictionaryTest, convert_to_predicate_column_if_dictionary) { - ColumnDictI32::MutablePtr tmp_column_dict = - ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_CHAR); + ColumnDictI32::MutablePtr tmp_column_dict = ColumnDictI32::create(); auto dict_data_row_count = column_dict_data->size(); auto dict_indices_row_count = column_dict_indices->size(); tmp_column_dict->reserve(dict_indices_row_count); diff --git a/be/test/core/column/column_nullable_test.cpp b/be/test/core/column/column_nullable_test.cpp index 01e9e05b79334b..96db41296eff92 100644 --- a/be/test/core/column/column_nullable_test.cpp +++ b/be/test/core/column/column_nullable_test.cpp @@ -28,7 +28,7 @@ #include "core/block/block.h" #include "core/column/column_decimal.h" #include "core/column/column_nullable_test.h" -#include "core/column/predicate_column.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" @@ -98,8 +98,7 @@ TEST(ColumnNullableTest, CreateRejectsMismatchedNestedAndNullMapSizes) { } TEST(ColumnNullableTest, PredicateTest) { - auto nullable_pred = - ColumnNullable::create(PredicateColumnType::create(), ColumnUInt8::create()); + auto nullable_pred = ColumnNullable::create(ColumnDate::create(), ColumnUInt8::create()); nullable_pred->insert_many_defaults(3); EXPECT_TRUE(nullable_pred->has_null()); nullable_pred->insert_many_defaults(10); diff --git a/be/test/core/column/common_column_test.h b/be/test/core/column/common_column_test.h index 7088893b895faf..93c5f966c5c551 100644 --- a/be/test/core/column/common_column_test.h +++ b/be/test/core/column/common_column_test.h @@ -28,6 +28,7 @@ #include "core/column/column.h" #include "core/column/column_array.h" +#include "core/column/column_decimal.h" #include "core/column/column_dictionary.h" #include "core/column/column_map.h" #include "core/cow.h" diff --git a/be/test/core/column/predicate_column_test.cpp b/be/test/core/column/predicate_column_test.cpp deleted file mode 100644 index 9d1b4d36140188..00000000000000 --- a/be/test/core/column/predicate_column_test.cpp +++ /dev/null @@ -1,1082 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "core/column/predicate_column.h" - -#include -#include -#include - -#include "common/status.h" -#include "core/column/column_decimal.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/define_primitive_type.h" -#include "core/decimal12.h" -#include "core/field.h" -#include "core/types.h" -#include "core/uint24.h" -#include "core/value/vdatetime_value.h" -#include "testutil/column_helper.h" - -namespace doris { - -// ============================================================================ -// Test size() for all types -// ============================================================================ -TEST(PredicateColumnTest, SizeBoolean) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); - col->insert_default(); - EXPECT_EQ(col->size(), 2); -} - -TEST(PredicateColumnTest, SizeTinyInt) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeSmallInt) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeInt) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeBigInt) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeLargeInt) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeFloat) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDouble) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDate) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDateV2) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDateTime) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDateTimeV2) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeChar) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeVarchar) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeString) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDecimalV2) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDecimal32) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDecimal64) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDecimal128I) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeDecimal256) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeIPv4) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, SizeIPv6) { - auto col = PredicateColumnType::create(); - EXPECT_EQ(col->size(), 0); - col->insert_default(); - EXPECT_EQ(col->size(), 1); -} - -// ============================================================================ -// Test insert_data() for all types -// ============================================================================ -TEST(PredicateColumnTest, InsertDataBoolean) { - auto col = PredicateColumnType::create(); - col->reserve(3); - UInt8 vals[] = {0, 1, 1}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], 0); - EXPECT_EQ(col->get_data()[1], 1); - EXPECT_EQ(col->get_data()[2], 1); -} - -TEST(PredicateColumnTest, InsertDataTinyInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int8 vals[] = {-128, 0, 127}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], -128); - EXPECT_EQ(col->get_data()[1], 0); - EXPECT_EQ(col->get_data()[2], 127); -} - -TEST(PredicateColumnTest, InsertDataSmallInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int16 vals[] = {-32768, 0, 32767}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], -32768); - EXPECT_EQ(col->get_data()[1], 0); - EXPECT_EQ(col->get_data()[2], 32767); -} - -TEST(PredicateColumnTest, InsertDataInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int32 vals[] = {INT32_MIN, 0, INT32_MAX}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], INT32_MIN); - EXPECT_EQ(col->get_data()[1], 0); - EXPECT_EQ(col->get_data()[2], INT32_MAX); -} - -TEST(PredicateColumnTest, InsertDataBigInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int64 vals[] = {INT64_MIN, 0, INT64_MAX}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], INT64_MIN); - EXPECT_EQ(col->get_data()[1], 0); - EXPECT_EQ(col->get_data()[2], INT64_MAX); -} - -TEST(PredicateColumnTest, InsertDataLargeInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int128 vals[] = {-(Int128(1) << 100), Int128(0), Int128(1) << 100}; - for (auto& v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], vals[0]); - EXPECT_EQ(col->get_data()[1], vals[1]); - EXPECT_EQ(col->get_data()[2], vals[2]); -} - -TEST(PredicateColumnTest, InsertDataFloat) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Float32 vals[] = {-3.14f, 0.0f, 2.718f}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_FLOAT_EQ(col->get_data()[0], -3.14f); - EXPECT_FLOAT_EQ(col->get_data()[1], 0.0f); - EXPECT_FLOAT_EQ(col->get_data()[2], 2.718f); -} - -TEST(PredicateColumnTest, InsertDataDouble) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Float64 vals[] = {-3.14159265358979, 0.0, 2.71828182845904}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_DOUBLE_EQ(col->get_data()[0], -3.14159265358979); - EXPECT_DOUBLE_EQ(col->get_data()[1], 0.0); - EXPECT_DOUBLE_EQ(col->get_data()[2], 2.71828182845904); -} - -TEST(PredicateColumnTest, InsertDataDateV2) { - auto col = PredicateColumnType::create(); - col->reserve(3); - uint32_t vals[] = {20230115, 20230620, 20231231}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); -} - -TEST(PredicateColumnTest, InsertDataDateTimeV2) { - auto col = PredicateColumnType::create(); - col->reserve(3); - uint64_t vals[] = {1234567890123ULL, 2345678901234ULL, 3456789012345ULL}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); -} - -TEST(PredicateColumnTest, InsertDataString) { - auto col = PredicateColumnType::create(); - col->reserve(3); - std::string s1 = "hello"; - std::string s2 = "world"; - std::string s3 = ""; - col->insert_data(s1.data(), s1.size()); - col->insert_data(s2.data(), s2.size()); - col->insert_data(s3.data(), s3.size()); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "hello"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "world"); - EXPECT_EQ(col->get_data()[2].size, 0); -} - -TEST(PredicateColumnTest, InsertDataDecimal32) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Decimal32 vals[] = {Decimal32(12345), Decimal32(-67890), Decimal32(0)}; - for (auto& v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], Decimal32(12345)); - EXPECT_EQ(col->get_data()[1], Decimal32(-67890)); - EXPECT_EQ(col->get_data()[2], Decimal32(0)); -} - -TEST(PredicateColumnTest, InsertDataDecimal64) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Decimal64 vals[] = {Decimal64(Int64(123456789012LL)), Decimal64(Int64(-987654321098LL)), - Decimal64(Int64(0))}; - for (auto& v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], Decimal64(Int64(123456789012LL))); - EXPECT_EQ(col->get_data()[1], Decimal64(Int64(-987654321098LL))); - EXPECT_EQ(col->get_data()[2], Decimal64(Int64(0))); -} - -TEST(PredicateColumnTest, InsertDataIPv4) { - auto col = PredicateColumnType::create(); - col->reserve(3); - IPv4 vals[] = {IPv4(0x7F000001), IPv4(0xC0A80001), IPv4(0x00000000)}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], IPv4(0x7F000001)); - EXPECT_EQ(col->get_data()[1], IPv4(0xC0A80001)); - EXPECT_EQ(col->get_data()[2], IPv4(0x00000000)); -} - -TEST(PredicateColumnTest, InsertDataIPv6) { - auto col = PredicateColumnType::create(); - col->reserve(2); - IPv6 vals[] = {IPv6(1), IPv6(0)}; - for (auto& v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - EXPECT_EQ(col->size(), 2); - EXPECT_EQ(col->get_data()[0], IPv6(1)); - EXPECT_EQ(col->get_data()[1], IPv6(0)); -} - -// ============================================================================ -// Test insert_many_fix_len_data() for numeric types -// ============================================================================ -TEST(PredicateColumnTest, InsertManyFixLenDataInt) { - auto col = PredicateColumnType::create(); - col->reserve(5); - Int32 vals[] = {1, 2, 3, 4, 5}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 5); - EXPECT_EQ(col->size(), 5); - for (int i = 0; i < 5; i++) { - EXPECT_EQ(col->get_data()[i], i + 1); - } -} - -TEST(PredicateColumnTest, InsertManyFixLenDataBigInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int64 vals[] = {100000000000LL, 200000000000LL, 300000000000LL}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], 100000000000LL); - EXPECT_EQ(col->get_data()[1], 200000000000LL); - EXPECT_EQ(col->get_data()[2], 300000000000LL); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataFloat) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Float32 vals[] = {1.1f, 2.2f, 3.3f}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); - EXPECT_FLOAT_EQ(col->get_data()[0], 1.1f); - EXPECT_FLOAT_EQ(col->get_data()[1], 2.2f); - EXPECT_FLOAT_EQ(col->get_data()[2], 3.3f); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataDouble) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Float64 vals[] = {1.11, 2.22, 3.33}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); - EXPECT_DOUBLE_EQ(col->get_data()[0], 1.11); - EXPECT_DOUBLE_EQ(col->get_data()[1], 2.22); - EXPECT_DOUBLE_EQ(col->get_data()[2], 3.33); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataDateV2) { - auto col = PredicateColumnType::create(); - col->reserve(3); - uint32_t vals[] = {20230115, 20230620, 20231231}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataDateTimeV2) { - auto col = PredicateColumnType::create(); - col->reserve(3); - uint64_t vals[] = {1234567890123ULL, 2345678901234ULL, 3456789012345ULL}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataDecimal32) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Decimal32 vals[] = {Decimal32(100), Decimal32(200), Decimal32(300)}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], Decimal32(100)); - EXPECT_EQ(col->get_data()[1], Decimal32(200)); - EXPECT_EQ(col->get_data()[2], Decimal32(300)); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataDecimal64) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Decimal64 vals[] = {Decimal64(Int64(1000)), Decimal64(Int64(2000)), Decimal64(Int64(3000))}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], Decimal64(Int64(1000))); - EXPECT_EQ(col->get_data()[1], Decimal64(Int64(2000))); - EXPECT_EQ(col->get_data()[2], Decimal64(Int64(3000))); -} - -TEST(PredicateColumnTest, InsertManyFixLenDataIPv4) { - auto col = PredicateColumnType::create(); - col->reserve(3); - IPv4 vals[] = {IPv4(1), IPv4(2), IPv4(3)}; - col->insert_many_fix_len_data(reinterpret_cast(vals), 3); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], IPv4(1)); - EXPECT_EQ(col->get_data()[1], IPv4(2)); - EXPECT_EQ(col->get_data()[2], IPv4(3)); -} - -// ============================================================================ -// Test insert_many_date() for TYPE_DATE -// ============================================================================ -TEST(PredicateColumnTest, InsertManyDate) { - auto col = PredicateColumnType::create(); - col->reserve(3); - uint24_t date_vals[3]; - date_vals[0] = (2023 << 9) | (1 << 5) | 15; - date_vals[1] = (2023 << 9) | (6 << 5) | 20; - date_vals[2] = (2023 << 9) | (12 << 5) | 31; - col->insert_many_date(reinterpret_cast(date_vals), 3); - EXPECT_EQ(col->size(), 3); -} - -// ============================================================================ -// Test insert_many_datetime() for TYPE_DATETIME -// ============================================================================ -TEST(PredicateColumnTest, InsertManyDateTime) { - auto col = PredicateColumnType::create(); - col->reserve(3); - uint64_t datetime_vals[] = {20230115120000ULL, 20230620153045ULL, 20231231235959ULL}; - col->insert_many_datetime(reinterpret_cast(datetime_vals), 3); - EXPECT_EQ(col->size(), 3); -} - -// ============================================================================ -// Test insert_many_decimalv2() for TYPE_DECIMALV2 -// ============================================================================ -TEST(PredicateColumnTest, InsertManyDecimalV2) { - auto col = PredicateColumnType::create(); - col->reserve(3); - decimal12_t decimals[3]; - decimals[0].integer = 123; - decimals[0].fraction = 456000000; - decimals[1].integer = -789; - decimals[1].fraction = 123000000; - decimals[2].integer = 0; - decimals[2].fraction = 999000000; - col->insert_many_fix_len_data(reinterpret_cast(decimals), 3); - EXPECT_EQ(col->size(), 3); -} - -// ============================================================================ -// Test insert_many_strings() for string types -// ============================================================================ -TEST(PredicateColumnTest, InsertManyStringsString) { - auto col = PredicateColumnType::create(); - col->reserve(5); - StringRef strings[] = {StringRef("one", 3), StringRef("two", 3), StringRef("three", 5)}; - col->insert_many_strings(strings, 3); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "one"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "two"); - EXPECT_EQ(std::string(col->get_data()[2].data, col->get_data()[2].size), "three"); -} - -TEST(PredicateColumnTest, InsertManyStringsVarchar) { - auto col = PredicateColumnType::create(); - col->reserve(3); - StringRef strings[] = {StringRef("abc", 3), StringRef("defgh", 5)}; - col->insert_many_strings(strings, 2); - EXPECT_EQ(col->size(), 2); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "abc"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "defgh"); -} - -TEST(PredicateColumnTest, InsertManyStringsChar) { - auto col = PredicateColumnType::create(); - col->reserve(3); - StringRef strings[] = {StringRef("ab", 2), StringRef("cd", 2)}; - col->insert_many_strings(strings, 2); - EXPECT_EQ(col->size(), 2); -} - -TEST(PredicateColumnTest, InsertManyStringsEmpty) { - auto col = PredicateColumnType::create(); - col->reserve(3); - col->insert_many_strings(nullptr, 0); - EXPECT_EQ(col->size(), 0); -} - -// ============================================================================ -// Test insert_many_continuous_binary_data() for string types -// ============================================================================ -TEST(PredicateColumnTest, InsertManyContinuousBinaryDataString) { - auto col = PredicateColumnType::create(); - col->reserve(3); - const char* data = "helloworld!"; - uint32_t offsets[] = {0, 5, 10, 11}; - col->insert_many_continuous_binary_data(data, offsets, 3); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "hello"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "world"); - EXPECT_EQ(std::string(col->get_data()[2].data, col->get_data()[2].size), "!"); -} - -TEST(PredicateColumnTest, InsertManyContinuousBinaryDataVarchar) { - auto col = PredicateColumnType::create(); - col->reserve(2); - const char* data = "abcdef"; - uint32_t offsets[] = {0, 3, 6}; - col->insert_many_continuous_binary_data(data, offsets, 2); - EXPECT_EQ(col->size(), 2); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "abc"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "def"); -} - -TEST(PredicateColumnTest, InsertManyContinuousBinaryDataEmpty) { - auto col = PredicateColumnType::create(); - col->reserve(3); - const char* data = "test"; - uint32_t offsets[] = {0}; - col->insert_many_continuous_binary_data(data, offsets, 0); - EXPECT_EQ(col->size(), 0); -} - -// ============================================================================ -// Test insert_many_dict_data() for string types -// ============================================================================ -TEST(PredicateColumnTest, InsertManyDictDataString) { - auto col = PredicateColumnType::create(); - col->reserve(5); - StringRef dict[] = {StringRef("apple", 5), StringRef("banana", 6), StringRef("cherry", 6)}; - int32_t codewords[] = {0, 1, 2, 0, 1}; - col->insert_many_dict_data(codewords, 0, dict, 5, 3); - EXPECT_EQ(col->size(), 5); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "apple"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "banana"); - EXPECT_EQ(std::string(col->get_data()[2].data, col->get_data()[2].size), "cherry"); - EXPECT_EQ(std::string(col->get_data()[3].data, col->get_data()[3].size), "apple"); - EXPECT_EQ(std::string(col->get_data()[4].data, col->get_data()[4].size), "banana"); -} - -TEST(PredicateColumnTest, InsertManyDictDataVarchar) { - auto col = PredicateColumnType::create(); - col->reserve(3); - StringRef dict[] = {StringRef("x", 1), StringRef("yy", 2)}; - int32_t codewords[] = {0, 1, 0}; - col->insert_many_dict_data(codewords, 0, dict, 3, 2); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(std::string(col->get_data()[0].data, col->get_data()[0].size), "x"); - EXPECT_EQ(std::string(col->get_data()[1].data, col->get_data()[1].size), "yy"); - EXPECT_EQ(std::string(col->get_data()[2].data, col->get_data()[2].size), "x"); -} - -// ============================================================================ -// Test insert_duplicate_fields() for all types -// ============================================================================ -TEST(PredicateColumnTest, InsertDuplicateFieldsString) { - auto col = PredicateColumnType::create(); - col->reserve(10); - std::string test_str = "hello"; - Field field = Field::create_field(test_str); - col->insert_duplicate_fields(field, 5); - EXPECT_EQ(col->get_data()[col->size() - 1].data, - col->get_data()[0].data + (col->size() - 1) * test_str.size()); - EXPECT_EQ(col->size(), 5); - for (size_t i = 0; i < 5; i++) { - StringRef ref = col->get_data()[i]; - EXPECT_EQ(std::string(ref.data, ref.size), test_str); - } - // Insert another batch to verify memory doesn't overlap - std::string str2 = "world"; - Field field2 = Field::create_field(str2); - col->insert_duplicate_fields(field2, 3); - EXPECT_EQ(col->size(), 8); - for (size_t i = 0; i < 5; i++) { - EXPECT_EQ(std::string(col->get_data()[i].data, col->get_data()[i].size), test_str); - } - for (size_t i = 5; i < 8; i++) { - EXPECT_EQ(std::string(col->get_data()[i].data, col->get_data()[i].size), str2); - } -} - -TEST(PredicateColumnTest, InsertDuplicateFieldsInt) { - auto col = PredicateColumnType::create(); - col->reserve(10); - Int32 val = 42; - Field field = Field::create_field(val); - col->insert_duplicate_fields(field, 5); - EXPECT_EQ(col->size(), 5); - for (size_t i = 0; i < 5; i++) { - EXPECT_EQ(col->get_data()[i], 42); - } -} - -TEST(PredicateColumnTest, InsertDuplicateFieldsLargeInt) { - auto col = PredicateColumnType::create(); - col->reserve(10); - Int128 val = Int128(123456789012345LL) * Int128(1000000000LL); - Field field = Field::create_field(val); - col->insert_duplicate_fields(field, 3); - EXPECT_EQ(col->size(), 3); - for (size_t i = 0; i < 3; i++) { - EXPECT_EQ(col->get_data()[i], val); - } -} - -TEST(PredicateColumnTest, InsertDuplicateFieldsZeroCount) { - auto col = PredicateColumnType::create(); - col->reserve(5); - Int32 val = 42; - Field field = Field::create_field(val); - col->insert_duplicate_fields(field, 0); - EXPECT_EQ(col->size(), 0); -} - -// ============================================================================ -// Test insert_default() for all types -// ============================================================================ -TEST(PredicateColumnTest, InsertDefaultInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - col->insert_default(); - col->insert_default(); - EXPECT_EQ(col->size(), 2); - EXPECT_EQ(col->get_data()[0], 0); - EXPECT_EQ(col->get_data()[1], 0); -} - -TEST(PredicateColumnTest, InsertDefaultDouble) { - auto col = PredicateColumnType::create(); - col->reserve(2); - col->insert_default(); - EXPECT_EQ(col->size(), 1); - EXPECT_DOUBLE_EQ(col->get_data()[0], 0.0); -} - -TEST(PredicateColumnTest, InsertDefaultString) { - auto col = PredicateColumnType::create(); - col->reserve(2); - col->insert_default(); - EXPECT_EQ(col->size(), 1); - EXPECT_EQ(col->get_data()[0].size, 0); -} - -// ============================================================================ -// Test clear() for all types -// ============================================================================ -TEST(PredicateColumnTest, ClearInt) { - auto col = PredicateColumnType::create(); - col->reserve(10); - Int32 val = 42; - col->insert_data(reinterpret_cast(&val), sizeof(val)); - col->insert_data(reinterpret_cast(&val), sizeof(val)); - EXPECT_EQ(col->size(), 2); - col->clear(); - EXPECT_EQ(col->size(), 0); - col->insert_data(reinterpret_cast(&val), sizeof(val)); - EXPECT_EQ(col->size(), 1); -} - -TEST(PredicateColumnTest, ClearString) { - auto col = PredicateColumnType::create(); - col->reserve(5); - std::string s = "test"; - col->insert_data(s.data(), s.size()); - EXPECT_EQ(col->size(), 1); - col->clear(); - EXPECT_EQ(col->size(), 0); -} - -// ============================================================================ -// Test reserve() for all types -// ============================================================================ -TEST(PredicateColumnTest, ReserveInt) { - auto col = PredicateColumnType::create(); - col->reserve(100); - EXPECT_EQ(col->size(), 0); - for (int i = 0; i < 100; i++) { - Int32 val = i; - col->insert_data(reinterpret_cast(&val), sizeof(val)); - } - EXPECT_EQ(col->size(), 100); -} - -TEST(PredicateColumnTest, ReserveString) { - auto col = PredicateColumnType::create(); - col->reserve(50); - EXPECT_EQ(col->size(), 0); -} - -// ============================================================================ -// Test byte_size() and allocated_bytes() for all types -// ============================================================================ -TEST(PredicateColumnTest, ByteSizeInt) { - auto col = PredicateColumnType::create(); - col->reserve(5); - EXPECT_EQ(col->byte_size(), 0); - Int32 val = 1; - col->insert_data(reinterpret_cast(&val), sizeof(val)); - EXPECT_EQ(col->byte_size(), sizeof(Int32)); - col->insert_data(reinterpret_cast(&val), sizeof(val)); - EXPECT_EQ(col->byte_size(), 2 * sizeof(Int32)); - EXPECT_EQ(col->allocated_bytes(), col->byte_size()); -} - -TEST(PredicateColumnTest, ByteSizeBigInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int64 val = 12345; - col->insert_data(reinterpret_cast(&val), sizeof(val)); - col->insert_data(reinterpret_cast(&val), sizeof(val)); - col->insert_data(reinterpret_cast(&val), sizeof(val)); - EXPECT_EQ(col->byte_size(), 3 * sizeof(Int64)); - EXPECT_EQ(col->allocated_bytes(), 3 * sizeof(Int64)); -} - -TEST(PredicateColumnTest, ByteSizeString) { - auto col = PredicateColumnType::create(); - col->reserve(3); - std::string s = "hello"; - col->insert_data(s.data(), s.size()); - EXPECT_EQ(col->byte_size(), sizeof(StringRef)); -} - -// ============================================================================ -// Test get_name() for all types -// ============================================================================ -TEST(PredicateColumnTest, GetNameAllTypes) { - EXPECT_EQ(PredicateColumnType::create()->get_name(), "BOOL"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "TINYINT"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "SMALLINT"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "INT"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "BIGINT"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "LARGEINT"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "FLOAT"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DOUBLE"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DATE"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DATEV2"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DATETIME"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DATETIMEV2"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "CHAR"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "VARCHAR"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "STRING"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DECIMALV2"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DECIMAL32"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DECIMAL64"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DECIMAL128I"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "DECIMAL256"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "IPV4"); - EXPECT_EQ(PredicateColumnType::create()->get_name(), "IPV6"); -} - -// ============================================================================ -// Test clone_resized() for all types -// ============================================================================ -TEST(PredicateColumnTest, CloneResizedInt) { - auto col = PredicateColumnType::create(); - col->reserve(5); - Int32 val = 42; - col->insert_data(reinterpret_cast(&val), sizeof(val)); - auto cloned = col->clone_resized(0); - EXPECT_EQ(cloned->size(), 0); -} - -TEST(PredicateColumnTest, CloneResizedString) { - auto col = PredicateColumnType::create(); - col->reserve(5); - std::string s = "test"; - col->insert_data(s.data(), s.size()); - auto cloned = col->clone_resized(0); - EXPECT_EQ(cloned->size(), 0); -} - -// ============================================================================ -// Test get_data_at() for string types -// ============================================================================ -TEST(PredicateColumnTest, GetDataAtString) { - auto col = PredicateColumnType::create(); - col->reserve(3); - std::string s1 = "abc"; - std::string s2 = "defgh"; - col->insert_data(s1.data(), s1.size()); - col->insert_data(s2.data(), s2.size()); - StringRef ref0 = col->get_data_at(0); - StringRef ref1 = col->get_data_at(1); - EXPECT_EQ(std::string(ref0.data, ref0.size), "abc"); - EXPECT_EQ(std::string(ref1.data, ref1.size), "defgh"); -} - -TEST(PredicateColumnTest, GetDataAtVarchar) { - auto col = PredicateColumnType::create(); - col->reserve(2); - std::string s1 = "test"; - col->insert_data(s1.data(), s1.size()); - StringRef ref = col->get_data_at(0); - EXPECT_EQ(std::string(ref.data, ref.size), "test"); -} - -// ============================================================================ -// Test get_data() for all types -// ============================================================================ -TEST(PredicateColumnTest, GetDataInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int32 vals[] = {1, 2, 3}; - for (auto v : vals) { - col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - auto& data = col->get_data(); - EXPECT_EQ(data.size(), 3); - EXPECT_EQ(data[0], 1); - EXPECT_EQ(data[1], 2); - EXPECT_EQ(data[2], 3); -} - -TEST(PredicateColumnTest, GetDataString) { - auto col = PredicateColumnType::create(); - col->reserve(2); - std::string s1 = "a"; - std::string s2 = "bb"; - col->insert_data(s1.data(), s1.size()); - col->insert_data(s2.data(), s2.size()); - const auto& data = col->get_data(); - EXPECT_EQ(data.size(), 2); - EXPECT_EQ(std::string(data[0].data, data[0].size), "a"); - EXPECT_EQ(std::string(data[1].data, data[1].size), "bb"); -} - -// ============================================================================ -// Test filter_by_selector() for all types -// ============================================================================ -TEST(PredicateColumnTest, FilterBySelectorInt) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(5); - Int32 vals[] = {10, 20, 30, 40, 50}; - for (auto v : vals) { - pred_col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - auto result_col = ColumnInt32::create(); - uint16_t selector[] = {0, 2, 4}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 3, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 3); - EXPECT_EQ(result_col->get_data()[0], 10); - EXPECT_EQ(result_col->get_data()[1], 30); - EXPECT_EQ(result_col->get_data()[2], 50); -} - -TEST(PredicateColumnTest, FilterBySelectorBigInt) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(4); - Int64 vals[] = {100LL, 200LL, 300LL, 400LL}; - for (auto v : vals) { - pred_col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - auto result_col = ColumnInt64::create(); - uint16_t selector[] = {1, 3}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 2, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 2); - EXPECT_EQ(result_col->get_data()[0], 200LL); - EXPECT_EQ(result_col->get_data()[1], 400LL); -} - -TEST(PredicateColumnTest, FilterBySelectorFloat) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(3); - Float32 vals[] = {1.1f, 2.2f, 3.3f}; - for (auto v : vals) { - pred_col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - auto result_col = ColumnFloat32::create(); - uint16_t selector[] = {0, 2}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 2, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 2); - EXPECT_FLOAT_EQ(result_col->get_data()[0], 1.1f); - EXPECT_FLOAT_EQ(result_col->get_data()[1], 3.3f); -} - -TEST(PredicateColumnTest, FilterBySelectorDouble) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(3); - Float64 vals[] = {1.11, 2.22, 3.33}; - for (auto v : vals) { - pred_col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - auto result_col = ColumnFloat64::create(); - uint16_t selector[] = {1}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 1, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 1); - EXPECT_DOUBLE_EQ(result_col->get_data()[0], 2.22); -} - -TEST(PredicateColumnTest, FilterBySelectorBoolean) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(5); - UInt8 vals[] = {0, 1, 0, 1, 1}; - for (auto v : vals) { - pred_col->insert_data(reinterpret_cast(&v), sizeof(v)); - } - auto result_col = ColumnUInt8::create(); - uint16_t selector[] = {1, 3, 4}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 3, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 3); - EXPECT_EQ(result_col->get_data()[0], 1); - EXPECT_EQ(result_col->get_data()[1], 1); - EXPECT_EQ(result_col->get_data()[2], 1); -} - -TEST(PredicateColumnTest, FilterBySelectorString) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(5); - std::vector strings = {"a", "bb", "ccc", "dddd", "eeeee"}; - for (const auto& s : strings) { - pred_col->insert_data(s.data(), s.size()); - } - auto result_col = ColumnString::create(); - uint16_t selector[] = {1, 3}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 2, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 2); - EXPECT_EQ(result_col->get_data_at(0).to_string(), "bb"); - EXPECT_EQ(result_col->get_data_at(1).to_string(), "dddd"); -} - -TEST(PredicateColumnTest, FilterBySelectorVarchar) { - auto pred_col = PredicateColumnType::create(); - pred_col->reserve(3); - std::vector strings = {"x", "yy", "zzz"}; - for (const auto& s : strings) { - pred_col->insert_data(s.data(), s.size()); - } - auto result_col = ColumnString::create(); - uint16_t selector[] = {0, 2}; - EXPECT_EQ(pred_col->filter_by_selector(selector, 2, result_col.get()), Status::OK()); - EXPECT_EQ(result_col->size(), 2); - EXPECT_EQ(result_col->get_data_at(0).to_string(), "x"); - EXPECT_EQ(result_col->get_data_at(1).to_string(), "zzz"); -} - -// ============================================================================ -// Test insert_string_value() for string types -// ============================================================================ -TEST(PredicateColumnTest, InsertStringValueString) { - auto col = PredicateColumnType::create(); - col->reserve(5); - std::vector strings = {"hello", "world", "", "test123", "long string"}; - for (const auto& s : strings) { - col->insert_string_value(s.data(), s.size()); - } - EXPECT_EQ(col->size(), 5); - for (size_t i = 0; i < strings.size(); i++) { - StringRef ref = col->get_data()[i]; - EXPECT_EQ(std::string(ref.data, ref.size), strings[i]); - } -} - -// ============================================================================ -// Test insert_in_copy_way() for LARGEINT -// ============================================================================ -TEST(PredicateColumnTest, InsertInCopyWayLargeInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int128 val1 = Int128(1) << 100; - Int128 val2 = -val1; - Int128 val3 = 0; - col->insert_in_copy_way(reinterpret_cast(&val1), sizeof(val1)); - col->insert_in_copy_way(reinterpret_cast(&val2), sizeof(val2)); - col->insert_in_copy_way(reinterpret_cast(&val3), sizeof(val3)); - EXPECT_EQ(col->size(), 3); - EXPECT_EQ(col->get_data()[0], val1); - EXPECT_EQ(col->get_data()[1], val2); - EXPECT_EQ(col->get_data()[2], val3); -} - -// ============================================================================ -// Test insert_default_type() for numeric types -// ============================================================================ -TEST(PredicateColumnTest, InsertDefaultTypeInt) { - auto col = PredicateColumnType::create(); - col->reserve(3); - Int32 val = 999; - col->insert_default_type(reinterpret_cast(&val), sizeof(val)); - EXPECT_EQ(col->size(), 1); - EXPECT_EQ(col->get_data()[0], 999); -} - -// ============================================================================ -// Test empty string handling -// ============================================================================ -TEST(PredicateColumnTest, EmptyStringHandling) { - auto col = PredicateColumnType::create(); - col->reserve(5); - std::vector strings = {"", "a", "", "bc", ""}; - for (const auto& s : strings) { - col->insert_data(s.data(), s.size()); - } - EXPECT_EQ(col->size(), 5); - EXPECT_EQ(col->get_data()[0].size, 0); - EXPECT_EQ(col->get_data()[1].size, 1); - EXPECT_EQ(col->get_data()[2].size, 0); - EXPECT_EQ(col->get_data()[3].size, 2); - EXPECT_EQ(col->get_data()[4].size, 0); -} - -} // namespace doris \ No newline at end of file diff --git a/be/test/exprs/bloom_filter_func_test.cpp b/be/test/exprs/bloom_filter_func_test.cpp index bcabdcc6a5a6a3..bc0ce276a37e7e 100644 --- a/be/test/exprs/bloom_filter_func_test.cpp +++ b/be/test/exprs/bloom_filter_func_test.cpp @@ -183,23 +183,17 @@ TEST_F(BloomFilterFuncTest, InsertFixedLen) { PODArray offsets(4); std::iota(offsets.begin(), offsets.end(), 0); - std::vector strings(4); - strings[0] = StringRef("aa"); - strings[1] = StringRef("bb"); - strings[2] = StringRef("cc"); - strings[3] = StringRef("dd"); + auto probe_column = ColumnHelper::create_column({"aa", "bb", "cc", "dd"}); auto find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(strings.data()), nullmap_column->get_data().data(), - offsets.data(), 4, false); + *probe_column, nullmap_column->get_data().data(), offsets.data(), 4, false); ASSERT_EQ(find_count, 4); nullmap_column->get_data()[1] = 0; nullmap_column->get_data()[3] = 0; find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(strings.data()), nullmap_column->get_data().data(), - offsets.data(), 4, false); + *probe_column, nullmap_column->get_data().data(), offsets.data(), 4, false); ASSERT_EQ(find_count, 2); ASSERT_EQ(offsets[0], 0); @@ -417,7 +411,7 @@ TEST_F(BloomFilterFuncTest, FindDictOlapEngine) { std::vector dicts = {StringRef("aa"), StringRef("bb"), StringRef("cc"), StringRef("dd"), StringRef("aab"), StringRef("bbc"), StringRef("ccd"), StringRef("dde")}; - auto column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + auto column = ColumnDictI32::create(); column->reserve(count); std::vector data(count); for (size_t i = 0; i != count; ++i) { @@ -483,9 +477,8 @@ TEST_F(BloomFilterFuncTest, FindFixedLenOlapEngine) { PODArray nullmap; uint8_t flag = 0; nullmap.assign(count, flag); - auto find_count = bloom_filter_func.find_fixed_len_olap_engine( - reinterpret_cast(decimal_column2->get_data().data()), nullmap.data(), - offsets.data(), count, true); + auto find_count = bloom_filter_func.find_fixed_len_olap_engine(*decimal_column2, nullmap.data(), + offsets.data(), count, true); ASSERT_EQ(find_count, count); BloomFilterFunc bloom_filter_func2(true); @@ -500,29 +493,28 @@ TEST_F(BloomFilterFuncTest, FindFixedLenOlapEngine) { // CHAR padding is stripped at the page decoder now, so the runtime BF // probe sees natural-length StringRefs; no trailing '\0' bytes here. - StringRef strings[] = {StringRef("aa"), StringRef("bb"), StringRef("cc"), StringRef("dd"), - StringRef("ef")}; + auto probe_column = ColumnHelper::create_column({"aa", "bb", "cc", "dd", "ef"}); PODArray offsets2(5); std::iota(offsets2.begin(), offsets2.end(), 0); - find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(&strings[0]), nullmap.data(), offsets2.data(), 5, false); + find_count = bloom_filter_func2.find_fixed_len_olap_engine(*probe_column, nullmap.data(), + offsets2.data(), 5, false); ASSERT_EQ(find_count, 4); std::iota(offsets2.begin(), offsets2.end(), 0); - find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(&strings[0]), nullmap.data(), offsets2.data(), 5, true); + find_count = bloom_filter_func2.find_fixed_len_olap_engine(*probe_column, nullmap.data(), + offsets2.data(), 5, true); ASSERT_EQ(find_count, 4); std::iota(offsets2.begin(), offsets2.end(), 0); - find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(&strings[0]), nullptr, offsets2.data(), 5, false); + find_count = bloom_filter_func2.find_fixed_len_olap_engine(*probe_column, nullptr, + offsets2.data(), 5, false); ASSERT_EQ(find_count, 4); std::iota(offsets2.begin(), offsets2.end(), 0); - find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(&strings[0]), nullptr, offsets2.data(), 5, true); + find_count = bloom_filter_func2.find_fixed_len_olap_engine(*probe_column, nullptr, + offsets2.data(), 5, true); ASSERT_EQ(find_count, 4); PODArray nullmap2; @@ -531,8 +523,8 @@ TEST_F(BloomFilterFuncTest, FindFixedLenOlapEngine) { nullmap2[2] = 1; std::iota(offsets2.begin(), offsets2.end(), 0); - find_count = bloom_filter_func2.find_fixed_len_olap_engine( - reinterpret_cast(&strings[0]), nullmap2.data(), offsets2.data(), 5, false); + find_count = bloom_filter_func2.find_fixed_len_olap_engine(*probe_column, nullmap2.data(), + offsets2.data(), 5, false); ASSERT_EQ(find_count, 2); ASSERT_EQ(offsets2[0], 0); ASSERT_EQ(offsets2[1], 3); diff --git a/be/test/format/parquet/byte_array_dict_decoder_test.cpp b/be/test/format/parquet/byte_array_dict_decoder_test.cpp index 29213246db3f92..2e3f398ef56e5a 100644 --- a/be/test/format/parquet/byte_array_dict_decoder_test.cpp +++ b/be/test/format/parquet/byte_array_dict_decoder_test.cpp @@ -188,7 +188,7 @@ TEST_F(ByteArrayDictDecoderTest, test_empty_dict) { // Test decoding with ColumnDictI32 TEST_F(ByteArrayDictDecoderTest, test_decode_with_column_dict_i32) { // Create ColumnDictI32 column - MutableColumnPtr column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + MutableColumnPtr column = ColumnDictI32::create(); DataTypePtr data_type = std::make_shared(); // RLE encoded data: 4 zeros followed by 1, 2, 1, padded to 8 values, [0 0 0 0 1 2 1] @@ -231,7 +231,7 @@ TEST_F(ByteArrayDictDecoderTest, test_decode_with_column_dict_i32) { // Test decoding with ColumnDictI32 and filter TEST_F(ByteArrayDictDecoderTest, test_decode_with_column_dict_i32_with_filter) { // Create ColumnDictI32 column - MutableColumnPtr column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + MutableColumnPtr column = ColumnDictI32::create(); DataTypePtr data_type = std::make_shared(); // RLE encoded data: 4 zeros followed by 1, 2, 1, padded to 8 values, [0 0 0 0 1 2 1] @@ -272,7 +272,7 @@ TEST_F(ByteArrayDictDecoderTest, test_decode_with_column_dict_i32_with_filter) { // Test decoding with ColumnDictI32 with filter and null TEST_F(ByteArrayDictDecoderTest, test_decode_with_column_dict_i32_with_filter_and_null) { // Create ColumnDictI32 column - MutableColumnPtr column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + MutableColumnPtr column = ColumnDictI32::create(); DataTypePtr data_type = std::make_shared(); // RLE encoded data: 4 zeros followed by 2, padded to 8 values, [0 0 0 0 2] diff --git a/be/test/format/parquet/fix_length_dict_decoder_test.cpp b/be/test/format/parquet/fix_length_dict_decoder_test.cpp index afd419c546954e..5c8854b665b3fc 100644 --- a/be/test/format/parquet/fix_length_dict_decoder_test.cpp +++ b/be/test/format/parquet/fix_length_dict_decoder_test.cpp @@ -287,7 +287,7 @@ TEST_F(FixLengthDictDecoderTest, test_empty_dict) { // Test decoding with ColumnDictI32 TEST_F(FixLengthDictDecoderTest, test_decode_with_column_dict_i32) { // Create ColumnDictI32 column - MutableColumnPtr column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + MutableColumnPtr column = ColumnDictI32::create(); DataTypePtr data_type = std::make_shared(); // RLE encoded data: 4 zeros followed by 1, 2, 1, padded to 8 values, [0 0 0 0 1 2 1] @@ -330,7 +330,7 @@ TEST_F(FixLengthDictDecoderTest, test_decode_with_column_dict_i32) { // Test decoding with ColumnDictI32 and filter TEST_F(FixLengthDictDecoderTest, test_decode_with_column_dict_i32_with_filter) { // Create ColumnDictI32 column - MutableColumnPtr column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + MutableColumnPtr column = ColumnDictI32::create(); DataTypePtr data_type = std::make_shared(); // RLE encoded data: 4 zeros followed by 1, 2, 1, padded to 8 values, [0 0 0 0 1 2 1] @@ -371,7 +371,7 @@ TEST_F(FixLengthDictDecoderTest, test_decode_with_column_dict_i32_with_filter) { // Test decoding with ColumnDictI32 with filter and null TEST_F(FixLengthDictDecoderTest, test_decode_with_column_dict_i32_with_filter_and_null) { // Create ColumnDictI32 column - MutableColumnPtr column = ColumnDictI32::create(FieldType::OLAP_FIELD_TYPE_VARCHAR); + MutableColumnPtr column = ColumnDictI32::create(); DataTypePtr data_type = std::make_shared(); // RLE encoded data: 4 zeros followed by 2, padded to 8 values, [0 0 0 0 2] diff --git a/be/test/storage/predicate/block_column_predicate_test.cpp b/be/test/storage/predicate/block_column_predicate_test.cpp index def48d67df75bc..83b7e36fd3f656 100644 --- a/be/test/storage/predicate/block_column_predicate_test.cpp +++ b/be/test/storage/predicate/block_column_predicate_test.cpp @@ -32,7 +32,7 @@ #include "common/status.h" #include "core/column/column.h" -#include "core/column/predicate_column.h" +#include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" #include "core/type_limit.h" @@ -77,7 +77,7 @@ class BlockColumnPredicateTest : public testing::Test { TEST_F(BlockColumnPredicateTest, SINGLE_COLUMN_VEC) { MutableColumns block; - block.push_back(PredicateColumnType::create()); + block.push_back(ColumnInt32::create()); auto value = Field::create_field(5); int rows = 10; @@ -97,13 +97,13 @@ TEST_F(BlockColumnPredicateTest, SINGLE_COLUMN_VEC) { selected_size = single_column_block_pred.evaluate(block, sel_idx.data(), selected_size); EXPECT_EQ(selected_size, 1); - auto* pred_col = reinterpret_cast*>(block[col_idx].get()); + auto* pred_col = reinterpret_cast(block[col_idx].get()); EXPECT_EQ(pred_col->get_data()[sel_idx[0]], value.template get()); } TEST_F(BlockColumnPredicateTest, AND_MUTI_COLUMN_VEC) { MutableColumns block; - block.push_back(PredicateColumnType::create()); + block.push_back(ColumnInt32::create()); auto less_value = Field::create_field(5); auto great_value = Field::create_field(3); @@ -131,13 +131,13 @@ TEST_F(BlockColumnPredicateTest, AND_MUTI_COLUMN_VEC) { selected_size = and_block_column_pred.evaluate(block, sel_idx.data(), selected_size); EXPECT_EQ(selected_size, 1); - auto* pred_col = reinterpret_cast*>(block[col_idx].get()); + auto* pred_col = reinterpret_cast(block[col_idx].get()); EXPECT_EQ(pred_col->get_data()[sel_idx[0]], 4); } TEST_F(BlockColumnPredicateTest, OR_MUTI_COLUMN_VEC) { MutableColumns block; - block.push_back(PredicateColumnType::create()); + block.push_back(ColumnInt32::create()); auto less_value = Field::create_field(5); auto great_value = Field::create_field(3); @@ -165,13 +165,13 @@ TEST_F(BlockColumnPredicateTest, OR_MUTI_COLUMN_VEC) { selected_size = or_block_column_pred.evaluate(block, sel_idx.data(), selected_size); EXPECT_EQ(selected_size, 10); - auto* pred_col = reinterpret_cast*>(block[col_idx].get()); + auto* pred_col = reinterpret_cast(block[col_idx].get()); EXPECT_EQ(pred_col->get_data()[sel_idx[0]], 0); } TEST_F(BlockColumnPredicateTest, OR_AND_MUTI_COLUMN_VEC) { MutableColumns block; - block.push_back(PredicateColumnType::create()); + block.push_back(ColumnInt32::create()); auto less_value = Field::create_field(5); auto great_value = Field::create_field(3); @@ -208,7 +208,7 @@ TEST_F(BlockColumnPredicateTest, OR_AND_MUTI_COLUMN_VEC) { selected_size = or_block_column_pred.evaluate(block, sel_idx.data(), selected_size); EXPECT_EQ(selected_size, 4); - auto* pred_col = reinterpret_cast*>(block[col_idx].get()); + auto* pred_col = reinterpret_cast(block[col_idx].get()); EXPECT_EQ(pred_col->get_data()[sel_idx[0]], 0); EXPECT_EQ(pred_col->get_data()[sel_idx[1]], 1); EXPECT_EQ(pred_col->get_data()[sel_idx[2]], 2); @@ -237,7 +237,7 @@ TEST_F(BlockColumnPredicateTest, OR_AND_MUTI_COLUMN_VEC) { TEST_F(BlockColumnPredicateTest, AND_OR_MUTI_COLUMN_VEC) { MutableColumns block; - block.push_back(PredicateColumnType::create()); + block.push_back(ColumnInt32::create()); auto less_value = Field::create_field(5); auto great_value = Field::create_field(3); @@ -274,7 +274,7 @@ TEST_F(BlockColumnPredicateTest, AND_OR_MUTI_COLUMN_VEC) { selected_size = and_block_column_pred.evaluate(block, sel_idx.data(), selected_size); - auto* pred_col = reinterpret_cast*>(block[col_idx].get()); + auto* pred_col = reinterpret_cast(block[col_idx].get()); EXPECT_EQ(selected_size, 1); EXPECT_EQ(pred_col->get_data()[sel_idx[0]], 4);