diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index 23e2e5a4a0feec..2658726fe00759 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -21,23 +21,32 @@ #include #include +#include #include +#include #include +#include "common/cast_set.h" #include "core/column/column.h" #include "core/column/column_string.h" #include "core/data_type/data_type.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" +#include "core/packed_int128.h" #include "core/string_ref.h" +#include "core/uint24.h" #include "core/value/decimalv2_value.h" #include "core/value/vdatetime_value.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" #include "storage/index/indexed_column_reader.h" #include "storage/index/indexed_column_writer.h" #include "storage/olap_common.h" #include "storage/segment/encoding_info.h" +#include "storage/segment/page_pointer.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" +#include "util/coding.h" #include "util/slice.h" #include "util/unaligned.h" @@ -45,6 +54,504 @@ namespace doris { struct uint24_t; namespace segment_v2 { +namespace { + +constexpr uint32_t ZONE_MAP_NATIVE_ENTRY_VERSION = 1; +constexpr uint32_t ZONE_MAP_NATIVE_PAGE_FORMAT_VERSION = 1; +constexpr uint32_t ZONE_MAP_NATIVE_OFFSET_WIDTH = 4; + +template +void append_as_little_endian(std::string* dst, T value) { + static_assert(std::is_integral_v || std::is_floating_point_v); + if constexpr (sizeof(T) == 1) { + dst->push_back(static_cast(value)); + } else if constexpr (sizeof(T) == 2) { + uint8_t buf[sizeof(uint16_t)]; + uint16_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + encode_fixed16_le(buf, bits); + dst->append(reinterpret_cast(buf), sizeof(buf)); + } else if constexpr (sizeof(T) == 4) { + uint8_t buf[sizeof(uint32_t)]; + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + encode_fixed32_le(buf, bits); + dst->append(reinterpret_cast(buf), sizeof(buf)); + } else if constexpr (sizeof(T) == 8) { + uint8_t buf[sizeof(uint64_t)]; + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + encode_fixed64_le(buf, bits); + dst->append(reinterpret_cast(buf), sizeof(buf)); + } +} + +template +T read_little_endian(const char* data) { + static_assert(std::is_integral_v || std::is_floating_point_v); + if constexpr (sizeof(T) == 1) { + return static_cast(*reinterpret_cast(data)); + } else if constexpr (sizeof(T) == 2) { + uint16_t bits = decode_fixed16_le(reinterpret_cast(data)); + T value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } else if constexpr (sizeof(T) == 4) { + uint32_t bits = decode_fixed32_le(reinterpret_cast(data)); + T value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } else { + uint64_t bits = decode_fixed64_le(reinterpret_cast(data)); + T value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } +} + +void append_int128_le(std::string* dst, Int128 value) { + uint8_t buf[sizeof(uint128_t)]; + uint128_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + encode_fixed128_le(buf, bits); + dst->append(reinterpret_cast(buf), sizeof(buf)); +} + +Int128 read_int128_le(const char* data) { + uint128_t bits = decode_fixed128_le(reinterpret_cast(data)); + Int128 value; + std::memcpy(&value, &bits, sizeof(value)); + return value; +} + +void append_int256_le(std::string* dst, const wide::Int256& value) { + for (size_t i = 0; i < wide::Int256::item_count; ++i) { + uint8_t buf[sizeof(uint64_t)]; + encode_fixed64_le(buf, value.items[wide::Int256::_impl::little(i)]); + dst->append(reinterpret_cast(buf), sizeof(buf)); + } +} + +wide::Int256 read_int256_le(const char* data) { + wide::Int256 value; + for (size_t i = 0; i < wide::Int256::item_count; ++i) { + value.items[wide::Int256::_impl::little(i)] = + decode_fixed64_le(reinterpret_cast(data + i * sizeof(uint64_t))); + } + return value; +} + +void append_uint24_le(std::string* dst, uint24_t value) { + const uint32_t raw = static_cast(value); + dst->push_back(static_cast(raw)); + dst->push_back(static_cast(raw >> 8)); + dst->push_back(static_cast(raw >> 16)); +} + +uint24_t read_uint24_le(const char* data) { + uint32_t raw = static_cast(data[0]); + raw |= static_cast(static_cast(data[1])) << 8; + raw |= static_cast(static_cast(data[2])) << 16; + return uint24_t(raw); +} + +template +constexpr uint32_t native_fixed_width() { + if constexpr (Type == TYPE_DATE) { + return 3; + } else if constexpr (Type == TYPE_DECIMALV2) { + return sizeof(int64_t) + sizeof(int32_t); + } else { + return sizeof(typename PrimitiveTypeTraits::StorageFieldType); + } +} + +template +void append_native_value(const Field& field, std::string* dst) { + if constexpr (Type == TYPE_DATE) { + append_uint24_le(dst, uint24_t(field.get().to_olap_date())); + } else if constexpr (Type == TYPE_DATETIME) { + append_as_little_endian(dst, field.get().to_olap_datetime()); + } else if constexpr (Type == TYPE_DATEV2) { + append_as_little_endian( + dst, static_cast(field.get().to_date_int_val())); + } else if constexpr (Type == TYPE_DATETIMEV2) { + append_as_little_endian(dst, field.get().to_date_int_val()); + } else if constexpr (Type == TYPE_TIMESTAMPTZ) { + append_as_little_endian(dst, field.get().to_date_int_val()); + } else if constexpr (Type == TYPE_DECIMALV2) { + const auto& value = field.get(); + append_as_little_endian(dst, value.int_value()); + append_as_little_endian(dst, value.frac_value()); + } else if constexpr (Type == TYPE_DECIMAL32 || Type == TYPE_DECIMAL64 || + Type == TYPE_DECIMAL128I || Type == TYPE_DECIMAL256) { + const auto& value = field.get().value; + if constexpr (Type == TYPE_DECIMAL128I) { + append_int128_le(dst, value); + } else if constexpr (Type == TYPE_DECIMAL256) { + append_int256_le(dst, value); + } else { + append_as_little_endian(dst, value); + } + } else if constexpr (Type == TYPE_LARGEINT) { + append_int128_le(dst, field.get()); + } else if constexpr (Type == TYPE_IPV6) { + uint8_t buf[sizeof(uint128_t)]; + encode_fixed128_le(buf, field.get()); + dst->append(reinterpret_cast(buf), sizeof(buf)); + } else if constexpr (is_string_type(Type)) { + const auto& value = field.get(); + dst->append(value.data(), value.size()); + } else { + append_as_little_endian(dst, field.get()); + } +} + +template +Status decode_native_value(std::string_view value, Field* field) { + DORIS_CHECK(field != nullptr); + if constexpr (is_string_type(Type)) { + *field = Field::create_field(String(value.data(), value.size())); + return Status::OK(); + } else { + if (value.size() != native_fixed_width()) { + return Status::Corruption("invalid zone map native value size {}, expected {}", + value.size(), native_fixed_width()); + } + const char* data = value.data(); + if constexpr (Type == TYPE_DATE) { + *field = Field::create_field_from_olap_value(read_uint24_le(data)); + } else if constexpr (Type == TYPE_DATETIME) { + *field = Field::create_field_from_olap_value(read_little_endian(data)); + } else if constexpr (Type == TYPE_DATEV2) { + *field = Field::create_field_from_olap_value(read_little_endian(data)); + } else if constexpr (Type == TYPE_DATETIMEV2) { + *field = Field::create_field_from_olap_value(read_little_endian(data)); + } else if constexpr (Type == TYPE_TIMESTAMPTZ) { + *field = Field::create_field_from_olap_value(read_little_endian(data)); + } else if constexpr (Type == TYPE_DECIMALV2) { + decimal12_t decimal {read_little_endian(data), + read_little_endian(data + sizeof(int64_t))}; + *field = Field::create_field_from_olap_value(decimal); + } else if constexpr (Type == TYPE_DECIMAL32) { + *field = Field::create_field_from_olap_value(read_little_endian(data)); + } else if constexpr (Type == TYPE_DECIMAL64) { + *field = Field::create_field_from_olap_value(read_little_endian(data)); + } else if constexpr (Type == TYPE_DECIMAL128I) { + *field = Field::create_field_from_olap_value(read_int128_le(data)); + } else if constexpr (Type == TYPE_DECIMAL256) { + *field = Field::create_field_from_olap_value(read_int256_le(data)); + } else if constexpr (Type == TYPE_LARGEINT) { + *field = Field::create_field_from_olap_value(read_int128_le(data)); + } else if constexpr (Type == TYPE_IPV6) { + *field = Field::create_field_from_olap_value( + decode_fixed128_le(reinterpret_cast(data))); + } else { + *field = Field::create_field_from_olap_value( + read_little_endian::StorageFieldType>(data)); + } + return Status::OK(); + } +} + +template +void append_value_array_value(const Field& field, ZoneMapValueArrayPB* values) { + DORIS_CHECK(values != nullptr); + if constexpr (is_string_type(Type)) { + if (!values->has_layout()) { + values->set_layout(ZoneMapValueArrayPB::VAR_BINARY); + values->set_offset_width(ZONE_MAP_NATIVE_OFFSET_WIDTH); + put_fixed32_le(values->mutable_offsets(), 0); + } + const auto& value = field.get(); + values->mutable_values()->append(value.data(), value.size()); + put_fixed32_le(values->mutable_offsets(), cast_set(values->values().size())); + } else { + if (!values->has_layout()) { + values->set_layout(ZoneMapValueArrayPB::FIXED_WIDTH); + values->set_fixed_width(native_fixed_width()); + } + append_native_value(field, values->mutable_fixed_values()); + } +} + +template +void init_empty_value_array(ZoneMapValueArrayPB* values) { + DORIS_CHECK(values != nullptr); + if constexpr (is_string_type(Type)) { + values->set_layout(ZoneMapValueArrayPB::VAR_BINARY); + values->set_offset_width(ZONE_MAP_NATIVE_OFFSET_WIDTH); + put_fixed32_le(values->mutable_offsets(), 0); + } else { + values->set_layout(ZoneMapValueArrayPB::FIXED_WIDTH); + values->set_fixed_width(native_fixed_width()); + } +} + +void set_bitmap_bit(std::string* bitmap, uint32_t index, bool value) { + if (value) { + (*bitmap)[index / 8] = + static_cast(static_cast((*bitmap)[index / 8]) | (1U << (index % 8))); + } +} + +bool get_bitmap_bit(const std::string& bitmap, uint32_t index) { + return (static_cast(bitmap[index / 8]) & (1U << (index % 8))) != 0; +} + +Status validate_bitmap(const std::string& bitmap, uint32_t num_pages, std::string_view name) { + const size_t min_size = (num_pages + 7) / 8; + if (bitmap.size() < min_size) { + return Status::Corruption("zone map {} bitmap too small: {}, expected at least {}", name, + bitmap.size(), min_size); + } + return Status::OK(); +} + +bool has_present_value(const ZoneMap& zone_map) { + return zone_map.has_not_null && !zone_map.pass_all; +} + +template +Status apply_special_float_values(ZoneMap* zone_map) { + DORIS_CHECK(zone_map != nullptr); + if (!zone_map->has_not_null) { + return Status::OK(); + } + if constexpr (Type == TYPE_FLOAT || Type == TYPE_DOUBLE) { + using CppType = typename PrimitiveTypeTraits::CppType; + if (zone_map->has_negative_inf) { + zone_map->min_value = + Field::create_field(-std::numeric_limits::infinity()); + } + if (zone_map->has_nan) { + zone_map->max_value = + Field::create_field(std::numeric_limits::quiet_NaN()); + } else if (zone_map->has_positive_inf) { + zone_map->max_value = + Field::create_field(std::numeric_limits::infinity()); + } + } else { + if (zone_map->has_negative_inf) { + return Status::InternalError("invalid zone map with negative Infinity"); + } + if (zone_map->has_nan) { + return Status::InternalError("invalid zone map with NaN"); + } + if (zone_map->has_positive_inf) { + return Status::InternalError("invalid zone map with positive Infinity"); + } + } + return Status::OK(); +} + +template +Status validate_value_array(const ZoneMapValueArrayPB& values, uint32_t value_count, + std::string_view name) { + if constexpr (is_string_type(Type)) { + if (values.layout() != ZoneMapValueArrayPB::VAR_BINARY) { + return Status::Corruption("zone map {} values use unsupported layout {}", name, + values.layout()); + } + if (values.offset_width() != ZONE_MAP_NATIVE_OFFSET_WIDTH) { + return Status::Corruption("zone map {} values use unsupported offset width {}", name, + values.offset_width()); + } + const size_t expected_offsets_size = + (static_cast(value_count) + 1) * ZONE_MAP_NATIVE_OFFSET_WIDTH; + if (values.offsets().size() != expected_offsets_size) { + return Status::Corruption("zone map {} offsets size {}, expected {}", name, + values.offsets().size(), expected_offsets_size); + } + uint32_t previous_offset = 0; + for (uint32_t i = 0; i <= value_count; ++i) { + uint32_t offset = decode_fixed32_le(reinterpret_cast( + values.offsets().data() + i * ZONE_MAP_NATIVE_OFFSET_WIDTH)); + if (i == 0 && offset != 0) { + return Status::Corruption("zone map {} first offset must be zero", name); + } + if (offset < previous_offset) { + return Status::Corruption("zone map {} offsets are not monotonic", name); + } + if (offset > values.values().size()) { + return Status::Corruption("zone map {} offset {} exceeds values size {}", name, + offset, values.values().size()); + } + previous_offset = offset; + } + if (previous_offset != values.values().size()) { + return Status::Corruption("zone map {} last offset {} does not match values size {}", + name, previous_offset, values.values().size()); + } + } else { + if (values.layout() != ZoneMapValueArrayPB::FIXED_WIDTH) { + return Status::Corruption("zone map {} values use unsupported layout {}", name, + values.layout()); + } + if (values.fixed_width() != native_fixed_width()) { + return Status::Corruption("zone map {} fixed width {}, expected {}", name, + values.fixed_width(), native_fixed_width()); + } + const size_t expected_values_size = + static_cast(value_count) * native_fixed_width(); + if (values.fixed_values().size() != expected_values_size) { + return Status::Corruption("zone map {} fixed values size {}, expected {}", name, + values.fixed_values().size(), expected_values_size); + } + } + return Status::OK(); +} + +template +Status decode_value_array_value(const ZoneMapValueArrayPB& values, uint32_t index, Field* field) { + if constexpr (is_string_type(Type)) { + const char* offsets = values.offsets().data(); + const uint32_t start = decode_fixed32_le( + reinterpret_cast(offsets + index * ZONE_MAP_NATIVE_OFFSET_WIDTH)); + const uint32_t end = decode_fixed32_le(reinterpret_cast( + offsets + (index + 1) * ZONE_MAP_NATIVE_OFFSET_WIDTH)); + return decode_native_value( + std::string_view(values.values().data() + start, static_cast(end - start)), + field); + } else { + const uint32_t width = native_fixed_width(); + return decode_native_value( + std::string_view(values.fixed_values().data() + static_cast(index) * width, + width), + field); + } +} + +template +Status build_zone_map_pages_pb(const std::vector& zone_maps, ZoneMapPagesPB* pages) { + DORIS_CHECK(pages != nullptr); + const auto num_pages = cast_set(zone_maps.size()); + const size_t bitmap_size = (num_pages + 7) / 8; + pages->set_num_pages(num_pages); + pages->set_entry_version(ZONE_MAP_NATIVE_ENTRY_VERSION); + pages->set_has_null_bitmap(std::string(bitmap_size, '\0')); + pages->set_has_not_null_bitmap(std::string(bitmap_size, '\0')); + pages->set_pass_all_bitmap(std::string(bitmap_size, '\0')); + pages->set_has_positive_inf_bitmap(std::string(bitmap_size, '\0')); + pages->set_has_negative_inf_bitmap(std::string(bitmap_size, '\0')); + pages->set_has_nan_bitmap(std::string(bitmap_size, '\0')); + init_empty_value_array(pages->mutable_min_values()); + init_empty_value_array(pages->mutable_max_values()); + + for (uint32_t i = 0; i < num_pages; ++i) { + const auto& zone_map = zone_maps[i]; + set_bitmap_bit(pages->mutable_has_null_bitmap(), i, zone_map.has_null); + set_bitmap_bit(pages->mutable_has_not_null_bitmap(), i, zone_map.has_not_null); + set_bitmap_bit(pages->mutable_pass_all_bitmap(), i, zone_map.pass_all); + set_bitmap_bit(pages->mutable_has_positive_inf_bitmap(), i, zone_map.has_positive_inf); + set_bitmap_bit(pages->mutable_has_negative_inf_bitmap(), i, zone_map.has_negative_inf); + set_bitmap_bit(pages->mutable_has_nan_bitmap(), i, zone_map.has_nan); + if (has_present_value(zone_map)) { + append_value_array_value(zone_map.min_value, pages->mutable_min_values()); + append_value_array_value(zone_map.max_value, pages->mutable_max_values()); + } + } + return Status::OK(); +} + +template +Status decode_zone_map_pages_pb(const ZoneMapPagesPB& pages, std::vector* zone_maps) { + DORIS_CHECK(zone_maps != nullptr); + if (pages.entry_version() != ZONE_MAP_NATIVE_ENTRY_VERSION) { + return Status::Corruption("unsupported zone map native entry version {}", + pages.entry_version()); + } + if (!pages.has_has_null_bitmap() || !pages.has_has_not_null_bitmap() || + !pages.has_pass_all_bitmap() || !pages.has_has_positive_inf_bitmap() || + !pages.has_has_negative_inf_bitmap() || !pages.has_has_nan_bitmap()) { + return Status::Corruption("zone map native page bitmaps are incomplete"); + } + if (!pages.has_min_values() || !pages.has_max_values()) { + return Status::Corruption("zone map native page values are incomplete"); + } + + const uint32_t num_pages = pages.num_pages(); + RETURN_IF_ERROR(validate_bitmap(pages.has_null_bitmap(), num_pages, "has_null")); + RETURN_IF_ERROR(validate_bitmap(pages.has_not_null_bitmap(), num_pages, "has_not_null")); + RETURN_IF_ERROR(validate_bitmap(pages.pass_all_bitmap(), num_pages, "pass_all")); + RETURN_IF_ERROR( + validate_bitmap(pages.has_positive_inf_bitmap(), num_pages, "has_positive_inf")); + RETURN_IF_ERROR( + validate_bitmap(pages.has_negative_inf_bitmap(), num_pages, "has_negative_inf")); + RETURN_IF_ERROR(validate_bitmap(pages.has_nan_bitmap(), num_pages, "has_nan")); + + uint32_t value_count = 0; + for (uint32_t i = 0; i < num_pages; ++i) { + if (get_bitmap_bit(pages.has_not_null_bitmap(), i) && + !get_bitmap_bit(pages.pass_all_bitmap(), i)) { + ++value_count; + } + } + RETURN_IF_ERROR(validate_value_array(pages.min_values(), value_count, "min")); + RETURN_IF_ERROR(validate_value_array(pages.max_values(), value_count, "max")); + + zone_maps->resize(num_pages); + uint32_t value_index = 0; + for (uint32_t i = 0; i < num_pages; ++i) { + auto& zone_map = (*zone_maps)[i]; + zone_map.has_null = get_bitmap_bit(pages.has_null_bitmap(), i); + zone_map.has_not_null = get_bitmap_bit(pages.has_not_null_bitmap(), i); + zone_map.pass_all = get_bitmap_bit(pages.pass_all_bitmap(), i); + zone_map.has_positive_inf = get_bitmap_bit(pages.has_positive_inf_bitmap(), i); + zone_map.has_negative_inf = get_bitmap_bit(pages.has_negative_inf_bitmap(), i); + zone_map.has_nan = get_bitmap_bit(pages.has_nan_bitmap(), i); + if (has_present_value(zone_map)) { + RETURN_IF_ERROR(decode_value_array_value(pages.min_values(), value_index, + &zone_map.min_value)); + RETURN_IF_ERROR(decode_value_array_value(pages.max_values(), value_index, + &zone_map.max_value)); + ++value_index; + } + RETURN_IF_ERROR(apply_special_float_values(&zone_map)); + } + return Status::OK(); +} + +#define APPLY_FOR_ZONEMAP_NATIVE_PRIMITIVE(M) \ + M(TYPE_BOOLEAN) \ + M(TYPE_TINYINT) \ + M(TYPE_SMALLINT) \ + M(TYPE_INT) \ + M(TYPE_BIGINT) \ + M(TYPE_LARGEINT) \ + M(TYPE_FLOAT) \ + M(TYPE_DOUBLE) \ + M(TYPE_CHAR) \ + M(TYPE_DATE) \ + M(TYPE_DATETIME) \ + M(TYPE_DATEV2) \ + M(TYPE_DATETIMEV2) \ + M(TYPE_TIMESTAMPTZ) \ + M(TYPE_IPV4) \ + M(TYPE_IPV6) \ + M(TYPE_VARCHAR) \ + M(TYPE_STRING) \ + M(TYPE_DECIMALV2) \ + M(TYPE_DECIMAL32) \ + M(TYPE_DECIMAL64) \ + M(TYPE_DECIMAL128I) \ + M(TYPE_DECIMAL256) + +template +Status dispatch_zonemap_native_type(PrimitiveType type, Callback&& callback) { + switch (type) { +#define M(NAME) \ + case NAME: \ + return callback.template operator()(); + APPLY_FOR_ZONEMAP_NATIVE_PRIMITIVE(M) +#undef M + default: + return Status::InvalidArgument("invalid zone map native type {}", type_to_string(type)); + } +} + +} // namespace + Status ZoneMap::from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_type, ZoneMap& zone_map_info) { zone_map_info.has_null = zone_map.has_null(); @@ -104,6 +611,95 @@ Status ZoneMap::from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_ty return Status::OK(); } +void ZoneMap::to_entry_pb_native(ZoneMapEntryPB* dst, const DataTypePtr& data_type) const { + DORIS_CHECK(dst != nullptr); + if (pass_all || !has_not_null) { + dst->set_min(""); + dst->set_max(""); + } else { + std::string min; + std::string max; + static_cast(dispatch_zonemap_native_type( + data_type->get_primitive_type(), [&]() -> Status { + append_native_value(min_value, &min); + append_native_value(max_value, &max); + return Status::OK(); + })); + dst->set_min(std::move(min)); + dst->set_max(std::move(max)); + } + dst->set_has_null(has_null); + dst->set_has_not_null(has_not_null); + dst->set_pass_all(pass_all); + dst->set_has_positive_inf(has_positive_inf); + dst->set_has_negative_inf(has_negative_inf); + dst->set_has_nan(has_nan); +} + +Status ZoneMap::from_entry_pb_native(const ZoneMapEntryPB& zone_map, const DataTypePtr& data_type, + ZoneMap& zone_map_info) { + zone_map_info.has_null = zone_map.has_null(); + zone_map_info.has_not_null = zone_map.has_not_null(); + zone_map_info.pass_all = zone_map.pass_all(); + zone_map_info.has_negative_inf = zone_map.has_negative_inf(); + zone_map_info.has_positive_inf = zone_map.has_positive_inf(); + zone_map_info.has_nan = zone_map.has_nan(); + + auto field_type = data_type->get_storage_field_type(); + if (zone_map.has_not_null()) { + if (zone_map.has_negative_inf()) { + if (FieldType::OLAP_FIELD_TYPE_FLOAT == field_type) { + static auto constexpr float_neg_inf = -std::numeric_limits::infinity(); + zone_map_info.min_value = Field::create_field(float_neg_inf); + } else if (FieldType::OLAP_FIELD_TYPE_DOUBLE == field_type) { + static auto constexpr double_neg_inf = -std::numeric_limits::infinity(); + zone_map_info.min_value = Field::create_field(double_neg_inf); + } else { + return Status::InternalError("invalid zone map with negative Infinity"); + } + } else if (!zone_map_info.pass_all) { + if (!zone_map.has_min()) { + return Status::Corruption("zone map native min is missing"); + } + RETURN_IF_ERROR(dispatch_zonemap_native_type( + data_type->get_primitive_type(), [&]() -> Status { + return decode_native_value(zone_map.min(), &zone_map_info.min_value); + })); + } + + if (zone_map.has_nan()) { + if (FieldType::OLAP_FIELD_TYPE_FLOAT == field_type) { + static auto constexpr float_nan = std::numeric_limits::quiet_NaN(); + zone_map_info.max_value = Field::create_field(float_nan); + } else if (FieldType::OLAP_FIELD_TYPE_DOUBLE == field_type) { + static auto constexpr double_nan = std::numeric_limits::quiet_NaN(); + zone_map_info.max_value = Field::create_field(double_nan); + } else { + return Status::InternalError("invalid zone map with NaN"); + } + } else if (zone_map.has_positive_inf()) { + if (FieldType::OLAP_FIELD_TYPE_FLOAT == field_type) { + static auto constexpr float_pos_inf = std::numeric_limits::infinity(); + zone_map_info.max_value = Field::create_field(float_pos_inf); + } else if (FieldType::OLAP_FIELD_TYPE_DOUBLE == field_type) { + static auto constexpr double_pos_inf = std::numeric_limits::infinity(); + zone_map_info.max_value = Field::create_field(double_pos_inf); + } else { + return Status::InternalError("invalid zone map with positive Infinity"); + } + } else if (!zone_map_info.pass_all) { + if (!zone_map.has_max()) { + return Status::Corruption("zone map native max is missing"); + } + RETURN_IF_ERROR(dispatch_zonemap_native_type( + data_type->get_primitive_type(), [&]() -> Status { + return decode_native_value(zone_map.max(), &zone_map_info.max_value); + })); + } + } + return Status::OK(); +} + template TypedZoneMapIndexWriter::TypedZoneMapIndexWriter(DataTypePtr&& data_type) : _data_type(std::move(data_type)) { @@ -253,18 +849,11 @@ Status TypedZoneMapIndexWriter::flush() { _segment_zone_map.has_nan = true; } - ZoneMapPB zone_map_pb; modify_index_before_flush(_page_zone_map); - _page_zone_map.to_proto(&zone_map_pb, _data_type); + _page_zone_maps.push_back(_page_zone_map); _reset_zone_map(&_page_zone_map); - std::string serialized_zone_map; - bool ret = zone_map_pb.SerializeToString(&serialized_zone_map); - if (!ret) { - return Status::InternalError("serialize zone map failed"); - } - _estimated_size += serialized_zone_map.size() + sizeof(uint32_t); - _values.push_back(std::move(serialized_zone_map)); + _estimated_size += sizeof(ZoneMap); return Status::OK(); } @@ -275,25 +864,25 @@ Status TypedZoneMapIndexWriter::finish(io::FileWriter* file_writer, ZoneMapIndexPB* meta = index_meta->mutable_zone_map_index(); // store segment zone map modify_index_before_flush(_segment_zone_map); - _segment_zone_map.to_proto(meta->mutable_segment_zone_map(), _data_type); + _segment_zone_map.to_entry_pb_native(meta->mutable_segment_zone_map_v2(), _data_type); - // write out zone map for each data pages - constexpr FieldType type = FieldType::OLAP_FIELD_TYPE_BITMAP; - IndexedColumnWriterOptions options; - options.write_ordinal_index = true; - options.write_value_index = false; - // Zone map page always uses PLAIN_ENCODING. Do not change. - options.encoding = PLAIN_ENCODING; - options.compression = NO_COMPRESSION; // currently not compressed + ZoneMapPagesPB pages; + RETURN_IF_ERROR(build_zone_map_pages_pb(_page_zone_maps, &pages)); + std::string serialized_pages; + if (!pages.SerializeToString(&serialized_pages)) { + return Status::InternalError("serialize zone map pages failed"); + } - IndexedColumnWriter writer(options, type, file_writer); - RETURN_IF_ERROR(writer.init()); + PagePointer page_pointer(file_writer->bytes_appended(), + cast_set(serialized_pages.size())); + Slice pages_slice(serialized_pages); + RETURN_IF_ERROR(file_writer->append(pages_slice)); - for (auto& value : _values) { - Slice value_slice(value); - RETURN_IF_ERROR(writer.add(&value_slice)); - } - return writer.finish(meta->mutable_page_zone_maps()); + ZoneMapPageIndexPB* page_index = meta->mutable_page_zone_maps_v2(); + page_pointer.to_proto(page_index->mutable_data()); + page_index->set_num_pages(cast_set(_page_zone_maps.size())); + page_index->set_format_version(ZONE_MAP_NATIVE_PAGE_FORMAT_VERSION); + return Status::OK(); } Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory, @@ -302,19 +891,37 @@ Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory, // TODO yyq: implement a new once flag to avoid status construct. return _load_once.call([this, use_page_cache, kept_in_memory, index_load_stats, io_ctx] { return _load(use_page_cache, kept_in_memory, std::move(_page_zone_maps_meta), - index_load_stats, io_ctx); + std::move(_page_zone_maps_v2_meta), index_load_stats, io_ctx); }); } Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, std::unique_ptr page_zone_maps_meta, + std::unique_ptr page_zone_maps_v2_meta, OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx) { + if (page_zone_maps_v2_meta != nullptr) { + return _load_v2(std::move(page_zone_maps_v2_meta), index_load_stats, io_ctx); + } + if (page_zone_maps_meta != nullptr) { + return _load_legacy(use_page_cache, kept_in_memory, std::move(page_zone_maps_meta), + index_load_stats, io_ctx); + } + _num_pages = 0; + update_metadata_size(); + return Status::OK(); +} + +Status ZoneMapIndexReader::_load_legacy(bool use_page_cache, bool kept_in_memory, + std::unique_ptr page_zone_maps_meta, + OlapReaderStatistics* index_load_stats, + const io::IOContext* io_ctx) { IndexedColumnReader reader(_file_reader, *page_zone_maps_meta); RETURN_IF_ERROR(reader.load(use_page_cache, kept_in_memory, index_load_stats, io_ctx)); IndexedColumnIterator iter(&reader, index_load_stats, io_ctx); _page_zone_maps.resize(reader.num_values()); + _num_pages = _page_zone_maps.size(); // read and cache all page zone maps for (int i = 0; i < reader.num_values(); ++i) { @@ -336,10 +943,68 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, _pb_meta_size += _page_zone_maps[i].ByteSizeLong(); } + if (_data_type != nullptr) { + _page_zone_map_infos.resize(_page_zone_maps.size()); + for (size_t i = 0; i < _page_zone_maps.size(); ++i) { + RETURN_IF_ERROR( + ZoneMap::from_proto(_page_zone_maps[i], _data_type, _page_zone_map_infos[i])); + } + } + update_metadata_size(); return Status::OK(); } +Status ZoneMapIndexReader::_load_v2(std::unique_ptr page_zone_maps_v2_meta, + OlapReaderStatistics* index_load_stats, + const io::IOContext* io_ctx) { + (void)index_load_stats; + DORIS_CHECK(_data_type != nullptr); + if (page_zone_maps_v2_meta->format_version() != ZONE_MAP_NATIVE_PAGE_FORMAT_VERSION) { + return Status::Corruption("unsupported zone map native page format version {}", + page_zone_maps_v2_meta->format_version()); + } + if (!page_zone_maps_v2_meta->has_data()) { + return Status::Corruption("zone map native page payload is missing"); + } + const PagePointer page_pointer(page_zone_maps_v2_meta->data()); + std::string payload(page_pointer.size, '\0'); + size_t bytes_read = 0; + RETURN_IF_ERROR(_file_reader->read_at( + page_pointer.offset, Slice(payload.data(), payload.size()), &bytes_read, io_ctx)); + if (bytes_read != page_pointer.size) { + return Status::Corruption("short read zone map native page payload: expect {}, got {}", + page_pointer.size, bytes_read); + } + + ZoneMapPagesPB pages; + if (!pages.ParseFromArray(payload.data(), cast_set(payload.size()))) { + return Status::Corruption("failed to parse zone map native page payload"); + } + if (page_zone_maps_v2_meta->num_pages() != pages.num_pages()) { + return Status::Corruption("zone map native page count mismatch: meta {}, payload {}", + page_zone_maps_v2_meta->num_pages(), pages.num_pages()); + } + RETURN_IF_ERROR(dispatch_zonemap_native_type( + _data_type->get_primitive_type(), [&]() -> Status { + return decode_zone_map_pages_pb(pages, &_page_zone_map_infos); + })); + _num_pages = _page_zone_map_infos.size(); + _pb_meta_size += pages.ByteSizeLong(); + update_metadata_size(); + return Status::OK(); +} + +const std::vector& ZoneMapIndexReader::page_zone_maps() const { + if (_page_zone_maps.empty() && !_page_zone_map_infos.empty() && _data_type != nullptr) { + _page_zone_maps.resize(_page_zone_map_infos.size()); + for (size_t i = 0; i < _page_zone_map_infos.size(); ++i) { + _page_zone_map_infos[i].to_proto(&_page_zone_maps[i], _data_type); + } + } + return _page_zone_maps; +} + int64_t ZoneMapIndexReader::get_metadata_size() const { return sizeof(ZoneMapIndexReader) + _pb_meta_size; } diff --git a/be/src/storage/index/zone_map/zone_map_index.h b/be/src/storage/index/zone_map/zone_map_index.h index 80c6928f60bb07..1517aad78ad05d 100644 --- a/be/src/storage/index/zone_map/zone_map_index.h +++ b/be/src/storage/index/zone_map/zone_map_index.h @@ -84,6 +84,11 @@ struct ZoneMap { static Status from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_type, ZoneMap& zone_map_info); + + void to_entry_pb_native(ZoneMapEntryPB* dst, const DataTypePtr& data_type) const; + + static Status from_entry_pb_native(const ZoneMapEntryPB& zone_map, const DataTypePtr& data_type, + ZoneMap& zone_map_info); }; class ZoneMapIndexWriter { @@ -166,8 +171,8 @@ class TypedZoneMapIndexWriter final : public ZoneMapIndexWriter { std::string _page_min_storage; std::string _page_max_storage; - // serialized ZoneMapPB for each data page - std::vector _values; + // ZoneMap for each data page. + std::vector _page_zone_maps; uint64_t _estimated_size = 0; }; @@ -179,6 +184,17 @@ class ZoneMapIndexReader : public MetadataAdder { _page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps)); } + explicit ZoneMapIndexReader(io::FileReaderSPtr file_reader, + const ZoneMapIndexPB& zone_map_index, DataTypePtr data_type) + : _file_reader(std::move(file_reader)), _data_type(std::move(data_type)) { + if (zone_map_index.has_page_zone_maps_v2()) { + _page_zone_maps_v2_meta.reset( + new ZoneMapPageIndexPB(zone_map_index.page_zone_maps_v2())); + } else if (zone_map_index.has_page_zone_maps()) { + _page_zone_maps_meta.reset(new IndexedColumnMetaPB(zone_map_index.page_zone_maps())); + } + } + virtual ~ZoneMapIndexReader(); // load all page zone maps into memory @@ -186,22 +202,37 @@ class ZoneMapIndexReader : public MetadataAdder { OlapReaderStatistics* index_load_stats = nullptr, const io::IOContext* io_ctx = nullptr); - const std::vector& page_zone_maps() const { return _page_zone_maps; } + const std::vector& page_zone_maps() const; + + const std::vector& page_zone_map_infos() const { return _page_zone_map_infos; } - size_t num_pages() const { return _page_zone_maps.size(); } + size_t num_pages() const { return _num_pages; } private: - Status _load(bool use_page_cache, bool kept_in_memory, std::unique_ptr, + Status _load(bool use_page_cache, bool kept_in_memory, + std::unique_ptr page_zone_maps_meta, + std::unique_ptr page_zone_maps_v2_meta, OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx); + Status _load_legacy(bool use_page_cache, bool kept_in_memory, + std::unique_ptr page_zone_maps_meta, + OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx); + + Status _load_v2(std::unique_ptr page_zone_maps_v2_meta, + OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx); + int64_t get_metadata_size() const override; private: DorisCallOnce _load_once; // TODO: yyq, we shoud remove file_reader from here. io::FileReaderSPtr _file_reader; + DataTypePtr _data_type; std::unique_ptr _page_zone_maps_meta; - std::vector _page_zone_maps; + std::unique_ptr _page_zone_maps_v2_meta; + mutable std::vector _page_zone_maps; + std::vector _page_zone_map_infos; + size_t _num_pages = 0; int64_t _pb_meta_size {0}; }; diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index f15e7462e6c11a..340fae5e0f1cee 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -110,6 +110,43 @@ void remove_current_level_meta_access_paths(TColumnAccessPaths& paths) { paths.erase(removed.begin(), removed.end()); } +Status read_or_fill_map_item_column(ColumnIterator* iterator, bool seek_to_start, + ordinal_t start_ordinal, size_t item_count, + MutableColumnPtr& dst) { + if (item_count == 0) { + return Status::OK(); + } + + if (iterator->need_to_read()) { + if (seek_to_start) { + RETURN_IF_ERROR(iterator->seek_to_ordinal(start_ordinal)); + } + size_t num_read = item_count; + bool has_null = false; + RETURN_IF_ERROR(iterator->next_batch(&num_read, dst, &has_null)); + DCHECK_EQ(num_read, item_count); + return Status::OK(); + } + + if (iterator->read_requirement() == ColumnIterator::ReadRequirement::SKIP) { + dst->insert_many_defaults(item_count); + return Status::OK(); + } + + if (iterator->read_requirement() == ColumnIterator::ReadRequirement::PREDICATE) { + return Status::OK(); + } + + if (seek_to_start) { + RETURN_IF_ERROR(iterator->seek_to_ordinal(start_ordinal)); + } + size_t num_read = item_count; + bool has_null = false; + RETURN_IF_ERROR(iterator->next_batch(&num_read, dst, &has_null)); + DCHECK_EQ(num_read, item_count); + return Status::OK(); +} + Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, const io::FileReaderSPtr& file_reader, std::shared_ptr* reader) { @@ -319,7 +356,8 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& ColumnReader::~ColumnReader() = default; int64_t ColumnReader::get_metadata_size() const { - return sizeof(ColumnReader) + (_segment_zone_map ? _segment_zone_map->ByteSizeLong() : 0); + return sizeof(ColumnReader) + (_segment_zone_map ? _segment_zone_map->ByteSizeLong() : 0) + + (_segment_zone_map_v2 ? _segment_zone_map_v2->ByteSizeLong() : 0); } #ifdef BE_TEST @@ -327,7 +365,7 @@ int64_t ColumnReader::get_metadata_size() const { /// See UT case 'SegCompactionMoWTest.SegCompactionInterleaveWithBig_ooooOOoOooooooooO' /// be/test/olap/segcompaction_mow_test.cpp void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) const { - if (!_segment_zone_map) { + if (!_segment_zone_map && !_segment_zone_map_v2) { return; } @@ -353,7 +391,7 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) } ZoneMap zone_map; - THROW_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map)); + THROW_IF_ERROR(_get_segment_zone_map(&zone_map)); if (zone_map.has_null) { return; @@ -392,10 +430,15 @@ Status ColumnReader::init(const ColumnMetaPB* meta) { new OrdinalIndexReader(_file_reader, _num_rows, index_meta.ordinal_index())); break; case ZONE_MAP_INDEX: - _segment_zone_map = - std::make_unique(index_meta.zone_map_index().segment_zone_map()); - _zone_map_index.reset(new ZoneMapIndexReader( - _file_reader, index_meta.zone_map_index().page_zone_maps())); + if (index_meta.zone_map_index().has_segment_zone_map_v2()) { + _segment_zone_map_v2 = std::make_unique( + index_meta.zone_map_index().segment_zone_map_v2()); + } else { + _segment_zone_map = + std::make_unique(index_meta.zone_map_index().segment_zone_map()); + } + _zone_map_index.reset( + new ZoneMapIndexReader(_file_reader, index_meta.zone_map_index(), _data_type)); break; case BLOOM_FILTER_INDEX: _bloom_filter_index.reset( @@ -469,12 +512,12 @@ Status ColumnReader::get_row_ranges_by_zone_map( } Status ColumnReader::next_batch_of_zone_map(size_t* n, MutableColumnPtr& dst) const { - if (_segment_zone_map == nullptr) { + if (_segment_zone_map == nullptr && _segment_zone_map_v2 == nullptr) { return Status::InternalError("segment zonemap not exist"); } // TODO: this work to get min/max value seems should only do once ZoneMap zone_map; - RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map)); + RETURN_IF_ERROR(_get_segment_zone_map(&zone_map)); dst->reserve(*n); if (!zone_map.has_not_null) { @@ -495,7 +538,7 @@ Status ColumnReader::match_condition(const AndBlockColumnPredicate* col_predicat return Status::OK(); } ZoneMap zone_map; - RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map)); + RETURN_IF_ERROR(_get_segment_zone_map(&zone_map)); *matched = _zone_map_match_condition(zone_map, col_predicates); return Status::OK(); @@ -522,7 +565,7 @@ Status ColumnReader::prune_predicates_by_zone_map( } ZoneMap zone_map; - RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map)); + RETURN_IF_ERROR(_get_segment_zone_map(&zone_map)); if (zone_map.pass_all) { return Status::OK(); } @@ -554,21 +597,19 @@ Status ColumnReader::_get_filtered_pages( std::vector* page_indexes, const ColumnIteratorOptions& iter_opts) { RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); - const std::vector& zone_maps = _zone_map_index->page_zone_maps(); + const std::vector& zone_maps = _zone_map_index->page_zone_map_infos(); size_t page_size = _zone_map_index->num_pages(); for (size_t i = 0; i < page_size; ++i) { - if (zone_maps[i].pass_all()) { + if (zone_maps[i].pass_all) { page_indexes->push_back(cast_set(i)); } else { - segment_v2::ZoneMap zone_map; - RETURN_IF_ERROR(ZoneMap::from_proto(zone_maps[i], _data_type, zone_map)); - if (_zone_map_match_condition(zone_map, col_predicates)) { + if (_zone_map_match_condition(zone_maps[i], col_predicates)) { bool should_read = true; if (delete_predicates != nullptr) { for (auto del_pred : *delete_predicates) { // TODO: Both `min_value` and `max_value` should be 0 or neither should be 0. // So nullable only need to judge once. - if (del_pred->evaluate_del(zone_map)) { + if (del_pred->evaluate_del(zone_maps[i])) { should_read = false; break; } @@ -656,8 +697,7 @@ Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memo Status ColumnReader::get_segment_zone_map(segment_v2::ZoneMap* zone_map) const { DORIS_CHECK(zone_map != nullptr); - DORIS_CHECK(_segment_zone_map != nullptr); - return ZoneMap::from_proto(*_segment_zone_map, _data_type, *zone_map); + return _get_segment_zone_map(zone_map); } Status ConstantColumnReader::get_segment_zone_map(segment_v2::ZoneMap* zone_map) const { @@ -679,6 +719,27 @@ Status ColumnReader::get_page_zone_maps(const ColumnIteratorOptions& iter_opts, return Status::OK(); } +Status ColumnReader::get_page_zone_map_infos(const ColumnIteratorOptions& iter_opts, + const std::vector** zone_maps) { + DORIS_CHECK(zone_maps != nullptr); + if (_zone_map_index == nullptr) { + *zone_maps = nullptr; + return Status::OK(); + } + RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); + *zone_maps = &_zone_map_index->page_zone_map_infos(); + return Status::OK(); +} + +Status ColumnReader::_get_segment_zone_map(segment_v2::ZoneMap* zone_map) const { + DORIS_CHECK(zone_map != nullptr); + if (_segment_zone_map_v2 != nullptr) { + return ZoneMap::from_entry_pb_native(*_segment_zone_map_v2, _data_type, *zone_map); + } + DORIS_CHECK(_segment_zone_map != nullptr); + return ZoneMap::from_proto(*_segment_zone_map, _data_type, *zone_map); +} + Status ColumnReader::get_row_range_for_page(uint32_t page_index, const ColumnIteratorOptions& iter_opts, RowRange* row_range) { @@ -1171,12 +1232,10 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* key_ptr->insert_many_defaults(num_items); val_ptr->insert_many_defaults(num_items); } else { - size_t num_read = num_items; - bool key_has_null = false; - bool val_has_null = false; - RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null)); - RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null)); - DCHECK(num_read == num_items); + RETURN_IF_ERROR(read_or_fill_map_item_column(_key_iterator.get(), false, 0, num_items, + key_ptr)); + RETURN_IF_ERROR(read_or_fill_map_item_column(_val_iterator.get(), false, 0, num_items, + val_ptr)); } } @@ -1377,18 +1436,11 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t } auto start = static_cast(starts_data[i]); if (start != last_idx) { - size_t n = this_run; - bool dummy_has_null = false; - if (this_run != 0) { - RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); - DCHECK(n == this_run); - - n = this_run; - RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); - DCHECK(n == this_run); + RETURN_IF_ERROR(read_or_fill_map_item_column(_key_iterator.get(), true, start_idx, + this_run, keys_ptr)); + RETURN_IF_ERROR(read_or_fill_map_item_column(_val_iterator.get(), true, start_idx, + this_run, vals_ptr)); } start_idx = start; this_run = sz; @@ -1400,17 +1452,11 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t last_idx += sz; } - size_t n = this_run; - bool dummy_has_null = false; if (this_run != 0) { - RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); - DCHECK(n == this_run); - - n = this_run; - RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); - DCHECK(n == this_run); + RETURN_IF_ERROR(read_or_fill_map_item_column(_key_iterator.get(), true, start_idx, this_run, + keys_ptr)); + RETURN_IF_ERROR(read_or_fill_map_item_column(_val_iterator.get(), true, start_idx, this_run, + vals_ptr)); } return Status::OK(); } diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index 777f8eddf0247b..5af3b244b9961a 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -225,6 +225,8 @@ class ColumnReader : public MetadataAdder, virtual Status get_segment_zone_map(segment_v2::ZoneMap* zone_map) const; Status get_page_zone_maps(const ColumnIteratorOptions& iter_opts, const std::vector** zone_maps); + Status get_page_zone_map_infos(const ColumnIteratorOptions& iter_opts, + const std::vector** zone_maps); Status get_row_range_for_page(uint32_t page_index, const ColumnIteratorOptions& iter_opts, RowRange* row_range); @@ -275,6 +277,7 @@ class ColumnReader : public MetadataAdder, bool _zone_map_match_condition(const segment_v2::ZoneMap& zone_map, const AndBlockColumnPredicate* col_predicates) const; + Status _get_segment_zone_map(segment_v2::ZoneMap* zone_map) const; Status _get_filtered_pages( const AndBlockColumnPredicate* col_predicates, @@ -310,6 +313,7 @@ class ColumnReader : public MetadataAdder, // meta for various column indexes (null if the index is absent) std::unique_ptr _segment_zone_map; + std::unique_ptr _segment_zone_map_v2; mutable std::shared_mutex _load_index_lock; std::unique_ptr _zone_map_index; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index f667efc0db14cc..ee8bdeb6ab207e 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -3427,8 +3427,8 @@ Status SegmentIterator::_apply_expr_zonemap_to_row_ranges(const VExprContextSPtr if (reader == nullptr || !reader->has_zone_map()) { continue; } - const std::vector* page_zone_maps = nullptr; - RETURN_IF_ERROR(reader->get_page_zone_maps(iter_opts, &page_zone_maps)); + const std::vector* page_zone_maps = nullptr; + RETURN_IF_ERROR(reader->get_page_zone_map_infos(iter_opts, &page_zone_maps)); if (page_zone_maps == nullptr || page_zone_maps->empty()) { continue; } @@ -3448,10 +3448,7 @@ Status SegmentIterator::_apply_expr_zonemap_to_row_ranges(const VExprContextSPtr ZoneMapEvalContext ctx; ZoneMapEvalContext::SlotZoneMap slot_zone_map; slot_zone_map.data_type = data_type; - ZoneMap zone_map; - RETURN_IF_ERROR( - ZoneMap::from_proto((*page_zone_maps)[page_index], data_type, zone_map)); - slot_zone_map.zone_map = std::make_shared(std::move(zone_map)); + slot_zone_map.zone_map = std::make_shared((*page_zone_maps)[page_index]); ctx.slots.emplace(slot_index, std::move(slot_zone_map)); const auto result = VExprContext::evaluate_zonemap_filter(slot_conjuncts, ctx); page_stats.merge_page_eval_stats(ctx.stats); diff --git a/be/test/storage/metadata_adder_test.cpp b/be/test/storage/metadata_adder_test.cpp index b436a6c670cb2d..b151372e29f7fe 100644 --- a/be/test/storage/metadata_adder_test.cpp +++ b/be/test/storage/metadata_adder_test.cpp @@ -148,8 +148,8 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(file1, &file_reader).ok()); - segment_v2::ZoneMapIndexReader zonemap_col_reader( - file_reader, index_meta1.zone_map_index().page_zone_maps()); + segment_v2::ZoneMapIndexReader zonemap_col_reader(file_reader, index_meta1.zone_map_index(), + int_data_type_ptr); Status status = zonemap_col_reader.load(true, false); int mem_size = zonemap_col_reader.get_metadata_size(); @@ -185,7 +185,7 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { io::FileReaderSPtr file_reader2; EXPECT_TRUE(fs->open_file(file2, &file_reader2).ok()); segment_v2::ZoneMapIndexReader zonemap_col_reader2( - file_reader2, index_meta2.zone_map_index().page_zone_maps()); + file_reader2, index_meta2.zone_map_index(), str_data_type_ptr); Status status2 = zonemap_col_reader2.load(true, false); int mem_size2 = zonemap_col_reader2.get_metadata_size(); @@ -197,4 +197,4 @@ TEST_F(MetadataAdderTest, meta_load_with_pb_test) { ASSERT_TRUE(MetadataAdder::get_all_segments_size() == 0); } -}; // namespace doris \ No newline at end of file +}; // namespace doris diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index 3ee511a5980632..fa4d5b2259c7fa 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -192,6 +192,17 @@ class TrackingFileColumnIterator final : public FileColumnIterator { Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override { next_batch_sizes.emplace_back(*n); + if (!offset_values_by_ordinal.empty()) { + auto& offsets = assert_cast(*dst); + for (size_t i = 0; i < *n; ++i) { + offsets.insert_value(offset_values_by_ordinal[_current_ordinal + i]); + } + _current_ordinal += *n; + if (has_null != nullptr) { + *has_null = false; + } + return Status::OK(); + } dst->insert_many_defaults(*n); _current_ordinal += *n; if (has_null != nullptr) { @@ -203,6 +214,13 @@ class TrackingFileColumnIterator final : public FileColumnIterator { Status read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) override { read_by_rowids_batches.emplace_back(rowids, rowids + count); + if (!offset_values_by_ordinal.empty()) { + auto& offsets = assert_cast(*dst); + for (size_t i = 0; i < count; ++i) { + offsets.insert_value(offset_values_by_ordinal[rowids[i]]); + } + return Status::OK(); + } dst->insert_many_defaults(count); return Status::OK(); } @@ -224,6 +242,7 @@ class TrackingFileColumnIterator final : public FileColumnIterator { std::vector next_batch_sizes; std::vector> read_by_rowids_batches; std::vector collect_methods; + std::vector offset_values_by_ordinal; private: void record_collect_method(PrefetcherInitMethod init_method) { @@ -265,6 +284,11 @@ MutableColumnPtr create_nullable_int_map_column() { ColumnUInt8::create()); } +MutableColumnPtr create_int_map_column() { + return ColumnMap::create(ColumnInt32::create(), ColumnInt32::create(), + ColumnArray::ColumnOffsets::create()); +} + struct TrackingOffsetIterator { OffsetFileColumnIteratorUPtr iterator; TrackingFileColumnIterator* tracker = nullptr; @@ -2167,6 +2191,102 @@ TEST_F(ColumnReaderTest, MapNullMapOnlyNextBatchAndReadByRowidsSkipKeysAndValues EXPECT_TRUE(offset_iterator.tracker->next_batch_sizes.empty()); } +TEST_F(ColumnReaderTest, MapNextBatchLazyFillsSkippedKeyOrValueColumn) { + auto assert_lazy_next_batch = [](const std::string& access_component, bool reads_key) { + auto key_iterator = std::make_unique(); + auto* key_iterator_ptr = key_iterator.get(); + auto value_iterator = std::make_unique(); + auto* value_iterator_ptr = value_iterator.get(); + auto offset_iterator = create_tracking_offset_iterator(); + offset_iterator.tracker->get_current_page()->num_rows = 0; + offset_iterator.tracker->get_current_page()->offset_in_page = 0; + offset_iterator.tracker->get_current_page()->next_array_item_ordinal = 2; + + MapFileColumnIterator map_iterator( + create_test_reader(false, 1, FieldType::OLAP_FIELD_TYPE_MAP), nullptr, + std::move(offset_iterator.iterator), std::move(key_iterator), + std::move(value_iterator)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths all_access_paths {create_access_path({"m", access_component})}; + auto st = map_iterator.set_access_paths(all_access_paths, {}); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + map_iterator.set_read_requirement_self(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + map_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + + MutableColumnPtr dst = create_int_map_column(); + size_t rows = 1; + bool has_null = false; + st = map_iterator.next_batch(&rows, dst, &has_null); + ASSERT_TRUE(st.ok()) << "map lazy next_batch failed: " << st.to_string(); + EXPECT_EQ(rows, 1); + EXPECT_EQ(dst->size(), 1); + + const auto& map_column = assert_cast(*dst); + ASSERT_EQ(map_column.get_offsets().size(), 1); + EXPECT_EQ(map_column.get_offsets()[0], 2); + EXPECT_EQ(map_column.get_keys().size(), 2); + EXPECT_EQ(map_column.get_values().size(), 2); + if (reads_key) { + EXPECT_THAT(key_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(2)); + EXPECT_TRUE(value_iterator_ptr->next_batch_sizes.empty()); + } else { + EXPECT_TRUE(key_iterator_ptr->next_batch_sizes.empty()); + EXPECT_THAT(value_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(2)); + } + }; + + assert_lazy_next_batch(ColumnIterator::ACCESS_MAP_KEYS, true); + assert_lazy_next_batch(ColumnIterator::ACCESS_MAP_VALUES, false); +} + +TEST_F(ColumnReaderTest, MapReadByRowidsLazyFillsSkippedKeyOrValueColumn) { + auto assert_lazy_read_by_rowids = [](const std::string& access_component, bool reads_key) { + auto key_iterator = std::make_unique(); + auto* key_iterator_ptr = key_iterator.get(); + auto value_iterator = std::make_unique(); + auto* value_iterator_ptr = value_iterator.get(); + auto offset_iterator = create_tracking_offset_iterator(); + offset_iterator.tracker->offset_values_by_ordinal = {0, 2, 4}; + + MapFileColumnIterator map_iterator( + create_test_reader(false, 3, FieldType::OLAP_FIELD_TYPE_MAP), nullptr, + std::move(offset_iterator.iterator), std::move(key_iterator), + std::move(value_iterator)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths all_access_paths {create_access_path({"m", access_component})}; + auto st = map_iterator.set_access_paths(all_access_paths, {}); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + map_iterator.set_read_requirement_self(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + map_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + + MutableColumnPtr dst = create_int_map_column(); + const rowid_t rowids[] = {1}; + st = map_iterator.read_by_rowids(rowids, std::size(rowids), dst); + ASSERT_TRUE(st.ok()) << "map lazy read_by_rowids failed: " << st.to_string(); + EXPECT_EQ(dst->size(), 1); + + const auto& map_column = assert_cast(*dst); + ASSERT_EQ(map_column.get_offsets().size(), 1); + EXPECT_EQ(map_column.get_offsets()[0], 2); + EXPECT_EQ(map_column.get_keys().size(), 2); + EXPECT_EQ(map_column.get_values().size(), 2); + if (reads_key) { + EXPECT_THAT(key_iterator_ptr->seek_ordinals, ::testing::ElementsAre(2)); + EXPECT_THAT(key_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(2)); + EXPECT_TRUE(value_iterator_ptr->next_batch_sizes.empty()); + } else { + EXPECT_TRUE(key_iterator_ptr->next_batch_sizes.empty()); + EXPECT_THAT(value_iterator_ptr->seek_ordinals, ::testing::ElementsAre(2)); + EXPECT_THAT(value_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(2)); + } + }; + + assert_lazy_read_by_rowids(ColumnIterator::ACCESS_MAP_KEYS, true); + assert_lazy_read_by_rowids(ColumnIterator::ACCESS_MAP_VALUES, false); +} + TEST_F(ColumnReaderTest, CollectPrefetchersHonorsNestedReadRequirements) { auto null_iterator = std::make_unique(); auto* null_iterator_ptr = null_iterator.get(); diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index 51f8c068947c44..248badfc596bde 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -37,10 +37,12 @@ #include "exprs/function/cast/cast_to_string.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" +#include "storage/index/indexed_column_writer.h" #include "storage/olap_common.h" #include "storage/predicate/comparison_predicate.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" +#include "util/coding.h" #include "util/slice.h" namespace doris { @@ -59,6 +61,149 @@ class ColumnZoneMapTest : public testing::Test { } void TearDown() override { EXPECT_TRUE(_fs->delete_directory(kTestDir).ok()); } + ZoneMap load_segment_zone_map(const ColumnIndexMetaPB& index_meta, + const DataTypePtr& data_type) { + EXPECT_TRUE(index_meta.zone_map_index().has_segment_zone_map_v2()); + EXPECT_TRUE(index_meta.zone_map_index().has_page_zone_maps_v2()); + EXPECT_FALSE(index_meta.zone_map_index().has_segment_zone_map()); + EXPECT_FALSE(index_meta.zone_map_index().has_page_zone_maps()); + ZoneMap zone_map; + EXPECT_TRUE(ZoneMap::from_entry_pb_native(index_meta.zone_map_index().segment_zone_map_v2(), + data_type, zone_map) + .ok()); + return zone_map; + } + + static std::string one_byte_bitmap(uint8_t bits) { + return std::string(1, static_cast(bits)); + } + + static void init_fixed_int_values(ZoneMapValueArrayPB* values) { + values->set_layout(ZoneMapValueArrayPB::FIXED_WIDTH); + values->set_fixed_width(sizeof(int32_t)); + } + + static void append_fixed_int_value(ZoneMapValueArrayPB* values, int32_t value) { + put_fixed32_le(values->mutable_fixed_values(), static_cast(value)); + } + + static TabletColumnPtr create_scalar_key(FieldType type, int32_t length, int32_t precision = 0, + int32_t scale = 0) { + auto column = std::make_shared(); + column->_unique_id = 0; + column->_col_name = "0"; + column->_type = type; + column->_is_key = true; + column->_is_nullable = true; + column->_length = length; + column->_index_length = length; + column->_precision = precision; + column->_frac = scale; + column->_is_bf_column = false; + return column; + } + + static ZoneMapPagesPB one_present_int_page(int32_t min, int32_t max) { + ZoneMapPagesPB pages; + pages.set_num_pages(1); + pages.set_entry_version(1); + pages.set_has_null_bitmap(one_byte_bitmap(0)); + pages.set_has_not_null_bitmap(one_byte_bitmap(1)); + pages.set_pass_all_bitmap(one_byte_bitmap(0)); + pages.set_has_positive_inf_bitmap(one_byte_bitmap(0)); + pages.set_has_negative_inf_bitmap(one_byte_bitmap(0)); + pages.set_has_nan_bitmap(one_byte_bitmap(0)); + init_fixed_int_values(pages.mutable_min_values()); + init_fixed_int_values(pages.mutable_max_values()); + append_fixed_int_value(pages.mutable_min_values(), min); + append_fixed_int_value(pages.mutable_max_values(), max); + return pages; + } + + Status write_native_pages_payload(const std::string& filename, const ZoneMapPagesPB& pages, + uint32_t meta_num_pages, ZoneMapIndexPB* zone_map_index, + io::FileReaderSPtr* file_reader) { + std::string payload; + if (!pages.SerializeToString(&payload)) { + return Status::InternalError("serialize test zone map pages failed"); + } + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(_fs->create_file(filename, &file_writer)); + RETURN_IF_ERROR(file_writer->append(Slice(payload))); + RETURN_IF_ERROR(file_writer->close()); + + ZoneMapPageIndexPB* page_index = zone_map_index->mutable_page_zone_maps_v2(); + page_index->mutable_data()->set_offset(0); + page_index->mutable_data()->set_size(static_cast(payload.size())); + page_index->set_num_pages(meta_num_pages); + page_index->set_format_version(1); + return _fs->open_file(filename, file_reader); + } + + template + void test_native_storage_value_round_trip( + const std::string& testname, FieldType field_type, + typename PrimitiveTypeTraits::StorageFieldType min_value, + typename PrimitiveTypeTraits::StorageFieldType max_value, int32_t length, + int32_t precision = 0, int32_t scale = 0) { + auto column = create_scalar_key(field_type, length, precision, scale); + auto data_type = + DataTypeFactory::instance().create_data_type(Type, false, precision, scale, length); + + std::unique_ptr writer; + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, column.get(), writer).ok()); + + using StorageType = typename PrimitiveTypeTraits::StorageFieldType; + std::vector page0_values {max_value, min_value}; + writer->add_values(page0_values.data(), page0_values.size()); + ASSERT_TRUE(writer->flush().ok()); + + writer->add_nulls(1); + writer->add_values(&min_value, 1); + ASSERT_TRUE(writer->flush().ok()); + + writer->add_nulls(2); + ASSERT_TRUE(writer->flush().ok()); + + std::string filename = kTestDir + "/" + testname; + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(filename, &file_writer).ok()); + ColumnIndexMetaPB index_meta; + ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); + ASSERT_TRUE(file_writer->close().ok()); + + const Field expected_min = Field::create_field_from_olap_value(min_value); + const Field expected_max = Field::create_field_from_olap_value(max_value); + ZoneMap segment_zone_map = load_segment_zone_map(index_meta, data_type); + EXPECT_EQ(expected_min, segment_zone_map.min_value); + EXPECT_EQ(expected_max, segment_zone_map.max_value); + EXPECT_TRUE(segment_zone_map.has_null); + EXPECT_TRUE(segment_zone_map.has_not_null); + + io::FileReaderSPtr file_reader; + ASSERT_TRUE(_fs->open_file(filename, &file_reader).ok()); + ZoneMapIndexReader reader(file_reader, index_meta.zone_map_index(), data_type); + ASSERT_TRUE(reader.load(true, false).ok()); + ASSERT_EQ(3, reader.num_pages()); + ASSERT_EQ(3, reader.page_zone_map_infos().size()); + + const auto& page0 = reader.page_zone_map_infos()[0]; + EXPECT_EQ(expected_min, page0.min_value); + EXPECT_EQ(expected_max, page0.max_value); + EXPECT_FALSE(page0.has_null); + EXPECT_TRUE(page0.has_not_null); + + const auto& page1 = reader.page_zone_map_infos()[1]; + EXPECT_EQ(expected_min, page1.min_value); + EXPECT_EQ(expected_min, page1.max_value); + EXPECT_TRUE(page1.has_null); + EXPECT_TRUE(page1.has_not_null); + + const auto& page2 = reader.page_zone_map_infos()[2]; + EXPECT_TRUE(page2.has_null); + EXPECT_FALSE(page2.has_not_null); + } + void test_string(std::string testname, const TabletColumn* field, DataTypePtr data_type_ptr) { std::string filename = kTestDir + "/" + testname; auto fs = io::global_local_filesystem(); @@ -94,8 +239,7 @@ class ColumnZoneMapTest : public testing::Test { io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, - index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type_ptr); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(3, column_zone_map.num_pages()); @@ -177,16 +321,14 @@ class ColumnZoneMapTest : public testing::Test { ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); + const auto& seg_zm = index_meta.zone_map_index().segment_zone_map_v2(); // Min/Max should be truncated to MAX_ZONE_MAP_INDEX_SIZE and last byte of Max is incremented EXPECT_EQ(seg_zm.min().size(), MAX_ZONE_MAP_INDEX_SIZE); EXPECT_EQ(seg_zm.min(), small_long_string1_expect); EXPECT_EQ(seg_zm.max().size(), MAX_ZONE_MAP_INDEX_SIZE); EXPECT_EQ(seg_zm.max(), big_long_string2_expect); - // Verify ZoneMap::from_proto can correctly parse the truncated zone map - ZoneMap seg_zone_map; - ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type, seg_zone_map).ok()); + ZoneMap seg_zone_map = load_segment_zone_map(index_meta, data_type); EXPECT_EQ(seg_zone_map.has_null, false); EXPECT_EQ(seg_zone_map.has_not_null, true); EXPECT_EQ(seg_zone_map.pass_all, false); @@ -201,8 +343,7 @@ class ColumnZoneMapTest : public testing::Test { io::FileReaderSPtr file_reader; EXPECT_TRUE(_fs->open_file(file_path, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, - index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(2, column_zone_map.num_pages()); @@ -273,16 +414,14 @@ class ColumnZoneMapTest : public testing::Test { ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); + const auto& seg_zm = index_meta.zone_map_index().segment_zone_map_v2(); // On-disk min/max preserve the raw (unpadded) bytes. EXPECT_EQ(seg_zm.min().size(), s_less_than_char_len1.size()); EXPECT_EQ(seg_zm.min(), s_less_than_char_len1); EXPECT_EQ(seg_zm.max().size(), s_less_than_char_len2.size()); EXPECT_EQ(seg_zm.max(), s_less_than_char_len2); - // Verify ZoneMap::from_proto materializes the unpadded Field. - ZoneMap seg_zone_map; - ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type, seg_zone_map).ok()); + ZoneMap seg_zone_map = load_segment_zone_map(index_meta, data_type); EXPECT_EQ(seg_zone_map.has_null, false); EXPECT_EQ(seg_zone_map.has_not_null, true); EXPECT_EQ(seg_zone_map.pass_all, false); @@ -297,8 +436,7 @@ class ColumnZoneMapTest : public testing::Test { io::FileReaderSPtr file_reader; EXPECT_TRUE(_fs->open_file(file_path, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, - index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(1, column_zone_map.num_pages()); @@ -368,13 +506,7 @@ class ColumnZoneMapTest : public testing::Test { ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); - EXPECT_EQ(seg_zm.min(), DecimalV2Value(decimal1.integer, decimal1.fraction)); - EXPECT_EQ(seg_zm.max(), DecimalV2Value(decimal3.integer, decimal3.fraction)); - - // Verify ZoneMap::from_proto can correctly parse the truncated zone map - ZoneMap seg_zone_map; - ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type, seg_zone_map).ok()); + ZoneMap seg_zone_map = load_segment_zone_map(index_meta, data_type); EXPECT_EQ(seg_zone_map.has_null, false); EXPECT_EQ(seg_zone_map.has_not_null, true); EXPECT_EQ(seg_zone_map.pass_all, false); @@ -389,8 +521,7 @@ class ColumnZoneMapTest : public testing::Test { io::FileReaderSPtr file_reader; EXPECT_TRUE(_fs->open_file(file_path, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, - index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(1, column_zone_map.num_pages()); @@ -455,12 +586,7 @@ class ColumnZoneMapTest : public testing::Test { ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); - // EXPECT_EQ(seg_zm.min(), value1); - // EXPECT_EQ(seg_zm.max(), value3); - - ZoneMap seg_zone_map; - ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type, seg_zone_map).ok()); + ZoneMap seg_zone_map = load_segment_zone_map(index_meta, data_type); EXPECT_EQ(seg_zone_map.has_null, false); EXPECT_EQ(seg_zone_map.has_not_null, true); EXPECT_EQ(seg_zone_map.pass_all, false); @@ -473,8 +599,7 @@ class ColumnZoneMapTest : public testing::Test { io::FileReaderSPtr file_reader; EXPECT_TRUE(_fs->open_file(file_path, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, - index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(1, column_zone_map.num_pages()); @@ -537,10 +662,7 @@ class ColumnZoneMapTest : public testing::Test { ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); - - ZoneMap seg_zone_map; - ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type, seg_zone_map).ok()); + ZoneMap seg_zone_map = load_segment_zone_map(index_meta, data_type); EXPECT_EQ(seg_zone_map.has_null, false); EXPECT_EQ(seg_zone_map.has_not_null, true); EXPECT_EQ(seg_zone_map.pass_all, false); @@ -553,8 +675,7 @@ class ColumnZoneMapTest : public testing::Test { io::FileReaderSPtr file_reader; EXPECT_TRUE(_fs->open_file(file_path, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, - index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(1, column_zone_map.num_pages()); @@ -615,7 +736,7 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type_ptr); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(3, column_zone_map.num_pages()); @@ -677,6 +798,47 @@ TEST_F(ColumnZoneMapTest, CharColumnPadding) { test_char_padding(true); } +TEST_F(ColumnZoneMapTest, NativeVariableWidthEmptyStringRoundTrip) { + std::string filename = kTestDir + "/NativeVariableWidthEmptyStringRoundTrip"; + auto column = create_string_key(0); + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false); + + std::unique_ptr writer; + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, column.get(), writer).ok()); + std::string empty; + std::string z = "z"; + std::vector page0_values {StringRef(empty.data(), empty.size()), + StringRef(z.data(), z.size())}; + writer->add_values(page0_values.data(), page0_values.size()); + ASSERT_TRUE(writer->flush().ok()); + + writer->add_nulls(1); + writer->add_values(&page0_values[0], 1); + ASSERT_TRUE(writer->flush().ok()); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(filename, &file_writer).ok()); + ColumnIndexMetaPB index_meta; + ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); + ASSERT_TRUE(file_writer->close().ok()); + + ZoneMap segment_zone_map = load_segment_zone_map(index_meta, data_type); + EXPECT_EQ("", segment_zone_map.min_value.get()); + EXPECT_EQ("z", segment_zone_map.max_value.get()); + + io::FileReaderSPtr file_reader; + ASSERT_TRUE(_fs->open_file(filename, &file_reader).ok()); + ZoneMapIndexReader reader(file_reader, index_meta.zone_map_index(), data_type); + ASSERT_TRUE(reader.load(true, false).ok()); + ASSERT_EQ(2, reader.page_zone_map_infos().size()); + EXPECT_EQ("", reader.page_zone_map_infos()[0].min_value.get()); + EXPECT_EQ("z", reader.page_zone_map_infos()[0].max_value.get()); + EXPECT_EQ("", reader.page_zone_map_infos()[1].min_value.get()); + EXPECT_EQ("", reader.page_zone_map_infos()[1].max_value.get()); + EXPECT_TRUE(reader.page_zone_map_infos()[1].has_null); + EXPECT_TRUE(reader.page_zone_map_infos()[1].has_not_null); +} + TEST_F(ColumnZoneMapTest, DecimalV2) { test_decimalv2(false); test_decimalv2(true); @@ -692,6 +854,61 @@ TEST_F(ColumnZoneMapTest, DateTimeV1) { test_datetimev1(true); } +TEST_F(ColumnZoneMapTest, NativeFixedWidthTypeRoundTrip) { + test_native_storage_value_round_trip( + "native_bool_round_trip", FieldType::OLAP_FIELD_TYPE_BOOL, UInt8(0), UInt8(1), 1); + test_native_storage_value_round_trip("native_tinyint_round_trip", + FieldType::OLAP_FIELD_TYPE_TINYINT, + int8_t(-12), int8_t(42), 1); + test_native_storage_value_round_trip("native_smallint_round_trip", + FieldType::OLAP_FIELD_TYPE_SMALLINT, + int16_t(-1234), int16_t(2345), 2); + test_native_storage_value_round_trip( + "native_bigint_round_trip", FieldType::OLAP_FIELD_TYPE_BIGINT, int64_t(-1234567890123), + int64_t(4567890123456), 8); + test_native_storage_value_round_trip( + "native_largeint_round_trip", FieldType::OLAP_FIELD_TYPE_LARGEINT, + static_cast(-123456789012345678LL), static_cast(456789012345678901LL), + 16); + + DateV2Value date_min; + DateV2Value date_max; + ASSERT_TRUE(date_min.from_date_int64(20200102)); + ASSERT_TRUE(date_max.from_date_int64(20251231)); + test_native_storage_value_round_trip( + "native_datev2_round_trip", FieldType::OLAP_FIELD_TYPE_DATEV2, + date_min.to_date_int_val(), date_max.to_date_int_val(), 4); + + DateV2Value datetime_min; + DateV2Value datetime_max; + ASSERT_TRUE(datetime_min.from_date_int64(20200102030405)); + ASSERT_TRUE(datetime_max.from_date_int64(20251231235959)); + test_native_storage_value_round_trip( + "native_datetimev2_round_trip", FieldType::OLAP_FIELD_TYPE_DATETIMEV2, + datetime_min.to_date_int_val(), datetime_max.to_date_int_val(), 8, 0, 6); + + test_native_storage_value_round_trip("native_decimal32_round_trip", + FieldType::OLAP_FIELD_TYPE_DECIMAL32, + int32_t(-12345), int32_t(67890), 4, 9, 3); + test_native_storage_value_round_trip( + "native_decimal64_round_trip", FieldType::OLAP_FIELD_TYPE_DECIMAL64, + int64_t(-1234567890123), int64_t(6789012345678), 8, 18, 6); + test_native_storage_value_round_trip( + "native_decimal128_round_trip", FieldType::OLAP_FIELD_TYPE_DECIMAL128I, + static_cast(-123456789012345678LL), static_cast(456789012345678901LL), + 16, 38, 9); + test_native_storage_value_round_trip( + "native_decimal256_round_trip", FieldType::OLAP_FIELD_TYPE_DECIMAL256, + wide::Int256(-123456789012345678LL), wide::Int256(456789012345678901LL), 32, 76, 18); + + test_native_storage_value_round_trip("native_ipv4_round_trip", + FieldType::OLAP_FIELD_TYPE_IPV4, + IPv4(0x01020304), IPv4(0x7F000001), 4); + test_native_storage_value_round_trip("native_ipv6_round_trip", + FieldType::OLAP_FIELD_TYPE_IPV6, IPv6(1), + (static_cast(1) << 80) + IPv6(42), 16); +} + // Test for float/double template TabletColumnPtr create_float_column(int32_t id, bool is_nullable) { @@ -761,25 +978,41 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - auto segment_zone_map = index_meta.zone_map_index().segment_zone_map(); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), - segment_zone_map.min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), segment_zone_map.max()); - EXPECT_EQ(true, segment_zone_map.has_null()); - EXPECT_EQ(true, segment_zone_map.has_not_null()); - EXPECT_EQ(true, segment_zone_map.has_positive_inf()); - EXPECT_EQ(true, segment_zone_map.has_negative_inf()); - EXPECT_EQ(true, segment_zone_map.has_nan()); - - ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index().page_zone_maps()); + auto segment_zone_map = load_segment_zone_map(index_meta, data_type_ptr); + EXPECT_EQ(-std::numeric_limits::infinity(), + segment_zone_map.min_value.get()); + EXPECT_TRUE(std::isnan(segment_zone_map.max_value.get())); + EXPECT_EQ(true, segment_zone_map.has_null); + EXPECT_EQ(true, segment_zone_map.has_not_null); + EXPECT_EQ(true, segment_zone_map.has_positive_inf); + EXPECT_EQ(true, segment_zone_map.has_negative_inf); + EXPECT_EQ(true, segment_zone_map.has_nan); + + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type_ptr); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(3, column_zone_map.num_pages()); + const std::vector& zone_map_infos = column_zone_map.page_zone_map_infos(); + ASSERT_EQ(3, zone_map_infos.size()); + EXPECT_EQ(-std::numeric_limits::infinity(), + zone_map_infos[0].min_value.get()); + EXPECT_TRUE(std::isnan(zone_map_infos[0].max_value.get())); + EXPECT_EQ(false, zone_map_infos[0].has_null); + EXPECT_EQ(true, zone_map_infos[0].has_not_null); + EXPECT_EQ(true, zone_map_infos[0].has_positive_inf); + EXPECT_EQ(true, zone_map_infos[0].has_negative_inf); + EXPECT_EQ(true, zone_map_infos[0].has_nan); + + EXPECT_EQ(-1234.56F, zone_map_infos[1].min_value.get()); + EXPECT_EQ(1234.56F, zone_map_infos[1].max_value.get()); + EXPECT_EQ(true, zone_map_infos[1].has_null); + EXPECT_EQ(true, zone_map_infos[1].has_not_null); + + EXPECT_EQ(true, zone_map_infos[2].has_null); + EXPECT_EQ(false, zone_map_infos[2].has_not_null); + const std::vector& zone_maps = column_zone_map.page_zone_maps(); EXPECT_EQ(3, zone_maps.size()); - - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), zone_maps[0].min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), zone_maps[0].max()); EXPECT_EQ(false, zone_maps[0].has_null()); EXPECT_EQ(true, zone_maps[0].has_not_null()); EXPECT_EQ(true, zone_maps[0].has_positive_inf()); @@ -850,26 +1083,41 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - auto segment_zone_map = index_meta.zone_map_index().segment_zone_map(); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), - segment_zone_map.min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), - segment_zone_map.max()); - EXPECT_EQ(true, segment_zone_map.has_null()); - EXPECT_EQ(true, segment_zone_map.has_not_null()); - EXPECT_EQ(true, segment_zone_map.has_positive_inf()); - EXPECT_EQ(true, segment_zone_map.has_negative_inf()); - EXPECT_EQ(true, segment_zone_map.has_nan()); - - ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index().page_zone_maps()); + auto segment_zone_map = load_segment_zone_map(index_meta, data_type_ptr); + EXPECT_EQ(-std::numeric_limits::infinity(), + segment_zone_map.min_value.get()); + EXPECT_TRUE(std::isnan(segment_zone_map.max_value.get())); + EXPECT_EQ(true, segment_zone_map.has_null); + EXPECT_EQ(true, segment_zone_map.has_not_null); + EXPECT_EQ(true, segment_zone_map.has_positive_inf); + EXPECT_EQ(true, segment_zone_map.has_negative_inf); + EXPECT_EQ(true, segment_zone_map.has_nan); + + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type_ptr); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(3, column_zone_map.num_pages()); + const std::vector& zone_map_infos = column_zone_map.page_zone_map_infos(); + ASSERT_EQ(3, zone_map_infos.size()); + EXPECT_EQ(-std::numeric_limits::infinity(), + zone_map_infos[0].min_value.get()); + EXPECT_TRUE(std::isnan(zone_map_infos[0].max_value.get())); + EXPECT_EQ(false, zone_map_infos[0].has_null); + EXPECT_EQ(true, zone_map_infos[0].has_not_null); + EXPECT_EQ(true, zone_map_infos[0].has_positive_inf); + EXPECT_EQ(true, zone_map_infos[0].has_negative_inf); + EXPECT_EQ(true, zone_map_infos[0].has_nan); + + EXPECT_EQ(-1234.56789012345, zone_map_infos[1].min_value.get()); + EXPECT_EQ(1234.56789012345, zone_map_infos[1].max_value.get()); + EXPECT_EQ(true, zone_map_infos[1].has_null); + EXPECT_EQ(true, zone_map_infos[1].has_not_null); + + EXPECT_EQ(true, zone_map_infos[2].has_null); + EXPECT_EQ(false, zone_map_infos[2].has_not_null); + const std::vector& zone_maps = column_zone_map.page_zone_maps(); EXPECT_EQ(3, zone_maps.size()); - - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), zone_maps[0].min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), zone_maps[0].max()); EXPECT_EQ(false, zone_maps[0].has_null()); EXPECT_EQ(true, zone_maps[0].has_not_null()); EXPECT_EQ(true, zone_maps[0].has_positive_inf()); @@ -993,17 +1241,17 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); UInt32 scale = 6; - auto segment_zone_map = index_meta.zone_map_index().segment_zone_map(); - EXPECT_EQ(CastToString::from_timestamptz(type_limit::min(), scale), - segment_zone_map.min()); + auto segment_zone_map = load_segment_zone_map(index_meta, data_type_ptr); + EXPECT_EQ(type_limit::min(), + segment_zone_map.min_value.get()); std::string max_str = "9999-12-31 23:59:59"; TimestampTzValue tz_max {}; EXPECT_TRUE(tz_max.from_string(StringRef {max_str}, &time_zone, params, 0)); - EXPECT_EQ(CastToString::from_timestamptz(tz_max, scale), segment_zone_map.max()); - EXPECT_EQ(true, segment_zone_map.has_null()); - EXPECT_EQ(true, segment_zone_map.has_not_null()); + EXPECT_EQ(tz_max, segment_zone_map.max_value.get()); + EXPECT_EQ(true, segment_zone_map.has_null); + EXPECT_EQ(true, segment_zone_map.has_not_null); - ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index().page_zone_maps()); + ZoneMapIndexReader column_zone_map(file_reader, index_meta.zone_map_index(), data_type_ptr); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); EXPECT_EQ(5, column_zone_map.num_pages()); @@ -1096,11 +1344,11 @@ TEST_F(ColumnZoneMapTest, AllNullPageAfterIntValues_SegmentMinMaxPreserved) { ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); - EXPECT_TRUE(seg_zm.has_null()); - EXPECT_TRUE(seg_zm.has_not_null()); - EXPECT_EQ(std::to_string(100), seg_zm.min()); - EXPECT_EQ(std::to_string(200), seg_zm.max()); + ZoneMap seg_zm = load_segment_zone_map(index_meta, data_type_ptr); + EXPECT_TRUE(seg_zm.has_null); + EXPECT_TRUE(seg_zm.has_not_null); + EXPECT_EQ(100, seg_zm.min_value.get()); + EXPECT_EQ(200, seg_zm.max_value.get()); } // Demonstrates the actual bug fixed by the `if (_page_has_minmax)` guard. @@ -1147,9 +1395,10 @@ TEST_F(ColumnZoneMapTest, AllNullPageAfterMaxLenStringPage_NoSegmentMaxDoubleInc ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); ASSERT_TRUE(file_writer->close().ok()); - const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); - EXPECT_TRUE(seg_zm.has_null()); - EXPECT_TRUE(seg_zm.has_not_null()); + const auto& seg_zm = index_meta.zone_map_index().segment_zone_map_v2(); + ZoneMap decoded_seg_zm = load_segment_zone_map(index_meta, data_type); + EXPECT_TRUE(decoded_seg_zm.has_null); + EXPECT_TRUE(decoded_seg_zm.has_not_null); ASSERT_EQ(seg_zm.min().size(), MAX_ZONE_MAP_INDEX_SIZE); EXPECT_EQ(seg_zm.min(), long_x); @@ -1166,5 +1415,181 @@ TEST_F(ColumnZoneMapTest, AllNullPageAfterMaxLenStringPage_NoSegmentMaxDoubleInc EXPECT_EQ(static_cast(seg_zm.max().back()), static_cast('y')); } +TEST_F(ColumnZoneMapTest, NativePagePayloadRoundTrip) { + ZoneMapPagesPB pages = one_present_int_page(10, 20); + ZoneMapIndexPB zone_map_index; + io::FileReaderSPtr file_reader; + ASSERT_TRUE(write_native_pages_payload(kTestDir + "/native_page_payload_round_trip", pages, 1, + &zone_map_index, &file_reader) + .ok()); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, false); + ZoneMapIndexReader reader(file_reader, zone_map_index, data_type); + ASSERT_TRUE(reader.load(true, false).ok()); + + EXPECT_EQ(1, reader.num_pages()); + ASSERT_EQ(1, reader.page_zone_map_infos().size()); + const ZoneMap& zone_map = reader.page_zone_map_infos()[0]; + EXPECT_FALSE(zone_map.has_null); + EXPECT_TRUE(zone_map.has_not_null); + EXPECT_EQ(10, zone_map.min_value.get()); + EXPECT_EQ(20, zone_map.max_value.get()); + + const std::vector& legacy_view = reader.page_zone_maps(); + ASSERT_EQ(1, legacy_view.size()); + EXPECT_EQ("10", legacy_view[0].min()); + EXPECT_EQ("20", legacy_view[0].max()); +} + +TEST_F(ColumnZoneMapTest, NativePagePayloadRejectsPageCountMismatch) { + ZoneMapPagesPB pages = one_present_int_page(10, 20); + ZoneMapIndexPB zone_map_index; + io::FileReaderSPtr file_reader; + ASSERT_TRUE(write_native_pages_payload(kTestDir + "/native_page_count_mismatch", pages, 2, + &zone_map_index, &file_reader) + .ok()); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, false); + ZoneMapIndexReader reader(file_reader, zone_map_index, data_type); + Status status = reader.load(true, false); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("page count mismatch")); +} + +TEST_F(ColumnZoneMapTest, NativePagePayloadRejectsShortBitmap) { + ZoneMapPagesPB pages; + pages.set_num_pages(9); + pages.set_entry_version(1); + pages.set_has_null_bitmap(one_byte_bitmap(0)); + pages.set_has_not_null_bitmap(one_byte_bitmap(0)); + pages.set_pass_all_bitmap(one_byte_bitmap(0)); + pages.set_has_positive_inf_bitmap(one_byte_bitmap(0)); + pages.set_has_negative_inf_bitmap(one_byte_bitmap(0)); + pages.set_has_nan_bitmap(one_byte_bitmap(0)); + init_fixed_int_values(pages.mutable_min_values()); + init_fixed_int_values(pages.mutable_max_values()); + + ZoneMapIndexPB zone_map_index; + io::FileReaderSPtr file_reader; + ASSERT_TRUE(write_native_pages_payload(kTestDir + "/native_short_bitmap", pages, 9, + &zone_map_index, &file_reader) + .ok()); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, false); + ZoneMapIndexReader reader(file_reader, zone_map_index, data_type); + Status status = reader.load(true, false); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("bitmap too small")); +} + +TEST_F(ColumnZoneMapTest, NativePagePayloadRejectsInvalidFixedValues) { + ZoneMapPagesPB pages = one_present_int_page(10, 20); + pages.mutable_min_values()->set_fixed_values(std::string(sizeof(int16_t), '\0')); + + ZoneMapIndexPB zone_map_index; + io::FileReaderSPtr file_reader; + ASSERT_TRUE(write_native_pages_payload(kTestDir + "/native_invalid_fixed_values", pages, 1, + &zone_map_index, &file_reader) + .ok()); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, false); + ZoneMapIndexReader reader(file_reader, zone_map_index, data_type); + Status status = reader.load(true, false); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("fixed values size")); +} + +TEST_F(ColumnZoneMapTest, NativeStringPagePayloadRejectsInvalidOffsets) { + ZoneMapPagesPB pages; + pages.set_num_pages(1); + pages.set_entry_version(1); + pages.set_has_null_bitmap(one_byte_bitmap(0)); + pages.set_has_not_null_bitmap(one_byte_bitmap(1)); + pages.set_pass_all_bitmap(one_byte_bitmap(0)); + pages.set_has_positive_inf_bitmap(one_byte_bitmap(0)); + pages.set_has_negative_inf_bitmap(one_byte_bitmap(0)); + pages.set_has_nan_bitmap(one_byte_bitmap(0)); + + pages.mutable_min_values()->set_layout(ZoneMapValueArrayPB::VAR_BINARY); + pages.mutable_min_values()->set_offset_width(4); + put_fixed32_le(pages.mutable_min_values()->mutable_offsets(), 1); + put_fixed32_le(pages.mutable_min_values()->mutable_offsets(), 1); + + pages.mutable_max_values()->set_layout(ZoneMapValueArrayPB::VAR_BINARY); + pages.mutable_max_values()->set_offset_width(4); + put_fixed32_le(pages.mutable_max_values()->mutable_offsets(), 0); + put_fixed32_le(pages.mutable_max_values()->mutable_offsets(), 1); + pages.mutable_max_values()->set_values("z"); + + ZoneMapIndexPB zone_map_index; + io::FileReaderSPtr file_reader; + ASSERT_TRUE(write_native_pages_payload(kTestDir + "/native_invalid_string_offsets", pages, 1, + &zone_map_index, &file_reader) + .ok()); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false); + ZoneMapIndexReader reader(file_reader, zone_map_index, data_type); + Status status = reader.load(true, false); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("first offset must be zero")); +} + +TEST_F(ColumnZoneMapTest, NativeSegmentEntryRejectsMissingPresentValue) { + ZoneMapEntryPB entry; + entry.set_has_not_null(true); + entry.set_has_null(false); + entry.set_pass_all(false); + put_fixed32_le(entry.mutable_max(), 20); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, false); + ZoneMap zone_map; + Status status = ZoneMap::from_entry_pb_native(entry, data_type, zone_map); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("min is missing")); +} + +TEST_F(ColumnZoneMapTest, LegacyPageZoneMapsFallbackPopulatesDecodedInfos) { + std::string filename = kTestDir + "/legacy_page_zone_maps_fallback"; + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(filename, &file_writer).ok()); + + IndexedColumnWriterOptions options; + options.write_ordinal_index = true; + options.write_value_index = false; + options.encoding = PLAIN_ENCODING; + IndexedColumnWriter legacy_writer(options, FieldType::OLAP_FIELD_TYPE_VARCHAR, + file_writer.get()); + ASSERT_TRUE(legacy_writer.init().ok()); + + ZoneMapPB legacy_page; + legacy_page.set_min("10"); + legacy_page.set_max("20"); + legacy_page.set_has_null(false); + legacy_page.set_has_not_null(true); + std::string serialized_page; + ASSERT_TRUE(legacy_page.SerializeToString(&serialized_page)); + Slice serialized_page_slice(serialized_page); + ASSERT_TRUE(legacy_writer.add(&serialized_page_slice).ok()); + + ZoneMapIndexPB zone_map_index; + ASSERT_TRUE(legacy_writer.finish(zone_map_index.mutable_page_zone_maps()).ok()); + ASSERT_TRUE(file_writer->close().ok()); + + io::FileReaderSPtr file_reader; + ASSERT_TRUE(_fs->open_file(filename, &file_reader).ok()); + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_INT, false); + ZoneMapIndexReader reader(file_reader, zone_map_index, data_type); + ASSERT_TRUE(reader.load(true, false).ok()); + + EXPECT_EQ(1, reader.num_pages()); + ASSERT_EQ(1, reader.page_zone_maps().size()); + EXPECT_EQ("10", reader.page_zone_maps()[0].min()); + EXPECT_EQ("20", reader.page_zone_maps()[0].max()); + ASSERT_EQ(1, reader.page_zone_map_infos().size()); + EXPECT_EQ(10, reader.page_zone_map_infos()[0].min_value.get()); + EXPECT_EQ(20, reader.page_zone_map_infos()[0].max_value.get()); +} + } // namespace segment_v2 } // namespace doris diff --git a/gensrc/proto/segment_v2.proto b/gensrc/proto/segment_v2.proto index 6fb6fa5fe037a5..abde02d2c340fa 100644 --- a/gensrc/proto/segment_v2.proto +++ b/gensrc/proto/segment_v2.proto @@ -145,6 +145,61 @@ message ZoneMapPB { optional bool has_nan = 8; } +message ZoneMapEntryPB { + // Native binary minimum not-null value, interpreted by column type. + optional bytes min = 1; + // Native binary maximum not-null value, interpreted by column type. + optional bytes max = 2; + + optional bool has_null = 3; + optional bool has_not_null = 4; + optional bool pass_all = 5 [default = false]; + optional bool has_positive_inf = 6; + optional bool has_negative_inf = 7; + optional bool has_nan = 8; +} + +message ZoneMapValueArrayPB { + enum Layout { + UNKNOWN = 0; + FIXED_WIDTH = 1; + VAR_BINARY = 2; + } + + optional Layout layout = 1; + + // FIXED_WIDTH stores values consecutively for pages whose value is present. + optional uint32 fixed_width = 2; + optional bytes fixed_values = 3; + + // VAR_BINARY stores offset_width-byte little-endian offsets followed by concatenated bytes. + optional uint32 offset_width = 4 [default = 4]; + optional bytes offsets = 5; + optional bytes values = 6; +} + +message ZoneMapPagesPB { + optional uint32 num_pages = 1; + + optional bytes has_null_bitmap = 2; + optional bytes has_not_null_bitmap = 3; + optional bytes pass_all_bitmap = 4; + optional bytes has_positive_inf_bitmap = 5; + optional bytes has_negative_inf_bitmap = 6; + optional bytes has_nan_bitmap = 7; + + optional ZoneMapValueArrayPB min_values = 8; + optional ZoneMapValueArrayPB max_values = 9; + + optional uint32 entry_version = 10 [default = 1]; +} + +message ZoneMapPageIndexPB { + optional PagePointerPB data = 1; + optional uint32 num_pages = 2; + optional uint32 format_version = 3 [default = 1]; +} + message ColumnPathPartInfo { // key represent a part of key in full parts info optional string key = 1; @@ -353,6 +408,10 @@ message ZoneMapIndexPB { optional ZoneMapPB segment_zone_map = 1; // required: zone map for each data page is stored in an IndexedColumn with ordinal index optional IndexedColumnMetaPB page_zone_maps = 2; + // Native binary segment-level zone map. + optional ZoneMapEntryPB segment_zone_map_v2 = 3; + // Native array-oriented page-level zone maps. + optional ZoneMapPageIndexPB page_zone_maps_v2 = 4; } message BitmapIndexPB { diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_struct_map_lazy.out b/regression-test/data/datatype_p0/complex_types/test_pruned_struct_map_lazy.out new file mode 100644 index 00000000000000..16e7eea72a028d --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_pruned_struct_map_lazy.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !map_values_lazy_pruned -- +2 ["val3"] + +-- !map_keys_lazy_pruned -- +2 [3] diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_struct_map_lazy.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_struct_map_lazy.groovy new file mode 100644 index 00000000000000..80c757034865fc --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_struct_map_lazy.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pruned_struct_map_lazy") { + sql "set enable_prune_nested_column = true" + sql "DROP TABLE IF EXISTS struct_pruned_map_lazy_tbl" + sql """ + CREATE TABLE struct_pruned_map_lazy_tbl ( + id INT, + heavy_struct STRUCT< + big_string:VARCHAR(20), + small_int:INT, + nested_map:MAP + > + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO struct_pruned_map_lazy_tbl VALUES + (1, named_struct('big_string', 'Alpha', 'small_int', 100, + 'nested_map', map(1, 'val1', 2, 'val2'))), + (2, named_struct('big_string', 'Beta', 'small_int', 200, + 'nested_map', map(3, 'val3'))), + (3, named_struct('big_string', 'Gamma', 'small_int', 300, + 'nested_map', map(4, 'val4'))) + """ + + order_qt_map_values_lazy_pruned """ + SELECT id, array_sort(map_values(struct_element(heavy_struct, 'nested_map'))) + FROM struct_pruned_map_lazy_tbl + WHERE struct_element(heavy_struct, 'big_string') LIKE 'B%' + ORDER BY id + """ + + order_qt_map_keys_lazy_pruned """ + SELECT id, array_sort(map_keys(struct_element(heavy_struct, 'nested_map'))) + FROM struct_pruned_map_lazy_tbl + WHERE struct_element(heavy_struct, 'small_int') = 200 + ORDER BY id + """ +}