diff --git a/be/src/core/block/block.cpp b/be/src/core/block/block.cpp index 2059466f9fa3a1..4fc97d5a6631f9 100644 --- a/be/src/core/block/block.cpp +++ b/be/src/core/block/block.cpp @@ -308,7 +308,7 @@ void Block::check_number_of_rows(bool allow_null_columns) const { } } -Status Block::check_type_and_column() const { +Status Block::check_type_and_column() { #ifndef NDEBUG for (const auto& elem : data) { if (!elem.column) { @@ -336,6 +336,14 @@ Status Block::check_type_and_column() const { elem.name, column->get_name(), type->get_name(), st.msg()); } } + + for (size_t i = 0; i < data.size(); ++i) { + if (!data[i].column || !data[i].column->is_nullable()) { + continue; + } + auto column_guard = mutate_column_scoped(i); + column_guard.mutable_column()->inject_debug_nullable_payload(); + } #endif return Status::OK(); } diff --git a/be/src/core/block/block.h b/be/src/core/block/block.h index b45d4bd7d18a45..63e601d8820760 100644 --- a/be/src/core/block/block.h +++ b/be/src/core/block/block.h @@ -181,7 +181,7 @@ class Block { /// Checks that every column in block is not nullptr and has same number of elements. void check_number_of_rows(bool allow_null_columns = false) const; - Status check_type_and_column() const; + Status check_type_and_column(); Status check_column_and_type_not_null() const; diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index bde5b4786c161d..e7fb2492fe335b 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -735,6 +735,13 @@ class IColumn : public COW { Status column_self_check() const; + /** Replace physical values selected by null_map with conspicuous debug-only values. + * A top-level call passes nullptr. Wrapper columns forward the applicable null map to their + * nested value column. The caller must own a recursively mutable column tree. Implementations + * must not change column sizes or structural metadata. + */ + virtual void inject_debug_nullable_payload(const uint8_t* null_map = nullptr) {} + // only used in agg value replace for column which is not variable length, eg.BlockReader::_copy_value_data // usage: self_column.replace_column_data(other_column, other_column's row index, self_column's row index) virtual void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) = 0; diff --git a/be/src/core/column/column_const.h b/be/src/core/column/column_const.h index 44b1f9d477a966..07a1145b7c8595 100644 --- a/be/src/core/column/column_const.h +++ b/be/src/core/column/column_const.h @@ -281,6 +281,11 @@ class ColumnConst final : public COWHelper { bool only_null() const override { return data->is_null_at(0); } StringRef get_raw_data() const override { return data->get_raw_data(); } + void inject_debug_nullable_payload(const uint8_t* null_map = nullptr) override { + auto mutable_data = data->assert_mutable(); + mutable_data->inject_debug_nullable_payload(null_map); + } + /// Not part of the common interface. const IColumn& get_data_column() const { return *data; } const ColumnPtr& get_data_column_ptr() const { return data; } diff --git a/be/src/core/column/column_decimal.h b/be/src/core/column/column_decimal.h index 8cd7be991a783f..8e2417f8c030dd 100644 --- a/be/src/core/column/column_decimal.h +++ b/be/src/core/column/column_decimal.h @@ -148,6 +148,17 @@ class ColumnDecimal final : public COWHelper> { memset(data.data() + old_size, 0, length * sizeof(data[0])); } + void inject_debug_nullable_payload(const uint8_t* null_map = nullptr) override { + if (null_map == nullptr) { + return; + } + for (size_t i = 0; i < data.size(); ++i) { + if (null_map[i] != 0) { + memset(&data[i], i % 2 == 0 ? 0x80 : 0x7F, sizeof(value_type)); + } + } + } + Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) const override { Self* output = assert_cast(col_ptr); diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index bc4bc9358a1302..84c08e92813e32 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -275,6 +275,11 @@ class ColumnNullable final : public COWHelper { bool is_nullable() const override { return true; } bool is_column_string() const override { return get_nested_column().is_column_string(); } + void inject_debug_nullable_payload(const uint8_t* null_map = nullptr) override { + DCHECK(null_map == nullptr); + get_nested_column().inject_debug_nullable_payload(get_null_map_data().data()); + } + bool is_exclusive() const override { return IColumn::is_exclusive() && _nested_column->is_exclusive() && get_null_map_column().is_exclusive(); diff --git a/be/src/core/column/column_vector.h b/be/src/core/column/column_vector.h index 05be617e4373e9..ff682fa15ad815 100644 --- a/be/src/core/column/column_vector.h +++ b/be/src/core/column/column_vector.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +187,37 @@ class ColumnVector final : public COWHelper> { std::fill(data.data() + old_size, data.data() + old_size + length, default_value()); } + void inject_debug_nullable_payload(const uint8_t* null_map = nullptr) override { + if (null_map == nullptr) { + return; + } + for (size_t i = 0; i < data.size(); ++i) { + if (null_map[i] == 0) { + continue; + } + if constexpr (T == TYPE_BOOLEAN) { + data[i] = static_cast(0xA5); + } else if constexpr (is_int(T)) { + data[i] = i % 2 == 0 ? std::numeric_limits::lowest() + : std::numeric_limits::max(); + } else if constexpr (is_float_or_double(T) || T == TYPE_TIMEV2) { + switch (i % 3) { + case 0: + data[i] = std::numeric_limits::quiet_NaN(); + break; + case 1: + data[i] = std::numeric_limits::infinity(); + break; + default: + data[i] = -std::numeric_limits::infinity(); + break; + } + } else { + memset(&data[i], 0xFF, sizeof(value_type)); + } + } + } + void pop_back(size_t n) override { data.resize_assume_reserved(data.size() - n); } StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override; diff --git a/be/test/core/data_type/block_check_type.cpp b/be/test/core/data_type/block_check_type.cpp index af3afbe9ae702c..260f0842c70891 100644 --- a/be/test/core/data_type/block_check_type.cpp +++ b/be/test/core/data_type/block_check_type.cpp @@ -18,7 +18,11 @@ #include +#include +#include + #include "core/block/block.h" +#include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" @@ -79,4 +83,63 @@ TEST(BlockCheckType, CheckNoColumnString64) { st = block.check_no_column_string64(); EXPECT_FALSE(st.ok()); } + +#ifndef NDEBUG +TEST(BlockCheckType, InjectDebugNullablePayload) { + auto original = ColumnHelper::create_nullable_column({1, 2, 3, 4}, {1, 1, 0, 0}); + Block block {{original, make_nullable(std::make_shared()), "nullable_int"}}; + + ASSERT_TRUE(block.check_type_and_column().ok()); + + const auto& nullable = assert_cast(*block.get_by_position(0).column); + const auto& values = assert_cast(nullable.get_nested_column()).get_data(); + EXPECT_EQ(values[0], std::numeric_limits::lowest()); + EXPECT_EQ(values[1], std::numeric_limits::max()); + EXPECT_EQ(values[2], 3); + EXPECT_EQ(values[3], 4); + + const auto& original_nullable = assert_cast(*original); + const auto& original_values = + assert_cast(original_nullable.get_nested_column()).get_data(); + EXPECT_EQ(original_values[0], 1); + EXPECT_EQ(original_values[1], 2); +} + +TEST(BlockCheckType, InjectDebugNullableBooleanAndFloatPayload) { + Block block {{ColumnHelper::create_nullable_column({0, 1}, {1, 0}), + make_nullable(std::make_shared()), "nullable_bool"}, + {ColumnHelper::create_nullable_column({0.0, 1.0}, {1, 0}), + make_nullable(std::make_shared()), "nullable_double"}}; + + ASSERT_TRUE(block.check_type_and_column().ok()); + + const auto& nullable_bool = + assert_cast(*block.get_by_position(0).column); + const auto& bool_values = + assert_cast(nullable_bool.get_nested_column()).get_data(); + EXPECT_EQ(bool_values[0], 0xA5); + EXPECT_EQ(bool_values[1], 1); + + const auto& nullable_double = + assert_cast(*block.get_by_position(1).column); + const auto& double_values = + assert_cast(nullable_double.get_nested_column()).get_data(); + EXPECT_TRUE(std::isnan(double_values[0])); + EXPECT_EQ(double_values[1], 1.0); +} + +TEST(BlockCheckType, InjectDebugNullablePayloadThroughConstColumn) { + auto nullable = ColumnHelper::create_nullable_column({0}, {1}); + Block block {{ColumnConst::create(std::move(nullable), 3), + make_nullable(std::make_shared()), "const_null"}}; + + ASSERT_TRUE(block.check_type_and_column().ok()); + + const auto& column_const = assert_cast(*block.get_by_position(0).column); + const auto& nullable_data = assert_cast(column_const.get_data_column()); + const auto& values = + assert_cast(nullable_data.get_nested_column()).get_data(); + EXPECT_EQ(values[0], std::numeric_limits::lowest()); +} +#endif } // namespace doris