From cf9a13f2620eccc0bec4b553ae39acdced307d7e Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Thu, 17 Sep 2026 17:30:08 +0800 Subject: [PATCH 1/6] [fix](inverted index) Skip BM25 norms for variant subcolumn indexes ### What problem does this PR solve? Issue Number: None Related PR: #68039 Problem Summary: Analyzed inverted indexes inherited by sparse VARIANT subcolumns write dense norms. Add the variant norms switch and skip norms for those indexes by default while retaining norms for ordinary analyzed indexes and an opt-in compatibility setting. ### Release note Variant subcolumn inverted indexes no longer write BM25 norms by default. ### Check List (For Author) - Test: Source-level verification only in this commit; focused build/test follows. - Regression test / Unit Test / Manual test / No need to test (with reason) - Behavior changed: Yes (variant subcolumn norms default to omitted) - Does this need documentation: No --- be/src/common/config.cpp | 4 + be/src/common/config.h | 2 + .../index/inverted/inverted_index_writer.cpp | 10 +- .../inverted/similarity/bm25_similarity.cpp | 8 ++ .../similarity/bm25_similarity_test.cpp | 19 +++ .../segment/inverted_index_writer_test.cpp | 42 ++++++- .../test_variant_subcolumn_index_norms.out | 9 ++ .../test_variant_subcolumn_index_norms.groovy | 109 ++++++++++++++++++ 8 files changed, 199 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out create mode 100644 regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5c2e0ebf8d63f1..dfd3c0ae78d316 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1355,6 +1355,10 @@ DEFINE_mDouble(inverted_index_ram_buffer_size, "512"); // -1 indicates not working. // Normally we should not change this, it's useful for testing. DEFINE_mInt32(inverted_index_max_buffered_docs, "-1"); +// Norms of a variant subcolumn index are dense even when the path is sparse, so a segment with +// thousands of indexed paths pays rows * paths bytes. Off by default; enable it only when BM25 on +// variant subcolumns needs document-length normalization. +DEFINE_mBool(inverted_index_write_norms_for_variant_subcolumn, "false"); // dict path for chinese analyzer DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict"); DEFINE_Int32(inverted_index_read_buffer_size, "4096"); diff --git a/be/src/common/config.h b/be/src/common/config.h index fca3d1c6534cf5..8f5d504e2604b0 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1375,6 +1375,8 @@ DECLARE_Int16(condition_cache_limit); // inverted index DECLARE_mDouble(inverted_index_ram_buffer_size); DECLARE_mInt32(inverted_index_max_buffered_docs); +// Whether analyzed inverted indexes on variant subcolumns write BM25 norms (one byte per row). +DECLARE_mBool(inverted_index_write_norms_for_variant_subcolumn); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); DECLARE_Int32(inverted_index_read_buffer_size); diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 8e4730cc063a73..339c6c02246dbd 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -162,7 +162,13 @@ Status InvertedIndexColumnWriter::create_field(lucene::document::Fie (*field)->setOmitTermFreqAndPositions( !(get_parser_phrase_support_string_from_properties(_index_meta->properties()) == INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES)); - (*field)->setOmitNorms(false); + // Norms cost one byte per segment row, including rows without a value. Every variant + // subcolumn (non-empty index suffix) gets its own index, so a segment may hold thousands of + // them and their norms can dwarf the data. BM25 on such an index scores without length norms. + if (_should_analyzer && (_index_meta->get_index_suffix().empty() || + config::inverted_index_write_norms_for_variant_subcolumn)) { + (*field)->setOmitNorms(false); + } DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_field_v3", { if (_index_file_writer->get_storage_format() != InvertedIndexStorageFormatPB::V3) { return Status::Error( @@ -677,4 +683,4 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/inverted/similarity/bm25_similarity.cpp b/be/src/storage/index/inverted/similarity/bm25_similarity.cpp index d3e1ba5ac6323a..daf6bf5d3c3397 100644 --- a/be/src/storage/index/inverted/similarity/bm25_similarity.cpp +++ b/be/src/storage/index/inverted/similarity/bm25_similarity.cpp @@ -17,6 +17,7 @@ #include "storage/index/inverted/similarity/bm25_similarity.h" +#include #include namespace doris::segment_v2 { @@ -44,6 +45,13 @@ BM25Similarity::BM25Similarity(float idf, float avgdl) : _idf(idf), _avgdl(avgdl } void BM25Similarity::compute_tf_cache() { + // CLucene keeps a field's token count in its .nrm header, so avgdl is 0 when no segment of the + // field stores norms (e.g. variant subcolumn indexes). Every document then has an unknown length: + // score without length normalization instead of computing 0 / 0. + if (_avgdl <= 0.0F) { + std::fill(_cache.begin(), _cache.end(), 1.0F / _k1); + return; + } for (int i = 0; i < _cache.size(); i++) { _cache[i] = 1.0F / (_k1 * ((1 - _b) + _b * LENGTH_TABLE[i] / _avgdl)); } diff --git a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp index 24fdfbd2e62709..c6b1ddb2eec274 100644 --- a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp +++ b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp @@ -19,6 +19,7 @@ #include +#include #include #include "common/be_mock_util.h" @@ -289,3 +290,21 @@ TEST_F(BM25SimilarityTest, CacheConsistencyTest) { ASSERT_FLOAT_EQ(similarity_->_cache[i], expected); } } + +// Indexes without norms report no token count, so avgdl is 0: scores must stay finite and ignore +// document length instead of turning into NaN. +TEST_F(BM25SimilarityTest, ZeroAvgDlScoresWithoutLengthNorm) { + mock_stats_->set_mock_idf(2.0f); + mock_stats_->set_mock_avg_dl(0.0f); + + similarity_->for_one_term(context_, L"field", L"term"); + + for (int i = 0; i < 256; ++i) { + ASSERT_FLOAT_EQ(similarity_->_cache[i], 1.0f / similarity_->_k1); + } + float score = similarity_->score(1.0f, 0); + ASSERT_FALSE(std::isnan(score)); + ASSERT_FLOAT_EQ(score, + similarity_->_weight - similarity_->_weight / (1.0f + 1.0f / similarity_->_k1)); + ASSERT_GT(similarity_->score(2.0f, 0), score); +} diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index c5bdf4c5547391..6b225fe6bf1a3f 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -362,7 +362,8 @@ class InvertedIndexWriterTest : public testing::Test { } // Helper method to create an inverted index with tokenization enabled - void create_tokenized_index(std::string_view rowset_id, int seg_id, bool enable_analyzer) { + void create_tokenized_index(std::string_view rowset_id, int seg_id, bool enable_analyzer, + const std::string& index_suffix = "") { auto tablet_schema = create_schema(); // Create index meta with tokenization setting @@ -386,6 +387,9 @@ class InvertedIndexWriterTest : public testing::Test { TabletIndex idx_meta; idx_meta.init_from_pb(*index_meta_pb.get()); + if (!index_suffix.empty()) { + idx_meta.set_escaped_escaped_index_suffix_path(index_suffix); + } std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -1478,4 +1482,38 @@ TEST_F(InvertedIndexWriterTest, FileCreationAndOutputErrorHandling) { // but it should not crash } -} // namespace doris::segment_v2 \ No newline at end of file +// A variant subcolumn index carries a non-empty index suffix. Its norms take one byte per segment +// row even when the path is sparse, so they are written only when +// inverted_index_write_norms_for_variant_subcolumn is enabled. +TEST_F(InvertedIndexWriterTest, NormsFileSkippedForVariantSubcolumn) { + const bool original_config_value = config::inverted_index_write_norms_for_variant_subcolumn; + Defer restore_config {[&]() { + config::inverted_index_write_norms_for_variant_subcolumn = original_config_value; + }}; + + TabletIndexPB index_meta_pb; + index_meta_pb.set_index_type(IndexType::INVERTED); + index_meta_pb.set_index_id(1); + index_meta_pb.set_index_name("test"); + index_meta_pb.add_col_unique_id(1); // c2 column id + (*index_meta_pb.mutable_properties())["parser"] = "standard"; + TabletIndex subcolumn_index_meta; + subcolumn_index_meta.init_from_pb(index_meta_pb); + subcolumn_index_meta.set_escaped_escaped_index_suffix_path("v.s_host"); + + config::inverted_index_write_norms_for_variant_subcolumn = false; + create_tokenized_index("test_variant_subcolumn_without_norms", 0, true, "v.s_host"); + std::string prefix_without_norms {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, "test_variant_subcolumn_without_norms", 0))}; + EXPECT_FALSE(check_norms_file_exists(prefix_without_norms, &subcolumn_index_meta)) + << "a tokenized variant subcolumn index must not write .nrm by default"; + + config::inverted_index_write_norms_for_variant_subcolumn = true; + create_tokenized_index("test_variant_subcolumn_with_norms", 1, true, "v.s_host"); + std::string prefix_with_norms {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, "test_variant_subcolumn_with_norms", 1))}; + EXPECT_TRUE(check_norms_file_exists(prefix_with_norms, &subcolumn_index_meta)) + << "inverted_index_write_norms_for_variant_subcolumn=true must restore .nrm"; +} + +} // namespace doris::segment_v2 diff --git a/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out b/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out new file mode 100644 index 00000000000000..2988072d642e32 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_subcolumn_score -- +2 0.6931 +3 0.9531 + +-- !plain_column_score -- +1 0.5754 +3 0.8714 + diff --git a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy new file mode 100644 index 00000000000000..61783c2e5ac215 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Analyzed indexes on variant subcolumns must not write dense BM25 norms (.nrm, one byte per row), +// while analyzed indexes on ordinary columns still do. BM25 scoring keeps working on both. +suite("test_variant_subcolumn_index_norms", "p0") { + if (isCloudMode()) { + return + } + + sql """ set enable_segment_limit_pushdown = true """ + sql """ set enable_match_without_inverted_index = false """ + sql """ set default_variant_enable_typed_paths_to_sparse = false """ + sql """ set default_variant_enable_doc_mode = false """ + + sql "DROP TABLE IF EXISTS test_variant_subcolumn_index_norms" + sql """ + CREATE TABLE test_variant_subcolumn_index_norms ( + id INT, + content TEXT, + v variant< + 's_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + >, + INDEX idx_content (content) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true" + ), + INDEX idx_v_s (v) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "field_pattern"="s_*" + ) + ) ENGINE=OLAP DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ insert into test_variant_subcolumn_index_norms values + (1, 'alpha database server', parse_to_variant('{"s_host":"alpha database server"}')), + (2, 'beta server cluster', parse_to_variant('{"s_host":"beta server cluster", "s_note":"alpha"}')), + (3, 'alpha', parse_to_variant('{"s_note":"alpha alpha beta"}')), + (4, 'gamma', parse_to_variant('{"other":"alpha"}')) + """ + sql " sync " + + // scores are printed so that a NaN (which compares greater than 0) cannot slip through + order_qt_variant_subcolumn_score """ + select id, round(score(), 4) + from test_variant_subcolumn_index_norms + where cast(v["s_note"] as string) match_phrase "alpha" + order by score() desc + limit 10 + """ + order_qt_plain_column_score """ + select id, round(score(), 4) + from test_variant_subcolumn_index_norms + where content match_phrase "alpha" + order by score() desc + limit 10 + """ + + def backendIdToIp = [:] + def backendIdToHttpPort = [:] + getBackendIpHttpPort(backendIdToIp, backendIdToHttpPort) + def tablet = sql_return_maparray("show tablets from test_variant_subcolumn_index_norms")[0] + def (code, out, err) = http_client("GET", String.format( + "http://%s:%s/api/show_nested_index_file?tablet_id=%s", + backendIdToIp.get(tablet.BackendId), backendIdToHttpPort.get(tablet.BackendId), + tablet.TabletId)) + logger.info("show_nested_index_file code=${code}, out=${out}, err=${err}") + assertEquals(0, code) + + def subcolumnIndexes = [] + def plainIndexes = [] + for (def rowset in parseJson(out.trim()).rowsets) { + for (def segment in rowset.segments) { + for (def index in segment.indices) { + def hasNorms = index.files.any { file -> file.name.endsWith(".nrm") } + if (index.index_suffix.isEmpty()) { + plainIndexes.add(hasNorms) + } else { + subcolumnIndexes.add(hasNorms) + } + } + } + } + logger.info("norms of plain indexes: ${plainIndexes}, subcolumn indexes: ${subcolumnIndexes}") + // idx_content on the single segment writes norms; idx_v_s on s_host and s_note does not + assertEquals([true], plainIndexes) + assertEquals([false, false], subcolumnIndexes) +} From 06c268cc14c0b7907d34605875d66dc65d354231 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Thu, 17 Sep 2026 17:36:45 +0800 Subject: [PATCH 2/6] [fix](inverted index) Make BM25 norms a per-index property ### What problem does this PR solve? Issue Number: None Related PR: #68039 Problem Summary: Make norms an inverted-index property. Variant path indexes omit norms by default, while the norms property can restore them per index; ordinary column indexes keep norms by default. ### Release note Inverted indexes accept the norms property. ### Check List (For Author) - Test: Pending focused build and test verification - Behavior changed: Yes (norms are controlled per index) - Does this need documentation: Yes --- be/src/common/config.cpp | 4 - be/src/common/config.h | 2 - .../index/inverted/inverted_index_parser.cpp | 8 ++ .../index/inverted/inverted_index_parser.h | 9 ++ .../index/inverted/inverted_index_writer.cpp | 13 ++- .../segment/inverted_index_writer_test.cpp | 92 ++++++++++++------- .../doris/analysis/InvertedIndexUtil.java | 9 ++ .../test_variant_subcolumn_index_norms.groovy | 37 +++++--- 8 files changed, 115 insertions(+), 59 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index dfd3c0ae78d316..5c2e0ebf8d63f1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1355,10 +1355,6 @@ DEFINE_mDouble(inverted_index_ram_buffer_size, "512"); // -1 indicates not working. // Normally we should not change this, it's useful for testing. DEFINE_mInt32(inverted_index_max_buffered_docs, "-1"); -// Norms of a variant subcolumn index are dense even when the path is sparse, so a segment with -// thousands of indexed paths pays rows * paths bytes. Off by default; enable it only when BM25 on -// variant subcolumns needs document-length normalization. -DEFINE_mBool(inverted_index_write_norms_for_variant_subcolumn, "false"); // dict path for chinese analyzer DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict"); DEFINE_Int32(inverted_index_read_buffer_size, "4096"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 8f5d504e2604b0..fca3d1c6534cf5 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1375,8 +1375,6 @@ DECLARE_Int16(condition_cache_limit); // inverted index DECLARE_mDouble(inverted_index_ram_buffer_size); DECLARE_mInt32(inverted_index_max_buffered_docs); -// Whether analyzed inverted indexes on variant subcolumns write BM25 norms (one byte per row). -DECLARE_mBool(inverted_index_write_norms_for_variant_subcolumn); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); DECLARE_Int32(inverted_index_read_buffer_size); diff --git a/be/src/storage/index/inverted/inverted_index_parser.cpp b/be/src/storage/index/inverted/inverted_index_parser.cpp index 47819cc62f6397..dcc412a2c40b77 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.cpp +++ b/be/src/storage/index/inverted/inverted_index_parser.cpp @@ -104,6 +104,14 @@ std::string get_parser_phrase_support_string_from_properties( return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO; } +bool get_index_norms_from_properties(const std::map& properties, + bool default_value) { + if (auto it = properties.find(INVERTED_INDEX_NORMS_KEY); it != properties.end()) { + return it->second == INVERTED_INDEX_PARSER_TRUE; + } + return default_value; +} + CharFilterMap get_parser_char_filter_map_from_properties( const std::map& properties) { if (!properties.contains(INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE)) { diff --git a/be/src/storage/index/inverted/inverted_index_parser.h b/be/src/storage/index/inverted/inverted_index_parser.h index d2d3df47abd0a3..46d75bcbcb59dd 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.h +++ b/be/src/storage/index/inverted/inverted_index_parser.h @@ -82,6 +82,9 @@ const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY = "support_phrase"; const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES = "true"; const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO = "false"; +// Whether an analyzed index stores BM25 norms, which take one byte per row of the segment. +const std::string INVERTED_INDEX_NORMS_KEY = "norms"; + const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE = "char_filter_type"; const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN = "char_filter_pattern"; const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT = "char_filter_replacement"; @@ -138,6 +141,12 @@ std::string get_parser_mode_string_from_properties( std::string get_parser_phrase_support_string_from_properties( const std::map& properties); +// Whether this index writes BM25 norms. Norms cost one byte per row of the segment, including rows +// that have no value for the field, so callers pass a default of false for indexes on variant paths, +// where one segment holds one index per path. "norms" = "true" / "false" overrides the default. +bool get_index_norms_from_properties(const std::map& properties, + bool default_value); + CharFilterMap get_parser_char_filter_map_from_properties( const std::map& properties); diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 339c6c02246dbd..ebb0c126d31259 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -162,11 +162,14 @@ Status InvertedIndexColumnWriter::create_field(lucene::document::Fie (*field)->setOmitTermFreqAndPositions( !(get_parser_phrase_support_string_from_properties(_index_meta->properties()) == INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES)); - // Norms cost one byte per segment row, including rows without a value. Every variant - // subcolumn (non-empty index suffix) gets its own index, so a segment may hold thousands of - // them and their norms can dwarf the data. BM25 on such an index scores without length norms. - if (_should_analyzer && (_index_meta->get_index_suffix().empty() || - config::inverted_index_write_norms_for_variant_subcolumn)) { + // Norms cost one byte per segment row, including rows without a value. A variant path index + // (a field_pattern index, or the copy inherited by one extracted subcolumn, which carries the + // path as its index suffix) is one of possibly thousands in a segment, so its norms can dwarf + // the data: those default to no norms, and "norms" = "true" brings them back per index. + const bool variant_path_index = + !_index_meta->get_index_suffix().empty() || !_index_meta->field_pattern().empty(); + if (_should_analyzer && + get_index_norms_from_properties(_index_meta->properties(), !variant_path_index)) { (*field)->setOmitNorms(false); } DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_field_v3", { diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 6b225fe6bf1a3f..65d0beb48869e9 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -362,8 +362,10 @@ class InvertedIndexWriterTest : public testing::Test { } // Helper method to create an inverted index with tokenization enabled - void create_tokenized_index(std::string_view rowset_id, int seg_id, bool enable_analyzer, - const std::string& index_suffix = "") { + void create_tokenized_index( + std::string_view rowset_id, int seg_id, bool enable_analyzer, + const std::string& index_suffix = "", + const std::map& extra_properties = {}) { auto tablet_schema = create_schema(); // Create index meta with tokenization setting @@ -384,6 +386,9 @@ class InvertedIndexWriterTest : public testing::Test { // This will make should_analyzer() return true (*properties)["parser"] = "standard"; } + for (const auto& [key, value] : extra_properties) { + (*properties)[key] = value; + } TabletIndex idx_meta; idx_meta.init_from_pb(*index_meta_pb.get()); @@ -1482,38 +1487,57 @@ TEST_F(InvertedIndexWriterTest, FileCreationAndOutputErrorHandling) { // but it should not crash } -// A variant subcolumn index carries a non-empty index suffix. Its norms take one byte per segment -// row even when the path is sparse, so they are written only when -// inverted_index_write_norms_for_variant_subcolumn is enabled. -TEST_F(InvertedIndexWriterTest, NormsFileSkippedForVariantSubcolumn) { - const bool original_config_value = config::inverted_index_write_norms_for_variant_subcolumn; - Defer restore_config {[&]() { - config::inverted_index_write_norms_for_variant_subcolumn = original_config_value; - }}; - - TabletIndexPB index_meta_pb; - index_meta_pb.set_index_type(IndexType::INVERTED); - index_meta_pb.set_index_id(1); - index_meta_pb.set_index_name("test"); - index_meta_pb.add_col_unique_id(1); // c2 column id - (*index_meta_pb.mutable_properties())["parser"] = "standard"; - TabletIndex subcolumn_index_meta; - subcolumn_index_meta.init_from_pb(index_meta_pb); - subcolumn_index_meta.set_escaped_escaped_index_suffix_path("v.s_host"); - - config::inverted_index_write_norms_for_variant_subcolumn = false; - create_tokenized_index("test_variant_subcolumn_without_norms", 0, true, "v.s_host"); - std::string prefix_without_norms {InvertedIndexDescriptor::get_index_file_path_prefix( - local_segment_path(kTestDir, "test_variant_subcolumn_without_norms", 0))}; - EXPECT_FALSE(check_norms_file_exists(prefix_without_norms, &subcolumn_index_meta)) - << "a tokenized variant subcolumn index must not write .nrm by default"; - - config::inverted_index_write_norms_for_variant_subcolumn = true; - create_tokenized_index("test_variant_subcolumn_with_norms", 1, true, "v.s_host"); - std::string prefix_with_norms {InvertedIndexDescriptor::get_index_file_path_prefix( - local_segment_path(kTestDir, "test_variant_subcolumn_with_norms", 1))}; - EXPECT_TRUE(check_norms_file_exists(prefix_with_norms, &subcolumn_index_meta)) - << "inverted_index_write_norms_for_variant_subcolumn=true must restore .nrm"; +// Norms take one byte per segment row for every indexed path, so an index on a variant path (a +// field_pattern index, or the copy inherited by one extracted subcolumn, which carries the path as +// its index suffix) writes none by default. The "norms" property overrides that per index. +TEST_F(InvertedIndexWriterTest, NormsFollowIndexNormsProperty) { + auto make_index_meta = [](const std::string& index_suffix, + const std::map& extra_properties) { + TabletIndexPB index_meta_pb; + index_meta_pb.set_index_type(IndexType::INVERTED); + index_meta_pb.set_index_id(1); + index_meta_pb.set_index_name("test"); + index_meta_pb.add_col_unique_id(1); // c2 column id + (*index_meta_pb.mutable_properties())["parser"] = "standard"; + for (const auto& [key, value] : extra_properties) { + (*index_meta_pb.mutable_properties())[key] = value; + } + TabletIndex index_meta; + index_meta.init_from_pb(index_meta_pb); + if (!index_suffix.empty()) { + index_meta.set_escaped_escaped_index_suffix_path(index_suffix); + } + return index_meta; + }; + auto path_prefix = [this](const std::string& rowset_id, int seg_id) { + return std::string {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, rowset_id, seg_id))}; + }; + + create_tokenized_index("variant_subcolumn_default", 0, true, "v.s_host"); + TabletIndex subcolumn_default = make_index_meta("v.s_host", {}); + EXPECT_FALSE(check_norms_file_exists(path_prefix("variant_subcolumn_default", 0), + &subcolumn_default)) + << "a variant subcolumn index must not write .nrm by default"; + + create_tokenized_index("variant_subcolumn_norms_on", 1, true, "v.s_host", + {{"norms", "true"}}); + TabletIndex subcolumn_norms_on = make_index_meta("v.s_host", {{"norms", "true"}}); + EXPECT_TRUE(check_norms_file_exists(path_prefix("variant_subcolumn_norms_on", 1), + &subcolumn_norms_on)) + << "norms = true must restore .nrm for a variant subcolumn index"; + + create_tokenized_index("field_pattern_default", 2, true, "", {{"field_pattern", "s_*"}}); + TabletIndex field_pattern_default = make_index_meta("", {{"field_pattern", "s_*"}}); + EXPECT_FALSE(check_norms_file_exists(path_prefix("field_pattern_default", 2), + &field_pattern_default)) + << "a field_pattern index must not write .nrm by default"; + + create_tokenized_index("plain_column_norms_off", 3, true, "", {{"norms", "false"}}); + TabletIndex plain_norms_off = make_index_meta("", {{"norms", "false"}}); + EXPECT_FALSE( + check_norms_file_exists(path_prefix("plain_column_norms_off", 3), &plain_norms_off)) + << "norms = false must drop .nrm for an ordinary column index"; } } // namespace doris::segment_v2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java index b7c31c33b7d249..146a0da30c1de5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java @@ -68,6 +68,8 @@ public class InvertedIndexUtil { public static String INVERTED_INDEX_SUPPORT_PHRASE_KEY = "support_phrase"; + public static String INVERTED_INDEX_NORMS_KEY = "norms"; + public static String INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY = "ignore_above"; public static String INVERTED_INDEX_PARSER_LOWERCASE_KEY = "lower_case"; @@ -251,6 +253,7 @@ public static void checkInvertedIndexProperties(Map properties, INVERTED_INDEX_PARSER_KEY_ALIAS, INVERTED_INDEX_PARSER_MODE_KEY, INVERTED_INDEX_SUPPORT_PHRASE_KEY, + INVERTED_INDEX_NORMS_KEY, INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE, INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN, INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, @@ -334,6 +337,12 @@ public static void checkInvertedIndexProperties(Map properties, + ", support_phrase must be true or false"); } + String norms = properties.get(INVERTED_INDEX_NORMS_KEY); + if (norms != null && !norms.matches("true|false")) { + throw new AnalysisException("Invalid inverted index 'norms' value: " + norms + + ", norms must be true or false"); + } + if (charFilterType != null) { if (!INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE.equals(charFilterType)) { throw new AnalysisException("Invalid 'char_filter_type', only '" diff --git a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy index 61783c2e5ac215..4e22ba2e96ad7a 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy @@ -34,6 +34,7 @@ suite("test_variant_subcolumn_index_norms", "p0") { content TEXT, v variant< 's_*' : text, + 't_*' : text, PROPERTIES("variant_max_subcolumns_count"="0") >, INDEX idx_content (content) USING INVERTED PROPERTIES( @@ -44,6 +45,12 @@ suite("test_variant_subcolumn_index_norms", "p0") { "parser"="english", "support_phrase"="true", "field_pattern"="s_*" + ), + INDEX idx_v_t (v) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "field_pattern"="t_*", + "norms"="true" ) ) ENGINE=OLAP DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 @@ -55,8 +62,8 @@ suite("test_variant_subcolumn_index_norms", "p0") { """ sql """ insert into test_variant_subcolumn_index_norms values (1, 'alpha database server', parse_to_variant('{"s_host":"alpha database server"}')), - (2, 'beta server cluster', parse_to_variant('{"s_host":"beta server cluster", "s_note":"alpha"}')), - (3, 'alpha', parse_to_variant('{"s_note":"alpha alpha beta"}')), + (2, 'beta server cluster', parse_to_variant('{"s_host":"beta server cluster", "s_note":"alpha", "t_note":"alpha"}')), + (3, 'alpha', parse_to_variant('{"s_note":"alpha alpha beta", "t_note":"alpha beta"}')), (4, 'gamma', parse_to_variant('{"other":"alpha"}')) """ sql " sync " @@ -88,22 +95,24 @@ suite("test_variant_subcolumn_index_norms", "p0") { logger.info("show_nested_index_file code=${code}, out=${out}, err=${err}") assertEquals(0, code) - def subcolumnIndexes = [] - def plainIndexes = [] + def normsBySuffix = [:] for (def rowset in parseJson(out.trim()).rowsets) { for (def segment in rowset.segments) { for (def index in segment.indices) { - def hasNorms = index.files.any { file -> file.name.endsWith(".nrm") } - if (index.index_suffix.isEmpty()) { - plainIndexes.add(hasNorms) - } else { - subcolumnIndexes.add(hasNorms) - } + normsBySuffix[index.index_suffix] = index.files.any { file -> file.name.endsWith(".nrm") } } } } - logger.info("norms of plain indexes: ${plainIndexes}, subcolumn indexes: ${subcolumnIndexes}") - // idx_content on the single segment writes norms; idx_v_s on s_host and s_note does not - assertEquals([true], plainIndexes) - assertEquals([false, false], subcolumnIndexes) + logger.info("norms by index suffix: ${normsBySuffix}") + // the suffix is the escaped variant path, e.g. v%2Es%5Fhost for v.s_host + def normsOf = { path -> + normsBySuffix.find { suffix, hasNorms -> + suffix.replace("%2E", ".").replace("%5F", "_").contains(path) + }?.value + } + // idx_content on an ordinary column keeps norms, idx_v_s drops them, idx_v_t asks for them back + assertEquals(true, normsBySuffix[""]) + assertEquals(false, normsOf("s_host")) + assertEquals(false, normsOf("s_note")) + assertEquals(true, normsOf("t_note")) } From 2726d94d4558bb9cfbae17b1f58de4da99456de8 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Thu, 17 Sep 2026 17:37:29 +0800 Subject: [PATCH 3/6] [test](inverted index) Cover whole-column VARIANT norms ### What problem does this PR solve? Issue Number: None Related PR: #68039 Problem Summary: Extend the norms regression coverage to whole-column VARIANT indexes and verify that per-subcolumn copies inherit the default or explicit norms property. ### Release note None ### Check List (For Author) - Test: Regression test added; execution pending - Behavior changed: No - Does this need documentation: No --- .../test_variant_subcolumn_index_norms.groovy | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy index 4e22ba2e96ad7a..fb4916cb953b28 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy @@ -17,6 +17,8 @@ // Analyzed indexes on variant subcolumns must not write dense BM25 norms (.nrm, one byte per row), // while analyzed indexes on ordinary columns still do. BM25 scoring keeps working on both. +// This holds both for an index declared with a field_pattern and for a whole-column index on a +// VARIANT column, whose per-subcolumn copies inherit the properties of the index they come from. suite("test_variant_subcolumn_index_norms", "p0") { if (isCloudMode()) { return @@ -37,6 +39,14 @@ suite("test_variant_subcolumn_index_norms", "p0") { 't_*' : text, PROPERTIES("variant_max_subcolumns_count"="0") >, + vd variant< + 'a_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + >, + vn variant< + 'b_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + >, INDEX idx_content (content) USING INVERTED PROPERTIES( "parser"="english", "support_phrase"="true" @@ -51,6 +61,15 @@ suite("test_variant_subcolumn_index_norms", "p0") { "support_phrase"="true", "field_pattern"="t_*", "norms"="true" + ), + INDEX idx_vd (vd) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true" + ), + INDEX idx_vn (vn) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "norms"="true" ) ) ENGINE=OLAP DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 @@ -61,10 +80,18 @@ suite("test_variant_subcolumn_index_norms", "p0") { ) """ sql """ insert into test_variant_subcolumn_index_norms values - (1, 'alpha database server', parse_to_variant('{"s_host":"alpha database server"}')), - (2, 'beta server cluster', parse_to_variant('{"s_host":"beta server cluster", "s_note":"alpha", "t_note":"alpha"}')), - (3, 'alpha', parse_to_variant('{"s_note":"alpha alpha beta", "t_note":"alpha beta"}')), - (4, 'gamma', parse_to_variant('{"other":"alpha"}')) + (1, 'alpha database server', parse_to_variant('{"s_host":"alpha database server"}'), + parse_to_variant('{"a_host":"alpha database server"}'), + parse_to_variant('{"b_host":"alpha database server"}')), + (2, 'beta server cluster', parse_to_variant('{"s_host":"beta server cluster", "s_note":"alpha", "t_note":"alpha"}'), + parse_to_variant('{"a_host":"beta server cluster"}'), + parse_to_variant('{"b_host":"beta server cluster"}')), + (3, 'alpha', parse_to_variant('{"s_note":"alpha alpha beta", "t_note":"alpha beta"}'), + parse_to_variant('{"a_host":"alpha"}'), + parse_to_variant('{"b_host":"alpha"}')), + (4, 'gamma', parse_to_variant('{"other":"alpha"}'), + parse_to_variant('{"other":"alpha"}'), + parse_to_variant('{"other":"alpha"}')) """ sql " sync " @@ -115,4 +142,9 @@ suite("test_variant_subcolumn_index_norms", "p0") { assertEquals(false, normsOf("s_host")) assertEquals(false, normsOf("s_note")) assertEquals(true, normsOf("t_note")) + // a whole-column index on a VARIANT column has no suffix of its own, but every subcolumn copy + // inherits its properties: idx_vd drops norms by default, idx_vn keeps them because it asks to + // keep them. + assertEquals(false, normsOf("a_host")) + assertEquals(true, normsOf("b_host")) } From aad5f0c0303ac51d136f50eb535c6cff8f7ab4c5 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Thu, 17 Sep 2026 17:39:32 +0800 Subject: [PATCH 4/6] [fix](inverted index) Add BE config to skip variant norms ### What problem does this PR solve? Issue Number: None Related PR: #68039 Problem Summary: Add an opt-in BE config that omits BM25 norms for indexes on variant paths, regardless of their per-index norms property, while preserving ordinary-column behavior. ### Release note Add inverted_index_skip_norms_for_variant for reducing dense norms on variant paths. ### Check List (For Author) - Test: Focused unit and regression coverage added; execution pending - Behavior changed: Yes (opt-in BE config) - Does this need documentation: Yes --- be/src/common/config.cpp | 4 + be/src/common/config.h | 4 + .../index/inverted/inverted_index_parser.cpp | 5 +- .../index/inverted/inverted_index_parser.h | 8 +- .../index/inverted/inverted_index_writer.cpp | 16 ++- .../segment/inverted_index_writer_test.cpp | 77 ++++++++--- .../test_variant_subcolumn_index_norms.out | 6 +- .../test_variant_subcolumn_index_norms.groovy | 125 +++++++++++++----- 8 files changed, 182 insertions(+), 63 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5c2e0ebf8d63f1..0197be3e9c1471 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1368,6 +1368,10 @@ DEFINE_mBool(debug_inverted_index_compaction, "false"); DEFINE_mBool(inverted_index_ram_dir_enable, "true"); // wheather index by RAM directory when base compaction DEFINE_mBool(inverted_index_ram_dir_enable_when_base_compaction, "true"); +// Norms cost one byte per segment row, including rows that hold no value for the field. A segment +// holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on +// to leave norms out of indexes on a variant path, regardless of their "norms" property. +DEFINE_mBool(inverted_index_skip_norms_for_variant, "false"); // use num_broadcast_buffer blocks as buffer to do broadcast DEFINE_Int32(num_broadcast_buffer, "32"); diff --git a/be/src/common/config.h b/be/src/common/config.h index fca3d1c6534cf5..666b9d47248e2b 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1388,6 +1388,10 @@ DECLARE_mBool(debug_inverted_index_compaction); DECLARE_mBool(inverted_index_ram_dir_enable); // wheather index by RAM directory when base compaction DECLARE_mBool(inverted_index_ram_dir_enable_when_base_compaction); +// Norms cost one byte per segment row, including rows that hold no value for the field. A segment +// holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on +// to leave norms out of indexes on a variant path, regardless of their "norms" property. +DECLARE_mBool(inverted_index_skip_norms_for_variant); // use num_broadcast_buffer blocks as buffer to do broadcast DECLARE_Int32(num_broadcast_buffer); diff --git a/be/src/storage/index/inverted/inverted_index_parser.cpp b/be/src/storage/index/inverted/inverted_index_parser.cpp index dcc412a2c40b77..e9573ce2739ac3 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.cpp +++ b/be/src/storage/index/inverted/inverted_index_parser.cpp @@ -104,12 +104,11 @@ std::string get_parser_phrase_support_string_from_properties( return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO; } -bool get_index_norms_from_properties(const std::map& properties, - bool default_value) { +bool get_index_norms_from_properties(const std::map& properties) { if (auto it = properties.find(INVERTED_INDEX_NORMS_KEY); it != properties.end()) { return it->second == INVERTED_INDEX_PARSER_TRUE; } - return default_value; + return true; } CharFilterMap get_parser_char_filter_map_from_properties( diff --git a/be/src/storage/index/inverted/inverted_index_parser.h b/be/src/storage/index/inverted/inverted_index_parser.h index 46d75bcbcb59dd..8aee17b16decb0 100644 --- a/be/src/storage/index/inverted/inverted_index_parser.h +++ b/be/src/storage/index/inverted/inverted_index_parser.h @@ -141,11 +141,9 @@ std::string get_parser_mode_string_from_properties( std::string get_parser_phrase_support_string_from_properties( const std::map& properties); -// Whether this index writes BM25 norms. Norms cost one byte per row of the segment, including rows -// that have no value for the field, so callers pass a default of false for indexes on variant paths, -// where one segment holds one index per path. "norms" = "true" / "false" overrides the default. -bool get_index_norms_from_properties(const std::map& properties, - bool default_value); +// Whether this index writes BM25 norms, which it does unless "norms" = "false" says otherwise. +// Norms cost one byte per row of the segment, including rows that have no value for the field. +bool get_index_norms_from_properties(const std::map& properties); CharFilterMap get_parser_char_filter_map_from_properties( const std::map& properties); diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index ebb0c126d31259..01d48afb285607 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -162,14 +162,18 @@ Status InvertedIndexColumnWriter::create_field(lucene::document::Fie (*field)->setOmitTermFreqAndPositions( !(get_parser_phrase_support_string_from_properties(_index_meta->properties()) == INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES)); - // Norms cost one byte per segment row, including rows without a value. A variant path index - // (a field_pattern index, or the copy inherited by one extracted subcolumn, which carries the - // path as its index suffix) is one of possibly thousands in a segment, so its norms can dwarf - // the data: those default to no norms, and "norms" = "true" brings them back per index. + // An analyzed index writes norms unless its "norms" property says otherwise. Norms cost one byte + // per segment row, including rows without a value, and a variant path index (a field_pattern + // index, or the copy inherited by one extracted subcolumn, which carries the path as its index + // suffix) is one of possibly thousands in a segment, so their norms can dwarf the data. + // inverted_index_skip_norms_for_variant drops norms for those indexes whatever their property + // says, so that a cluster can reclaim that space without rewriting its index definitions. const bool variant_path_index = !_index_meta->get_index_suffix().empty() || !_index_meta->field_pattern().empty(); - if (_should_analyzer && - get_index_norms_from_properties(_index_meta->properties(), !variant_path_index)) { + const bool skipped_by_config = + variant_path_index && config::inverted_index_skip_norms_for_variant; + if (_should_analyzer && !skipped_by_config && + get_index_norms_from_properties(_index_meta->properties())) { (*field)->setOmitNorms(false); } DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_field_v3", { diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 65d0beb48869e9..4ff67464a28a66 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -1514,30 +1514,71 @@ TEST_F(InvertedIndexWriterTest, NormsFollowIndexNormsProperty) { local_segment_path(kTestDir, rowset_id, seg_id))}; }; - create_tokenized_index("variant_subcolumn_default", 0, true, "v.s_host"); - TabletIndex subcolumn_default = make_index_meta("v.s_host", {}); - EXPECT_FALSE(check_norms_file_exists(path_prefix("variant_subcolumn_default", 0), - &subcolumn_default)) - << "a variant subcolumn index must not write .nrm by default"; + bool original_skip_norms_for_variant = config::inverted_index_skip_norms_for_variant; - create_tokenized_index("variant_subcolumn_norms_on", 1, true, "v.s_host", - {{"norms", "true"}}); - TabletIndex subcolumn_norms_on = make_index_meta("v.s_host", {{"norms", "true"}}); - EXPECT_TRUE(check_norms_file_exists(path_prefix("variant_subcolumn_norms_on", 1), - &subcolumn_norms_on)) - << "norms = true must restore .nrm for a variant subcolumn index"; + // an analyzed index writes norms wherever it sits, and only "norms" = "false" drops them + config::inverted_index_skip_norms_for_variant = false; - create_tokenized_index("field_pattern_default", 2, true, "", {{"field_pattern", "s_*"}}); - TabletIndex field_pattern_default = make_index_meta("", {{"field_pattern", "s_*"}}); - EXPECT_FALSE(check_norms_file_exists(path_prefix("field_pattern_default", 2), - &field_pattern_default)) - << "a field_pattern index must not write .nrm by default"; + create_tokenized_index("plain_column_default", 0, true, ""); + TabletIndex plain_default = make_index_meta("", {}); + EXPECT_TRUE(check_norms_file_exists(path_prefix("plain_column_default", 0), &plain_default)) + << "an analyzed index must write .nrm by default"; - create_tokenized_index("plain_column_norms_off", 3, true, "", {{"norms", "false"}}); + create_tokenized_index("plain_column_norms_off", 1, true, "", {{"norms", "false"}}); TabletIndex plain_norms_off = make_index_meta("", {{"norms", "false"}}); EXPECT_FALSE( - check_norms_file_exists(path_prefix("plain_column_norms_off", 3), &plain_norms_off)) + check_norms_file_exists(path_prefix("plain_column_norms_off", 1), &plain_norms_off)) << "norms = false must drop .nrm for an ordinary column index"; + + create_tokenized_index("variant_subcolumn_default", 2, true, "v.s_host"); + TabletIndex subcolumn_default = make_index_meta("v.s_host", {}); + EXPECT_TRUE(check_norms_file_exists(path_prefix("variant_subcolumn_default", 2), + &subcolumn_default)) + << "a variant subcolumn index must write .nrm by default too"; + + create_tokenized_index("variant_subcolumn_norms_off", 3, true, "v.s_host", + {{"norms", "false"}}); + TabletIndex subcolumn_norms_off = make_index_meta("v.s_host", {{"norms", "false"}}); + EXPECT_FALSE(check_norms_file_exists(path_prefix("variant_subcolumn_norms_off", 3), + &subcolumn_norms_off)) + << "norms = false must drop .nrm for a variant subcolumn index"; + + create_tokenized_index("field_pattern_default", 4, true, "", {{"field_pattern", "s_*"}}); + TabletIndex field_pattern_default = make_index_meta("", {{"field_pattern", "s_*"}}); + EXPECT_TRUE(check_norms_file_exists(path_prefix("field_pattern_default", 4), + &field_pattern_default)) + << "a field_pattern index must write .nrm by default too"; + + // the config drops norms for a variant path index whatever its property says, and leaves every + // other index alone + config::inverted_index_skip_norms_for_variant = true; + + create_tokenized_index("variant_subcolumn_skipped", 5, true, "v.s_host"); + TabletIndex subcolumn_skipped = make_index_meta("v.s_host", {}); + EXPECT_FALSE(check_norms_file_exists(path_prefix("variant_subcolumn_skipped", 5), + &subcolumn_skipped)) + << "the config must drop .nrm for a variant subcolumn index"; + + create_tokenized_index("variant_subcolumn_norms_on_skipped", 6, true, "v.s_host", + {{"norms", "true"}}); + TabletIndex subcolumn_norms_on_skipped = make_index_meta("v.s_host", {{"norms", "true"}}); + EXPECT_FALSE(check_norms_file_exists(path_prefix("variant_subcolumn_norms_on_skipped", 6), + &subcolumn_norms_on_skipped)) + << "the config must win over norms = true on a variant subcolumn index"; + + create_tokenized_index("field_pattern_skipped", 7, true, "", {{"field_pattern", "s_*"}}); + TabletIndex field_pattern_skipped = make_index_meta("", {{"field_pattern", "s_*"}}); + EXPECT_FALSE(check_norms_file_exists(path_prefix("field_pattern_skipped", 7), + &field_pattern_skipped)) + << "the config must drop .nrm for a field_pattern index"; + + create_tokenized_index("plain_column_not_skipped", 8, true, ""); + TabletIndex plain_not_skipped = make_index_meta("", {}); + EXPECT_TRUE( + check_norms_file_exists(path_prefix("plain_column_not_skipped", 8), &plain_not_skipped)) + << "the config must leave an ordinary column index alone"; + + config::inverted_index_skip_norms_for_variant = original_skip_norms_for_variant; } } // namespace doris::segment_v2 diff --git a/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out b/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out index 2988072d642e32..e1747721be6b9f 100644 --- a/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out +++ b/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out @@ -1,7 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !variant_subcolumn_score -- 2 0.6931 -3 0.9531 +3 0.61 + +-- !variant_subcolumn_score_no_norms -- +2 0.6931 +3 0.6931 -- !plain_column_score -- 1 0.5754 diff --git a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy index fb4916cb953b28..31a2ecc350ce9a 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -// Analyzed indexes on variant subcolumns must not write dense BM25 norms (.nrm, one byte per row), -// while analyzed indexes on ordinary columns still do. BM25 scoring keeps working on both. -// This holds both for an index declared with a field_pattern and for a whole-column index on a -// VARIANT column, whose per-subcolumn copies inherit the properties of the index they come from. +// An analyzed index writes dense BM25 norms (.nrm, one byte per segment row), on a variant path as +// on any other column, and "norms" = "false" drops them. Norms on a variant path cost rows * paths +// bytes, so inverted_index_skip_norms_for_variant leaves them out there whatever the property says. +// This covers both an index declared with a field_pattern and a whole-column +// index on a VARIANT column, whose per-subcolumn copies inherit the properties of the index they +// come from. BM25 scoring keeps working with and without norms. suite("test_variant_subcolumn_index_norms", "p0") { if (isCloudMode()) { return @@ -60,7 +62,7 @@ suite("test_variant_subcolumn_index_norms", "p0") { "parser"="english", "support_phrase"="true", "field_pattern"="t_*", - "norms"="true" + "norms"="false" ), INDEX idx_vd (vd) USING INVERTED PROPERTIES( "parser"="english", @@ -69,7 +71,7 @@ suite("test_variant_subcolumn_index_norms", "p0") { INDEX idx_vn (vn) USING INVERTED PROPERTIES( "parser"="english", "support_phrase"="true", - "norms"="true" + "norms"="false" ) ) ENGINE=OLAP DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 @@ -103,6 +105,13 @@ suite("test_variant_subcolumn_index_norms", "p0") { order by score() desc limit 10 """ + order_qt_variant_subcolumn_score_no_norms """ + select id, round(score(), 4) + from test_variant_subcolumn_index_norms + where cast(v["t_note"] as string) match_phrase "alpha" + order by score() desc + limit 10 + """ order_qt_plain_column_score """ select id, round(score(), 4) from test_variant_subcolumn_index_norms @@ -114,37 +123,93 @@ suite("test_variant_subcolumn_index_norms", "p0") { def backendIdToIp = [:] def backendIdToHttpPort = [:] getBackendIpHttpPort(backendIdToIp, backendIdToHttpPort) - def tablet = sql_return_maparray("show tablets from test_variant_subcolumn_index_norms")[0] - def (code, out, err) = http_client("GET", String.format( - "http://%s:%s/api/show_nested_index_file?tablet_id=%s", - backendIdToIp.get(tablet.BackendId), backendIdToHttpPort.get(tablet.BackendId), - tablet.TabletId)) - logger.info("show_nested_index_file code=${code}, out=${out}, err=${err}") - assertEquals(0, code) - - def normsBySuffix = [:] - for (def rowset in parseJson(out.trim()).rowsets) { - for (def segment in rowset.segments) { - for (def index in segment.indices) { - normsBySuffix[index.index_suffix] = index.files.any { file -> file.name.endsWith(".nrm") } + def normsBySuffixOf = { tableName -> + def tablet = sql_return_maparray("show tablets from ${tableName}")[0] + def (code, out, err) = http_client("GET", String.format( + "http://%s:%s/api/show_nested_index_file?tablet_id=%s", + backendIdToIp.get(tablet.BackendId), backendIdToHttpPort.get(tablet.BackendId), + tablet.TabletId)) + logger.info("show_nested_index_file of ${tableName} code=${code}, out=${out}, err=${err}") + assertEquals(0, code) + def norms = [:] + for (def rowset in parseJson(out.trim()).rowsets) { + for (def segment in rowset.segments) { + for (def index in segment.indices) { + norms[index.index_suffix] = index.files.any { file -> file.name.endsWith(".nrm") } + } } } + logger.info("norms by index suffix of ${tableName}: ${norms}") + return norms } - logger.info("norms by index suffix: ${normsBySuffix}") // the suffix is the escaped variant path, e.g. v%2Es%5Fhost for v.s_host - def normsOf = { path -> - normsBySuffix.find { suffix, hasNorms -> + def normsOf = { norms, path -> + norms.find { suffix, hasNorms -> suffix.replace("%2E", ".").replace("%5F", "_").contains(path) }?.value } - // idx_content on an ordinary column keeps norms, idx_v_s drops them, idx_v_t asks for them back + + def normsBySuffix = normsBySuffixOf("test_variant_subcolumn_index_norms") + // an analyzed index writes norms wherever it sits, and "norms" = "false" drops them assertEquals(true, normsBySuffix[""]) - assertEquals(false, normsOf("s_host")) - assertEquals(false, normsOf("s_note")) - assertEquals(true, normsOf("t_note")) + assertEquals(true, normsOf(normsBySuffix, "s_host")) + assertEquals(true, normsOf(normsBySuffix, "s_note")) + assertEquals(false, normsOf(normsBySuffix, "t_note")) // a whole-column index on a VARIANT column has no suffix of its own, but every subcolumn copy - // inherits its properties: idx_vd drops norms by default, idx_vn keeps them because it asks to - // keep them. - assertEquals(false, normsOf("a_host")) - assertEquals(true, normsOf("b_host")) + // inherits its properties: idx_vd keeps norms, idx_vn drops them because it asks to + // keep them off. + assertEquals(true, normsOf(normsBySuffix, "a_host")) + assertEquals(false, normsOf(normsBySuffix, "b_host")) + + // the skip is a dynamic BE config: with it turned on, every index on a variant path leaves norms + // out, even one that asks for them, while an ordinary column index is untouched + setBeConfigTemporary([inverted_index_skip_norms_for_variant: true]) { + sql "DROP TABLE IF EXISTS test_variant_subcolumn_index_norms_config" + sql """ + CREATE TABLE test_variant_subcolumn_index_norms_config ( + id INT, + content TEXT, + v variant< + 's_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + >, + vf variant< + 'c_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + >, + INDEX idx_content (content) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true" + ), + INDEX idx_v_s (v) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "field_pattern"="s_*" + ), + INDEX idx_vf (vf) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "norms"="true" + ) + ) ENGINE=OLAP DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ insert into test_variant_subcolumn_index_norms_config values + (1, 'alpha database server', parse_to_variant('{"s_host":"alpha database server"}'), + parse_to_variant('{"c_host":"alpha database server"}')), + (2, 'beta server cluster', parse_to_variant('{"s_host":"beta server cluster"}'), + parse_to_variant('{"c_host":"beta server cluster"}')) + """ + sql " sync " + + def configNorms = normsBySuffixOf("test_variant_subcolumn_index_norms_config") + assertEquals(false, normsOf(configNorms, "s_host")) + assertEquals(false, normsOf(configNorms, "c_host")) + assertEquals(true, configNorms[""]) + } } From 399ef2659237028163bdef11fe6870b8b41a8e9a Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Thu, 17 Sep 2026 18:14:06 +0800 Subject: [PATCH 5/6] [fix](inverted index) Reject BM25 scoring without norms ### What problem does this PR solve?\n\nIssue Number: None\n\nRelated PR: #68039\n\nProblem Summary: BM25 scoring used a zero average document length when a CLucene segment omitted norms, producing invalid or misleading scores. Reject scoring when an analyzed index has a segment without norms, while keeping MATCH filtering available and preserving the VARIANT skip-norms configuration behavior. This selective backport excludes all SNII changes.\n\n### Release note\n\nBM25 scoring now reports an error for analyzed indexes whose segments were written without norms.\n\n### Check List (For Author)\n\n- Test: FE build, FE unit test, format check; BE build and regression are blocked by the local ASAN/toolchain environment.\n- Behavior changed: Yes (BM25 scoring without norms is rejected)\n- Does this need documentation: No --- be/src/common/config.cpp | 3 +- be/src/common/config.h | 3 +- .../compaction/collection_statistics.cpp | 20 ++++- .../inverted/similarity/bm25_similarity.cpp | 10 +-- .../similarity/bm25_similarity_test.cpp | 19 ----- .../segment/inverted_index_writer_test.cpp | 7 +- .../InvertedIndexNormsPropertyTest.java | 56 +++++++++++++ .../test_variant_subcolumn_index_norms.out | 9 +- .../test_variant_subcolumn_index_norms.groovy | 83 +++++++++++++++++-- 9 files changed, 164 insertions(+), 46 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexNormsPropertyTest.java diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 0197be3e9c1471..4b0ead5a7a879e 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1370,7 +1370,8 @@ DEFINE_mBool(inverted_index_ram_dir_enable, "true"); DEFINE_mBool(inverted_index_ram_dir_enable_when_base_compaction, "true"); // Norms cost one byte per segment row, including rows that hold no value for the field. A segment // holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on -// to leave norms out of indexes on a variant path, regardless of their "norms" property. +// to leave norms out of every index on a variant path, whatever its "norms" property says; BM25 +// scoring (score()) on those indexes then fails. DEFINE_mBool(inverted_index_skip_norms_for_variant, "false"); // use num_broadcast_buffer blocks as buffer to do broadcast DEFINE_Int32(num_broadcast_buffer, "32"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 666b9d47248e2b..94a9fd88f86037 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1390,7 +1390,8 @@ DECLARE_mBool(inverted_index_ram_dir_enable); DECLARE_mBool(inverted_index_ram_dir_enable_when_base_compaction); // Norms cost one byte per segment row, including rows that hold no value for the field. A segment // holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on -// to leave norms out of indexes on a variant path, regardless of their "norms" property. +// to leave norms out of every index on a variant path, whatever its "norms" property says; BM25 +// scoring (score()) on those indexes then fails. DECLARE_mBool(inverted_index_skip_norms_for_variant); // use num_broadcast_buffer blocks as buffer to do broadcast DECLARE_Int32(num_broadcast_buffer); diff --git a/be/src/storage/compaction/collection_statistics.cpp b/be/src/storage/compaction/collection_statistics.cpp index 6a680ad036c696..8f21b18e34fbe7 100644 --- a/be/src/storage/compaction/collection_statistics.cpp +++ b/be/src/storage/compaction/collection_statistics.cpp @@ -205,8 +205,22 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, int3 #endif total_seg_num_docs = std::max(total_seg_num_docs, index_reader->maxDoc()); - _total_num_tokens[ws_field_name] += - index_reader->sumTotalTermFreq(ws_field_name.c_str()).value_or(0); + // BM25 on an analyzed index needs the record length of every row, and CLucene keeps + // them, together with the field's token count, in the norms. A segment written without + // norms would feed a zero avgdl, or rank its rows as zero-length documents next to the + // segments that have norms, so refuse to score the collection. An index that is not + // analyzed never writes norms and is left as it is. + const auto token_count = index_reader->sumTotalTermFreq(ws_field_name.c_str()); + if (!token_count.has_value() && + segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer( + collect_info.index_meta->properties())) { + return Status::Error( + "BM25 scoring requires norms, but segment {} was written without norms for " + "field {}. Norms are left out when the index sets \"norms\" = \"false\" or, " + "for a variant path, when inverted_index_skip_norms_for_variant is on", + seg_path, StringHelper::to_string(ws_field_name)); + } + _total_num_tokens[ws_field_name] += token_count.value_or(0); for (const auto& term_info : collect_info.term_infos) { auto iter = TermIterator::create(io_ctx, false, index_reader, ws_field_name, @@ -289,4 +303,4 @@ float CollectionStatistics::get_or_calculate_idf(const std::wstring& lucene_col_ } #include "common/compile_check_end.h" -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/index/inverted/similarity/bm25_similarity.cpp b/be/src/storage/index/inverted/similarity/bm25_similarity.cpp index daf6bf5d3c3397..e02242985b38f0 100644 --- a/be/src/storage/index/inverted/similarity/bm25_similarity.cpp +++ b/be/src/storage/index/inverted/similarity/bm25_similarity.cpp @@ -17,7 +17,6 @@ #include "storage/index/inverted/similarity/bm25_similarity.h" -#include #include namespace doris::segment_v2 { @@ -45,13 +44,6 @@ BM25Similarity::BM25Similarity(float idf, float avgdl) : _idf(idf), _avgdl(avgdl } void BM25Similarity::compute_tf_cache() { - // CLucene keeps a field's token count in its .nrm header, so avgdl is 0 when no segment of the - // field stores norms (e.g. variant subcolumn indexes). Every document then has an unknown length: - // score without length normalization instead of computing 0 / 0. - if (_avgdl <= 0.0F) { - std::fill(_cache.begin(), _cache.end(), 1.0F / _k1); - return; - } for (int i = 0; i < _cache.size(); i++) { _cache[i] = 1.0F / (_k1 * ((1 - _b) + _b * LENGTH_TABLE[i] / _avgdl)); } @@ -156,4 +148,4 @@ int32_t BM25Similarity::byte4_to_int(uint8_t b) { } #include "common/compile_check_end.h" -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp index c6b1ddb2eec274..24fdfbd2e62709 100644 --- a/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp +++ b/be/test/storage/index/inverted/similarity/bm25_similarity_test.cpp @@ -19,7 +19,6 @@ #include -#include #include #include "common/be_mock_util.h" @@ -290,21 +289,3 @@ TEST_F(BM25SimilarityTest, CacheConsistencyTest) { ASSERT_FLOAT_EQ(similarity_->_cache[i], expected); } } - -// Indexes without norms report no token count, so avgdl is 0: scores must stay finite and ignore -// document length instead of turning into NaN. -TEST_F(BM25SimilarityTest, ZeroAvgDlScoresWithoutLengthNorm) { - mock_stats_->set_mock_idf(2.0f); - mock_stats_->set_mock_avg_dl(0.0f); - - similarity_->for_one_term(context_, L"field", L"term"); - - for (int i = 0; i < 256; ++i) { - ASSERT_FLOAT_EQ(similarity_->_cache[i], 1.0f / similarity_->_k1); - } - float score = similarity_->score(1.0f, 0); - ASSERT_FALSE(std::isnan(score)); - ASSERT_FLOAT_EQ(score, - similarity_->_weight - similarity_->_weight / (1.0f + 1.0f / similarity_->_k1)); - ASSERT_GT(similarity_->score(2.0f, 0), score); -} diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 4ff67464a28a66..6af028c13d1efe 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -362,10 +362,9 @@ class InvertedIndexWriterTest : public testing::Test { } // Helper method to create an inverted index with tokenization enabled - void create_tokenized_index( - std::string_view rowset_id, int seg_id, bool enable_analyzer, - const std::string& index_suffix = "", - const std::map& extra_properties = {}) { + void create_tokenized_index(std::string_view rowset_id, int seg_id, bool enable_analyzer, + const std::string& index_suffix = "", + const std::map& extra_properties = {}) { auto tablet_schema = create_schema(); // Create index meta with tokenization setting diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexNormsPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexNormsPropertyTest.java new file mode 100644 index 00000000000000..9e44b7c631af04 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexNormsPropertyTest.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +public class InvertedIndexNormsPropertyTest { + + @Test + public void testNormsPropertyAccepted() throws AnalysisException { + for (String value : new String[] {"true", "false"}) { + Map properties = new HashMap<>(); + properties.put("parser", "english"); + properties.put("norms", value); + + InvertedIndexUtil.checkInvertedIndexParser("col1", PrimitiveType.STRING, properties, + TInvertedIndexFileStorageFormat.V2); + } + } + + @Test + public void testNormsPropertyRejectsOtherValues() { + Map properties = new HashMap<>(); + properties.put("parser", "english"); + properties.put("norms", "yes"); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> InvertedIndexUtil.checkInvertedIndexParser("col1", PrimitiveType.STRING, + properties, TInvertedIndexFileStorageFormat.V2)); + Assertions.assertTrue(exception.getMessage().contains("norms must be true or false"), + exception.getMessage()); + } +} diff --git a/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out b/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out index e1747721be6b9f..a22a1151ed2df9 100644 --- a/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out +++ b/regression-test/data/inverted_index_p0/test_variant_subcolumn_index_norms.out @@ -3,11 +3,14 @@ 2 0.6931 3 0.61 --- !variant_subcolumn_score_no_norms -- -2 0.6931 -3 0.6931 +-- !variant_subcolumn_match_no_norms -- +2 +3 -- !plain_column_score -- 1 0.5754 3 0.8714 +-- !mixed_match -- +1 +2 diff --git a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy index 31a2ecc350ce9a..8f36e872f3617b 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy @@ -20,8 +20,10 @@ // bytes, so inverted_index_skip_norms_for_variant leaves them out there whatever the property says. // This covers both an index declared with a field_pattern and a whole-column // index on a VARIANT column, whose per-subcolumn copies inherit the properties of the index they -// come from. BM25 scoring keeps working with and without norms. -suite("test_variant_subcolumn_index_norms", "p0") { +// come from. BM25 scoring needs norms: score() on an index without them fails, also while only +// some segments lack them, and MATCH filtering keeps working. +// It flips a BE config, so it must not share the cluster with other suites. +suite("test_variant_subcolumn_index_norms", "p0,nonConcurrent") { if (isCloudMode()) { return } @@ -105,13 +107,22 @@ suite("test_variant_subcolumn_index_norms", "p0") { order by score() desc limit 10 """ - order_qt_variant_subcolumn_score_no_norms """ - select id, round(score(), 4) + // without norms MATCH still filters, and score() is refused + order_qt_variant_subcolumn_match_no_norms """ + select id from test_variant_subcolumn_index_norms where cast(v["t_note"] as string) match_phrase "alpha" - order by score() desc - limit 10 """ + test { + sql """ + select id, score() + from test_variant_subcolumn_index_norms + where cast(v["t_note"] as string) match_phrase "alpha" + order by score() desc + limit 10 + """ + exception "written without norms" + } order_qt_plain_column_score """ select id, round(score(), 4) from test_variant_subcolumn_index_norms @@ -212,4 +223,64 @@ suite("test_variant_subcolumn_index_norms", "p0") { assertEquals(false, normsOf(configNorms, "c_host")) assertEquals(true, configNorms[""]) } + test { + sql """ + select id, score() + from test_variant_subcolumn_index_norms_config + where cast(vf["c_host"] as string) match_phrase "alpha" + order by score() desc + limit 10 + """ + exception "written without norms" + } + + // segments with and without norms side by side, as while the config is being turned on: + // MATCH still filters, and score() is refused rather than ranking the rows of the newer + // segment as zero-length documents + sql "DROP TABLE IF EXISTS test_variant_subcolumn_index_norms_mixed" + sql """ + CREATE TABLE test_variant_subcolumn_index_norms_mixed ( + id INT, + v variant< + 's_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + >, + INDEX idx_v_s (v) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "field_pattern"="s_*" + ) + ) ENGINE=OLAP DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ insert into test_variant_subcolumn_index_norms_mixed values + (1, parse_to_variant('{"s_note":"alpha database server"}')) + """ + setBeConfigTemporary([inverted_index_skip_norms_for_variant: true]) { + sql """ insert into test_variant_subcolumn_index_norms_mixed values + (2, parse_to_variant('{"s_note":"alpha"}')) + """ + } + sql " sync " + + order_qt_mixed_match """ + select id + from test_variant_subcolumn_index_norms_mixed + where cast(v["s_note"] as string) match_phrase "alpha" + """ + test { + sql """ + select id, score() + from test_variant_subcolumn_index_norms_mixed + where cast(v["s_note"] as string) match_phrase "alpha" + order by score() desc + limit 10 + """ + exception "written without norms" + } } From 28d31cb4e89ebab139b78e8117b941c4fee10e76 Mon Sep 17 00:00:00 2001 From: eldenmoon Date: Fri, 18 Sep 2026 10:41:20 +0800 Subject: [PATCH 6/6] [fix](regression) Remove unsupported session variable --- .../inverted_index_p0/test_variant_subcolumn_index_norms.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy index 8f36e872f3617b..1a82e6bbba1bef 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_subcolumn_index_norms.groovy @@ -28,7 +28,6 @@ suite("test_variant_subcolumn_index_norms", "p0,nonConcurrent") { return } - sql """ set enable_segment_limit_pushdown = true """ sql """ set enable_match_without_inverted_index = false """ sql """ set default_variant_enable_typed_paths_to_sparse = false """ sql """ set default_variant_enable_doc_mode = false """