apache · eldenmoon · Sep 17, 2026 · Sep 15, 2026 · Sep 16, 2026 · Sep 16, 2026
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1408,6 +1408,11 @@ DEFINE_mBool(debug_inverted_index_compaction, "false");
 DEFINE_mBool(inverted_index_ram_dir_enable, "true");
 // wheather index by RAM directory when base compaction
 DEFINE_mBool(inverted_index_ram_dir_enable_when_base_compaction, "true");
+// Norms cost one byte per segment row, including rows that hold no value for the field. A segment
+// holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on
+// to leave norms out of every index on a variant path, whatever its "norms" property says; BM25
+// scoring (score()) on those indexes then fails.
+DEFINE_mBool(inverted_index_skip_norms_for_variant, "false");
 // use num_broadcast_buffer blocks as buffer to do broadcast
 DEFINE_Int32(num_broadcast_buffer, "32");
 

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1462,6 +1462,11 @@ DECLARE_mBool(debug_inverted_index_compaction);
 DECLARE_mBool(inverted_index_ram_dir_enable);
 // wheather index by RAM directory when base compaction
 DECLARE_mBool(inverted_index_ram_dir_enable_when_base_compaction);
+// Norms cost one byte per segment row, including rows that hold no value for the field. A segment
+// holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on
+// to leave norms out of every index on a variant path, whatever its "norms" property says; BM25
+// scoring (score()) on those indexes then fails.
+DECLARE_mBool(inverted_index_skip_norms_for_variant);
 // use num_broadcast_buffer blocks as buffer to do broadcast
 DECLARE_Int32(num_broadcast_buffer);
 

@@ -17,6 +17,8 @@
 
 #include "storage/index/inverted/inverted_index_parser.h"
 
+#include "common/config.h"
+#include "storage/tablet/tablet_schema.h"
 #include "util/string_util.h"
 
 namespace doris {
@@ -117,6 +119,23 @@ std::string get_parser_phrase_support_string_from_properties(
     return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
 }
 
+bool should_write_index_norms(const TabletIndex& index_meta) {
+    // A variant path index (a field_pattern index, or the copy inherited by one extracted
+    // subcolumn, which carries the path as its index suffix) is one of possibly thousands in a
+    // segment, so its norms can dwarf the data. The config drops them whatever the property says,
+    // so that a cluster can reclaim that space without rewriting its index definitions.
+    const bool variant_path_index =
+            !index_meta.get_index_suffix().empty() || !index_meta.field_pattern().empty();
+    if (variant_path_index && config::inverted_index_skip_norms_for_variant) {
+        return false;
+    }
+    const auto& properties = index_meta.properties();
+    if (auto it = properties.find(INVERTED_INDEX_NORMS_KEY); it != properties.end()) {
+        return it->second == INVERTED_INDEX_PARSER_TRUE;
+    }
+    return true;
+}
+
 CharFilterMap get_parser_char_filter_map_from_properties(
         const std::map<std::string, std::string>& properties) {
     if (!properties.contains(INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE)) {

@@ -34,6 +34,8 @@ class Analyzer;
 
 namespace doris {
 
+class TabletIndex;
+
 enum class InvertedIndexParserType {
     PARSER_UNKNOWN = 0,
     PARSER_NONE = 1,
@@ -89,6 +91,9 @@ const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY = "support_phrase";
 const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES = "true";
 const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO = "false";
 
+// Whether an analyzed index stores BM25 norms, which take one byte per row of the segment.
+const std::string INVERTED_INDEX_NORMS_KEY = "norms";
+
 const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE = "char_filter_type";
 const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN = "char_filter_pattern";
 const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT = "char_filter_replacement";
@@ -152,6 +157,13 @@ std::string get_parser_mode_string_from_properties(
 std::string get_parser_phrase_support_string_from_properties(
         const std::map<std::string, std::string>& properties);
 
+// Whether an analyzed index writes BM25 norms: the one policy shared by every index storage format
+// and by index compaction. Norms cost one byte per row of the segment, including rows that have no
+// value for the field. An index writes them unless its "norms" property is "false", or unless it
+// is on a variant path while inverted_index_skip_norms_for_variant is on, which wins over the
+// property.
+bool should_write_index_norms(const TabletIndex& index_meta);
+
 CharFilterMap get_parser_char_filter_map_from_properties(
         const std::map<std::string, std::string>& properties);
 

@@ -162,7 +162,7 @@ Status InvertedIndexColumnWriter<field_type>::create_field(lucene::document::Fie
     (*field)->setOmitTermFreqAndPositions(
             !(get_parser_phrase_support_string_from_properties(_index_meta->properties()) ==
               INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES));
-    if (_should_analyzer) {
+    if (_should_analyzer && should_write_index_norms(*_index_meta)) {
         (*field)->setOmitNorms(false);
     }
 

@@ -47,7 +47,9 @@ Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t index_doc_
     if (!has_positions || !has_norms) {
         return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, false>(
                 "SNII scoring requires positions and norms; this segment was written without "
-                "norms -- rebuild the index or wait for compaction to rewrite it"));
+                "positions or without norms. Norms are left out when the index sets \"norms\" = "
+                "\"false\" or, for a variant path, when inverted_index_skip_norms_for_variant is "
+                "on"));
     }
     return SniiScoringSegmentStats {.doc_count = index_doc_count,
                                     .token_count = sum_total_term_freq};
@@ -313,8 +315,22 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
         index_reader = index_searcher->getReader();
 #endif
         total_segment_docs = std::max(total_segment_docs, index_reader->maxDoc());
-        _total_num_tokens[ws_field_name] +=
-                index_reader->sumTotalTermFreq(ws_field_name.c_str()).value_or(0);
+        // BM25 on an analyzed index needs the record length of every row, and CLucene keeps
+        // them, together with the field's token count, in the norms. A segment written without
+        // norms would feed a zero avgdl, or rank its rows as zero-length documents next to the
+        // segments that have norms, so refuse to score the collection, as SNII does. An index
+        // that is not analyzed never writes norms and is left as it is.
+        const auto token_count = index_reader->sumTotalTermFreq(ws_field_name.c_str());
+        if (!token_count.has_value() &&
+            segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
+                    collect_info.index_meta->properties())) {
+            return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+                    "BM25 scoring requires norms, but segment {} was written without norms for "
+                    "field {}. Norms are left out when the index sets \"norms\" = \"false\" or, "
+                    "for a variant path, when inverted_index_skip_norms_for_variant is on",
+                    seg_path, StringHelper::to_string(ws_field_name));
+        }
+        _total_num_tokens[ws_field_name] += token_count.value_or(0);
 
         for (const auto& logical_term_bytes : collect_info.unique_terms) {
             const auto logical_term =

@@ -27,6 +27,7 @@
 #include "common/config.h"
 #include "common/exception.h"
 #include "storage/index/inverted/analyzer/analyzer.h"
+#include "storage/index/inverted/inverted_index_parser.h"
 #include "storage/index/snii/format/format_constants.h"
 #include "storage/index/snii/format/phrase_bigram.h"
 #include "storage/index/snii/reader/logical_index_reader.h"
@@ -244,8 +245,11 @@ Status validate_snii_compaction_eligibility(
                                                              source_ordinal));
     }
     RETURN_IF_ERROR(validate_destination_policy(destination_index, analyzer_provider_factory));
-    out->destination_writes_norms =
-            inverted_index::InvertedIndexAnalyzer::should_analyzer(destination_index.properties());
+    // Merged norms are rebuilt from the postings, so the destination follows the same norms
+    // policy as a fresh write whether or not the sources carry norms.
+    out->destination_writes_norms = inverted_index::InvertedIndexAnalyzer::should_analyzer(
+                                            destination_index.properties()) &&
+                                    should_write_index_norms(destination_index);
     return Status::OK();
 }
 

@@ -29,6 +29,7 @@
 #include "common/logging.h"
 #include "storage/index/index_file_writer.h"
 #include "storage/index/inverted/analyzer/analyzer.h"
+#include "storage/index/inverted/inverted_index_parser.h"
 #include "storage/index/inverted/query/query_info.h"
 #include "storage/index/snii/query/bm25_scorer.h"
 #include "storage/index/snii/writer/global_memory_limiter.h"
@@ -115,10 +116,11 @@ Status SniiIndexColumnWriter::init() {
         return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
                 "SNII create analyzer failed: {}", e.what());
     }
-    // A2: Analyzed indexes with positions always write norms (tokens per document, clamped to
-    // 1..255), matching CLucene's scoring capabilities. Keyword or positionless indexes omit
-    // them. Norms are an optional core-metadata region ignored by older readers.
-    _writes_norms = _should_analyzer && _has_positions;
+    // A2: Analyzed indexes with positions write norms (tokens per document, clamped to 1..255),
+    // matching CLucene's scoring capabilities, unless the shared norms policy turns them off.
+    // Keyword or positionless indexes omit them. Norms are an optional core-metadata region
+    // ignored by older readers.
+    _writes_norms = _should_analyzer && _has_positions && should_write_index_norms(*_index_meta);
     return Status::OK();
 }
 

@@ -93,7 +93,8 @@ class SniiIndexColumnWriter final : public IndexColumnWriter {
     bool _should_analyzer = false;
     bool _has_positions = false;
     const bool _is_char;
-    // A2: Analyzed indexes with positions always write BM25 norms, matching CLucene.
+    // A2: Analyzed indexes with positions write BM25 norms, matching CLucene, unless the shared
+    // norms policy (should_write_index_norms) turns them off.
     bool _writes_norms = false;
     // Latch: set_direct_load() ran. The first call wins; a repeat or late call
     // is ignored (and logged) so one index keeps one stable compression-tier

@@ -375,7 +375,9 @@ class CollectionStatisticsTest : public ::testing::Test {
         return splits;
     }
 
-    TabletSchemaSPtr create_legacy_v3_schema() {
+    TabletSchemaSPtr create_legacy_v3_schema(std::map<std::string, std::string> properties = {
+                                                     {"parser", "standard"},
+                                                     {"support_phrase", "true"}}) {
         TabletSchemaPB schema_pb;
         schema_pb.set_keys_type(DUP_KEYS);
         schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
@@ -392,8 +394,7 @@ class CollectionStatisticsTest : public ::testing::Test {
         index._index_id = 1;
         index._index_type = IndexType::INVERTED;
         index._col_unique_ids.push_back(1);
-        index._properties["parser"] = "standard";
-        index._properties["support_phrase"] = "true";
+        index._properties = std::move(properties);
         tablet_schema->append_index(std::move(index));
         return tablet_schema;
     }
@@ -910,6 +911,63 @@ TEST_F(CollectionStatisticsTest, LegacyV3SkipsEmptySegmentAfterCollectingAvailab
     expect_collected_term(L"1", L"alpha", 1);
 }
 
+// BM25 needs norms from every segment of an analyzed index: a segment written without them makes
+// the whole collection refuse to score, on its own or next to segments that have norms.
+TEST_F(CollectionStatisticsTest, LegacyV3RejectsSegmentWrittenWithoutNorms) {
+    auto with_norms_schema = create_legacy_v3_schema();
+    auto without_norms_schema = create_legacy_v3_schema(
+            {{"parser", "standard"}, {"support_phrase", "true"}, {"norms", "false"}});
+    const std::string with_norms_path = test_dir_ + "/legacy_v3_with_norms_0.dat";
+    const std::string without_norms_path = test_dir_ + "/legacy_v3_without_norms_1.dat";
+    ASSERT_TRUE(write_legacy_v3_segment(with_norms_schema, with_norms_path).ok());
+    ASSERT_TRUE(write_legacy_v3_segment(without_norms_schema, without_norms_path).ok());
+
+    auto collect = [&](const std::vector<std::string>& segment_paths) {
+        auto rowset_meta = std::make_shared<collection_statistics::MockRowsetMeta>();
+        auto rowset = std::make_shared<collection_statistics::MockRowset>(without_norms_schema,
+                                                                          rowset_meta);
+        rowset->set_num_segments(static_cast<int>(segment_paths.size()));
+        for (size_t i = 0; i < segment_paths.size(); ++i) {
+            rowset->set_segment_path(static_cast<int>(i), segment_paths[i]);
+        }
+        auto reader = std::make_shared<collection_statistics::MockRowsetReader>(rowset);
+        std::vector<RowSetSplits> splits {RowSetSplits(reader)};
+        return stats_->collect(runtime_state_.get(), splits, without_norms_schema,
+                               create_match_expr_contexts("alpha"), nullptr);
+    };
+
+    Status status = collect({without_norms_path});
+    EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED) << status;
+    EXPECT_NE(status.to_string().find("written without norms"), std::string::npos) << status;
+    expect_no_collected_tokens(L"1");
+
+    status = collect({with_norms_path, without_norms_path});
+    EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED) << status;
+    expect_no_collected_tokens(L"1");
+}
+
+// An index that is not analyzed never writes norms, and its scoring statistics are collected as
+// before.
+TEST_F(CollectionStatisticsTest, LegacyV3KeywordIndexWithoutNormsIsStillCollected) {
+    auto tablet_schema = create_legacy_v3_schema({});
+    const std::string segment_path = test_dir_ + "/legacy_v3_keyword_0.dat";
+    ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, segment_path).ok());
+
+    auto rowset_meta = std::make_shared<collection_statistics::MockRowsetMeta>();
+    auto rowset = std::make_shared<collection_statistics::MockRowset>(tablet_schema, rowset_meta);
+    rowset->set_num_segments(1);
+    rowset->set_segment_path(0, segment_path);
+    auto reader = std::make_shared<collection_statistics::MockRowsetReader>(rowset);
+    std::vector<RowSetSplits> splits {RowSetSplits(reader)};
+
+    const Status status = stats_->collect(runtime_state_.get(), splits, tablet_schema,
+                                          create_search_contexts("TERM", "alpha beta"), nullptr);
+
+    ASSERT_TRUE(status.ok()) << status;
+    expect_collected_stats(L"1", 1, 0);
+    expect_collected_term(L"1", L"alpha beta", 1);
+}
+
 TEST_F(CollectionStatisticsTest, SniiScoringUsesPhysicalStatistics) {
     auto tablet_schema = create_snii_schema();
     auto expr_contexts = create_match_expr_contexts("alpha");

@@ -248,6 +248,42 @@ TEST(SniiCompactionEligibilityTest, DestinationWritesNormsExactlyWhenAnalyzed) {
     EXPECT_FALSE(keyword.destination_writes_norms);
 }
 
+// The destination follows the norms policy shared with fresh writes: the "norms" property, and on
+// a variant path inverted_index_skip_norms_for_variant, which wins over the property.
+TEST(SniiCompactionEligibilityTest, DestinationWritesNormsFollowSharedNormsPolicy) {
+    const bool original_skip_norms_for_variant =
+            doris::config::inverted_index_skip_norms_for_variant;
+    auto legacy = open_index({});
+    auto writes_norms = [&legacy](const std::map<std::string, std::string>& properties,
+                                  const std::string& index_suffix) {
+        auto source_meta = make_index(properties, doris::IndexType::INVERTED, 7, index_suffix);
+        auto destination = make_index(properties, doris::IndexType::INVERTED, 7, index_suffix);
+        std::vector sources {source(*legacy, *source_meta)};
+        compaction::SniiCompactionEligibility eligibility;
+        const Status status = compaction::validate_snii_compaction_eligibility(
+                sources, *destination, &eligibility);
+        EXPECT_TRUE(status.ok()) << status.to_string();
+        return eligibility.destination_writes_norms;
+    };
+    auto norms_off = plain_properties();
+    norms_off["norms"] = "false";
+    auto norms_on = plain_properties();
+    norms_on["norms"] = "true";
+
+    doris::config::inverted_index_skip_norms_for_variant = false;
+    EXPECT_TRUE(writes_norms(plain_properties(), ""));
+    EXPECT_FALSE(writes_norms(norms_off, ""));
+    EXPECT_TRUE(writes_norms(plain_properties(), "v.s_host"));
+    EXPECT_FALSE(writes_norms(norms_off, "v.s_host"));
+
+    doris::config::inverted_index_skip_norms_for_variant = true;
+    EXPECT_TRUE(writes_norms(plain_properties(), ""));
+    EXPECT_FALSE(writes_norms(plain_properties(), "v.s_host"));
+    EXPECT_FALSE(writes_norms(norms_on, "v.s_host"));
+
+    doris::config::inverted_index_skip_norms_for_variant = original_skip_norms_for_variant;
+}
+
 TEST(SniiCompactionEligibilityTest, RejectsLegacyBigramMarkerBeforeMergeExecution) {
     snii_test::MemoryFile file;
     writer::SniiIndexInput input;

@@ -333,6 +333,50 @@ TEST(SniiWriterFailureLatch, AnalyzerFailureDiscardsStateAndBlocksFinish) {
     EXPECT_EQ(writer.memory_reporter_for_test(), nullptr);
 }
 
+// The writer follows the norms policy it shares with the CLucene writer and SNII compaction: the
+// "norms" property, and on a variant path inverted_index_skip_norms_for_variant, which wins over
+// the property.
+TEST(SniiWriterNorms, WritesNormsFollowSharedNormsPolicy) {
+    const bool original_skip_norms_for_variant =
+            doris::config::inverted_index_skip_norms_for_variant;
+    auto writes_norms = [](const std::map<std::string, std::string>& extra_properties,
+                           const std::string& index_suffix) {
+        doris::TabletIndexPB index_pb;
+        index_pb.set_index_type(doris::IndexType::INVERTED);
+        index_pb.set_index_id(92);
+        index_pb.set_index_name("norms_policy");
+        index_pb.add_col_unique_id(0);
+        index_pb.set_index_suffix_name(index_suffix);
+        index_pb.mutable_properties()->insert({"parser", "english"});
+        index_pb.mutable_properties()->insert({"support_phrase", "true"});
+        for (const auto& [key, value] : extra_properties) {
+            index_pb.mutable_properties()->insert({key, value});
+        }
+        doris::TabletIndex index_meta;
+        index_meta.init_from_pb(index_pb);
+        doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta,
+                                                        doris::FieldType::OLAP_FIELD_TYPE_VARCHAR);
+        const doris::Status status = writer.init();
+        EXPECT_TRUE(status.ok()) << status.to_string();
+        return writer.writes_norms_for_test();
+    };
+
+    doris::config::inverted_index_skip_norms_for_variant = false;
+    EXPECT_TRUE(writes_norms({}, ""));
+    EXPECT_FALSE(writes_norms({{"norms", "false"}}, ""));
+    EXPECT_TRUE(writes_norms({}, "v.s_host"));
+    EXPECT_FALSE(writes_norms({{"norms", "false"}}, "v.s_host"));
+    EXPECT_TRUE(writes_norms({{"field_pattern", "s_*"}}, ""));
+
+    doris::config::inverted_index_skip_norms_for_variant = true;
+    EXPECT_TRUE(writes_norms({}, ""));
+    EXPECT_FALSE(writes_norms({}, "v.s_host"));
+    EXPECT_FALSE(writes_norms({{"norms", "true"}}, "v.s_host"));
+    EXPECT_FALSE(writes_norms({{"field_pattern", "s_*"}}, ""));
+
+    doris::config::inverted_index_skip_norms_for_variant = original_skip_norms_for_variant;
+}
+
 TEST(SniiDocIdSinkGrowth, AppendRangeGrowsGeometrically) {
     std::vector<uint32_t> docids;
     doris::snii::query::VectorDocIdSink sink(docids);