From cc097fb7d12ce8fc80b355285b0347515a2d17f2 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 19 Jun 2026 23:05:53 +0800 Subject: [PATCH 1/4] branch-4.2 [fix](NestedColumnPruning) Rewrite the NestedColumnPruning meta-path handling to properly eliminate redundant NULL/OFFSET access paths and prevent unsafe BE reader mode combinations. (#64535) 1. Map-star path normalization (normalizeMapValueMetaOnlyAccessPaths) - Rewrites [m, *, OFFSET] -> [m, KEYS] + [m, VALUES, OFFSET], and [m, *, NULL] -> [m, KEYS] + [m, VALUES, NULL] - Unified OFFSET and NULL normalization via collectMapValueMetaOnlyAccessPaths(String metaSuffix) 2. Meta path stripping (MetaPathStriper) - Two-level approach: - Level 1 (same-prefix): [prefix] > [prefix, OFFSET] > [prefix, NULL] - Level 2 (deeper-prefix): deeper path covers shallower meta path, merged into single stripMetaPathsByDeeperPrefix(metaSuffix) function - Type-aware */VALUES/KEYS equivalence for map columns via compareMetaPathPrefixCoverage - Depth guard prevents same-depth cross-type from being removed in Level 2 - Supplemental KEYS path handling when removing OFFSET paths 3. MV fragment meta path handling - skipMetaPath flag in AccessPathExpressionCollector -- MV fragments skip meta path collection at the source, falling through to default visitor for data-only access branch-4.2 adaptations: - master-only FE access-path model types are mapped onto this branch's thrift types (ColumnAccessPath -> TColumnAccessPath, ColumnAccessPathType -> TAccessPathType, AccessPathInfo path components read via getAccessPathList). - Kept branch-4.2's segment-wise path comparator (comparePathSegments) and the variant terminal branch in DataTypeAccessTree.setAccessByPath. - Kept branch-4.2's struct_element(...) spelling in the ported unit tests. Co-authored-by: Claude --- .../AccessPathExpressionCollector.java | 41 +- .../rewrite/AccessPathPlanCollector.java | 13 +- .../rules/rewrite/MetaPathStriper.java | 300 +++++++ .../rules/rewrite/NestedColumnPruning.java | 822 ++++-------------- .../rules/rewrite/PruneNestedColumnTest.java | 202 ++++- .../column_pruning/null_column_pruning.groovy | 17 +- .../string_length_column_pruning.groovy | 53 +- 7 files changed, 732 insertions(+), 716 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java index 74a9934e249d7e..a325fab81455ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java @@ -84,15 +84,17 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor { private StatementContext statementContext; private boolean bottomPredicate; + private boolean skipMetaPath; private Multimap slotToAccessPaths; private Stack> nameToLambdaArguments = new Stack<>(); public AccessPathExpressionCollector( StatementContext statementContext, Multimap slotToAccessPaths, - boolean bottomPredicate) { + boolean bottomPredicate, boolean skipMetaPath) { this.statementContext = statementContext; this.slotToAccessPaths = slotToAccessPaths; this.bottomPredicate = bottomPredicate; + this.skipMetaPath = skipMetaPath; } public void collect(Expression expression) { @@ -199,10 +201,27 @@ public Void visitLength(Length length, CollectorContext context) { // length() only needs the offset array, not the chars data. // Add ACCESS_STRING_OFFSET as a suffix so the path builder accumulates // e.g. ["str_col", "OFFSET"] or ["c_struct", "f3", "OFFSET"]. - if (arg.getDataType().isStringLikeType() && context.accessPathBuilder.isEmpty()) { + // + // CHAR is excluded: CHAR(N) is stored padded to N bytes per row (see BE + // OlapColumnDataConvertorChar::clone_and_padding), so the per-row length + // information available without reading the chars buffer is the padded + // length (always N), not the logical post-trim length expected by + // length(). There is no way to recover the logical length from offsets + // alone — the chars buffer must be scanned with strnlen() (BE + // shrink_padding_chars). Falling through to the default visit causes + // length() to read the column normally, which is correct. + // NOTE: arg.getDataType() is the resolved type at the leaf of any + // chained access (struct field, map subscript, array index), so this + // single check covers nested CHAR cases too. + if (arg.getDataType().isStringLikeType() && !arg.getDataType().isCharType() + && context.accessPathBuilder.isEmpty()) { + if (skipMetaPath) { + return arg.accept(this, + new CollectorContext(context.statementContext, false)); + } CollectorContext offsetContext = new CollectorContext(context.statementContext, context.bottomFilter); - offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_OFFSET); return arg.accept(this, offsetContext); } // fall through to default (recurse into children with fresh contexts) @@ -214,9 +233,13 @@ public Void visitMapSize(MapSize mapSize, CollectorContext context) { Expression arg = mapSize.child(); DataType argType = arg.getDataType(); if (argType.isMapType() && context.accessPathBuilder.isEmpty()) { + if (skipMetaPath) { + return arg.accept(this, + new CollectorContext(context.statementContext, false)); + } CollectorContext offsetContext = new CollectorContext(context.statementContext, context.bottomFilter); - offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_OFFSET); return arg.accept(this, offsetContext); } return visit(mapSize, context); @@ -229,9 +252,13 @@ public Void visitCardinality(Cardinality cardinality, CollectorContext context) // Arrays and maps share the same offset-array + data storage layout as strings on the BE. DataType argType = arg.getDataType(); if ((argType.isArrayType() || argType.isMapType()) && context.accessPathBuilder.isEmpty()) { + if (skipMetaPath) { + return arg.accept(this, + new CollectorContext(context.statementContext, false)); + } CollectorContext offsetContext = new CollectorContext(context.statementContext, context.bottomFilter); - offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_OFFSET); // cardinality(map_keys(m)) == cardinality(m) == cardinality(map_values(m)): // all three count map entries, so emit the same [map_col, OFFSET] path. Expression effectiveArg = (arg instanceof MapKeys || arg instanceof MapValues) @@ -558,6 +585,10 @@ public Void visitIsNull(IsNull isNull, CollectorContext context) { // and nested access (struct_element(s, 'city') IS NULL → [s, city, NULL]). // For unrecognized expressions, the default visitor resets context, safely discarding NULL. if (arg.nullable() && context.accessPathBuilder.isEmpty()) { + if (skipMetaPath) { + return arg.accept(this, + new CollectorContext(context.statementContext, false)); + } CollectorContext nullContext = new CollectorContext(context.statementContext, context.bottomFilter); nullContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_NULL); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java index f3e92216afc57f..adb5e5eebe12d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java @@ -67,6 +67,11 @@ public class AccessPathPlanCollector extends DefaultPlanVisitor { private Multimap allSlotToAccessPaths = LinkedHashMultimap.create(); private Map> scanSlotToAccessPaths = new LinkedHashMap<>(); + private boolean skipMetaPath; + + public void setSkipMetaPath(boolean skipMetaPath) { + this.skipMetaPath = skipMetaPath; + } public Map> collect(Plan root, StatementContext context) { root.accept(this, context); @@ -86,7 +91,7 @@ public Void visitLogicalGenerate(LogicalGenerate generate, State List output = generate.getGeneratorOutput(); AccessPathExpressionCollector exprCollector - = new AccessPathExpressionCollector(context, allSlotToAccessPaths, false); + = new AccessPathExpressionCollector(context, allSlotToAccessPaths, false, skipMetaPath); for (int i = 0; i < output.size(); i++) { Slot generatorOutput = output.get(i); Function function = generators.get(i); @@ -240,7 +245,7 @@ public Void visitLogicalGenerate(LogicalGenerate generate, State @Override public Void visitLogicalProject(LogicalProject project, StatementContext context) { AccessPathExpressionCollector exprCollector - = new AccessPathExpressionCollector(context, allSlotToAccessPaths, false); + = new AccessPathExpressionCollector(context, allSlotToAccessPaths, false, skipMetaPath); for (NamedExpression output : project.getProjects()) { // e.g. select element_at(s, 'city') from (select s from tbl)a; // we will not treat the inner `s` access all path @@ -402,7 +407,7 @@ private void collectByExpressions(Plan plan, StatementContext context) { private void collectByExpressions(Plan plan, StatementContext context, boolean bottomPredicate) { AccessPathExpressionCollector exprCollector - = new AccessPathExpressionCollector(context, allSlotToAccessPaths, bottomPredicate); + = new AccessPathExpressionCollector(context, allSlotToAccessPaths, bottomPredicate, skipMetaPath); for (Expression expression : plan.getExpressions()) { exprCollector.collect(expression); } @@ -434,6 +439,6 @@ private static boolean isDataSkippingOnlyAccessPath(List path) { } String lastComponent = path.get(path.size() - 1); return AccessPathInfo.ACCESS_NULL.equals(lastComponent) - || AccessPathInfo.ACCESS_STRING_OFFSET.equals(lastComponent); + || AccessPathInfo.ACCESS_OFFSET.equals(lastComponent); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java new file mode 100644 index 00000000000000..8f08699ec232d3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.ColumnAccessPathType; +import org.apache.doris.common.Pair; + +import com.google.common.collect.Multimap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Strips redundant metadata-only (NULL/OFFSET) access paths using pure + * string-prefix comparison. Map-level wildcards must already have been + * expanded into {@code KEYS} + {@code VALUES} by the caller. + * + *

Single entry point: {@link #strip(int, Multimap, Multimap)}. + */ +public final class MetaPathStriper { + + private MetaPathStriper() {} + + /** + * Strip redundant metadata-only NULL/OFFSET paths using pure string-prefix + * comparison, keeping enough real paths for BE readers to avoid + * OFFSET_ONLY / NULL_MAP_ONLY modes that skip required child data. + * + *

Stripping is organised in two levels: + * + *

Level 1 — Same-prefix priority: when two paths share the same + * prefix and differ only in the final meta suffix, the higher-priority one + * eliminates the lower. + * 

{@code
+     *   Data  >  OFFSET  >  NULL
+     * }
+ *
    + *
  • {@code Data} strips {@code OFFSET}: {@code [a]} strips {@code [a, OFFSET]}.
    • + *
  • {@code Data} strips {@code NULL}: {@code [a]} strips {@code [a, NULL]}.
    • + *
  • {@code OFFSET} strips {@code NULL}: {@code [a, OFFSET]} strips + * {@code [a, NULL]}.
    • + *
+ * + *

Level 2 — Deeper-prefix coverage: when a covering path goes + * deeper into the type tree, its data reader already materialises the + * container, making a shallower meta-only path redundant. + *

    + *
  • Target suffix {@code OFFSET}, covered by deeper: + *
      + *
    • {@code Data}: {@code [a, *, field]} strips {@code [a, OFFSET]}.
      • + *
    • {@code OFFSET}: {@code [a, *, OFFSET]} strips {@code [a, OFFSET]}.
      • + *
    • {@code NULL}: {@code [a, *, NULL]} strips {@code [a, OFFSET]}.
      • + *
    + *
    • + *
  • Target suffix {@code NULL}, covered by deeper: + *
      + *
    • {@code Data}: {@code [a, b, c]} strips {@code [a, b, NULL]}.
      • + *
    • {@code OFFSET}: {@code [a, *, OFFSET]} strips {@code [a, NULL]}.
      • + *
    • {@code NULL}: {@code [a, *, NULL]} strips {@code [a, NULL]}.
      • + *
    + *
    • + *
+ * + *

Pre-condition: map-level {@code *} wildcards must already have been + * expanded into {@code KEYS} + {@code VALUES} by the caller. This class + * uses pure string-prefix comparison and is type-unaware. + */ + public static void strip( + int slotId, + Multimap>> targetAccessPaths, + Multimap>> coveringAccessPaths) { + stripExactPrefixCoveredMetaPaths(slotId, targetAccessPaths, coveringAccessPaths); + stripNullBySamePrefixOffset(slotId, targetAccessPaths); + + stripMetaPathsByDeeperPrefix(slotId, AccessPathInfo.ACCESS_OFFSET, + targetAccessPaths, coveringAccessPaths); + stripMetaPathsByDeeperPrefix(slotId, AccessPathInfo.ACCESS_NULL, + targetAccessPaths, coveringAccessPaths); + } + + // ======================================================================== + // Path helpers + // ======================================================================== + + private static boolean isMetaPath(List path) { + if (path.isEmpty()) { + return false; + } + String lastComponent = path.get(path.size() - 1); + return AccessPathInfo.ACCESS_NULL.equals(lastComponent) + || AccessPathInfo.ACCESS_OFFSET.equals(lastComponent); + } + + private static List> collectPaths( + Collection>> a, + Collection>> b, boolean meta) { + List> result = new ArrayList<>(); + for (Pair> p : a) { + if (!p.second.isEmpty() && isMetaPath(p.second) == meta) { + result.add(p.second); + } + } + for (Pair> p : b) { + if (!p.second.isEmpty() && isMetaPath(p.second) == meta) { + result.add(p.second); + } + } + return result; + } + + private static boolean isPrefixCovered(List prefix, List coveringPath) { + if (coveringPath.isEmpty()) { + return true; + } + int minLen = Math.min(prefix.size(), coveringPath.size()); + for (int i = 0; i < minLen; i++) { + if (!prefix.get(i).equals(coveringPath.get(i))) { + return false; + } + } + return true; + } + + // ======================================================================== + // Level 1 — same-prefix priority + // ======================================================================== + + /** + * {@code [prefix]} strips {@code [prefix, OFFSET]} and {@code [prefix, NULL]}. + */ + private static void stripExactPrefixCoveredMetaPaths( + int slotId, + Multimap>> targetAccessPaths, + Multimap>> coveringAccessPaths) { + Collection>> targetPaths = targetAccessPaths.get(slotId); + if (targetPaths.isEmpty()) { + return; + } + + List> fullAccessPaths = collectPaths( + coveringAccessPaths.get(slotId), targetPaths, false); + + List>> pathsToRemove = new ArrayList<>(); + for (Pair> p : targetPaths) { + List path = p.second; + if (!isMetaPath(path)) { + continue; + } + List prefix = path.subList(0, path.size() - 1); + for (List fullAccessPath : fullAccessPaths) { + if (isPrefixCovered(prefix, fullAccessPath) + && prefix.size() >= fullAccessPath.size()) { + pathsToRemove.add(p); + break; + } + } + } + targetPaths.removeAll(pathsToRemove); + } + + /** + * {@code [prefix, OFFSET]} strips {@code [prefix, NULL]}. + */ + private static void stripNullBySamePrefixOffset( + int slotId, Multimap>> allAccessPaths) { + Collection>> slotPaths = allAccessPaths.get(slotId); + if (slotPaths.isEmpty()) { + return; + } + + List>> toRemove = new ArrayList<>(); + for (Pair> p : slotPaths) { + List path = p.second; + if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { + continue; + } + List prefix = path.subList(0, path.size() - 1); + for (Pair> q : slotPaths) { + List other = q.second; + if (other == path || other.isEmpty()) { + continue; + } + if (other.size() != path.size() + || !AccessPathInfo.ACCESS_OFFSET.equals(other.get(other.size() - 1))) { + continue; + } + List otherPrefix = other.subList(0, other.size() - 1); + if (isPrefixCovered(prefix, otherPrefix)) { + toRemove.add(p); + break; + } + } + } + slotPaths.removeAll(toRemove); + } + + // ======================================================================== + // Level 2 — deeper-prefix coverage + // ======================================================================== + + private static void stripMetaPathsByDeeperPrefix( + int slotId, String metaSuffix, + Multimap>> targetAccessPaths, + Multimap>> coveringAccessPaths) { + Collection>> targetPaths = targetAccessPaths.get(slotId); + if (targetPaths.isEmpty()) { + return; + } + Collection>> coveringPaths = + coveringAccessPaths.get(slotId); + + List> dataPaths = collectPaths(coveringPaths, targetPaths, false); + stripMetaByDeeperDataPaths(slotId, targetAccessPaths, dataPaths, metaSuffix); + + List> metaPaths = collectPaths(coveringPaths, targetPaths, true); + stripMetaByDeeperMetaPaths(slotId, metaSuffix, metaPaths, targetAccessPaths); + } + + /** + * Strip target meta paths covered by a data path. + */ + private static void stripMetaByDeeperDataPaths( + int slotId, + Multimap>> targetAccessPaths, + List> dataPaths, String metaSuffix) { + Collection>> targetPaths = + targetAccessPaths.get(slotId); + if (targetPaths.isEmpty() || dataPaths.isEmpty()) { + return; + } + + List>> toRemove = new ArrayList<>(); + for (Pair> p : new ArrayList<>(targetPaths)) { + List path = p.second; + if (path.isEmpty() || !metaSuffix.equals(path.get(path.size() - 1))) { + continue; + } + List prefix = path.subList(0, path.size() - 1); + for (List dataPath : dataPaths) { + if (isPrefixCovered(prefix, dataPath)) { + toRemove.add(p); + break; + } + } + } + targetPaths.removeAll(toRemove); + } + + /** + * Strip target meta paths covered by a strictly deeper meta path. + */ + private static void stripMetaByDeeperMetaPaths( + int slotId, String metaSuffix, + List> coveringPaths, + Multimap>> targetAccessPaths) { + Collection>> targetPaths = + targetAccessPaths.get(slotId); + if (targetPaths.isEmpty() || coveringPaths.isEmpty()) { + return; + } + + List>> toRemove = new ArrayList<>(); + for (Pair> p : targetPaths) { + List targetPath = p.second; + if (targetPath.isEmpty() || !metaSuffix.equals(targetPath.get(targetPath.size() - 1))) { + continue; + } + List targetPrefix = targetPath.subList(0, targetPath.size() - 1); + for (List coveringPath : coveringPaths) { + if (coveringPath == targetPath || coveringPath.isEmpty()) { + continue; + } + if (coveringPath.size() - 1 <= targetPath.size() - 1) { + continue; + } + if (isPrefixCovered(targetPrefix, coveringPath)) { + toRemove.add(p); + break; + } + } + } + targetPaths.removeAll(toRemove); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index 8056ebd747c214..4518a77a3c8355 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -93,9 +93,13 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) { return plan; } AccessPathPlanCollector collector = new AccessPathPlanCollector(); - Map> slotToAccessPaths = collector.collect(plan, statementContext); - Map slotToResult = pruneDataType(slotToAccessPaths, + // MV rewrite fragments must not be perturbed by metadata-only paths + // (NULL/OFFSET) that would otherwise prune the slot to a narrower type. + // Skip collecting them at the source instead of working around them downstream. + collector.setSkipMetaPath( jobContext.getCascadesContext().isMaterializedViewRewritePlanFragment()); + Map> slotToAccessPaths = collector.collect(plan, statementContext); + Map slotToResult = pruneDataType(slotToAccessPaths); if (!slotToResult.isEmpty()) { Map slotIdToPruneType = Maps.newLinkedHashMap(); @@ -205,8 +209,7 @@ private static boolean expressionContainsNullCheck(Expression expr) { } private static Map pruneDataType( - Map> slotToAccessPaths, - boolean skipDataSkippingOnlyAccessPath) { + Map> slotToAccessPaths) { Map result = new LinkedHashMap<>(); Map slotIdToAllAccessTree = new LinkedHashMap<>(); Map slotIdToPredicateAccessTree = new LinkedHashMap<>(); @@ -244,19 +247,6 @@ private static Map pruneDataType( } continue; } - if (skipDataSkippingOnlyAccessPath - && containsDataSkippingOnlyAccessPath(collectAccessPathResults)) { - // An MV rewrite child context optimizes a temporary plan fragment rather - // than the final plan. A nested metadata-only path such as - // [s, city, NULL] or [s, city, OFFSET] would otherwise prune the scan slot - // to only that nested field, while the final MV rewritten plan may still - // reuse the same slot as a full complex value or need another child. Drop - // access-info for the whole slot instead of just removing that path: - // predicate expressions inside this fragment still reference the original - // slot shape, so partial pruning after deleting the predicate-only path - // could make the fragment itself inconsistent. - continue; - } for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { List path = collectAccessPathResult.getPath(); TAccessPathType pathType = collectAccessPathResult.getType(); @@ -278,105 +268,30 @@ && containsDataSkippingOnlyAccessPath(collectAccessPathResults)) { } } - // second: build non-predicate access paths + // phase 1.5: for slots with meta paths, expand map-star paths and strip + // redundant meta paths. Strip predicate first using the COMPLETE + // allAccessPaths as covering, then strip allAccessPaths self-covering. for (Entry kv : slotIdToAllAccessTree.entrySet()) { Slot slot = kv.getKey(); DataTypeAccessTree accessTree = kv.getValue(); - DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType()); - - if (slot.getDataType().isStringLikeType()) { - if (accessTree.hasStringOffsetOnlyAccess()) { - if (skipDataSkippingOnlyAccessPath) { - continue; - } - // Offset-only access (e.g. length(str_col)): type stays varchar, - // but we must still send the access path to BE so it skips the char data. - stripExactCoveredDataSkippingSuffixPaths(slot, allAccessPaths, allAccessPaths); - stripNullSuffixPaths(slot, allAccessPaths); - List allPaths = buildColumnAccessPaths(slot, allAccessPaths); - result.put(slot.getExprId().asInt(), - new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); - } else if (accessTree.hasNullCheckOnlyAccess()) { - if (skipDataSkippingOnlyAccessPath) { - continue; - } - // Null-check-only access (e.g. str_col IS NULL): type stays varchar, - // but we send [col, NULL] access path so BE only reads the null flag. - List allPaths = buildColumnAccessPaths(slot, allAccessPaths); - result.put(slot.getExprId().asInt(), - new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); - } - // direct access (accessAll=true) or other: skip — no type change, no access paths needed. - continue; - } - - if ((slot.getDataType().isArrayType() || slot.getDataType().isMapType()) - && accessTree.hasStringOffsetOnlyAccess()) { - if (skipDataSkippingOnlyAccessPath) { - continue; - } - // Offset-only access (e.g. length(arr_col) / length(map_col)): type stays unchanged, - // but we must send the OFFSET access path to BE so it skips element/key-value data. - List allPaths = buildColumnAccessPaths(slot, allAccessPaths); - result.put(slot.getExprId().asInt(), - new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); - continue; - } - - // Null-check-only access (e.g. col IS NULL / col IS NOT NULL): type stays unchanged, - // but we must send the [col, NULL] access path to BE so it only reads the null flag. - if (accessTree.hasNullCheckOnlyAccess()) { - if (skipDataSkippingOnlyAccessPath) { - continue; - } - List allPaths = buildColumnAccessPaths(slot, allAccessPaths); - result.put(slot.getExprId().asInt(), - new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); + if (!accessTree.hasOffsetPath() && !accessTree.hasNullPath()) { continue; } + int slotId = slot.getExprId().asInt(); + // Expand both sets before stripping so covering is complete. + expandMapStarPaths(slot, allAccessPaths); + expandMapStarPaths(slot, predicateAccessPaths); + MetaPathStriper.strip(slotId, predicateAccessPaths, allAccessPaths); + MetaPathStriper.strip(slotId, allAccessPaths, allAccessPaths); + } - if (slot.getDataType().isMapType() && accessTree.hasMapValueOffsetOnlyAccess()) { - if (skipDataSkippingOnlyAccessPath) { - continue; - } - // length(map_col['key']): keys read in full (element lookup) + values offset-only. - // Emit [col, KEYS] and [col, VALUES, OFFSET] directly instead of the collected - // [col, *, OFFSET] path which the BE cannot interpret for split key/value access. - String colName = slot.getName().toLowerCase(); - TDataAccessPath keysDataPath = new TDataAccessPath(); - keysDataPath.setPath( - new ArrayList<>(ImmutableList.of(colName, AccessPathInfo.ACCESS_MAP_KEYS))); - TColumnAccessPath keysColumnPath = new TColumnAccessPath(TAccessPathType.DATA); - keysColumnPath.setDataAccessPath(keysDataPath); - - TDataAccessPath valsOffsetDataPath = new TDataAccessPath(); - valsOffsetDataPath.setPath(new ArrayList<>(ImmutableList.of( - colName, AccessPathInfo.ACCESS_MAP_VALUES, AccessPathInfo.ACCESS_STRING_OFFSET))); - TColumnAccessPath valsOffsetColumnPath = new TColumnAccessPath(TAccessPathType.DATA); - valsOffsetColumnPath.setDataAccessPath(valsOffsetDataPath); - - result.put(slot.getExprId().asInt(), new AccessPathInfo( - slot.getDataType(), - ImmutableList.of(keysColumnPath, valsOffsetColumnPath), - new ArrayList<>())); - continue; - } - - // If a field is read in full, its metadata-only NULL/OFFSET access is redundant - // for any data type: e.g. [s] covers both [s.NULL] and [s.OFFSET]. - stripExactCoveredDataSkippingSuffixPaths(slot, allAccessPaths, allAccessPaths); - - // Strip OFFSET-suffix paths when a non-OFFSET path covers the same nested field or - // container. The overlapping array/map container may live under the root slot itself - // or under a nested struct field, so compare against the actual nested prefix instead - // of gating this logic on the root slot type. - stripCoveredOffsetSuffixPaths(slot, allAccessPaths, allAccessPaths); - - // Strip NULL-suffix paths when a non-NULL path also exists for the same slot. - // E.g. `SELECT col FROM t WHERE col IS NULL` — full data is needed, NULL path is redundant. - stripNullSuffixPaths(slot, allAccessPaths); + // second: build non-predicate access paths + for (Entry kv : slotIdToAllAccessTree.entrySet()) { + Slot slot = kv.getKey(); + DataTypeAccessTree accessTree = kv.getValue(); + DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType()); List allPaths = buildColumnAccessPaths(slot, allAccessPaths); - if (shouldSkipAccessInfo(slot, prunedDataType, allPaths, predicateAccessPaths)) { + if (shouldSkipAccessInfo(slot, prunedDataType, allPaths)) { continue; } result.put(slot.getExprId().asInt(), @@ -390,13 +305,9 @@ && containsDataSkippingOnlyAccessPath(collectAccessPathResults)) { new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); } - // third: build predicate access path + // third: build predicate access path (strip already done in phase 1.5) for (Entry kv : slotIdToPredicateAccessTree.entrySet()) { Slot slot = kv.getKey(); - stripExactCoveredDataSkippingSuffixPaths(slot, predicateAccessPaths, allAccessPaths); - stripCoveredOffsetSuffixPaths(slot, predicateAccessPaths, allAccessPaths); - stripCoveredArrayNullSuffixPaths(slot, predicateAccessPaths, allAccessPaths); - stripNullSuffixPaths(slot, predicateAccessPaths); List predicatePaths = buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); @@ -422,444 +333,6 @@ && containsDataSkippingOnlyAccessPath(collectAccessPathResults)) { return result; } - private static boolean containsDataSkippingOnlyAccessPath( - List collectAccessPathResults) { - for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { - if (isDataSkippingOnlyAccessPath(collectAccessPathResult.getPath())) { - return true; - } - } - return false; - } - - private static boolean isDataSkippingOnlyAccessPath(List path) { - if (path.isEmpty()) { - return false; - } - String lastComponent = path.get(path.size() - 1); - return AccessPathInfo.ACCESS_NULL.equals(lastComponent) - || AccessPathInfo.ACCESS_STRING_OFFSET.equals(lastComponent); - } - - /** - * Decide whether an OFFSET-suffix path can be removed because another non-OFFSET path - * already covers the same container. - * - *

For map element_at paths, {@code *} means "read keys fully, then follow the rest of - * the path on the value side". So a VALUES path can cover the value-side OFFSET access, - * but it does NOT cover the key lookup requirement. In that case we remove the OFFSET path - * and add a KEYS-only path instead. - */ - private static OffsetPathRewrite analyzeOffsetPathRewrite( - DataType slotType, List path, List> nonOffsetPaths) { - if (path.isEmpty() - || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) { - return OffsetPathRewrite.keep(); - } - List prefix = path.subList(0, path.size() - 1); - return analyzePrefixCoverage(slotType, prefix, nonOffsetPaths); - } - - private static OffsetPathRewrite analyzePrefixCoverage( - DataType slotType, List prefix, List> nonOffsetPaths) { - List> supplementalPaths = new ArrayList<>(); - for (List nonOffset : nonOffsetPaths) { - OffsetPathRewrite candidate = compareOffsetPrefixCoverage(slotType, prefix, nonOffset); - if (!candidate.shouldRemoveOffsetPath()) { - continue; - } - if (candidate.getSupplementalPaths().isEmpty()) { - return OffsetPathRewrite.remove(); - } - supplementalPaths.addAll(candidate.getSupplementalPaths()); - } - if (supplementalPaths.isEmpty()) { - return OffsetPathRewrite.keep(); - } - return OffsetPathRewrite.rewriteWithSupplementalPaths(supplementalPaths); - } - - /** - * Remove OFFSET-only paths from {@code targetAccessPaths} when data paths in - * {@code coveringAccessPaths} already read the same array/map/string container or a child - * under it. - * - *

Examples: - *

    - *
  • {@code [arr.OFFSET, arr.*.field]} becomes {@code [arr.*.field]} because the array - * child read must keep BE on the normal data iterator path.
    • - *
  • {@code [map.*.OFFSET, map.VALUES]} becomes {@code [map.KEYS, map.VALUES]} because - * {@code map['k']} still needs full keys for lookup, while values cover the offset.
    • - *
- */ - private static void stripCoveredOffsetSuffixPaths( - Slot slot, Multimap>> targetAccessPaths, - Multimap>> coveringAccessPaths) { - int slotId = slot.getExprId().asInt(); - Collection>> targetPaths = targetAccessPaths.get(slotId); - if (targetPaths.isEmpty()) { - return; - } - - List> nonOffsetPaths = new ArrayList<>(); - for (Pair> p : coveringAccessPaths.get(slotId)) { - List path = p.second; - if (path.isEmpty() - || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) { - nonOffsetPaths.add(path); - } - } - for (Pair> p : targetPaths) { - List path = p.second; - if (path.isEmpty() - || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) { - nonOffsetPaths.add(path); - } - } - - List>> pathsToRemove = new ArrayList<>(); - List>> pathsToAdd = new ArrayList<>(); - for (Pair> p : new ArrayList<>(targetPaths)) { - OffsetPathRewrite rewrite = analyzeOffsetPathRewrite( - slot.getDataType(), p.second, nonOffsetPaths); - if (!rewrite.shouldRemoveOffsetPath()) { - continue; - } - pathsToRemove.add(p); - for (List supplementalPath : rewrite.getSupplementalPaths()) { - pathsToAdd.add(Pair.of(p.first, supplementalPath)); - } - } - targetPaths.removeAll(pathsToRemove); - targetPaths.addAll(pathsToAdd); - } - - /** - * Remove array NULL-only paths from {@code targetAccessPaths} when another path already reads - * the same array container or data under it. This mirrors OFFSET coverage because an array - * element/data read must not be combined with an array NULL_MAP_ONLY read for the same prefix. - * - *

Examples: - *

    - *
  • {@code [map.VALUES.NULL, map.VALUES.*.field]} becomes - * {@code [map.VALUES.*.field]}.
    • - *
  • {@code [map.*.NULL, map.VALUES.*.field]} becomes - * {@code [map.KEYS, map.VALUES.*.field]} so map lookup keys are still available.
    • - *
- */ - private static void stripCoveredArrayNullSuffixPaths( - Slot slot, Multimap>> targetAccessPaths, - Multimap>> coveringAccessPaths) { - int slotId = slot.getExprId().asInt(); - Collection>> targetPaths = targetAccessPaths.get(slotId); - if (targetPaths.isEmpty()) { - return; - } - - List> nonNullPaths = new ArrayList<>(); - for (Pair> p : coveringAccessPaths.get(slotId)) { - List path = p.second; - if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { - nonNullPaths.add(path); - } - } - for (Pair> p : targetPaths) { - List path = p.second; - if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { - nonNullPaths.add(path); - } - } - - List>> pathsToRemove = new ArrayList<>(); - List>> pathsToAdd = new ArrayList<>(); - for (Pair> p : new ArrayList<>(targetPaths)) { - List path = p.second; - if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { - continue; - } - List prefix = path.subList(0, path.size() - 1); - Optional prefixType = dataTypeAtPath(slot.getDataType(), prefix); - if (!prefixType.isPresent() || !prefixType.get().isArrayType()) { - continue; - } - OffsetPathRewrite rewrite = analyzePrefixCoverage(slot.getDataType(), prefix, nonNullPaths); - if (!rewrite.shouldRemoveOffsetPath()) { - continue; - } - pathsToRemove.add(p); - for (List supplementalPath : rewrite.getSupplementalPaths()) { - pathsToAdd.add(Pair.of(p.first, supplementalPath)); - } - } - targetPaths.removeAll(pathsToRemove); - targetPaths.addAll(pathsToAdd); - } - - /** - * Remove exact metadata-only NULL/OFFSET paths when the same field is read in full. - * This rule is type-agnostic: once {@code s} itself is accessed, {@code s.NULL} and - * {@code s.OFFSET} are redundant and unsafe to keep with the full data path. - * - *

Examples: - *

    - *
  • {@code [str_col, str_col.NULL]} becomes {@code [str_col]}.
    • - *
  • {@code [arr, arr.OFFSET]} becomes {@code [arr]}.
    • - *
  • {@code [map.*, map.*.OFFSET]} becomes {@code [map.*]}.
    • - *
- */ - private static void stripExactCoveredDataSkippingSuffixPaths( - Slot slot, Multimap>> targetAccessPaths, - Multimap>> coveringAccessPaths) { - int slotId = slot.getExprId().asInt(); - Collection>> targetPaths = targetAccessPaths.get(slotId); - if (targetPaths.isEmpty()) { - return; - } - - List> fullAccessPaths = new ArrayList<>(); - for (Pair> p : coveringAccessPaths.get(slotId)) { - if (!isDataSkippingOnlyAccessPath(p.second)) { - fullAccessPaths.add(p.second); - } - } - for (Pair> p : targetPaths) { - if (!isDataSkippingOnlyAccessPath(p.second)) { - fullAccessPaths.add(p.second); - } - } - - List>> pathsToRemove = new ArrayList<>(); - for (Pair> p : targetPaths) { - List path = p.second; - if (!isDataSkippingOnlyAccessPath(path)) { - continue; - } - List prefix = path.subList(0, path.size() - 1); - for (List fullAccessPath : fullAccessPaths) { - if (pathCoversPrefix(fullAccessPath, prefix)) { - pathsToRemove.add(p); - break; - } - } - } - targetPaths.removeAll(pathsToRemove); - } - - private static Optional dataTypeAtPath(DataType slotType, List path) { - if (path.isEmpty()) { - return Optional.empty(); - } - DataType currentType = slotType; - for (int i = 1; i < path.size(); i++) { - String component = path.get(i); - if (currentType.isStructType()) { - StructField field = ((StructType) currentType).getField(component); - if (field == null) { - return Optional.empty(); - } - currentType = field.getDataType(); - } else if (currentType.isArrayType()) { - if (!AccessPathInfo.ACCESS_ALL.equals(component)) { - return Optional.empty(); - } - currentType = ((ArrayType) currentType).getItemType(); - } else if (currentType.isMapType()) { - currentType = descendMapType((MapType) currentType, component); - } else { - return Optional.empty(); - } - } - return Optional.of(currentType); - } - - private static OffsetPathRewrite compareOffsetPrefixCoverage( - DataType slotType, List prefix, List nonOffset) { - if (nonOffset.isEmpty()) { - return OffsetPathRewrite.remove(); - } - int minLen = Math.min(prefix.size(), nonOffset.size()); - List> supplementalPaths = new ArrayList<>(); - DataType currentType = slotType; - for (int i = 0; i < minLen; i++) { - String prefixComponent = prefix.get(i); - String nonOffsetComponent = nonOffset.get(i); - if (i == 0) { - if (!prefixComponent.equals(nonOffsetComponent)) { - return OffsetPathRewrite.keep(); - } - continue; - } - if (currentType.isStructType()) { - if (!prefixComponent.equals(nonOffsetComponent)) { - return OffsetPathRewrite.keep(); - } - StructField field = ((StructType) currentType).getField(prefixComponent); - if (field == null) { - return OffsetPathRewrite.keep(); - } - currentType = field.getDataType(); - continue; - } - if (currentType.isArrayType()) { - if (!prefixComponent.equals(nonOffsetComponent) - || !AccessPathInfo.ACCESS_ALL.equals(prefixComponent)) { - return OffsetPathRewrite.keep(); - } - currentType = ((ArrayType) currentType).getItemType(); - continue; - } - if (currentType.isMapType()) { - MapType mapType = (MapType) currentType; - if (prefixComponent.equals(nonOffsetComponent)) { - currentType = descendMapType(mapType, prefixComponent); - continue; - } - if (AccessPathInfo.ACCESS_ALL.equals(prefixComponent) - && AccessPathInfo.ACCESS_MAP_VALUES.equals(nonOffsetComponent)) { - supplementalPaths.add(buildMapKeysOnlyPath(prefix, i)); - currentType = mapType.getValueType(); - continue; - } - if (AccessPathInfo.ACCESS_MAP_VALUES.equals(prefixComponent) - && AccessPathInfo.ACCESS_ALL.equals(nonOffsetComponent)) { - currentType = mapType.getValueType(); - continue; - } - if (AccessPathInfo.ACCESS_MAP_KEYS.equals(prefixComponent) - && AccessPathInfo.ACCESS_ALL.equals(nonOffsetComponent)) { - currentType = mapType.getKeyType(); - continue; - } - return OffsetPathRewrite.keep(); - } - if (!prefixComponent.equals(nonOffsetComponent)) { - return OffsetPathRewrite.keep(); - } - } - if (supplementalPaths.isEmpty()) { - return OffsetPathRewrite.remove(); - } - return OffsetPathRewrite.rewriteWithSupplementalPaths(supplementalPaths); - } - - private static DataType descendMapType(MapType mapType, String component) { - if (AccessPathInfo.ACCESS_MAP_KEYS.equals(component)) { - return mapType.getKeyType(); - } - return mapType.getValueType(); - } - - private static List buildMapKeysOnlyPath(List prefix, int mapTokenIndex) { - List keyPath = new ArrayList<>(prefix.subList(0, mapTokenIndex)); - keyPath.add(AccessPathInfo.ACCESS_MAP_KEYS); - return keyPath; - } - - private static final class OffsetPathRewrite { - private static final OffsetPathRewrite KEEP = new OffsetPathRewrite(false, ImmutableList.of()); - private static final OffsetPathRewrite REMOVE = new OffsetPathRewrite(true, ImmutableList.of()); - - private final boolean removeOffsetPath; - private final List> supplementalPaths; - - private OffsetPathRewrite(boolean removeOffsetPath, List> supplementalPaths) { - this.removeOffsetPath = removeOffsetPath; - this.supplementalPaths = supplementalPaths; - } - - private static OffsetPathRewrite keep() { - return KEEP; - } - - private static OffsetPathRewrite remove() { - return REMOVE; - } - - private static OffsetPathRewrite rewriteWithSupplementalPaths(List> supplementalPaths) { - return new OffsetPathRewrite(true, ImmutableList.copyOf(supplementalPaths)); - } - - private boolean shouldRemoveOffsetPath() { - return removeOffsetPath; - } - - private List> getSupplementalPaths() { - return supplementalPaths; - } - } - - /** - * Strip NULL-suffix paths that are redundant because a non-NULL path reads child - * data below the same prefix or reads an OFFSET path over the same prefix. - * - *

Examples: - *

    - *
  • {@code [struct_col.NULL, struct_col.city]} becomes {@code [struct_col.city]}.
    • - *
  • {@code [str_col.NULL, str_col.OFFSET]} becomes {@code [str_col.OFFSET]} because - * the offset read can provide nullness for variable-length columns.
    • - *
- * - *

A parent NULL path must also be removed when any child path is required under the - * same prefix, e.g. [struct_col, NULL] with [struct_col, city]. This looks like the - * parent null map may still be useful for predicates, but it cannot be kept in - * allAccessPaths with the current BE iterator contract: Struct/Array/Map iterators - * treat a leading NULL sub-path as NULL_MAP_ONLY and skip all children. If FE kept - * [struct_col.NULL, struct_col.city] in allAccessPaths, BE would read only the - * struct null map and default-fill city instead of routing the city child iterator. - * When the NULL path is removed from allAccessPaths, it must also be removed from - * predicateAccessPaths so the BE can rely on predicate paths being a subset of all - * paths. The normal nullable container read materializes the parent null map - * together with required children. - */ - private static void stripNullSuffixPaths( - Slot slot, Multimap>> allAccessPaths) { - int slotId = slot.getExprId().asInt(); - Collection>> slotPaths = allAccessPaths.get(slotId); - - List>> toRemove = new ArrayList<>(); - for (Pair> p : slotPaths) { - List path = p.second; - if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { - continue; - } - // Prefix is the column/subcolumn path without the trailing NULL suffix. - // A non-NULL path that equals this prefix means the same column/subcolumn - // is read in full, making the NULL-only path redundant. - // An OFFSET-suffix path over the same prefix is also enough for the BE to - // derive null-ness for variable-length columns, so [col.NULL] is redundant - // when [col.OFFSET] already exists. - List prefix = path.subList(0, path.size() - 1); - boolean covered = false; - for (Pair> q : slotPaths) { - List other = q.second; - if (other.isEmpty() - || AccessPathInfo.ACCESS_NULL.equals(other.get(other.size() - 1))) { - continue; - } - if (other.equals(prefix)) { - covered = true; - break; - } - if (hasStrictPrefix(other, prefix)) { - covered = true; - break; - } - if (other.size() == prefix.size() + 1 - && AccessPathInfo.ACCESS_STRING_OFFSET.equals(other.get(other.size() - 1)) - && other.subList(0, prefix.size()).equals(prefix)) { - covered = true; - break; - } - } - if (covered) { - toRemove.add(p); - } - } - for (Pair> r : toRemove) { - allAccessPaths.remove(slotId, r); - } - } - /** * Keep predicate access paths as a subset of final all access paths after NULL/OFFSET cleanup. * Predicate paths are built from filter expressions first, but later all-path rewrites may drop @@ -889,14 +362,6 @@ private static void retainPredicatePathsInFinalAllAccessPaths( predicatePaths.removeAll(toRemove); } - private static boolean hasStrictPrefix(List path, List prefix) { - return path.size() > prefix.size() && path.subList(0, prefix.size()).equals(prefix); - } - - private static boolean pathCoversPrefix(List path, List prefix) { - return prefix.size() >= path.size() && prefix.subList(0, path.size()).equals(path); - } - private static List buildColumnAccessPaths( Slot slot, Multimap>> accessPaths) { List paths = new ArrayList<>(); @@ -955,17 +420,17 @@ private static int comparePathSegments(List left, List right) { } private static boolean shouldSkipAccessInfo( - Slot slot, DataType prunedDataType, List allPaths, - Multimap>> predicateAccessPaths) { + Slot slot, DataType prunedDataType, List allPaths) { if (!prunedDataType.equals(slot.getDataType())) { return false; } if (slot.getDataType() instanceof NestedColumnPrunable || slot.getDataType().isVariantType()) { return false; } - if (!predicateAccessPaths.get(slot.getExprId().asInt()).isEmpty()) { - return false; - } + + // Only scalar / string-like types reach here (NestedColumnPrunable and Variant + // returned false above). A single [col] path means the entire column is read; + // no access info needs to be sent to BE. if (allPaths.size() != 1) { return false; } @@ -989,15 +454,18 @@ public static class DataTypeAccessTree { // if access 's.a.b' the node 's' and 'a' has accessPartialChild, and node 'b' has accessAll private boolean accessPartialChild; private boolean accessAll; - // True when this string-typed node is accessed ONLY via the offset array - // (e.g. length(str_col) or length(element_at(c_struct,'f3'))). - // When this flag is set and accessAll is NOT set, pruneDataType() returns BigIntType - // to signal that the BE only needs to read the offset array, not the chars data. - private boolean isStringOffsetOnly; - // True when this column node is accessed ONLY via IS NULL / IS NOT NULL. - // When this flag is set and accessAll is NOT set, the BE only needs to read the null flag, - // not the actual column data. - private boolean isNullCheckOnly; + // Cached marker set by setAccessByPath() when a path component is OFFSET. + // Avoids scanning the multimap: hasStringOffsetOnlyAccess() reads this flag in + // O(1) instead of checking every path for an OFFSET suffix. Used for array, map, + // and string-like types (they share offset-based storage in BE). + // When set without accessAll, pruneDataType() keeps the node's type so that BE + // reads only the offset structure, skipping element / key-value / chars data. + private boolean hasOffsetPath; + // Cached marker set by setAccessByPath() when a path component is NULL. + // Same purpose as hasOffsetPath — O(1) flag read instead of multimap scan + // in hasNullCheckOnlyAccess(). When set without accessAll, BE reads only the + // null bitmap, skipping actual column data. + private boolean hasNullPath; // for the future, only access the meta of the column, // e.g. `is not null` can only access the column's offset, not need to read the data private TAccessPathType pathType; @@ -1040,80 +508,33 @@ public Map getChildren() { } /** - * True when a MAP column is accessed as {@code length(map_col['key'])}: the keys must - * be read in full (for the element lookup) while the values only need the offset array - * (since only their length, not their content, is used). - * Expected access paths: [col, KEYS] and [col, VALUES, OFFSET]. + * recursively search in the tree, if any node hasOffsetPath */ - public boolean hasMapValueOffsetOnlyAccess() { - if (!isRoot) { - return false; - } - DataTypeAccessTree child = children.values().iterator().next(); - if (!child.type.isMapType() || child.accessAll) { - return false; - } - DataTypeAccessTree keysChild = child.children.get(AccessPathInfo.ACCESS_MAP_KEYS); - DataTypeAccessTree valsChild = child.children.get(AccessPathInfo.ACCESS_MAP_VALUES); - // Keys must be fully accessed (element-at lookup). - if (!keysChild.accessAll) { - return false; - } - // Values must be accessed offset-only (no deeper element reads). - if (!valsChild.isStringOffsetOnly || valsChild.accessAll) { - return false; - } - if (valsChild.type.isStringLikeType()) { - // String value: accessAll check above is sufficient. + public boolean hasOffsetPath() { + if (hasOffsetPath) { return true; } - if (valsChild.type.isArrayType()) { - // Array value (e.g. MAP>): verify no element was read directly - // (e.g. map_col['k'][0] would set allChild.accessAll=true). - DataTypeAccessTree allChild = valsChild.children.get(AccessPathInfo.ACCESS_ALL); - return !allChild.accessAll && !allChild.accessPartialChild; - } - return true; - } - - /** True when the column is accessed ONLY via the offset array (e.g. length(str_col), - * length(arr_col), length(map_col)), meaning the type must not change but an access - * path still needs to be sent to BE so it can skip the char/element data. */ - public boolean hasStringOffsetOnlyAccess() { - if (isRoot) { - DataTypeAccessTree child = children.values().iterator().next(); - if (!child.isStringOffsetOnly || child.accessAll) { - return false; - } - if (child.type.isStringLikeType()) { + for (DataTypeAccessTree child : children.values()) { + if (child.hasOffsetPath()) { return true; } - if (child.type.isArrayType()) { - // True only if no element was accessed (element_at / explode etc.) - DataTypeAccessTree allChild = child.children.get(AccessPathInfo.ACCESS_ALL); - return !allChild.accessAll && !allChild.accessPartialChild; - } - if (child.type.isMapType()) { - // True only if neither keys nor values were accessed directly - DataTypeAccessTree keysChild = child.children.get(AccessPathInfo.ACCESS_MAP_KEYS); - DataTypeAccessTree valsChild = child.children.get(AccessPathInfo.ACCESS_MAP_VALUES); - return !keysChild.accessAll && !keysChild.accessPartialChild - && !valsChild.accessAll && !valsChild.accessPartialChild; - } - return false; } - return type.isStringLikeType() && isStringOffsetOnly && !accessAll; + return false; } - /** True when the column is accessed ONLY via IS NULL / IS NOT NULL, - * meaning the BE only needs to read the null flag, not the actual data. */ - public boolean hasNullCheckOnlyAccess() { - if (isRoot) { - DataTypeAccessTree child = children.values().iterator().next(); - return child.isNullCheckOnly && !child.accessAll - && !child.isStringOffsetOnly && !child.accessPartialChild; + /** + * recursively search in the tree, if any node hasNullPath + */ + public boolean hasNullPath() { + if (hasNullPath) { + return true; } - return isNullCheckOnly && !accessAll && !isStringOffsetOnly && !accessPartialChild; + for (DataTypeAccessTree child : children.values()) { + if (child.hasNullPath()) { + return true; + } + } + return false; } /** pruneCastType */ @@ -1211,7 +632,7 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType // Mark null-check-only and return without setting accessAll or accessPartialChild, // so that parent nodes can distinguish "null-only leaf" from "has real sub-access". if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_NULL)) { - isNullCheckOnly = true; + hasNullPath = true; return; } @@ -1228,9 +649,9 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType } return; } else if (this.type.isArrayType()) { - if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { + if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_OFFSET)) { // length(array_col) — only the offset array is needed, not element data. - isStringOffsetOnly = true; + hasOffsetPath = true; return; } DataTypeAccessTree child = children.get(AccessPathInfo.ACCESS_ALL); @@ -1241,9 +662,9 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType return; } else if (this.type.isMapType()) { String fieldName = path.get(accessIndex); - if (fieldName.equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { + if (fieldName.equals(AccessPathInfo.ACCESS_OFFSET)) { // length(map_col) — only the offset array is needed, not key/value data. - isStringOffsetOnly = true; + hasOffsetPath = true; return; } if (fieldName.equals(AccessPathInfo.ACCESS_ALL)) { @@ -1279,8 +700,8 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType } else if (type.isStringLikeType()) { // String leaf accessed via the offset array (e.g. path ends in "offset"). // Mark offset-only so pruneDataType() can return BigIntType instead of full data. - if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { - isStringOffsetOnly = true; + if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_OFFSET)) { + hasOffsetPath = true; return; // do NOT set accessAll — offset-only is distinguishable from full access } // Any other sub-path on a string column means full data is needed. @@ -1323,12 +744,11 @@ public Optional pruneDataType() { return children.values().iterator().next().pruneDataType(); } else if (accessAll) { return Optional.of(type); - } else if (isStringOffsetOnly && !accessPartialChild) { + } else if (hasOffsetPath && !accessPartialChild) { // Only the offset array is accessed (e.g. length(str_col)). - // The slot type stays unchanged (varchar); the access path tells BE to skip char data. - return Optional.empty(); - } else if (isNullCheckOnly && !accessPartialChild) { - // Only the null flag is accessed (e.g. col IS NULL / struct_element(s,'f') IS NULL). + return Optional.of(type); + } else if (hasNullPath && !accessPartialChild) { + // Only the null flag is accessed (e.g. col IS NULL / element_at(s,'f') IS NULL). // Return the node's type so that parent nodes include this child in their pruned type, // while the access path (ending in NULL) tells BE to skip actual data reading. return Optional.of(type); @@ -1390,4 +810,104 @@ private DataType pruneDataType(DataType dataType, List> n } } } + + /** + * Expand map-level {@code *} wildcards into {@code KEYS} + {@code VALUES} + * variants. For n map-level stars in a single path, n+1 paths are + * emitted: one all-VALUES path plus one KEYS-terminating path per star + * position. Array-level stars are left unchanged. + * + *

Paths with no map-level star are kept as-is. + */ + private static void expandMapStarPaths( + Slot slot, + Multimap>> accessPaths) { + int slotId = slot.getExprId().asInt(); + Collection>> slotPaths = accessPaths.get(slotId); + if (slotPaths.isEmpty()) { + return; + } + DataType slotType = slot.getDataType(); + + List>> toAdd = new ArrayList<>(); + List>> toRemove = new ArrayList<>(); + + for (Pair> p : slotPaths) { + List path = p.second; + List positions = new ArrayList<>(); + findMapStarPositions(path, slotType, positions); + if (positions.isEmpty()) { + continue; + } + toRemove.add(p); + toAdd.addAll(expandOnePath(p.first, path, positions)); + } + + slotPaths.removeAll(toRemove); + slotPaths.addAll(toAdd); + } + + private static void findMapStarPositions( + List path, DataType slotType, List positions) { + DataType current = slotType; + for (int i = 1; i < path.size(); i++) { + String component = path.get(i); + if (current.isStructType()) { + StructField field = ((StructType) current).getField(component); + if (field == null) { + break; + } + current = field.getDataType(); + } else if (current.isArrayType()) { + if (!AccessPathInfo.ACCESS_ALL.equals(component)) { + break; + } + current = ((ArrayType) current).getItemType(); + } else if (current.isMapType()) { + MapType mapType = (MapType) current; + if (AccessPathInfo.ACCESS_ALL.equals(component)) { + positions.add(i); + current = mapType.getValueType(); + } else if (AccessPathInfo.ACCESS_MAP_KEYS.equals(component)) { + current = mapType.getKeyType(); + } else if (AccessPathInfo.ACCESS_MAP_VALUES.equals(component)) { + current = mapType.getValueType(); + } else { + current = mapType.getValueType(); + } + } else { + break; + } + } + } + + private static List>> expandOnePath( + TAccessPathType type, List path, List positions) { + int n = positions.size(); + List>> result = new ArrayList<>(n + 1); + + // All-VALUES path: replace every map * with VALUES + List allValues = new ArrayList<>(path); + for (int pos : positions) { + allValues.set(pos, AccessPathInfo.ACCESS_MAP_VALUES); + } + result.add(Pair.of(type, allValues)); + + // KEYS-terminating path for each position + for (int i = 0; i < n; i++) { + int keysPos = positions.get(i); + List keysPath = new ArrayList<>(); + for (int j = 0; j < keysPos; j++) { + String component = path.get(j); + if (positions.contains(j)) { + component = AccessPathInfo.ACCESS_MAP_VALUES; + } + keysPath.add(component); + } + keysPath.add(AccessPathInfo.ACCESS_MAP_KEYS); + result.add(Pair.of(type, keysPath)); + } + + return result; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index cd0e08327b29cc..7748e9c2b46e03 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -204,6 +204,53 @@ public void testCardinalityArrayElementKeepsOffsetPath() throws Exception { ImmutableList.of(path("a", "*"))); } + @Test + public void testDeeperOffsetPathCoversShallowerOffsetPath() throws Exception { + // cardinality(a) on ARRAY> generates [a, OFFSET] + // cardinality(element_at(a, 1)) generates [a, *, OFFSET] + // [a, *, OFFSET] traverses into the outer array's items, so it must read + // through the outer array structure. The shallower [a, OFFSET] is therefore + // redundant and should be stripped. + assertAllAccessPathsContain( + "select cardinality(a), cardinality(element_at(a, 1)) from nested_array_tbl", + ImmutableList.of(path("a", "*", "OFFSET")), + ImmutableList.of(path("a", "OFFSET"))); + } + + @Test + public void testOffsetPathCoversNullPathWithSamePrefix() throws Exception { + assertAllAccessPathsContain( + "select cardinality(a), a is null from nested_array_tbl", + ImmutableList.of(path("a", "OFFSET")), + ImmutableList.of(path("a", "NULL"))); + } + + @Test + public void testDeeperNullPathCoversShallowerNullPath() throws Exception { + // a IS NULL on ARRAY> generates [a, NULL] + // element_at(a, 1) IS NULL generates [a, *, NULL] + // [a, *, NULL] traverses into the outer array's items, which requires + // reading the outer array's null bitmap. Therefore [a, NULL] is redundant + // and should be stripped. + assertAllAccessPathsContain( + "select a is null, element_at(a, 1) is null from nested_array_tbl", + ImmutableList.of(path("a", "*", "NULL")), + ImmutableList.of(path("a", "NULL"))); + } + + @Test + public void testDeeperOffsetPathCoversShallowerNullPathForArray() throws Exception { + // cardinality(element_at(a, 1)) on ARRAY> generates [a, *, OFFSET] + // a IS NULL generates [a, NULL] + // [a, *, OFFSET] reads the offset of inner-array elements, which requires + // reading through the outer array's null bitmap. Therefore [a, NULL] is + // redundant and should be stripped. + assertAllAccessPathsContain( + "select cardinality(element_at(a, 1)), a is null from nested_array_tbl", + ImmutableList.of(path("a", "*", "OFFSET")), + ImmutableList.of(path("a", "NULL"))); + } + @Test public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception { assertColumn("select cardinality(map_arr_col['a']) from map_array_tbl", @@ -212,6 +259,103 @@ public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception { ImmutableList.of()); } + @Test + public void testDeeperNullCoversValueOffsetForMapArray() throws Exception { + // cardinality(map_arr_col['a']) -> normalized to KEYS + VALUES.OFFSET + // element_at(map_arr_col['a'], 1) IS NULL -> *.*.NULL + // *.*.NULL goes deeper into the value-side array, so VALUES.OFFSET + // at the map value level is redundant. Without type-aware comparison + // * is not lexically equal to VALUES, so VALUES.OFFSET would survive + // and cause BE to skip the item iterator. + assertAllAccessPathsContain( + "select cardinality(map_arr_col['a']), " + + "element_at(map_arr_col['a'], 1) is null from map_array_tbl", + ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES", "*", "NULL")), + ImmutableList.of(path("map_arr_col", "VALUES", "OFFSET"))); + } + + @Test + public void testDataPathCoversNullPathWithMapAwareComparison() throws Exception { + // element_at(map_col, 'a') -> [map_col, KEYS] + [map_col, VALUES] + // element_at(map_values(map_col), 1) IS NULL -> [map_col, VALUES, NULL] + // Level 1 strips [map_col, VALUES, NULL] because [map_col, VALUES] covers it. + // + // NOT: map_values(map_col) IS NULL — visitMapValues special-cases a lone NULL + // suffix as a parent-map null check (isFunctionNullCheckPath), producing + // [map_col, NULL] instead of [map_col, VALUES, NULL]. + assertAllAccessPathsContain( + "select element_at(map_col, 'a') from str_tbl" + + " where element_at(map_values(map_col), 1) is null", + ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), + ImmutableList.of(path("map_col", "VALUES", "NULL"))); + } + + @Test + public void testDataPathCoversOffsetPathWithMapAwareComparison() throws Exception { + // element_at(map_col, 'a') -> [map_col, KEYS] + [map_col, VALUES] + // length(element_at(map_values(map_col), 1)) > 0 -> [map_col, VALUES, OFFSET] + // Level 2 strips [map_col, VALUES, OFFSET] because [map_col, VALUES] covers it. + // + // NOT: cardinality(map_values(map_col)) — visitCardinality unwraps MapValues + // and produces [map_col, OFFSET] instead of [map_col, VALUES, OFFSET]. + assertAllAccessPathsContain( + "select element_at(map_col, 'a') from str_tbl" + + " where length(element_at(map_values(map_col), 1)) > 0", + ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), + ImmutableList.of(path("map_col", "VALUES", "OFFSET"))); + } + + @Test + public void testMapValuesCoversStarNullPreservesKeysPath() throws Exception { + // After map-* normalization: + // element_at(map_col, 'a') -> [map_col, KEYS] + [map_col, VALUES] + // element_at(map_col, 'a') IS NULL -> [map_col, VALUES, NULL] + [map_col, KEYS] + // element_at(map_values(map_col), 1) -> [map_col, VALUES] + // Level 1 strips [map_col, VALUES, NULL] because [map_col, VALUES] covers it. + // The KEYS path is preserved for key lookup. + assertAllAccessPathsContain( + "select element_at(map_values(map_col), 1) from str_tbl" + + " where element_at(map_col, 'a') is null", + ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), + ImmutableList.of(path("map_col", "VALUES", "NULL"))); + } + + @Test + public void testSupplementalKeyPathShouldStripExistingKeyNullPath() throws Exception { + // After map-* normalization: + // projection: [map_arr_col, VALUES] + // filter: [map_arr_col, KEYS, NULL] + // OR [map_arr_col, *, *, NULL] → [map_arr_col, VALUES, *, NULL] + // + [map_arr_col, KEYS] + // Level 1 strips [map_arr_col, KEYS, NULL] because [map_arr_col, KEYS] now + // exists before the strip phase (produced by normalization, not supplemental). + // Level 2 strips [map_arr_col, VALUES, *, NULL] because [map_arr_col, VALUES] + // covers the value-side prefix. + assertAllAccessPathsContain( + "select element_at(map_values(map_arr_col), 1) from map_array_tbl" + + " where element_at(map_keys(map_arr_col), 1) is null" + + " or element_at(element_at(map_arr_col, 'a'), 1) is null", + ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES")), + ImmutableList.of(path("map_arr_col", "VALUES", "*", "NULL"), + path("map_arr_col", "KEYS", "NULL"))); + } + + @Test + public void testNestedMapElementLengthKeepsValueOffsetPath() throws Exception { + assertColumn("select length(element_at(element_at(s, 'm'), 'a')) from nested_container_tbl", + "struct>", + ImmutableList.of(path("s", "m", "KEYS"), path("s", "m", "VALUES", "OFFSET")), + ImmutableList.of()); + } + + @Test + public void testNestedMapElementIsNullKeepsValueIsNullPath() throws Exception { + assertColumn("select (element_at(element_at(s, 'm'), 'a')) is null from nested_container_tbl", + "struct>", + ImmutableList.of(path("s", "m", "KEYS"), path("s", "m", "VALUES", "NULL")), + ImmutableList.of()); + } + @Test public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { assertColumn("select struct_element(s, 'city') from tbl " @@ -232,12 +376,13 @@ public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { assertColumn("select cardinality(map_arr_col['a']), map_arr_col['a'] from map_array_tbl", "map>", - ImmutableList.of(path("map_arr_col", "*")), + ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES")), ImmutableList.of()); } @Test public void testCardinalityMapElementOffsetCoveredByValueFieldAccess() throws Exception { + // [s, m, *, *, verified] strips [s, m, *, OFFSET] (pure string prefix). Pair> result = collectComplexSlots( "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') " + "from map_array_value_tbl " @@ -248,13 +393,14 @@ public void testCardinalityMapElementOffsetCoveredByValueFieldAccess() throws Ex allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); } - Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "*", "*", "verified"))); - Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "OFFSET"))); - Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "*", "OFFSET"))); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified"))); + Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "VALUES", "OFFSET"))); + Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "VALUES", "OFFSET"))); } @Test public void testMapElementArrayNullPathCoveredByValueFieldAccess() throws Exception { + // [s, m, *, *, verified] strips [s, m, *, NULL] (pure string prefix). Pair> result = collectComplexSlots( "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') " + "from map_array_value_tbl " @@ -265,9 +411,9 @@ public void testMapElementArrayNullPathCoveredByValueFieldAccess() throws Except allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); } - Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "*", "*", "verified"))); - Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "NULL"))); - Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "*", "NULL"))); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified"))); + Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "VALUES", "NULL"))); + Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "VALUES", "NULL"))); } @Test @@ -691,18 +837,18 @@ public void testFilter() throws Throwable { ); assertColumn("select 100 from tbl where element_at(s, 'data')[1][1] is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "*", "NULL")), - ImmutableList.of(path("s", "data", "*", "*", "NULL")) + ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "NULL")), + ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "NULL")) ); assertColumn("select 100 from tbl where element_at(element_at(s, 'data')[1][1], 'a') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "*", "a", "NULL")), - ImmutableList.of(path("s", "data", "*", "*", "a", "NULL")) + ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "a", "NULL")), + ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "a", "NULL")) ); assertColumn("select 100 from tbl where element_at(element_at(s, 'data')[1][1], 'b') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "*", "b", "NULL")), - ImmutableList.of(path("s", "data", "*", "*", "b", "NULL")) + ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "b", "NULL")), + ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "b", "NULL")) ); } @@ -1373,10 +1519,12 @@ private void assertAllAccessPathsContain(String sql, List exp allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); } for (TColumnAccessPath accessPath : expectContainAllAccessPaths) { - Assertions.assertTrue(allAccessPaths.contains(accessPath)); + Assertions.assertTrue(allAccessPaths.contains(accessPath), + "expected " + accessPath + " but allAccessPaths=" + allAccessPaths); } for (TColumnAccessPath accessPath : expectNotContainAllAccessPaths) { - Assertions.assertFalse(allAccessPaths.contains(accessPath)); + Assertions.assertFalse(allAccessPaths.contains(accessPath), + "expected NOT " + accessPath + " but allAccessPaths=" + allAccessPaths); } } @@ -1572,6 +1720,8 @@ public void testMvRewritePlanFragmentSkipsNullOnlyAccessPath() { new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))), new TreeSet<>(normalSlot.getPredicateAccessPaths().get())); + // MV fragment: IS NULL degrades to full column read via default visitor. + // [str_col] full-access path passes shouldSkipAccessInfo → no pruning. SlotReference fragmentSlot = rewriteAndFindScanSlot( "select 1 from str_tbl where str_col is not null", "str_col", true); assertNoAccessPaths(fragmentSlot); @@ -1585,14 +1735,16 @@ public void testMvRewritePlanFragmentSkipsNullOnlyAccessPath() { new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))), new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get())); - // MV rewrite optimizes temporary fragments whose later consumers are not visible. - // If the fragment only needs nested null metadata, e.g. [s.city.NULL], pruning the - // scan slot to struct can break the final rewritten MV plan when it still - // needs the full struct or another child. The fragment marker therefore suppresses - // nested null-only access info too, not just top-level [col.NULL]. + // MV fragment: IS NULL degrades to element_at via default visitor, + // producing [s, city] data path. struct is NestedColumnPrunable so + // pruning to struct is safe — no meta suffix remains. SlotReference nestedFragmentSlot = rewriteAndFindScanSlot( "select 1 from tbl where struct_element(s, 'city') is not null", "s", true); - assertNoAccessPaths(nestedFragmentSlot); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("s", "city"))), + new TreeSet<>(nestedFragmentSlot.getAllAccessPaths().get())); + Assertions.assertTrue(!nestedFragmentSlot.getPredicateAccessPaths().isPresent() + || nestedFragmentSlot.getPredicateAccessPaths().get().isEmpty()); } @Test @@ -1620,10 +1772,16 @@ public void testMvRewritePlanFragmentSkipsOffsetOnlyAccessPath() { new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))), new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get())); + // MV fragment: length() degrades to element_at via default visitor, + // producing [c_struct, f3] data path without OFFSET suffix. SlotReference nestedFragmentSlot = rewriteAndFindScanSlot( "select 1 from str_tbl where length(struct_element(c_struct, 'f3')) > 0", "c_struct", true); - assertNoAccessPaths(nestedFragmentSlot); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("c_struct", "f3"))), + new TreeSet<>(nestedFragmentSlot.getAllAccessPaths().get())); + Assertions.assertTrue(!nestedFragmentSlot.getPredicateAccessPaths().isPresent() + || nestedFragmentSlot.getPredicateAccessPaths().get().isEmpty()); } /** diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy index cbbeac2935701f..7cba3bc1c40a89 100644 --- a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy +++ b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy @@ -180,9 +180,7 @@ suite("null_column_pruning") { // covers the same prefix and inherently includes the null flag. explain { sql "select int_col from ncp_tbl where int_col is null" - contains "nested columns" - contains "all access paths: [int_col]" - notContains "predicate access paths:" + notContains "nested columns" } order_qt_10 "select int_col from ncp_tbl where int_col is null"; @@ -351,7 +349,15 @@ suite("null_column_pruning") { explain { sql "select count(1) from ncp_tbl where map_col['a'] is null" contains "nested columns" - contains "map_col.*.NULL" + contains "map_col.KEYS" + contains "map_col.VALUES.NULL" + // expectedPlan + // nested columns: + // map_col: + // origin type: map + // all access paths: [map_col.KEYS, map_col.VALUES.NULL] + // predicate access paths: [map_col.KEYS, map_col.VALUES.NULL] + } order_qt_20 "select count(1) from ncp_tbl where map_col['a'] is null"; @@ -360,7 +366,8 @@ suite("null_column_pruning") { explain { sql "select count(1) from ncp_tbl where map_col['a'] is not null" contains "nested columns" - contains "map_col.*.NULL" + contains "map_col.KEYS" + contains "map_col.VALUES.NULL" } order_qt_21 "select count(1) from ncp_tbl where map_col['a'] is not null"; diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy index 8549d128ccd95f..afb83a401bd1d9 100644 --- a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy +++ b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy @@ -71,8 +71,7 @@ suite("string_length_column_pruning") { } sql "select length(str_col) from slcp_str_tbl" - // length(str_col) in IF plus ORDER BY on a plain primitive column: - // only str_col should appear in nested columns, and NULL is redundant when OFFSET exists. + // [str_col, OFFSET] strips [str_col, NULL]. explain { sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id" contains "nested columns" @@ -154,7 +153,7 @@ suite("string_length_column_pruning") { notContains "type=bigint" } - // Full access to the same array field covers its OFFSET metadata for any data type. + // [arr_col] strips [arr_col, OFFSET]. explain { sql "select id, cardinality(arr_col), arr_col from slcp_str_tbl" contains "nested columns" @@ -243,9 +242,7 @@ suite("string_length_column_pruning") { // ─── Map with complex value cases ──────────────────────────────────────────── - // cardinality(map_arr_col['a']): value is ARRAY. - // Keys read in full (element lookup); values need only the OFFSET array (array size). - // Expected paths: map_arr_col.KEYS + map_arr_col.VALUES.OFFSET + // Expected paths: [map_arr_col, KEYS] + [map_arr_col, VALUES, OFFSET] explain { sql "select cardinality(map_arr_col['a']) from slcp_str_tbl" contains "nested columns" @@ -272,20 +269,22 @@ suite("string_length_column_pruning") { notContains "type=bigint" } - // value array item also accessed directly → full VALUES item path covers value OFFSET. + // [map_arr_struct_col, VALUES, *, verified] strips [map_arr_struct_col, VALUES, OFFSET]. explain { sql "select cardinality(map_arr_struct_col['a']), map_arr_struct_col['a'][1].verified from slcp_str_tbl" contains "nested columns" - contains "map_arr_struct_col.*.*.verified" - notContains "map_arr_struct_col.*.OFFSET" + contains "map_arr_struct_col.KEYS" + contains "map_arr_struct_col.VALUES.*.verified" + notContains "map_arr_struct_col.VALUES.OFFSET" notContains "type=bigint" } + // [map_arr_col, VALUES] strips [map_arr_col, VALUES, OFFSET]. explain { sql "select id, cardinality(map_arr_col['a']), map_arr_col['a'] from slcp_str_tbl" contains "nested columns" - contains "all access paths: [map_arr_col.*]" - notContains "map_arr_col.*.OFFSET" + contains "all access paths: [map_arr_col.KEYS, map_arr_col.VALUES]" + notContains "map_arr_col.VALUES.OFFSET" notContains "predicate access paths:" notContains "type=bigint" } @@ -295,15 +294,14 @@ suite("string_length_column_pruning") { order by id """ - // Predicate OFFSET path must also be removed when the projected value field already - // makes the corresponding array data path available. predicateAccessPaths remains a - // subset of allAccessPaths. + // [map_arr_struct_col, VALUES, *, verified] strips [map_arr_struct_col, VALUES, OFFSET]. + // KEYS (data path) remains in predicateAccessPaths. explain { sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where cardinality(map_arr_struct_col['a']) > 0" contains "nested columns" - contains "all access paths: [map_arr_struct_col.*.*.verified]" - notContains "map_arr_struct_col.*.OFFSET" - notContains "predicate access paths:" + contains "all access paths: [map_arr_struct_col.KEYS, map_arr_struct_col.VALUES.*.verified]" + contains "predicate access paths: [map_arr_struct_col.KEYS]" + notContains "map_arr_struct_col.VALUES.OFFSET" notContains "type=bigint" } @@ -313,13 +311,15 @@ suite("string_length_column_pruning") { order by 1 """ - // value array item also accessed directly → full VALUES item path covers value NULL. + // [map_arr_struct_col, VALUES, *, verified] strips [map_arr_struct_col, VALUES, NULL]. + // KEYS (data path) remains in predicateAccessPaths. explain { sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where map_arr_struct_col['a'] is null" contains "nested columns" - contains "map_arr_struct_col.*.*.verified" - notContains "map_arr_struct_col.*.NULL" - notContains "predicate access paths:" + contains "map_arr_struct_col.KEYS" + contains "map_arr_struct_col.VALUES.*.verified" + contains "predicate access paths: [map_arr_struct_col.KEYS]" + notContains "map_arr_struct_col.VALUES.NULL" } // ─── Non-optimizable cases ────────────────────────────────────────────────── @@ -414,8 +414,7 @@ suite("string_length_column_pruning") { notContains "bigint" } - // length(map_col['a']): keys read fully for element lookup, values accessed offset-only. - // Expect access paths: map_col.KEYS (full) + map_col.VALUES.OFFSET + // Expected paths: [map_col, KEYS] + [map_col, VALUES, OFFSET] explain { sql "select length(map_col['a']) from slcp_str_tbl" contains "nested columns" @@ -538,9 +537,7 @@ suite("string_length_column_pruning") { // ─── Map element_at + map_values mixed access ───────────────────────────────── - // length(map_col['a']) needs keys for the element_at lookup and value offsets for length(). - // map_values(map_col)[1] needs full value data. The mixed query must therefore keep a KEYS - // path for element_at lookup while dropping the redundant value-side OFFSET path. + // [map_col, VALUES] strips [map_col, VALUES, OFFSET]. KEYS is kept for element_at lookup. order_qt_map_element_with_map_values """ select length(map_col['a']), map_values(map_col)[1] from slcp_str_tbl """ @@ -554,9 +551,7 @@ suite("string_length_column_pruning") { notContains "bigint" } - // Reverse direction: length(map_values(map_col)[1]) produces [map_col, VALUES, OFFSET] - // while map_col['a'] produces [map_col, *]. The * path reads full values, so OFFSET - // must be suppressed here as well. + // [map_col, VALUES] strips [map_col, VALUES, OFFSET]. explain { sql "select length(map_values(map_col)[1]), map_col['a'] from slcp_str_tbl" notContains "OFFSET" From b0fe572b9241d69fa2065c1f4f5b93d592bbdd89 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Thu, 2 Jul 2026 22:34:35 +0800 Subject: [PATCH 2/4] branch-4.2 [feat](olap) Support lazy reading mode for pruned complex columns (#59263) The subcolumns of a pruned complex-type column can fall into two categories: 1. Predicate columns -- columns required to evaluate filter predicates, which need to be read upfront. 2. Non-predicate columns -- columns that are not needed when evaluating filter predicates. For non-predicate columns, Doris can defer reading them until after predicate evaluation, reducing unnecessary I/O for nested columns. This update also preserves predicate metadata paths separately from final lazy-materialization data paths. Predicate evaluation may still need current-level metadata, such as OFFSET for cardinality()/length() and NULL for IS NULL, even when a covering data path is also needed later. The BE nested column iterators consume those current-level metadata paths at the correct iterator level without forwarding them to child iterators or incorrectly switching mixed data reads into metadata-only mode. This PR also handles predicate-only nested access paths. A complex iterator should skip access-path setup only when both final access paths and predicate access paths are empty. Otherwise predicate-only child or metadata paths may be ignored in lazy read mode, causing predicates such as array element IS NULL to evaluate on unread placeholder data. branch-4.2 adaptations: - gensrc/thrift/PaloInternalService.thrift: master uses id 226 for enable_prune_nested_column, but branch-4.2 already uses 226-228, so the option was moved to the next free id (229). - segment_iterator.cpp keeps branch-4.2's _non_predicate_column_ids naming and drops the master-only _update_lsn_col_if_needed/_update_tso_col_if_needed calls from the ported hunk; branch-4.2 keeps its own common/compile_check_begin.h include in front of the new ScopedColumnIteratorReadPhase helper. - column_reader.cpp keeps branch-4.2's dst->is_nullable()/zone_map.pass_all idioms and its AccessPathInfo/zone-map handling where the ported hunks only differed by master-side refactors. - regression suites lambda_null_pruning and map_contains_arg_pruning do not exist on branch-4.2 and were not added. Co-authored-by: Jerry Hu Co-authored-by: Claude --- be/src/exec/operator/olap_scan_operator.cpp | 2 + be/src/exec/operator/olap_scan_operator.h | 1 + be/src/exec/scan/olap_scanner.cpp | 1 + be/src/runtime/runtime_state.h | 5 + be/src/storage/olap_common.h | 1 + be/src/storage/segment/column_reader.cpp | 892 +++++-- be/src/storage/segment/column_reader.h | 219 +- be/src/storage/segment/segment_iterator.cpp | 128 +- be/src/storage/segment/segment_iterator.h | 7 +- .../segment/variant/variant_column_reader.cpp | 31 +- .../storage/segment/column_reader_test.cpp | 2134 ++++++++++++++++- .../segment_iterator_lazy_pruned_test.cpp | 186 ++ .../nereids/processor/post/TopNScanOpt.java | 21 +- .../rules/rewrite/MetaPathStriper.java | 300 --- .../rules/rewrite/NestedColumnPruning.java | 123 +- .../org/apache/doris/qe/SessionVariable.java | 2 + .../postprocess/TopNRuntimeFilterTest.java | 20 + .../rules/rewrite/PruneNestedColumnTest.java | 300 +-- gensrc/thrift/PaloInternalService.thrift | 2 + .../complex_types/test_pruned_columns.out | 417 +++- .../nested_container_offset_pruning.out | 12 + .../complex_types/test_pruned_columns.groovy | 266 +- .../nested_container_offset_pruning.groovy | 88 +- .../column_pruning/null_column_pruning.groovy | 71 +- .../string_length_column_pruning.groovy | 74 +- 25 files changed, 4244 insertions(+), 1059 deletions(-) create mode 100644 be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 051a8b5348ad5a..3f6e62084cf596 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -217,6 +217,8 @@ Status OlapScanLocalState::_init_profile() { _lazy_read_seek_timer = ADD_TIMER(_segment_profile, "LazyReadSeekTime"); _lazy_read_seek_counter = ADD_COUNTER(_segment_profile, "LazyReadSeekCount", TUnit::UNIT); + _lazy_read_pruned_timer = ADD_TIMER(_segment_profile, "LazyReadPrunedTime"); + _output_col_timer = ADD_TIMER(_segment_profile, "OutputColumnTime"); _stats_filtered_counter = ADD_COUNTER(_segment_profile, "RowsStatsFiltered", TUnit::UNIT); diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 57c4912b777f6b..aab56a084736ba 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -210,6 +210,7 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _lazy_read_timer = nullptr; RuntimeProfile::Counter* _lazy_read_seek_timer = nullptr; RuntimeProfile::Counter* _lazy_read_seek_counter = nullptr; + RuntimeProfile::Counter* _lazy_read_pruned_timer = nullptr; // total pages read // used by segment v2 diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 04ede793ed579c..ef879b96582f62 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -762,6 +762,7 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_predicate_column_read_seek_counter, stats.predicate_column_read_seek_num); COUNTER_UPDATE(local_state->_lazy_read_timer, stats.lazy_read_ns); + COUNTER_UPDATE(local_state->_lazy_read_pruned_timer, stats.lazy_read_pruned_ns); COUNTER_UPDATE(local_state->_lazy_read_seek_timer, stats.block_lazy_read_seek_ns); COUNTER_UPDATE(local_state->_lazy_read_seek_counter, stats.block_lazy_read_seek_num); COUNTER_UPDATE(local_state->_output_col_timer, stats.output_col_ns); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 3184b74a7445c7..f264851b37715a 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -642,6 +642,11 @@ class RuntimeState { _query_options.enable_aggregate_function_null_v2; } + bool enable_prune_nested_column() const { + return _query_options.__isset.enable_prune_nested_column && + _query_options.enable_prune_nested_column; + } + bool is_read_csv_empty_line_as_null() const { return _query_options.__isset.read_csv_empty_line_as_null && _query_options.read_csv_empty_line_as_null; diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index ee200a23c88b79..4b76905d345839 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -335,6 +335,7 @@ struct OlapReaderStatistics { int64_t lazy_read_ns = 0; int64_t block_lazy_read_seek_num = 0; int64_t block_lazy_read_seek_ns = 0; + int64_t lazy_read_pruned_ns = 0; int64_t raw_rows_read = 0; diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 7e2ee007f47474..7a993044f53099 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "common/compiler_util.h" // IWYU pragma: keep @@ -90,6 +91,26 @@ inline bool read_as_string(PrimitiveType type) { type == PrimitiveType::TYPE_BITMAP || type == PrimitiveType::TYPE_FIXED_LENGTH_OBJECT; } +bool is_current_level_meta_access_path(const TColumnAccessPath& path) { + if (path.data_access_path.path.size() != 1) { + return false; + } + const auto& component = path.data_access_path.path[0]; + return StringCaseEqual()(component, ColumnIterator::ACCESS_OFFSET) || + StringCaseEqual()(component, ColumnIterator::ACCESS_NULL); +} + +bool is_current_level_data_access_path(const TColumnAccessPath& path, + const std::string& column_name) { + return path.data_access_path.path.size() == 1 && + StringCaseEqual()(path.data_access_path.path[0], column_name); +} + +void remove_current_level_meta_access_paths(TColumnAccessPaths& paths) { + auto removed = std::ranges::remove_if(paths, is_current_level_meta_access_path); + paths.erase(removed.begin(), removed.end()); +} + Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, const io::FileReaderSPtr& file_reader, std::shared_ptr* reader) { @@ -886,14 +907,45 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator, return Status::OK(); } +void ColumnIterator::_convert_to_place_holder_column(MutableColumnPtr& dst, size_t count) { + if (_read_phase == ReadPhase::LAZY) { + return; + } else if (_read_requirement == ReadRequirement::LAZY_OUTPUT && + _read_phase == ReadPhase::PREDICATE) { + // This branch is for non-predicate columns that still have to appear in the + // predicate-phase block so row filtering can keep all block columns aligned. + // Columns already marked PREDICATE are read normally, and SKIP/NORMAL + // columns do not participate in lazy materialization. + _has_place_holder_column = true; + } + + dst->insert_many_defaults(count); +} + +void ColumnIterator::_recovery_from_place_holder_column(MutableColumnPtr& dst) { + if (_read_phase == ReadPhase::LAZY && _has_place_holder_column) { + dst->clear(); + _has_place_holder_column = false; + } +} + Result ColumnIterator::_get_sub_access_paths( - const TColumnAccessPaths& access_paths) { - TColumnAccessPaths sub_access_paths = access_paths; + TColumnAccessPaths sub_access_paths, bool is_predicate) { + // Access paths passed to a complex iterator always start with the current + // column name. Strip that component and return the remaining child-relative + // paths to the caller. For example, when this iterator is for column `s`, + // path `s.a.b` is converted to `a.b` and then dispatched to child `a`. + // + // If stripping the current column consumes the whole path, the current + // iterator itself is requested rather than one of its children. Mark the + // current iterator according to the path source: predicate paths must be read + // in the predicate phase, while all/output paths become lazy output targets. + // Empty or mismatched paths indicate an FE/BE access-path contract violation. for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) { TColumnAccessPath& name_path = *it; if (name_path.data_access_path.path.empty()) { - return ResultError( - Status::InternalError("Invalid access path for struct column: path is empty")); + return ResultError(Status::InternalError( + "Invalid access path for column '{}': path is empty", _column_name)); } if (!StringCaseEqual()(name_path.data_access_path.path[0], _column_name)) { @@ -906,7 +958,11 @@ Result ColumnIterator::_get_sub_access_paths( if (!name_path.data_access_path.path.empty()) { ++it; } else { - set_need_to_read(); + if (is_predicate) { + set_read_requirement(ReadRequirement::PREDICATE); + } else { + set_lazy_output_requirement(); + } it = sub_access_paths.erase(it); } } @@ -929,7 +985,7 @@ MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr reade } Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (_read_requirement == ReadRequirement::SKIP) { DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -943,7 +999,7 @@ Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) { } Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -961,7 +1017,7 @@ Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) { } RETURN_IF_ERROR(_offsets_iterator->seek_to_ordinal(ord)); if (read_offset_only()) { - // In OFFSET_ONLY mode, key/value iterators are SKIP_READING, no need to seek them + // In OFFSET_ONLY mode, key/value iterators are SKIP, no need to seek them return Status::OK(); } // here to use offset info @@ -985,23 +1041,37 @@ Status MapFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& param void MapFileColumnIterator::collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) { - _offsets_iterator->collect_prefetchers(prefetchers, init_method); + if (!need_to_read()) { + return; + } + if (!read_null_map_only()) { + _offsets_iterator->collect_prefetchers(prefetchers, init_method); + } if (_map_reader->is_nullable()) { _null_iterator->collect_prefetchers(prefetchers, init_method); } + if (read_offset_only() || read_null_map_only()) { + return; + } // the actual data pages to read of key/value column depends on the read result of offset column, // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here. - _key_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); - _val_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + if (_key_iterator->need_to_read()) { + _key_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + } + if (_val_iterator->need_to_read()) { + _val_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + } } Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; - dst->resize(dst->size() + *n); + _convert_to_place_holder_column(dst, *n); return Status::OK(); } + _recovery_from_place_holder_column(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnMap with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1033,14 +1103,35 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* } auto& column_map = assert_cast( - dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); - auto column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); + is_column_nullable(*dst) ? static_cast(*dst).get_nested_column() + : *dst); + const bool read_meta_columns = need_to_read_meta_columns(); + MutableColumnPtr column_offsets_ptr; + if (read_meta_columns) { + column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); + } else { + // The parent offsets were already materialized in the predicate phase, so + // they must not be appended to dst again. We still read offsets into a + // temporary column here: this sequential path may be serving a nested + // lazy read after seek_to_ordinal(), and the storage offsets are needed to + // compute how many key/value elements to read from the current source + // ordinal. The existing dst offsets only describe the filtered output + // shape and do not track the current source ordinal consumed by this + // iterator call. + const auto base_offset = + column_map.get_offsets().empty() ? 0 : column_map.get_offsets().back(); + column_offsets_ptr = ColumnMap::COffsets::create(); + assert_cast(*column_offsets_ptr) + .insert_value(base_offset); + } Defer defer_offsets {[&] { - auto typed_column_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr( - assert_cast( - column_offsets_ptr.get())); - column_offsets_ptr = nullptr; - column_map.get_offsets_ptr() = std::move(typed_column_offsets_ptr); + if (read_meta_columns) { + auto typed_column_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr( + assert_cast( + column_offsets_ptr.get())); + column_offsets_ptr = nullptr; + column_map.get_offsets_ptr() = std::move(typed_column_offsets_ptr); + } }}; bool offsets_has_null = false; ssize_t start = column_offsets_ptr->size(); @@ -1073,7 +1164,7 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* } } - if (dst->is_nullable()) { + if (is_column_nullable(*dst) && read_meta_columns) { size_t num_read = *n; auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); // in not-null to null linked-schemachange mode, @@ -1095,12 +1186,14 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; - dst->resize(count); + if (!need_to_read()) { + DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; + _convert_to_place_holder_column(dst, count); return Status::OK(); } + _recovery_from_place_holder_column(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map by rowids, fill nested ColumnMap with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1128,47 +1221,80 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t if (count == 0) { return Status::OK(); } + // resolve ColumnMap and nullable wrapper auto& column_map = assert_cast( - dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); - auto offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); + is_column_nullable(*dst) ? static_cast(*dst).get_nested_column() + : *dst); + const bool read_meta_columns = need_to_read_meta_columns(); + MutableColumnPtr offsets_ptr; + if (read_meta_columns) { + offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); + } else { + const auto base_offset = + column_map.get_offsets().empty() ? 0 : column_map.get_offsets().back(); + offsets_ptr = ColumnMap::COffsets::create(); + assert_cast(*offsets_ptr) + .insert_value(base_offset); + } Defer defer_offsets {[&] { - auto typed_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr( - assert_cast(offsets_ptr.get())); - offsets_ptr = nullptr; - column_map.get_offsets_ptr() = std::move(typed_offsets_ptr); + if (read_meta_columns) { + auto typed_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr( + assert_cast( + offsets_ptr.get())); + offsets_ptr = nullptr; + column_map.get_offsets_ptr() = std::move(typed_offsets_ptr); + } }}; auto& offsets = static_cast(*offsets_ptr); size_t base = offsets.get_data().empty() ? 0 : offsets.get_data().back(); // 1. bulk read null-map if nullable std::vector null_mask; // 0: not null, 1: null - if (_map_reader->is_nullable()) { - // For nullable map columns, the destination column must also be nullable. - if (UNLIKELY(!dst->is_nullable())) { + if (read_meta_columns) { + if (_map_reader->is_nullable()) { + // For nullable map columns, the destination column must also be nullable. + if (UNLIKELY(!is_column_nullable(*dst))) { + return Status::InternalError( + "unexpected non-nullable destination column for nullable map reader"); + } + MutableColumnPtr null_map_ptr = + static_cast(*dst).get_null_map_column_ptr(); + size_t null_before = null_map_ptr->size(); + RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_ptr)); + // extract a light-weight view to decide element reads + auto& null_map_col = assert_cast(*null_map_ptr); + const auto* src = null_map_col.get_data().data() + null_before; + null_mask.assign(src, src + count); + } else if (is_column_nullable(*dst)) { + // in not-null to null linked-schemachange mode, + // actually we do not change dat data include meta in footer, + // so may dst from changed meta which is nullable but old data is not nullable, + // if so, we should set null_map to all null by default + MutableColumnPtr null_map_ptr = + static_cast(*dst).get_null_map_column_ptr(); + auto& null_map = assert_cast(*null_map_ptr); + null_map.insert_many_vals(0, count); + } + } else if (_map_reader->is_nullable()) { + // In lazy mode the parent null map has already been materialized during + // predicate read and filtered together with the block. Reuse that dst + // null map to avoid re-reading the same meta column from storage. + if (UNLIKELY(!is_column_nullable(*dst))) { return Status::InternalError( "unexpected non-nullable destination column for nullable map reader"); } - auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); - size_t null_before = null_map_ptr->size(); - auto* null_map_col = null_map_ptr.get(); - MutableColumnPtr null_map_column = std::move(null_map_ptr); - RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_column)); - // extract a light-weight view to decide element reads - null_mask.reserve(count); - for (size_t i = 0; i < count; ++i) { - null_mask.push_back(null_map_col->get_element(null_before + i)); - } - } else if (dst->is_nullable()) { - // in not-null to null linked-schemachange mode, - // actually we do not change dat data include meta in footer, - // so may dst from changed meta which is nullable but old data is not nullable, - // if so, we should set null_map to all null by default - auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); - null_map_ptr->insert_many_vals(0, count); + const auto& null_map_col = static_cast(*dst).get_null_map_column(); + DORIS_CHECK(null_map_col.size() == count); + const auto* src = null_map_col.get_data().data(); + null_mask.assign(src, src + count); } - // 2. bulk read start ordinals for requested rows + // 2. Bulk read source start ordinals for requested rows. The offsets stored + // in dst already describe the filtered output shape when read_meta_columns is + // false, but they do not contain the source key/value ordinal for each + // selected rowid. We still need the storage offsets here to seek child + // iterators to the correct source element ranges. MutableColumnPtr starts_col = ColumnOffset64::create(); starts_col->reserve(count); RETURN_IF_ERROR(_offsets_iterator->read_by_rowids(rowids, count, starts_col)); @@ -1208,16 +1334,19 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t auto& next_starts_data = assert_cast(*next_starts_col).get_data(); std::vector sizes(count, 0); size_t acc = base; - const auto original_size = offsets.get_data().back(); - offsets.get_data().reserve(offsets.get_data().size() + count); + if (read_meta_columns) { + offsets.get_data().reserve(offsets.get_data().size() + count); + } for (size_t i = 0; i < count; ++i) { - size_t sz = static_cast(next_starts_data[i] - starts_data[i]); + auto sz = static_cast(next_starts_data[i] - starts_data[i]); if (_map_reader->is_nullable() && !null_mask.empty() && null_mask[i]) { sz = 0; // null rows do not consume elements } sizes[i] = sz; acc += sz; - offsets.get_data().push_back(acc); + if (read_meta_columns) { + offsets.get_data().push_back(acc); + } } // 6. read key/value elements for non-empty sizes @@ -1240,18 +1369,14 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t bool dummy_has_null = false; if (this_run != 0) { - if (_key_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } - - if (_val_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - n = this_run; - RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } + RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); + DCHECK(n == this_run); + + n = this_run; + RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); + DCHECK(n == this_run); } start_idx = start; this_run = sz; @@ -1264,36 +1389,24 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t } size_t n = this_run; - const size_t total_count = offsets.get_data().back() - original_size; bool dummy_has_null = false; - if (_key_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - if (this_run != 0) { - RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } - } else { - keys_ptr->insert_many_defaults(total_count); - } + if (this_run != 0) { + RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); + DCHECK(n == this_run); - if (_val_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - if (this_run != 0) { - n = this_run; - RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } - } else { - vals_ptr->insert_many_defaults(total_count); + n = this_run; + RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); + DCHECK(n == this_run); } - return Status::OK(); } -void MapFileColumnIterator::set_need_to_read() { - set_reading_flag(ReadingFlag::NEED_TO_READ); - _key_iterator->set_need_to_read(); - _val_iterator->set_need_to_read(); +void MapFileColumnIterator::set_lazy_output_requirement() { + set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); + _key_iterator->set_lazy_output_requirement(); + _val_iterator->set_lazy_output_requirement(); } void MapFileColumnIterator::remove_pruned_sub_iterators() { @@ -1303,37 +1416,65 @@ void MapFileColumnIterator::remove_pruned_sub_iterators() { Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { - if (all_access_paths.empty()) { + if (all_access_paths.empty() && predicate_access_paths.empty()) { return Status::OK(); } + const auto requirement_before_access_path = _read_requirement; if (!predicate_access_paths.empty()) { - set_reading_flag(ReadingFlag::READING_FOR_PREDICATE); - DLOG(INFO) << "Map column iterator set sub-column " << _column_name - << " to READING_FOR_PREDICATE"; + set_read_requirement_self(ReadRequirement::PREDICATE); + DLOG(INFO) << "Map column iterator set sub-column " << _column_name << " to PREDICATE"; } + const bool has_current_level_data_path = + std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { + return is_current_level_data_access_path(path, _column_name); + }); auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); - - if (sub_all_access_paths.empty()) { - return Status::OK(); + auto sub_predicate_access_paths = + DORIS_TRY(_get_sub_access_paths(predicate_access_paths, true)); + if (has_current_level_data_path) { + remove_current_level_meta_access_paths(sub_all_access_paths); + } + const bool has_current_level_predicate_meta_path = + std::ranges::any_of(sub_predicate_access_paths, is_current_level_meta_access_path); + // Current-level predicate metadata paths are consumed by this map iterator and must not be + // forwarded to key/value children. The FE keeps all_access_paths as a superset of predicate + // paths, so meta-only mode is still decided from sub_all_access_paths below. + remove_current_level_meta_access_paths(sub_predicate_access_paths); + + if (sub_predicate_access_paths.empty() && _read_requirement == ReadRequirement::PREDICATE && + !has_current_level_predicate_meta_path) { + // if no sub-column in predicate_access_paths, but current column is PREDICATE, + // then we should set key/value iterator to PREDICATE too. + _key_iterator->set_read_requirement(ReadRequirement::PREDICATE); + _val_iterator->set_read_requirement(ReadRequirement::PREDICATE); + } + + if (sub_predicate_access_paths.empty()) { + // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY). Only skip key/value + // iterators when no predicate sub-path needs them in the predicate phase. + _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); + if (read_offset_only()) { + _key_iterator->set_read_requirement(ReadRequirement::SKIP); + _val_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Map column iterator set column " << _column_name + << " to OFFSET_ONLY meta read mode, key/value columns set to SKIP"; + return Status::OK(); + } + if (read_null_map_only()) { + _key_iterator->set_read_requirement(ReadRequirement::SKIP); + _val_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Map column iterator set column " << _column_name + << " to NULL_MAP_ONLY meta read mode, key/value columns set to SKIP"; + return Status::OK(); + } } - // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY) - _check_and_set_meta_read_mode(sub_all_access_paths); - if (read_offset_only()) { - _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Map column iterator set column " << _column_name - << " to OFFSET_ONLY reading mode, key/value columns set to SKIP_READING"; - return Status::OK(); - } - if (read_null_map_only()) { - _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Map column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode, key/value columns set to SKIP_READING"; + // A current-level data path is consumed by _get_sub_access_paths() and leaves + // sub_all_access_paths empty after marking key/value as lazy-read targets. Predicate + // sub-paths still have to be routed to child iterators for the predicate phase. + if (sub_all_access_paths.empty() && sub_predicate_access_paths.empty()) { return Status::OK(); } @@ -1365,9 +1506,6 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc val_all_access_paths.emplace_back(paths); } } - const auto need_read_keys = !key_all_access_paths.empty(); - const auto need_read_values = !val_all_access_paths.empty(); - for (auto paths : sub_predicate_access_paths) { if (paths.data_access_path.path[0] == ACCESS_ALL) { // Same logic as above: key needs full data, value gets the sub-path. @@ -1388,26 +1526,65 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc } } + const auto need_read_keys = + !key_all_access_paths.empty() || !key_predicate_access_paths.empty(); + const auto need_read_values = + !val_all_access_paths.empty() || !val_predicate_access_paths.empty(); + if (need_read_keys) { - _key_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ); RETURN_IF_ERROR( _key_iterator->set_access_paths(key_all_access_paths, key_predicate_access_paths)); + // Apply LAZY_OUTPUT after child predicate paths have been handled. Read requirements are + // monotonic, so a predicate-only child already promoted to PREDICATE will not + // be downgraded, while a non-predicate child becomes a lazy materialization target. + _key_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); } else { - _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Map column iterator set key column to SKIP_READING"; + _key_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Map column iterator set key column to SKIP"; } if (need_read_values) { - _val_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ); RETURN_IF_ERROR( _val_iterator->set_access_paths(val_all_access_paths, val_predicate_access_paths)); + // Same as keys: predicate-only value paths stay PREDICATE because this + // post-processing update cannot lower a stronger child requirement. + _val_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); } else { - _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Map column iterator set value column to SKIP_READING"; + _val_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Map column iterator set value column to SKIP"; } return Status::OK(); } +void MapFileColumnIterator::set_read_phase(ReadPhase mode) { + ColumnIterator::set_read_phase(mode); + _key_iterator->set_read_phase(mode); + _val_iterator->set_read_phase(mode); +} + +void MapFileColumnIterator::finalize_lazy_phase(MutableColumnPtr& dst) { + _recovery_from_place_holder_column(dst); + auto& map_column = assert_cast( + dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + auto keys_ptr = IColumn::mutate(std::move(map_column.get_keys_ptr())); + auto vals_ptr = IColumn::mutate(std::move(map_column.get_values_ptr())); + _key_iterator->finalize_lazy_phase(keys_ptr); + _val_iterator->finalize_lazy_phase(vals_ptr); + map_column.get_keys_ptr() = std::move(keys_ptr); + map_column.get_values_ptr() = std::move(vals_ptr); +} + +void MapFileColumnIterator::set_read_requirement(ReadRequirement requirement) { + set_read_requirement_self(requirement); + _key_iterator->set_read_requirement(requirement); + _val_iterator->set_read_requirement(requirement); +} + +bool MapFileColumnIterator::has_lazy_read_target() const { + return _read_requirement == ReadRequirement::LAZY_OUTPUT || + _key_iterator->has_lazy_read_target() || _val_iterator->has_lazy_read_target(); +} + //////////////////////////////////////////////////////////////////////////////// StructFileColumnIterator::StructFileColumnIterator( @@ -1420,7 +1597,7 @@ StructFileColumnIterator::StructFileColumnIterator( } Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (_read_requirement == ReadRequirement::SKIP) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -1435,12 +1612,14 @@ Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) { } Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; - dst->resize(dst->size() + *n); + _convert_to_place_holder_column(dst, *n); return Status::OK(); } + _recovery_from_place_holder_column(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnStruct with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1484,7 +1663,7 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo DCHECK(num_read == *n); } - if (dst->is_nullable()) { + if (is_column_nullable(*dst) && need_to_read_meta_columns()) { size_t num_read = *n; auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); // in not-null to null linked-schemachange mode, @@ -1506,7 +1685,7 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo } Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -1522,7 +1701,8 @@ Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) { for (auto& column_iterator : _sub_column_iterators) { RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord)); } - if (_struct_reader->is_nullable()) { + + if (_struct_reader->is_nullable() && need_to_read_meta_columns()) { RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); } return Status::OK(); @@ -1541,22 +1721,32 @@ Status StructFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& pa void StructFileColumnIterator::collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) { - for (auto& column_iterator : _sub_column_iterators) { - column_iterator->collect_prefetchers(prefetchers, init_method); + if (!need_to_read()) { + return; } if (_struct_reader->is_nullable()) { _null_iterator->collect_prefetchers(prefetchers, init_method); } + if (read_null_map_only()) { + return; + } + for (auto& column_iterator : _sub_column_iterators) { + if (column_iterator->need_to_read()) { + column_iterator->collect_prefetchers(prefetchers, init_method); + } + } } Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; - dst->resize(count); + _convert_to_place_holder_column(dst, count); return Status::OK(); } + _recovery_from_place_holder_column(dst); + if (count == 0) { return Status::OK(); } @@ -1587,17 +1777,17 @@ Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const siz return Status::OK(); } -void StructFileColumnIterator::set_need_to_read() { - set_reading_flag(ReadingFlag::NEED_TO_READ); +void StructFileColumnIterator::set_lazy_output_requirement() { + set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); for (auto& sub_iterator : _sub_column_iterators) { - sub_iterator->set_need_to_read(); + sub_iterator->set_lazy_output_requirement(); } } void StructFileColumnIterator::remove_pruned_sub_iterators() { for (auto it = _sub_column_iterators.begin(); it != _sub_column_iterators.end();) { auto& sub_iterator = *it; - if (sub_iterator->reading_flag() == ReadingFlag::SKIP_READING) { + if (sub_iterator->read_requirement() == ReadRequirement::SKIP) { DLOG(INFO) << "Struct column iterator remove pruned sub-column " << sub_iterator->column_name(); it = _sub_column_iterators.erase(it); @@ -1611,27 +1801,45 @@ void StructFileColumnIterator::remove_pruned_sub_iterators() { Status StructFileColumnIterator::set_access_paths( const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { - if (all_access_paths.empty()) { + if (all_access_paths.empty() && predicate_access_paths.empty()) { return Status::OK(); } + const auto requirement_before_access_path = _read_requirement; if (!predicate_access_paths.empty()) { - set_reading_flag(ReadingFlag::READING_FOR_PREDICATE); - DLOG(INFO) << "Struct column iterator set sub-column " << _column_name - << " to READING_FOR_PREDICATE"; + set_read_requirement_self(ReadRequirement::PREDICATE); + DLOG(INFO) << "Struct column iterator set sub-column " << _column_name << " to PREDICATE"; } - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); - // Check for NULL_MAP_ONLY mode: only read null map, skip all sub-columns - _check_and_set_meta_read_mode(sub_all_access_paths); - if (read_null_map_only()) { - for (auto& sub_iterator : _sub_column_iterators) { - sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + const bool has_current_level_data_path = + std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { + return is_current_level_data_access_path(path, _column_name); + }); + auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); + auto sub_predicate_access_paths = + DORIS_TRY(_get_sub_access_paths(predicate_access_paths, true)); + if (has_current_level_data_path) { + remove_current_level_meta_access_paths(sub_all_access_paths); + } + const bool has_current_level_predicate_meta_path = + std::ranges::any_of(sub_predicate_access_paths, is_current_level_meta_access_path); + // Current-level predicate metadata paths are consumed by this struct iterator and must not be + // forwarded to child fields. The FE keeps all_access_paths as a superset of predicate paths, so + // NULL_MAP_ONLY is still decided from sub_all_access_paths below. + remove_current_level_meta_access_paths(sub_predicate_access_paths); + + if (sub_predicate_access_paths.empty()) { + // Check for NULL_MAP_ONLY mode: only read null map, skip all sub-columns. + // Do not take this early return when predicate child paths must still be read. + _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); + if (read_null_map_only()) { + for (auto& sub_iterator : _sub_column_iterators) { + sub_iterator->set_read_requirement(ReadRequirement::SKIP); + } + DLOG(INFO) << "Struct column iterator set column " << _column_name + << " to NULL_MAP_ONLY meta read mode, all sub-columns set to SKIP"; + return Status::OK(); } - DLOG(INFO) << "Struct column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode, all sub-columns set to SKIP_READING"; - return Status::OK(); } const auto no_sub_column_to_skip = sub_all_access_paths.empty(); @@ -1639,27 +1847,16 @@ Status StructFileColumnIterator::set_access_paths( for (auto& sub_iterator : _sub_column_iterators) { const auto name = sub_iterator->column_name(); - bool need_to_read = no_sub_column_to_skip; TColumnAccessPaths sub_all_access_paths_of_this; - if (!need_to_read) { + if (!no_sub_column_to_skip) { for (const auto& paths : sub_all_access_paths) { if (paths.data_access_path.path[0] == name) { sub_all_access_paths_of_this.emplace_back(paths); } } - need_to_read = !sub_all_access_paths_of_this.empty(); - } - - if (!need_to_read) { - sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Struct column iterator set sub-column " << name << " to SKIP_READING"; - continue; } - set_reading_flag(ReadingFlag::NEED_TO_READ); - sub_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ); TColumnAccessPaths sub_predicate_access_paths_of_this; - if (!no_predicate_sub_column) { for (const auto& paths : sub_predicate_access_paths) { if (StringCaseEqual()(paths.data_access_path.path[0], name)) { @@ -1668,12 +1865,73 @@ Status StructFileColumnIterator::set_access_paths( } } + // Predicate-only child paths still need to be routed to the child iterator + // even when the child is not requested by ordinary projection access paths. + const bool need_to_read = no_sub_column_to_skip || !sub_all_access_paths_of_this.empty() || + !sub_predicate_access_paths_of_this.empty(); + if (!need_to_read) { + set_read_requirement_self(ReadRequirement::SKIP); + sub_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Struct column iterator set sub-column " << name << " to SKIP"; + continue; + } + + if (no_predicate_sub_column && _read_requirement == ReadRequirement::PREDICATE && + !has_current_level_predicate_meta_path) { + // if no sub-column in predicate_access_paths, but current column is PREDICATE, + // then we should set sub iterator to PREDICATE too. + sub_iterator->set_read_requirement(ReadRequirement::PREDICATE); + } + RETURN_IF_ERROR(sub_iterator->set_access_paths(sub_all_access_paths_of_this, sub_predicate_access_paths_of_this)); + // Set LAZY_OUTPUT after routing child predicate paths. If the child was needed only for + // predicate evaluation, set_access_paths() has already promoted it to + // PREDICATE and this monotonic update will not downgrade it. Otherwise, this + // marks the child as a lazy materialization target. + set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); + sub_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); } return Status::OK(); } +void StructFileColumnIterator::set_read_phase(ReadPhase mode) { + ColumnIterator::set_read_phase(mode); + for (auto& sub_iterator : _sub_column_iterators) { + sub_iterator->set_read_phase(mode); + } +} + +void StructFileColumnIterator::finalize_lazy_phase(MutableColumnPtr& dst) { + _recovery_from_place_holder_column(dst); + auto& column_struct = assert_cast( + dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + + for (size_t i = 0; i < _sub_column_iterators.size(); ++i) { + auto& sub_column = column_struct.get_column_ptr(i); + MutableColumnPtr mutable_sub_column = IColumn::mutate(std::move(sub_column)); + _sub_column_iterators[i]->finalize_lazy_phase(mutable_sub_column); + sub_column = std::move(mutable_sub_column); + } +} + +void StructFileColumnIterator::set_read_requirement(ReadRequirement requirement) { + set_read_requirement_self(requirement); + for (const auto& sub_column_iterator : _sub_column_iterators) { + sub_column_iterator->set_read_requirement(requirement); + } +} + +bool StructFileColumnIterator::has_lazy_read_target() const { + if (_read_requirement == ReadRequirement::LAZY_OUTPUT) { + return true; + } + return std::any_of(_sub_column_iterators.begin(), _sub_column_iterators.end(), + [](const auto& sub_column_iterator) { + return sub_column_iterator->has_lazy_read_target(); + }); +} + //////////////////////////////////////////////////////////////////////////////// Status OffsetFileColumnIterator::init(const ColumnIteratorOptions& opts) { RETURN_IF_ERROR(_offset_iterator->init(opts)); @@ -1758,8 +2016,8 @@ ArrayFileColumnIterator::ArrayFileColumnIterator(std::shared_ptr r } Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) { - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "Array column iterator column " << _column_name << " skip readking."; + if (_read_requirement == ReadRequirement::SKIP) { + DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -1773,7 +2031,7 @@ Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) { Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) { if (read_offset_only()) { - // In OFFSET_ONLY mode, item iterator is SKIP_READING, no need to seek it + // In OFFSET_ONLY mode, item iterator is SKIP, no need to seek it return Status::OK(); } // using offsets info @@ -1784,7 +2042,7 @@ Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) { } Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -1805,12 +2063,16 @@ Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) { } Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; - dst->resize(dst->size() + *n); + if (!need_to_read()) { + DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading, read phase" + << static_cast(_read_phase) + << ", read requirement: " << static_cast(_read_requirement); + _convert_to_place_holder_column(dst, *n); return Status::OK(); } + _recovery_from_place_holder_column(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnArray with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1845,13 +2107,25 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); bool offsets_has_null = false; - auto column_offsets_ptr = std::move(*column_array.get_offsets_ptr()).mutate(); + const bool read_meta_columns = need_to_read_meta_columns(); + MutableColumnPtr column_offsets_ptr; + if (read_meta_columns) { + column_offsets_ptr = IColumn::mutate(std::move(column_array.get_offsets_ptr())); + } else { + const auto base_offset = + column_array.get_offsets().empty() ? 0 : column_array.get_offsets().back(); + column_offsets_ptr = ColumnArray::ColumnOffsets::create(); + assert_cast(*column_offsets_ptr) + .insert_value(base_offset); + } Defer defer_offsets {[&] { - auto typed_column_offsets_ptr = ColumnArray::ColumnOffsets::cast_to_column_mutptr( - assert_cast( - column_offsets_ptr.get())); - column_offsets_ptr = nullptr; - column_array.get_offsets_ptr() = std::move(typed_column_offsets_ptr); + if (read_meta_columns) { + auto typed_column_offsets_ptr = ColumnArray::ColumnOffsets::cast_to_column_mutptr( + assert_cast( + column_offsets_ptr.get())); + column_offsets_ptr = nullptr; + column_array.get_offsets_ptr() = std::move(typed_column_offsets_ptr); + } }}; ssize_t start = column_offsets_ptr->size(); RETURN_IF_ERROR(_offset_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null)); @@ -1877,7 +2151,7 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo } } - if (dst->is_nullable()) { + if (is_column_nullable(*dst) && read_meta_columns) { auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); size_t num_read = *n; // in not-null to null linked-schemachange mode, @@ -1910,23 +2184,35 @@ Status ArrayFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& par void ArrayFileColumnIterator::collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) { - _offset_iterator->collect_prefetchers(prefetchers, init_method); - // the actual data pages to read of item column depends on the read result of offset column, - // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here. - _item_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + if (!need_to_read()) { + return; + } + if (!read_null_map_only()) { + _offset_iterator->collect_prefetchers(prefetchers, init_method); + } if (_array_reader->is_nullable()) { _null_iterator->collect_prefetchers(prefetchers, init_method); } + if (read_offset_only() || read_null_map_only()) { + return; + } + // the actual data pages to read of item column depends on the read result of offset column, + // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here. + if (_item_iterator->need_to_read()) { + _item_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + } } Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; - dst->resize(count); + _convert_to_place_holder_column(dst, count); return Status::OK(); } + _recovery_from_place_holder_column(dst); + for (size_t i = 0; i < count; ++i) { // TODO(cambyszju): now read array one by one, need optimize later RETURN_IF_ERROR(seek_to_ordinal(rowids[i])); @@ -1936,44 +2222,93 @@ Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size return Status::OK(); } -void ArrayFileColumnIterator::set_need_to_read() { - set_reading_flag(ReadingFlag::NEED_TO_READ); - _item_iterator->set_need_to_read(); +void ArrayFileColumnIterator::set_lazy_output_requirement() { + set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); + _item_iterator->set_lazy_output_requirement(); } void ArrayFileColumnIterator::remove_pruned_sub_iterators() { _item_iterator->remove_pruned_sub_iterators(); } +void ArrayFileColumnIterator::set_read_phase(ReadPhase mode) { + ColumnIterator::set_read_phase(mode); + _item_iterator->set_read_phase(mode); +} + +void ArrayFileColumnIterator::finalize_lazy_phase(MutableColumnPtr& dst) { + _recovery_from_place_holder_column(dst); + auto& column_array = assert_cast( + dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + auto item_column_ptr = IColumn::mutate(std::move(column_array.get_data_ptr())); + _item_iterator->finalize_lazy_phase(item_column_ptr); + column_array.get_data_ptr() = std::move(item_column_ptr); +} + +void ArrayFileColumnIterator::set_read_requirement(ReadRequirement requirement) { + set_read_requirement_self(requirement); + _item_iterator->set_read_requirement(requirement); +} + +bool ArrayFileColumnIterator::has_lazy_read_target() const { + return _read_requirement == ReadRequirement::LAZY_OUTPUT || + _item_iterator->has_lazy_read_target(); +} + Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { - if (all_access_paths.empty()) { + if (all_access_paths.empty() && predicate_access_paths.empty()) { return Status::OK(); } + const auto requirement_before_access_path = _read_requirement; if (!predicate_access_paths.empty()) { - set_reading_flag(ReadingFlag::READING_FOR_PREDICATE); - DLOG(INFO) << "Array column iterator set sub-column " << _column_name - << " to READING_FOR_PREDICATE"; + set_read_requirement_self(ReadRequirement::PREDICATE); + DLOG(INFO) << "Array column iterator set sub-column " << _column_name << " to PREDICATE"; } + const bool has_current_level_data_path = + std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { + return is_current_level_data_access_path(path, _column_name); + }); auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); - - // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY) - _check_and_set_meta_read_mode(sub_all_access_paths); - if (read_offset_only()) { - _item_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Array column iterator set column " << _column_name - << " to OFFSET_ONLY reading mode, item column set to SKIP_READING"; - return Status::OK(); - } - if (read_null_map_only()) { - _item_iterator->set_reading_flag(ReadingFlag::SKIP_READING); - DLOG(INFO) << "Array column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode, item column set to SKIP_READING"; - return Status::OK(); + auto sub_predicate_access_paths = + DORIS_TRY(_get_sub_access_paths(predicate_access_paths, true)); + if (has_current_level_data_path) { + // A current-level data path already reads the array offsets while materializing the array. + // Do not let a redundant current-level OFFSET/NULL path switch this iterator into a + // meta-only mode that would skip item data. + remove_current_level_meta_access_paths(sub_all_access_paths); + } + const bool has_current_level_predicate_meta_path = + std::ranges::any_of(sub_predicate_access_paths, is_current_level_meta_access_path); + // Current-level predicate metadata paths are consumed by this array iterator and must not be + // forwarded to the item iterator. The FE keeps all_access_paths as a superset of predicate + // paths, so meta-only mode is still decided from sub_all_access_paths below. + auto removed = + std::ranges::remove_if(sub_predicate_access_paths, is_current_level_meta_access_path); + sub_predicate_access_paths.erase(removed.begin(), removed.end()); + + if (sub_predicate_access_paths.empty()) { + // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY). Only skip the item + // iterator when no predicate sub-path needs it in the predicate phase. + _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); + if (read_offset_only()) { + _item_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Array column iterator set column " << _column_name + << " to OFFSET_ONLY meta read mode, item column set to SKIP"; + return Status::OK(); + } + if (read_null_map_only()) { + _item_iterator->set_read_requirement(ReadRequirement::SKIP); + DLOG(INFO) << "Array column iterator set column " << _column_name + << " to NULL_MAP_ONLY meta read mode, item column set to SKIP"; + return Status::OK(); + } } + // OFFSET/NULL at the current array level is consumed by this iterator. After deciding that + // the array is not in a meta-only mode, do not forward those paths to the item iterator. + remove_current_level_meta_access_paths(sub_all_access_paths); const auto no_sub_column_to_skip = sub_all_access_paths.empty(); const auto no_predicate_sub_column = sub_predicate_access_paths.empty(); @@ -1986,7 +2321,17 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a } } - if (!no_predicate_sub_column) { + if (no_predicate_sub_column) { + // Current-level predicate meta paths (OFFSET/NULL) are consumed by the array itself and + // removed before forwarding paths to the item iterator. If they are the only predicate + // paths, the item iterator may still be needed later for lazy materialization, but it must + // not be promoted to PREDICATE. Only propagate the predicate requirement when the + // parent predicate really applies to the item/whole value instead of array metadata only. + if (_read_requirement == ReadRequirement::PREDICATE && + !has_current_level_predicate_meta_path) { + _item_iterator->set_read_requirement(ReadRequirement::PREDICATE); + } + } else { for (auto& path : sub_predicate_access_paths) { if (path.data_access_path.path[0] == ACCESS_ALL) { path.data_access_path.path[0] = _item_iterator->column_name(); @@ -1995,9 +2340,12 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a } if (!no_sub_column_to_skip || !no_predicate_sub_column) { - _item_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ); RETURN_IF_ERROR( _item_iterator->set_access_paths(sub_all_access_paths, sub_predicate_access_paths)); + // Predicate-only item paths stay PREDICATE because this update runs after + // child set_access_paths() and read requirements are monotonic. Non-predicate item paths are + // marked as lazy materialization targets. + _item_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); } return Status::OK(); } @@ -2022,24 +2370,54 @@ Status StringFileColumnIterator::init(const ColumnIteratorOptions& opts) { Status StringFileColumnIterator::set_access_paths( const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { - if (all_access_paths.empty()) { + if (all_access_paths.empty() && predicate_access_paths.empty()) { return Status::OK(); } + const auto requirement_before_access_path = _read_requirement; if (!predicate_access_paths.empty()) { - set_reading_flag(ReadingFlag::READING_FOR_PREDICATE); + set_read_requirement(ReadRequirement::PREDICATE); } + const bool has_current_level_data_path = + std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { + return is_current_level_data_access_path(path, _column_name); + }); // Strip the column name from path[0] before checking for meta-only modes. // Raw paths look like ["col_name", "OFFSET"] or ["col_name", "NULL"]. auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - _check_and_set_meta_read_mode(sub_all_access_paths); + if (has_current_level_data_path) { + remove_current_level_meta_access_paths(sub_all_access_paths); + } + _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); + // OFFSET_ONLY mode is fundamentally incompatible with CHAR columns: + // CHAR is stored padded to its declared length (see + // OlapColumnDataConvertorChar::clone_and_padding), so the per-row length + // recorded in dict word info / page headers is always the padded length + // (e.g. 25 for CHAR(25)) — never the logical length expected by length(). + // Recovering the logical length requires scanning the chars buffer with + // strnlen() (shrink_padding_chars), which OFFSET_ONLY by definition skips. + // There is no partial-benefit path: any optimization that still produces + // the correct length() result must read the chars buffer in full. + // + // FE (NestedColumnPruning) already filters CHAR slots out of the + // OFFSET-only access plan, so reaching this branch means an FE/BE + // contract violation. Fail loudly instead of silently falling back. + if (read_offset_only() && get_reader() != nullptr && + get_reader()->get_meta_type() == FieldType::OLAP_FIELD_TYPE_CHAR) { + return Status::InternalError( + "OFFSET_ONLY access path is not supported on CHAR column '{}': CHAR is stored " + "padded so the per-row length information available without reading the chars " + "buffer is always the padded length, not the logical length. The FE planner " + "must not emit an OFFSET access path for CHAR columns.", + _column_name); + } if (read_offset_only()) { DLOG(INFO) << "String column iterator set column " << _column_name - << " to OFFSET_ONLY reading mode"; + << " to OFFSET_ONLY meta read mode"; } else if (read_null_map_only()) { DLOG(INFO) << "String column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode"; + << " to NULL_MAP_ONLY meta read mode"; } return Status::OK(); @@ -2049,23 +2427,45 @@ Status StringFileColumnIterator::set_access_paths( FileColumnIterator::FileColumnIterator(std::shared_ptr reader) : _reader(reader) {} -void ColumnIterator::_check_and_set_meta_read_mode(const TColumnAccessPaths& sub_all_access_paths) { +void ColumnIterator::_check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, + const TColumnAccessPaths& sub_all_access_paths) { + _meta_read_mode = MetaReadMode::DEFAULT; + if (requirement_before_access_path != ReadRequirement::NORMAL && + requirement_before_access_path != ReadRequirement::SKIP) { + // A stronger requirement means a parent/full-data path already required this iterator + // to materialize data. In that case a later predicate NULL/OFFSET path is only + // an additional predicate requirement and must not downgrade the read to + // meta-only. + return; + } + + bool has_offset_path = false; + bool has_null_path = false; for (const auto& path : sub_all_access_paths) { - if (!path.data_access_path.path.empty()) { - if (StringCaseEqual()(path.data_access_path.path[0], ACCESS_OFFSET)) { - _read_mode = ReadMode::OFFSET_ONLY; - return; - } else if (StringCaseEqual()(path.data_access_path.path[0], ACCESS_NULL)) { - _read_mode = ReadMode::NULL_MAP_ONLY; - return; - } + if (!is_current_level_meta_access_path(path)) { + _meta_read_mode = MetaReadMode::DEFAULT; + return; + } + const auto& component = path.data_access_path.path[0]; + if (StringCaseEqual()(component, ACCESS_OFFSET)) { + has_offset_path = true; + } else { + has_null_path = true; } } - _read_mode = ReadMode::DEFAULT; + if (has_offset_path) { + // OFFSET_ONLY skips actual child/string data, but nullable complex iterators still + // materialize the current-level null map. So OFFSET covers OFFSET+NULL metadata. + _meta_read_mode = MetaReadMode::OFFSET_ONLY; + } else if (has_null_path) { + _meta_read_mode = MetaReadMode::NULL_MAP_ONLY; + } else { + _meta_read_mode = MetaReadMode::DEFAULT; + } } Status FileColumnIterator::init(const ColumnIteratorOptions& opts) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (_read_requirement == ReadRequirement::SKIP) { DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -2110,7 +2510,7 @@ void FileColumnIterator::_trigger_prefetch_if_eligible(ordinal_t ord) { } Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -2171,6 +2571,14 @@ Status FileColumnIterator::next_batch_of_zone_map(size_t* n, MutableColumnPtr& d } Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { + if (!need_to_read()) { + DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; + _convert_to_place_holder_column(dst, *n); + return Status::OK(); + } + + _recovery_from_place_holder_column(dst); + if (read_null_map_only()) { DLOG(INFO) << "File column iterator column " << _column_name << " in NULL_MAP_ONLY mode, reading only null map."; @@ -2219,12 +2627,6 @@ Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* ha return Status::OK(); } - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; - dst->resize(dst->size() + *n); - return Status::OK(); - } - size_t curr_size = dst->byte_size(); dst->reserve(*n); size_t remaining = *n; @@ -2284,6 +2686,14 @@ Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* ha Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { + if (!need_to_read()) { + DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; + _convert_to_place_holder_column(dst, count); + return Status::OK(); + } + + _recovery_from_place_holder_column(dst); + if (read_null_map_only()) { DLOG(INFO) << "File column iterator column " << _column_name << " in NULL_MAP_ONLY mode, reading only null map by rowids."; @@ -2338,9 +2748,21 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co total_read_count += nrows_to_read; remaining -= nrows_to_read; } else { - memset(null_map_data.data() + base_size + total_read_count, 0, nrows_to_read); - total_read_count += nrows_to_read; - remaining -= nrows_to_read; + rowid_t current_ordinal_in_page = + cast_set(_page.offset_in_page + _page.first_ordinal); + size_t rows_in_current_page = 0; + for (size_t i = 0; i < nrows_to_read; ++i) { + if (rowids[total_read_count + i] - current_ordinal_in_page >= nrows_to_read) { + break; + } + ++rows_in_current_page; + } + DCHECK_GT(rows_in_current_page, 0); + memset(null_map_data.data() + base_size + total_read_count, 0, + rows_in_current_page); + _page.offset_in_page += rows_in_current_page; + total_read_count += rows_in_current_page; + remaining -= rows_in_current_page; } } @@ -2349,12 +2771,6 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co return Status::OK(); } - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; - dst->resize(count); - return Status::OK(); - } - size_t remaining = count; size_t total_read_count = 0; size_t nrows_to_read = 0; diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index 8c1831aa7cb9c9..e7dd0c2869aaf7 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -19,6 +19,7 @@ #include #include +#include #include #include // for size_t @@ -374,7 +375,7 @@ class ColumnIterator { virtual Status set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { if (!predicate_access_paths.empty()) { - _reading_flag = ReadingFlag::READING_FOR_PREDICATE; + set_read_requirement_self(ReadRequirement::PREDICATE); } return Status::OK(); } @@ -383,33 +384,28 @@ class ColumnIterator { const std::string& column_name() const { return _column_name; } - // Since there may be multiple paths with conflicts or overlaps, - // we need to define several reading flags: + // Per-iterator read requirement derived from nested access paths. // - // NORMAL_READING — Default value, indicating that the column should be read. - // SKIP_READING — The column should not be read. - // NEED_TO_READ — The column must be read. - // READING_FOR_PREDICATE — The column is required for predicate evaluation. - // - // For example, suppose there are two paths: - // - Path 1 specifies that column A needs to be read, so it is marked as NEED_TO_READ. - // - Path 2 specifies that the column should not be read, but since it is already marked as NEED_TO_READ, - // it should not be changed to SKIP_READING. - enum class ReadingFlag : int { - NORMAL_READING, - SKIP_READING, - NEED_TO_READ, - READING_FOR_PREDICATE - }; - void set_reading_flag(ReadingFlag flag) { - if (static_cast(flag) > static_cast(_reading_flag)) { - _reading_flag = flag; - } + // The ordering is intentional and used by set_read_requirement_self(): requirements are + // monotonic and a weaker requirement must not downgrade a stronger one. + // - NORMAL: no pruning decision has been made yet. + // - SKIP: this iterator should not be read. + // - LAZY_OUTPUT: materialize this iterator in the lazy phase after predicate filtering. + // - PREDICATE: read this iterator in the predicate phase. This must stay stronger than + // LAZY_OUTPUT because parents may mark children as LAZY_OUTPUT after child set_access_paths() + // has already promoted predicate-only children to PREDICATE. + enum class ReadRequirement : int { NORMAL, SKIP, LAZY_OUTPUT, PREDICATE }; + + // Set the read requirement on this iterator and all nested child iterators. + virtual void set_read_requirement(ReadRequirement requirement) { + set_read_requirement_self(requirement); } - ReadingFlag reading_flag() const { return _reading_flag; } + ReadRequirement read_requirement() const { return _read_requirement; } - virtual void set_need_to_read() { set_reading_flag(ReadingFlag::NEED_TO_READ); } + virtual void set_lazy_output_requirement() { + set_read_requirement(ReadRequirement::LAZY_OUTPUT); + } virtual void remove_pruned_sub_iterators() {}; @@ -426,26 +422,96 @@ class ColumnIterator { static constexpr const char* ACCESS_NULL = "NULL"; // Meta-only read modes: - // - OFFSET_ONLY: only read offset information (e.g., for array_size/map_size/string_length) + // - OFFSET_ONLY: read offsets while skipping actual child/string data. For nullable + // complex columns, the parent null map is still materialized when needed. // - NULL_MAP_ONLY: only read null map (e.g., for IS NULL / IS NOT NULL predicates) // When these modes are enabled, actual content data is skipped. - enum class ReadMode : int { DEFAULT, OFFSET_ONLY, NULL_MAP_ONLY }; + enum class MetaReadMode : int { DEFAULT, OFFSET_ONLY, NULL_MAP_ONLY }; + + bool read_offset_only() const { return _meta_read_mode == MetaReadMode::OFFSET_ONLY; } + bool read_null_map_only() const { return _meta_read_mode == MetaReadMode::NULL_MAP_ONLY; } + + // The current scanner phase. This is intentionally separate from ReadRequirement + // (why this iterator is needed) and MetaReadMode (what physical metadata to read). + enum class ReadPhase : int { + NORMAL, // default full materialization without lazy read split + PREDICATE, // predicate evaluation before row filtering + LAZY // post-filter lazy materialization + }; + + virtual void set_read_phase(ReadPhase mode) { + _read_phase = mode; + if (mode == ReadPhase::PREDICATE) { + _has_place_holder_column = false; + } + } + + virtual bool need_to_read() const { + switch (_read_phase) { + case ReadPhase::NORMAL: + return _read_requirement != ReadRequirement::SKIP; + case ReadPhase::PREDICATE: + return _read_requirement == ReadRequirement::PREDICATE; + case ReadPhase::LAZY: + return _read_requirement == ReadRequirement::LAZY_OUTPUT; + default: + return false; + } + } - bool read_offset_only() const { return _read_mode == ReadMode::OFFSET_ONLY; } - bool read_null_map_only() const { return _read_mode == ReadMode::NULL_MAP_ONLY; } + // Whether the current iterator itself should materialize meta columns, such as + // the null-map column or the offset column, into the destination column. + // + // Do not use the virtual need_to_read() here. Complex iterators override + // need_to_read() in LAZY mode to keep the parent iterator active when only a + // nested child still has data to materialize. That parent-level control-flow + // decision is different from materializing the parent's own offsets/null-map: + // if the parent was already read for predicate evaluation, LAZY mode should + // only fill the missing children and must not append parent meta again. + bool need_to_read_meta_columns() const { return ColumnIterator::need_to_read(); } + + virtual void finalize_lazy_phase(MutableColumnPtr& dst) { + _recovery_from_place_holder_column(dst); + } + + // Set only this iterator's requirement without modifying requirements of any nested child + // iterators. Use this when the parent/wrapper state must be updated while child requirements + // are decided independently. + virtual void set_read_requirement_self(ReadRequirement requirement) { + if (static_cast(requirement) > static_cast(_read_requirement)) { + _read_requirement = requirement; + } + } + + // Whether this iterator or any nested iterator has data that must be materialized + // in lazy mode. Predicate-only branches are read before filtering and must not be + // re-read in the lazy phase. Meta-only access paths still become lazy targets when + // they appear only in all_access_paths, because OFFSET/NULL is the requested output. + virtual bool has_lazy_read_target() const { + return _read_requirement == ReadRequirement::LAZY_OUTPUT; + } protected: - // Checks sub access paths for OFFSET or NULL meta-only modes and - // updates _read_mode accordingly. Use the accessor helpers - // read_offset_only() / read_null_map_only() to query the current mode. - void _check_and_set_meta_read_mode(const TColumnAccessPaths& sub_all_access_paths); + void _convert_to_place_holder_column(MutableColumnPtr& dst, size_t count); + + void _recovery_from_place_holder_column(MutableColumnPtr& dst); + + // Derive current-level meta-only read mode from access paths. Meta-only is valid only when + // this iterator had no data-read requirement before applying the current paths, and every + // visible path at this level is NULL/OFFSET metadata. + void _check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, + const TColumnAccessPaths& sub_all_access_paths); - Result _get_sub_access_paths(const TColumnAccessPaths& access_paths); + Result _get_sub_access_paths(TColumnAccessPaths access_paths, + bool is_predicate = false); ColumnIteratorOptions _opts; - ReadingFlag _reading_flag {ReadingFlag::NORMAL_READING}; - ReadMode _read_mode = ReadMode::DEFAULT; + ReadRequirement _read_requirement {ReadRequirement::NORMAL}; + MetaReadMode _meta_read_mode = MetaReadMode::DEFAULT; + ReadPhase _read_phase {ReadPhase::NORMAL}; std::string _column_name; + + bool _has_place_holder_column {false}; }; // This iterator is used to read column data from file @@ -495,6 +561,11 @@ class FileColumnIterator : public ColumnIterator { std::map>& prefetchers, PrefetcherInitMethod init_method) override; +protected: + // Exposed to derived iterators (e.g. StringFileColumnIterator) so they can + // query column metadata such as the storage field type. + const std::shared_ptr& get_reader() const { return _reader; } + private: Status _seek_to_pos_in_page(ParsedPage* page, ordinal_t offset_in_page) const; Status _load_next_page(bool* eos); @@ -589,6 +660,11 @@ class OffsetFileColumnIterator final : public ColumnIterator { return _offset_iterator->read_by_rowids(rowids, count, dst); } + void set_read_requirement(ReadRequirement requirement) override { + set_read_requirement_self(requirement); + _offset_iterator->set_read_requirement(requirement); + } + Status init_prefetcher(const SegmentPrefetchParams& params) override; void collect_prefetchers( std::map>& prefetchers, @@ -634,10 +710,33 @@ class MapFileColumnIterator final : public ColumnIterator { Status set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) override; - void set_need_to_read() override; + void set_lazy_output_requirement() override; void remove_pruned_sub_iterators() override; + void set_read_phase(ReadPhase mode) override; + + bool need_to_read() const override { + switch (_read_phase) { + case ReadPhase::NORMAL: + return _read_requirement != ReadRequirement::SKIP; + case ReadPhase::PREDICATE: + return _read_requirement == ReadRequirement::PREDICATE; + case ReadPhase::LAZY: + // In lazy mode, read this map only when at least one key/value branch still + // has non-predicate data to materialize. + return has_lazy_read_target(); + default: + return false; + } + } + + void finalize_lazy_phase(MutableColumnPtr& dst) override; + + void set_read_requirement(ReadRequirement requirement) override; + + bool has_lazy_read_target() const override; + private: std::shared_ptr _map_reader = nullptr; ColumnIteratorUPtr _null_iterator; @@ -678,7 +777,7 @@ class StructFileColumnIterator final : public ColumnIterator { Status set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) override; - void set_need_to_read() override; + void set_lazy_output_requirement() override; void remove_pruned_sub_iterators() override; @@ -687,6 +786,27 @@ class StructFileColumnIterator final : public ColumnIterator { std::map>& prefetchers, PrefetcherInitMethod init_method) override; + void set_read_phase(ReadPhase mode) override; + + bool need_to_read() const override { + switch (_read_phase) { + case ReadPhase::NORMAL: + return _read_requirement != ReadRequirement::SKIP; + case ReadPhase::PREDICATE: + return _read_requirement == ReadRequirement::PREDICATE; + case ReadPhase::LAZY: + // In lazy mode, read this struct only when at least one nested branch still + // has non-predicate data to materialize. + return has_lazy_read_target(); + default: + return false; + } + } + + void finalize_lazy_phase(MutableColumnPtr& dst) override; + void set_read_requirement(ReadRequirement requirement) override; + bool has_lazy_read_target() const override; + private: std::shared_ptr _struct_reader = nullptr; ColumnIteratorUPtr _null_iterator; @@ -725,7 +845,7 @@ class ArrayFileColumnIterator final : public ColumnIterator { Status set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) override; - void set_need_to_read() override; + void set_lazy_output_requirement() override; void remove_pruned_sub_iterators() override; @@ -734,6 +854,29 @@ class ArrayFileColumnIterator final : public ColumnIterator { std::map>& prefetchers, PrefetcherInitMethod init_method) override; + void set_read_phase(ReadPhase mode) override; + + bool need_to_read() const override { + switch (_read_phase) { + case ReadPhase::NORMAL: + return _read_requirement != ReadRequirement::SKIP; + case ReadPhase::PREDICATE: + return _read_requirement == ReadRequirement::PREDICATE; + case ReadPhase::LAZY: + // In lazy mode, read this array only when its item branch still has + // non-predicate data to materialize. + return has_lazy_read_target(); + default: + return false; + } + } + + void finalize_lazy_phase(MutableColumnPtr& dst) override; + + void set_read_requirement(ReadRequirement requirement) override; + + bool has_lazy_read_target() const override; + private: std::shared_ptr _array_reader = nullptr; std::unique_ptr _offset_iterator; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 8198e00b674061..85f26f8c4f93d3 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -119,6 +120,30 @@ namespace segment_v2 { #include "common/compile_check_begin.h" +class ScopedColumnIteratorReadPhase { +public: + ScopedColumnIteratorReadPhase(ColumnIterator* column_iter, ColumnIterator::ReadPhase mode) + : _column_iter(column_iter) { + DORIS_CHECK(_column_iter != nullptr); + _column_iter->set_read_phase(mode); + } + + ScopedColumnIteratorReadPhase(const ScopedColumnIteratorReadPhase&) = delete; + ScopedColumnIteratorReadPhase& operator=(const ScopedColumnIteratorReadPhase&) = delete; + + ~ScopedColumnIteratorReadPhase() { + // ReadPhase is a per-read phase knob. SegmentIterator only needs a + // temporary PREDICATE/LAZY mode while reading one column in one phase; it + // must be restored before the next column or later normal reads reuse the + // same ColumnIterator. Keep the restoration in one scoped helper instead + // of open-coding the same Defer block at every call site. + _column_iter->set_read_phase(ColumnIterator::ReadPhase::NORMAL); + } + +private: + ColumnIterator* _column_iter = nullptr; +}; + SegmentIterator::~SegmentIterator() = default; void SegmentIterator::_init_row_bitmap_by_condition_cache() { @@ -419,6 +444,10 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _score_runtime = _opts.score_runtime; _ann_topn_runtime = _opts.ann_topn_runtime; + _enable_prune_nested_column = _opts.io_ctx.reader_type == ReaderType::READER_QUERY && + _opts.runtime_state && + _opts.runtime_state->enable_prune_nested_column(); + if (opts.output_columns != nullptr) { _output_columns = *(opts.output_columns); } @@ -657,8 +686,14 @@ void SegmentIterator::_init_segment_prefetchers() { ? PrefetcherInitMethod::FROM_ROWIDS : PrefetcherInitMethod::ALL_DATA_BLOCKS; std::map> prefetchers; - for (const auto& column_iter : _column_iterators) { + for (size_t idx = 0; idx < _column_iterators.size(); ++idx) { + auto cid = cast_set(idx); + auto* column_iter = _column_iterators[cid].get(); if (column_iter != nullptr) { + ScopedColumnIteratorReadPhase scoped_read_phase { + column_iter, _support_lazy_read_pruned_columns.contains(cid) + ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::NORMAL}; column_iter->collect_prefetchers(prefetchers, init_method); } } @@ -1984,6 +2019,25 @@ Status SegmentIterator::_vec_init_lazy_materialization() { if (_is_common_expr_column[cid] || _is_pred_column[cid]) { auto loc = _schema_block_id_map[cid]; _columns_to_filter.push_back(loc); + + const auto field_type = _schema->column(cid)->type(); + if (_is_common_expr_column[cid] && _enable_prune_nested_column && + (field_type == FieldType::OLAP_FIELD_TYPE_STRUCT || + field_type == FieldType::OLAP_FIELD_TYPE_ARRAY || + field_type == FieldType::OLAP_FIELD_TYPE_MAP)) { + DCHECK(_column_iterators[cid]); + if (_column_iterators[cid]->read_requirement() == + ColumnIterator::ReadRequirement::PREDICATE && + _column_iterators[cid]->has_lazy_read_target()) { + // Only split lazy recovery for complex common expr columns that have + // both predicate-only and non-predicate nested targets. The two requirement + // checks already imply that nested-column pruning happened: without an + // explicit predicate sub-path the parent would not be + // PREDICATE, and without a pruned non-predicate child there + // would be no lazy target to recover after filtering. + _support_lazy_read_pruned_columns.emplace(cid); + } + } } } @@ -2301,16 +2355,22 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 }) } + auto* column_iter = _column_iterators[cid].get(); + ScopedColumnIteratorReadPhase scoped_read_phase { + column_iter, _support_lazy_read_pruned_columns.contains(cid) + ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::NORMAL}; + if (is_continuous) { size_t rows_read = nrows_read; _opts.stats->predicate_column_read_seek_num += 1; if (_opts.runtime_state && _opts.runtime_state->enable_profile()) { SCOPED_RAW_TIMER(&_opts.stats->predicate_column_read_seek_ns); - RETURN_IF_ERROR(_column_iterators[cid]->seek_to_ordinal(_block_rowids[0])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[0])); } else { - RETURN_IF_ERROR(_column_iterators[cid]->seek_to_ordinal(_block_rowids[0])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[0])); } - RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, column)); + RETURN_IF_ERROR(column_iter->next_batch(&rows_read, column)); if (rows_read != nrows_read) { return Status::Error("nrows({}) != rows_read({})", nrows_read, rows_read); @@ -2330,20 +2390,18 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 _opts.stats->predicate_column_read_seek_num += 1; if (_opts.runtime_state && _opts.runtime_state->enable_profile()) { SCOPED_RAW_TIMER(&_opts.stats->predicate_column_read_seek_ns); - RETURN_IF_ERROR( - _column_iterators[cid]->seek_to_ordinal(_block_rowids[processed])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[processed])); } else { - RETURN_IF_ERROR( - _column_iterators[cid]->seek_to_ordinal(_block_rowids[processed])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[processed])); } - RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, column)); + RETURN_IF_ERROR(column_iter->next_batch(&rows_read, column)); if (rows_read != current_batch_size) { return Status::Error( "batch nrows({}) != rows_read({})", current_batch_size, rows_read); } } else { - RETURN_IF_ERROR(_column_iterators[cid]->read_by_rowids( - &_block_rowids[processed], current_batch_size, column)); + RETURN_IF_ERROR(column_iter->read_by_rowids(&_block_rowids[processed], + current_batch_size, column)); } processed += current_batch_size; } @@ -2483,7 +2541,8 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu std::vector& rowid_vector, uint16_t* sel_rowid_idx, size_t select_size, MutableColumns* mutable_columns, - bool init_condition_cache) { + bool init_condition_cache, + bool read_for_predicate) { SCOPED_RAW_TIMER(&_opts.stats->lazy_read_ns); std::vector rowids(select_size); @@ -2527,10 +2586,45 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu "SegmentIterator meet invalid column, return columns size {}, cid {}", _current_return_columns.size(), cid); } - RETURN_IF_ERROR(_column_iterators[cid]->read_by_rowids(rowids.data(), select_size, - _current_return_columns[cid])); + + auto* column_iter = _column_iterators[cid].get(); + ScopedColumnIteratorReadPhase scoped_read_phase { + column_iter, read_for_predicate && _support_lazy_read_pruned_columns.contains(cid) + ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::NORMAL}; + + RETURN_IF_ERROR(column_iter->read_by_rowids(rowids.data(), select_size, + _current_return_columns[cid])); + } + + return Status::OK(); +} + +Status SegmentIterator::_read_lazy_pruned_columns(Block* block) { + if (_support_lazy_read_pruned_columns.empty()) { + return Status::OK(); + } + + SCOPED_RAW_TIMER(&_opts.stats->lazy_read_pruned_ns); + DorisVector rowids(_selected_size); + for (size_t i = 0; i < _selected_size; ++i) { + rowids[i] = _block_rowids[_sel_rowid_idx[i]]; } + for (auto cid : _support_lazy_read_pruned_columns) { + // branch-4.2 keeps the column-id -> block-position map inside SegmentIterator; + // master's Schema::column_index() helper does not exist on this branch. + auto loc = cast_set(_schema_block_id_map[cid]); + auto column = IColumn::mutate(std::move(block->get_by_position(loc).column)); + auto* column_iter = _column_iterators[cid].get(); + ScopedColumnIteratorReadPhase scoped_read_phase {column_iter, + ColumnIterator::ReadPhase::LAZY}; + if (_selected_size > 0) { + RETURN_IF_ERROR(column_iter->read_by_rowids(rowids.data(), _selected_size, column)); + } + column_iter->finalize_lazy_phase(column); + block->get_by_position(loc).column = std::move(column); + } return Status::OK(); } @@ -2735,7 +2829,7 @@ Status SegmentIterator::_next_batch_internal(Block* block) { SCOPED_RAW_TIMER(&_opts.stats->non_predicate_read_ns); RETURN_IF_ERROR(_read_columns_by_rowids( _non_predicate_column_ids, _block_rowids, _sel_rowid_idx.data(), - _selected_size, &_current_return_columns)); + _selected_size, &_current_return_columns, false, true)); _replace_version_col_if_needed(_non_predicate_column_ids, _selected_size); RETURN_IF_ERROR(_process_columns(_non_predicate_column_ids, block)); } @@ -2766,7 +2860,7 @@ Status SegmentIterator::_next_batch_internal(Block* block) { RETURN_IF_ERROR(_read_columns_by_rowids( _non_predicate_columns, _block_rowids, _sel_rowid_idx.data(), _selected_size, &_current_return_columns, - _opts.condition_cache_digest && !_find_condition_cache)); + _opts.condition_cache_digest && !_find_condition_cache, false)); _replace_version_col_if_needed(_non_predicate_columns, _selected_size); } else { if (_opts.condition_cache_digest && !_find_condition_cache) { @@ -2778,6 +2872,8 @@ Status SegmentIterator::_next_batch_internal(Block* block) { } } } + + RETURN_IF_ERROR(_read_lazy_pruned_columns(block)); } // step5: output columns diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index d8f61daeba3553..5d849d438957f5 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -227,7 +227,9 @@ class SegmentIterator : public RowwiseIterator { std::vector& rowid_vector, uint16_t* sel_rowid_idx, size_t select_size, MutableColumns* mutable_columns, - bool init_condition_cache = false); + bool init_condition_cache = false, + bool read_for_predicate = false); + [[nodiscard]] Status _read_lazy_pruned_columns(Block* block); Status copy_column_data_by_selector(IColumn* input_col_ptr, MutableColumnPtr& output_col, uint16_t* sel_rowid_idx, uint16_t select_size, @@ -372,6 +374,9 @@ class SegmentIterator : public RowwiseIterator { bool _is_need_short_eval = false; bool _is_need_expr_eval = false; + std::set _support_lazy_read_pruned_columns; + bool _enable_prune_nested_column = false; + // fields for vectorization execution std::vector _vec_pred_column_ids; // keep columnId of columns for vectorized predicate evaluation diff --git a/be/src/storage/segment/variant/variant_column_reader.cpp b/be/src/storage/segment/variant/variant_column_reader.cpp index 8dca2d0034443a..6d0de2fb460732 100644 --- a/be/src/storage/segment/variant/variant_column_reader.cpp +++ b/be/src/storage/segment/variant/variant_column_reader.cpp @@ -86,7 +86,7 @@ class ReaderOwnedColumnIterator final : public ColumnIterator { : _inner(std::move(inner)), _owner(std::move(owner)) { DCHECK(_inner != nullptr); set_column_name(_inner->column_name()); - set_reading_flag(_inner->reading_flag()); + set_read_requirement(_inner->read_requirement()); } Status init(const ColumnIteratorOptions& opts) override { return _inner->init(opts); } @@ -130,15 +130,36 @@ class ReaderOwnedColumnIterator final : public ColumnIterator { Status set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) override { RETURN_IF_ERROR(_inner->set_access_paths(all_access_paths, predicate_access_paths)); - set_reading_flag(_inner->reading_flag()); + ColumnIterator::set_read_requirement_self(_inner->read_requirement()); return Status::OK(); } - void set_need_to_read() override { - _inner->set_need_to_read(); - set_reading_flag(_inner->reading_flag()); + void set_read_requirement(ReadRequirement requirement) override { + _inner->set_read_requirement(requirement); + ColumnIterator::set_read_requirement_self(_inner->read_requirement()); } + void set_read_requirement_self(ReadRequirement requirement) override { + _inner->set_read_requirement_self(requirement); + ColumnIterator::set_read_requirement_self(_inner->read_requirement()); + } + + void set_lazy_output_requirement() override { + _inner->set_lazy_output_requirement(); + ColumnIterator::set_read_requirement_self(_inner->read_requirement()); + } + + void set_read_phase(ReadPhase mode) override { + ColumnIterator::set_read_phase(mode); + _inner->set_read_phase(mode); + } + + void finalize_lazy_phase(MutableColumnPtr& dst) override { _inner->finalize_lazy_phase(dst); } + + bool has_lazy_read_target() const override { return _inner->has_lazy_read_target(); } + + bool need_to_read() const override { return _inner->need_to_read(); } + void remove_pruned_sub_iterators() override { _inner->remove_pruned_sub_iterators(); } Status init_prefetcher(const SegmentPrefetchParams& params) override { diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index de8b5aadf1f3de..2ca69b87aa63ce 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -23,27 +23,360 @@ #include #include +#include +#include #include +#include #include +#include #include #include "agent/be_exec_version_manager.h" #include "common/config.h" #include "io/fs/file_reader.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "storage/olap_common.h" #include "storage/segment/column_reader_cache.h" +#include "storage/segment/column_writer.h" #include "storage/segment/mock/mock_segment.h" #include "storage/segment/segment.h" #include "storage/segment/variant/variant_column_reader.h" #include "storage/tablet/tablet_schema.h" +#include "storage/types.h" #include "util/json/path_in_data.h" namespace doris::segment_v2 { +namespace { +class TestColumnIterator final : public ColumnIterator { +public: + Status seek_to_ordinal(ordinal_t /* ord */) override { return Status::OK(); } + + Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override { + dst->insert_many_defaults(*n); + if (has_null != nullptr) { + *has_null = false; + } + return Status::OK(); + } + + Status read_by_rowids(const rowid_t* /* rowids */, const size_t count, + MutableColumnPtr& dst) override { + dst->insert_many_defaults(count); + return Status::OK(); + } + + ordinal_t get_current_ordinal() const override { return 0; } + + void force_set_read_requirement(ReadRequirement requirement) { + _read_requirement = requirement; + } + + Result get_sub_access_paths(const TColumnAccessPaths& access_paths, + bool is_predicate = false) { + return _get_sub_access_paths(access_paths, is_predicate); + } + + void check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, + const TColumnAccessPaths& access_paths) { + _check_and_set_meta_read_mode(requirement_before_access_path, access_paths); + } + + void convert_to_place_holder_column(MutableColumnPtr& dst, size_t count) { + _convert_to_place_holder_column(dst, count); + } +}; + +TColumnAccessPath create_access_path(std::vector path) { + TColumnAccessPath access_path; + TDataAccessPath data_access_path; + data_access_path.__set_path(std::move(path)); + access_path.__set_data_access_path(std::move(data_access_path)); + return access_path; +} + +std::shared_ptr create_test_reader( + bool is_nullable = false, uint64_t num_rows = 0, + FieldType field_type = FieldType::OLAP_FIELD_TYPE_INT) { + auto reader = std::make_shared(); + reader->_meta_is_nullable = is_nullable; + reader->_num_rows = num_rows; + reader->_meta_type = field_type; + return reader; +} + +class TrackingColumnIterator final : public ColumnIterator { +public: + Status seek_to_ordinal(ordinal_t ord) override { + seek_ordinals.emplace_back(ord); + _current_ordinal = ord; + return Status::OK(); + } + + Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override { + next_batch_sizes.emplace_back(*n); + if (!need_to_read()) { + _convert_to_place_holder_column(dst, *n); + if (has_null != nullptr) { + *has_null = false; + } + return Status::OK(); + } + + _recovery_from_place_holder_column(dst); + dst->insert_many_defaults(*n); + _current_ordinal += *n; + if (has_null != nullptr) { + *has_null = false; + } + return Status::OK(); + } + + Status read_by_rowids(const rowid_t* rowids, const size_t count, + MutableColumnPtr& dst) override { + read_by_rowids_batches.emplace_back(rowids, rowids + count); + if (!need_to_read()) { + _convert_to_place_holder_column(dst, count); + return Status::OK(); + } + + _recovery_from_place_holder_column(dst); + dst->insert_many_defaults(count); + return Status::OK(); + } + + ordinal_t get_current_ordinal() const override { return _current_ordinal; } + + void collect_prefetchers( + std::map>& prefetchers, + PrefetcherInitMethod init_method) override { + record_collect_method(init_method); + prefetchers[init_method].emplace_back(prefetcher()); + } + + SegmentPrefetcher* prefetcher() const { + return reinterpret_cast(const_cast(this)); + } + + void clear_tracking() { + seek_ordinals.clear(); + next_batch_sizes.clear(); + read_by_rowids_batches.clear(); + collect_methods.clear(); + } + + std::vector seek_ordinals; + std::vector next_batch_sizes; + std::vector> read_by_rowids_batches; + std::vector collect_methods; + +private: + void record_collect_method(PrefetcherInitMethod init_method) { + collect_methods.emplace_back(init_method); + } + + ordinal_t _current_ordinal = 0; +}; + +class TrackingFileColumnIterator final : public FileColumnIterator { +public: + explicit TrackingFileColumnIterator(std::shared_ptr reader) + : FileColumnIterator(std::move(reader)) {} + + Status seek_to_ordinal(ordinal_t ord) override { + seek_ordinals.emplace_back(ord); + _current_ordinal = ord; + return Status::OK(); + } + + Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override { + next_batch_sizes.emplace_back(*n); + dst->insert_many_defaults(*n); + _current_ordinal += *n; + if (has_null != nullptr) { + *has_null = false; + } + return Status::OK(); + } + + Status read_by_rowids(const rowid_t* rowids, const size_t count, + MutableColumnPtr& dst) override { + read_by_rowids_batches.emplace_back(rowids, rowids + count); + dst->insert_many_defaults(count); + return Status::OK(); + } + + ordinal_t get_current_ordinal() const override { return _current_ordinal; } + + void collect_prefetchers( + std::map>& prefetchers, + PrefetcherInitMethod init_method) override { + record_collect_method(init_method); + prefetchers[init_method].emplace_back(prefetcher()); + } + + SegmentPrefetcher* prefetcher() const { + return reinterpret_cast(const_cast(this)); + } + + std::vector seek_ordinals; + std::vector next_batch_sizes; + std::vector> read_by_rowids_batches; + std::vector collect_methods; + +private: + void record_collect_method(PrefetcherInitMethod init_method) { + collect_methods.emplace_back(init_method); + } + + ordinal_t _current_ordinal = 0; +}; + +class NullMapOnlyFileColumnIterator final : public FileColumnIterator { +public: + explicit NullMapOnlyFileColumnIterator(std::shared_ptr reader) + : FileColumnIterator(std::move(reader)) {} + + void force_null_map_only() { _meta_read_mode = MetaReadMode::NULL_MAP_ONLY; } +}; + +MutableColumnPtr create_int_struct_column(size_t field_count) { + Columns columns; + for (size_t i = 0; i < field_count; ++i) { + columns.emplace_back(ColumnInt32::create()); + } + return ColumnStruct::create(std::move(columns)); +} + +MutableColumnPtr create_nullable_int_struct_column(size_t field_count) { + return ColumnNullable::create(create_int_struct_column(field_count), ColumnUInt8::create()); +} + +MutableColumnPtr create_nullable_int_array_column() { + return ColumnNullable::create( + ColumnArray::create(ColumnInt32::create(), ColumnArray::ColumnOffsets::create()), + ColumnUInt8::create()); +} + +MutableColumnPtr create_nullable_int_map_column() { + return ColumnNullable::create(ColumnMap::create(ColumnInt32::create(), ColumnInt32::create(), + ColumnArray::ColumnOffsets::create()), + ColumnUInt8::create()); +} + +struct TrackingOffsetIterator { + OffsetFileColumnIteratorUPtr iterator; + TrackingFileColumnIterator* tracker = nullptr; +}; + +TrackingOffsetIterator create_tracking_offset_iterator() { + auto file_iterator = std::make_unique(create_test_reader()); + auto* tracker = file_iterator.get(); + return {std::make_unique(std::move(file_iterator)), tracker}; +} +} // namespace + +static const std::string COLUMN_READER_FILE_TEST_DIR = "./ut_dir/column_reader_test"; + class ColumnReaderTest : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + void SetUp() override { + _old_disable_storage_page_cache = config::disable_storage_page_cache; + config::disable_storage_page_cache = true; + auto st = io::global_local_filesystem()->delete_directory(COLUMN_READER_FILE_TEST_DIR); + ASSERT_TRUE(st.ok()) << st.to_string(); + st = io::global_local_filesystem()->create_directory(COLUMN_READER_FILE_TEST_DIR); + ASSERT_TRUE(st.ok()) << st.to_string(); + } + + void TearDown() override { + EXPECT_TRUE( + io::global_local_filesystem()->delete_directory(COLUMN_READER_FILE_TEST_DIR).ok()); + config::disable_storage_page_cache = _old_disable_storage_page_cache; + } + +private: + bool _old_disable_storage_page_cache = false; }; +TEST_F(ColumnReaderTest, NullMapOnlyReadBySparseRowidsAcrossPages) { + ColumnMetaPB meta; + std::string fname = COLUMN_READER_FILE_TEST_DIR + "/null_map_only_sparse_rowids"; + auto fs = io::global_local_filesystem(); + + { + io::FileWriterPtr file_writer; + Status st = fs->create_file(fname, &file_writer); + ASSERT_TRUE(st.ok()) << st.to_string(); + + ColumnWriterOptions writer_opts; + writer_opts.meta = &meta; + writer_opts.meta->set_column_id(0); + writer_opts.meta->set_unique_id(0); + writer_opts.meta->set_type(static_cast(FieldType::OLAP_FIELD_TYPE_INT)); + writer_opts.meta->set_length(0); + writer_opts.meta->set_encoding(PLAIN_ENCODING); + writer_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F); + writer_opts.meta->set_is_nullable(true); + writer_opts.data_page_size = sizeof(int32_t) * 2; + writer_opts.need_zone_map = false; + + TabletColumn column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, + FieldType::OLAP_FIELD_TYPE_INT); + std::unique_ptr writer; + st = ColumnWriter::create(writer_opts, &column, file_writer.get(), &writer); + ASSERT_TRUE(st.ok()) << st.to_string(); + st = writer->init(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + for (int32_t i = 0; i < 6; ++i) { + st = writer->append(i == 2, &i); + ASSERT_TRUE(st.ok()) << st.to_string(); + } + + st = writer->finish(); + ASSERT_TRUE(st.ok()) << st.to_string(); + st = writer->write_data(); + ASSERT_TRUE(st.ok()) << st.to_string(); + st = writer->write_ordinal_index(); + ASSERT_TRUE(st.ok()) << st.to_string(); + st = file_writer->close(); + ASSERT_TRUE(st.ok()) << st.to_string(); + } + + io::FileReaderSPtr file_reader; + auto st = fs->open_file(fname, &file_reader); + ASSERT_TRUE(st.ok()) << st.to_string(); + + ColumnReaderOptions reader_opts; + std::shared_ptr reader; + st = ColumnReader::create(reader_opts, meta, 6, file_reader, &reader); + ASSERT_TRUE(st.ok()) << st.to_string(); + + NullMapOnlyFileColumnIterator iter(reader); + ColumnIteratorOptions iter_opts; + OlapReaderStatistics stats; + iter_opts.stats = &stats; + iter_opts.file_reader = file_reader.get(); + st = iter.init(iter_opts); + ASSERT_TRUE(st.ok()) << st.to_string(); + iter.force_null_map_only(); + + MutableColumnPtr dst = ColumnNullable::create(ColumnInt32::create(), ColumnUInt8::create()); + const rowid_t rowids[] = {0, 2}; + st = iter.read_by_rowids(rowids, std::size(rowids), dst); + ASSERT_TRUE(st.ok()) << st.to_string(); + + ASSERT_EQ(2, dst->size()); + const auto& nullable_col = assert_cast(*dst); + const auto& null_map = nullable_col.get_null_map_data(); + ASSERT_EQ(2, null_map.size()); + EXPECT_EQ(0, null_map[0]); + EXPECT_EQ(1, null_map[1]); + EXPECT_EQ(2, nullable_col.get_nested_column().size()); +} + TEST_F(ColumnReaderTest, StructAccessPaths) { auto create_struct_iterator = []() { auto null_reader = std::make_shared(); @@ -69,7 +402,7 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { auto st = iterator->set_access_paths(TColumnAccessPaths {}, TColumnAccessPaths {}); ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); - ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NORMAL_READING); + ASSERT_EQ(iterator->_read_requirement, ColumnIterator::ReadRequirement::NORMAL); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); @@ -82,7 +415,7 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { ASSERT_FALSE(st.ok()); // Only reading sub_col_1 - // sub_col_2 should be set to SKIP_READING + // sub_col_2 should be set to SKIP all_access_paths[0].data_access_path.path = {"self", "sub_col_1"}; predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"}; @@ -95,12 +428,12 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { // now column name is "self", should be ok st = iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); - ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + ASSERT_EQ(iterator->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); - ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag, - ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); - ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag, - ColumnIterator::ReadingFlag::SKIP_READING); + ASSERT_EQ(iterator->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + ASSERT_EQ(iterator->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); // Reading all sub columns all_access_paths[0].data_access_path.path = {"self"}; @@ -109,12 +442,1463 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { st = iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); - ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + ASSERT_EQ(iterator->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + + ASSERT_EQ(iterator->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + ASSERT_EQ(iterator->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); +} + +TEST_F(ColumnReaderTest, ReadPhaseMatrix) { + TestColumnIterator iterator; + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::SKIP); + iterator.set_read_phase(ColumnIterator::ReadPhase::NORMAL); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_TRUE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_TRUE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_TRUE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); +} + +TEST_F(ColumnReaderTest, ReadRequirementPriorityAndLazyOutput) { + TestColumnIterator iterator; + + iterator.set_read_requirement(ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(iterator.read_requirement(), ColumnIterator::ReadRequirement::SKIP); + + iterator.set_lazy_output_requirement(); + EXPECT_EQ(iterator.read_requirement(), ColumnIterator::ReadRequirement::LAZY_OUTPUT); + + iterator.set_read_requirement(ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(iterator.read_requirement(), ColumnIterator::ReadRequirement::LAZY_OUTPUT); + + iterator.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(iterator.read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + + iterator.set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(iterator.read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); +} + +TEST_F(ColumnReaderTest, MetaReadModePrefersOffsetOverNull) { + auto assert_meta_read_mode = [](TColumnAccessPaths access_paths, bool offset_only, + bool null_map_only) { + TestColumnIterator iterator; + iterator.check_and_set_meta_read_mode(ColumnIterator::ReadRequirement::NORMAL, + access_paths); + EXPECT_EQ(iterator.read_offset_only(), offset_only); + EXPECT_EQ(iterator.read_null_map_only(), null_map_only); + }; + + assert_meta_read_mode(TColumnAccessPaths {}, false, false); + assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_OFFSET})}, + true, false); + assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_NULL})}, + false, true); + assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_OFFSET}), + create_access_path({ColumnIterator::ACCESS_NULL})}, + true, false); + assert_meta_read_mode(TColumnAccessPaths {create_access_path({"child"})}, false, false); + assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_OFFSET}), + create_access_path({"child"})}, + false, false); + + { + TestColumnIterator iterator; + iterator.check_and_set_meta_read_mode( + ColumnIterator::ReadRequirement::LAZY_OUTPUT, + TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_NULL})}); + EXPECT_FALSE(iterator.read_null_map_only()); + EXPECT_FALSE(iterator.read_offset_only()); + } +} + +TEST_F(ColumnReaderTest, PlaceHolderLifecycleInLazyMode) { + TestColumnIterator iterator; + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + iterator.convert_to_place_holder_column(dst, 3); + + EXPECT_EQ(3, dst->size()); + EXPECT_TRUE(iterator._has_place_holder_column); + + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + iterator.finalize_lazy_phase(dst); + EXPECT_EQ(0, dst->size()); + EXPECT_FALSE(iterator._has_place_holder_column); + + MutableColumnPtr lazy_dst = ColumnInt32::create(); + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + iterator.convert_to_place_holder_column(lazy_dst, 4); + EXPECT_EQ(0, lazy_dst->size()); +} + +TEST_F(ColumnReaderTest, PlaceHolderRecoveryAfterColumnReplacement) { + TestColumnIterator iterator; + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + iterator.convert_to_place_holder_column(dst, 3); + EXPECT_TRUE(iterator._has_place_holder_column); + + IColumn::Filter filter; + filter.resize(3); + filter[0] = 1; + filter[1] = 0; + filter[2] = 1; + dst = IColumn::mutate(dst->filter(filter, 2)); + EXPECT_EQ(2, dst->size()); + + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + iterator.finalize_lazy_phase(dst); + EXPECT_EQ(0, dst->size()); + EXPECT_FALSE(iterator._has_place_holder_column); + + dst->insert_many_defaults(2); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + iterator.finalize_lazy_phase(dst); + EXPECT_EQ(2, dst->size()); +} + +TEST_F(ColumnReaderTest, SetReadRequirementPropagatesToNestedIterators) { + auto null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + + auto sub_col = std::make_unique(std::make_shared()); + sub_col->set_column_name("sub_col"); + struct_sub_iters.emplace_back(std::move(sub_col)); + + auto array_item = std::make_unique(std::make_shared()); + array_item->set_column_name("item"); + auto array_offsets = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null = std::make_unique(std::make_shared()); + auto array_iter = std::make_unique( + std::make_shared(), std::move(array_offsets), std::move(array_item), + std::move(array_null)); + array_iter->set_column_name("arr"); + struct_sub_iters.emplace_back(std::move(array_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), std::move(null_iter), + std::move(struct_sub_iters)); + struct_iter.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + + EXPECT_EQ(struct_iter.read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iter._sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* nested_array = + static_cast(struct_iter._sub_column_iterators[1].get()); + EXPECT_EQ(nested_array->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(nested_array->_item_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iter(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iter.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iter._key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iter._val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); +} + +TEST_F(ColumnReaderTest, SetReadRequirementSelfKeepsNestedIteratorRequirements) { + auto null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_iter = std::make_unique(std::make_shared()); + sub_iter->set_column_name("sub_col"); + struct_sub_iters.emplace_back(std::move(sub_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), std::move(null_iter), + std::move(struct_sub_iters)); + struct_iter.set_read_requirement_self(ColumnIterator::ReadRequirement::PREDICATE); + + EXPECT_EQ(struct_iter.read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iter._sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::NORMAL); +} + +TEST_F(ColumnReaderTest, RemovePrunedSubIterators) { + auto struct_null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_keep = std::make_unique(std::make_shared()); + sub_keep->set_column_name("keep"); + auto sub_prune = std::make_unique(std::make_shared()); + sub_prune->set_column_name("prune"); + sub_prune->set_read_requirement(ColumnIterator::ReadRequirement::SKIP); + struct_sub_iters.emplace_back(std::move(sub_keep)); + struct_sub_iters.emplace_back(std::move(sub_prune)); + + auto array_item_null = std::make_unique(std::make_shared()); + std::vector item_struct_sub_iters; + auto item_keep = std::make_unique(std::make_shared()); + item_keep->set_column_name("keep"); + auto item_prune = std::make_unique(std::make_shared()); + item_prune->set_column_name("prune"); + item_prune->set_read_requirement(ColumnIterator::ReadRequirement::SKIP); + item_struct_sub_iters.emplace_back(std::move(item_keep)); + item_struct_sub_iters.emplace_back(std::move(item_prune)); + auto item_struct = std::make_unique(std::make_shared(), + std::move(array_item_null), + std::move(item_struct_sub_iters)); + + auto array_offsets = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null = std::make_unique(std::make_shared()); + auto array_iter = std::make_unique( + std::make_shared(), std::move(array_offsets), std::move(item_struct), + std::move(array_null)); + struct_sub_iters.emplace_back(std::move(array_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), + std::move(struct_null_iter), std::move(struct_sub_iters)); + ASSERT_EQ(3, struct_iter._sub_column_iterators.size()); + struct_iter.remove_pruned_sub_iterators(); + ASSERT_EQ(2, struct_iter._sub_column_iterators.size()); + + auto* nested_array = + static_cast(struct_iter._sub_column_iterators[1].get()); + auto* nested_struct = + static_cast(nested_array->_item_iterator.get()); + ASSERT_EQ(1, nested_struct->_sub_column_iterators.size()); + EXPECT_EQ(nested_struct->_sub_column_iterators[0]->column_name(), "keep"); +} + +TEST_F(ColumnReaderTest, FinalizeLazyModeOnNestedStruct) { + auto sub_iter = std::make_unique(); + auto* sub_iter_ptr = sub_iter.get(); + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + sub_iters.emplace_back(std::move(sub_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), std::move(null_iter), + std::move(sub_iters)); + sub_iter_ptr->set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + struct_iter.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + sub_iter_ptr->set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr nested_column = ColumnInt32::create(); + MutableColumnPtr nested_mut = IColumn::mutate(std::move(nested_column)); + sub_iter_ptr->convert_to_place_holder_column(nested_mut, 5); + EXPECT_EQ(5, nested_mut->size()); + + Columns struct_columns; + struct_columns.emplace_back(std::move(nested_mut)); + auto struct_column = ColumnStruct::create(struct_columns); + MutableColumnPtr struct_mut = std::move(struct_column); + struct_iter.set_read_phase(ColumnIterator::ReadPhase::LAZY); + sub_iter_ptr->set_read_phase(ColumnIterator::ReadPhase::LAZY); + struct_iter.finalize_lazy_phase(struct_mut); + + auto& column_struct = assert_cast(*struct_mut); + auto nested_after = column_struct.get_column_ptr(0); + EXPECT_EQ(0, nested_after->size()); +} + +TEST_F(ColumnReaderTest, GetSubAccessPathsSetsPredicateFlag) { + TestColumnIterator iterator; + iterator.set_column_name("self"); + + TColumnAccessPaths access_paths; + access_paths.emplace_back(); + access_paths[0].data_access_path.path = {"self"}; + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); + auto sub_paths = TEST_TRY(iterator.get_sub_access_paths(access_paths)); + EXPECT_TRUE(sub_paths.empty()); + EXPECT_EQ(iterator._read_requirement, ColumnIterator::ReadRequirement::LAZY_OUTPUT); + + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); + sub_paths = TEST_TRY(iterator.get_sub_access_paths(access_paths, true)); + EXPECT_TRUE(sub_paths.empty()); + EXPECT_EQ(iterator._read_requirement, ColumnIterator::ReadRequirement::PREDICATE); +} + +TEST_F(ColumnReaderTest, NestedIteratorsPropagateReadPhase) { + auto struct_null_iterator = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + auto struct_iterator = std::make_unique( + std::make_shared(), std::move(struct_null_iterator), + std::move(struct_sub_iters)); + + struct_iterator->set_read_phase(ColumnIterator::ReadPhase::LAZY); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->_read_phase, + ColumnIterator::ReadPhase::LAZY); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->_read_phase, + ColumnIterator::ReadPhase::LAZY); + + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + EXPECT_EQ(array_iterator._item_iterator->_read_phase, ColumnIterator::ReadPhase::PREDICATE); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + EXPECT_EQ(map_iterator._key_iterator->_read_phase, ColumnIterator::ReadPhase::LAZY); + EXPECT_EQ(map_iterator._val_iterator->_read_phase, ColumnIterator::ReadPhase::LAZY); +} + +TEST_F(ColumnReaderTest, AccessPathsPropagatePredicateToChildren) { + auto struct_null_iterator = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + auto struct_iterator = std::make_unique( + std::make_shared(), std::move(struct_null_iterator), + std::move(struct_sub_iters)); + struct_iterator->set_column_name("s"); + + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"s"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + EXPECT_EQ(struct_iterator->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.set_column_name("a"); + TColumnAccessPaths array_access_paths; + array_access_paths.emplace_back(); + array_access_paths[0].data_access_path.path = {"a"}; + TColumnAccessPaths array_predicate_paths = array_access_paths; + st = array_iterator.set_access_paths(array_access_paths, array_predicate_paths); + ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); + EXPECT_EQ(array_iterator._read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(array_iterator._item_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iterator.set_column_name("m"); + TColumnAccessPaths map_access_paths; + map_access_paths.emplace_back(); + map_access_paths[0].data_access_path.path = {"m"}; + TColumnAccessPaths map_predicate_paths = map_access_paths; + st = map_iterator.set_access_paths(map_access_paths, map_predicate_paths); + ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); + EXPECT_EQ(map_iterator._read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iterator._key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iterator._val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); +} + +TEST_F(ColumnReaderTest, StructPredicateOnlyChildPathStillRoutesToChild) { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + StructFileColumnIterator struct_iterator(std::make_shared(), std::move(null_iter), + std::move(sub_iters)); + struct_iterator.set_column_name("s"); + + TColumnAccessPaths all_access_paths {create_access_path({"s", "a"})}; + TColumnAccessPaths predicate_access_paths {create_access_path({"s", "b"})}; + + auto st = struct_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + + EXPECT_EQ(struct_iterator._read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iterator._sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(struct_iterator._sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + struct_iterator.remove_pruned_sub_iterators(); + ASSERT_EQ(struct_iterator._sub_column_iterators.size(), 2); + EXPECT_EQ(struct_iterator._sub_column_iterators[0]->column_name(), "a"); + EXPECT_EQ(struct_iterator._sub_column_iterators[1]->column_name(), "b"); +} + +TEST_F(ColumnReaderTest, CurrentLevelPredicateNullPathUsesMetaOnlyMode) { + auto make_struct_iterator = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto struct_iterator = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + struct_iterator->set_column_name("s"); + return struct_iterator; + }; + + { + auto struct_iterator = make_struct_iterator(); + TColumnAccessPaths all_access_paths { + create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + + auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + + EXPECT_TRUE(struct_iterator->read_null_map_only()); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + + struct_iterator->remove_pruned_sub_iterators(); + EXPECT_TRUE(struct_iterator->_sub_column_iterators.empty()); + } + + { + auto struct_iterator = make_struct_iterator(); + TColumnAccessPaths all_access_paths { + create_access_path({"s"}), create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + + auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + + EXPECT_FALSE(struct_iterator->read_null_map_only()); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + } + + { + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = + std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.set_column_name("a"); + + TColumnAccessPaths all_access_paths { + create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + + auto st = array_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); + EXPECT_TRUE(array_iterator.read_null_map_only()); + EXPECT_EQ(array_iterator._item_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } + + { + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), + std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), std::move(map_val_iter)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths all_access_paths { + create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + + auto st = map_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); + EXPECT_TRUE(map_iterator.read_null_map_only()); + EXPECT_EQ(map_iterator._key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(map_iterator._val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } +} + +TEST_F(ColumnReaderTest, StructPredicateMetaPathDoesNotOverrideExistingDataNeed) { + auto make_struct_iterator = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto city_iter = + std::make_unique(std::make_shared()); + city_iter->set_column_name("city"); + auto data_iter = std::make_unique(std::make_shared()); + data_iter->set_column_name("data"); + sub_iters.emplace_back(std::move(city_iter)); + sub_iters.emplace_back(std::move(data_iter)); + auto struct_iterator = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + struct_iterator->set_column_name("s"); + return struct_iterator; + }; + + auto struct_iterator = make_struct_iterator(); + TColumnAccessPaths all_access_paths { + create_access_path({"s"}), + create_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + + auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + + auto* city_iter = + static_cast(struct_iterator->_sub_column_iterators[0].get()); + EXPECT_FALSE(city_iter->read_null_map_only()); + EXPECT_EQ(city_iter->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->read_requirement(), + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + + struct_iterator = make_struct_iterator(); + all_access_paths = {create_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + predicate_access_paths = all_access_paths; + st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set predicate-only struct access paths: " << st.to_string(); + city_iter = + static_cast(struct_iterator->_sub_column_iterators[0].get()); + EXPECT_TRUE(city_iter->read_null_map_only()); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); +} + +TEST_F(ColumnReaderTest, ArrayPredicateMetaPathDoesNotOverrideExistingDataNeed) { + auto make_array_iterator = []() { + auto item_iter = + std::make_unique(std::make_shared()); + item_iter->set_column_name("item"); + auto offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto null_iter = std::make_unique(std::make_shared()); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(offsets_iter), std::move(item_iter), + std::move(null_iter)); + array_iterator->set_column_name("a"); + return array_iterator; + }; + + auto array_iterator = make_array_iterator(); + TColumnAccessPaths all_access_paths { + create_access_path({"a"}), + create_access_path({"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; + + auto st = array_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); + auto* item_iter = static_cast(array_iterator->_item_iterator.get()); + EXPECT_FALSE(item_iter->read_null_map_only()); + EXPECT_EQ(item_iter->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); - ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag, - ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); - ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag, - ColumnIterator::ReadingFlag::NEED_TO_READ); + array_iterator = make_array_iterator(); + all_access_paths = { + create_access_path({"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; + predicate_access_paths = all_access_paths; + st = array_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set predicate-only array access paths: " << st.to_string(); + item_iter = static_cast(array_iterator->_item_iterator.get()); + EXPECT_TRUE(item_iter->read_null_map_only()); +} + +TEST_F(ColumnReaderTest, MapPredicateMetaPathDoesNotOverrideExistingDataNeed) { + auto make_map_iterator = []() { + auto null_iter = std::make_unique(std::make_shared()); + auto offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto key_iter = std::make_unique(std::make_shared()); + key_iter->set_column_name("key"); + auto value_iter = + std::make_unique(std::make_shared()); + value_iter->set_column_name("value"); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(offsets_iter), + std::move(key_iter), std::move(value_iter)); + map_iterator->set_column_name("m"); + return map_iterator; + }; + + auto map_iterator = make_map_iterator(); + TColumnAccessPaths all_access_paths {create_access_path({"m"}), + create_access_path({"m", ColumnIterator::ACCESS_MAP_VALUES, + ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths {create_access_path( + {"m", ColumnIterator::ACCESS_MAP_VALUES, ColumnIterator::ACCESS_NULL})}; + + auto st = map_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); + auto* value_iter = static_cast(map_iterator->_val_iterator.get()); + EXPECT_FALSE(value_iter->read_null_map_only()); + EXPECT_EQ(map_iterator->_key_iterator->read_requirement(), + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(value_iter->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + + map_iterator = make_map_iterator(); + all_access_paths = {create_access_path( + {"m", ColumnIterator::ACCESS_MAP_VALUES, ColumnIterator::ACCESS_NULL})}; + predicate_access_paths = all_access_paths; + st = map_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set predicate-only map access paths: " << st.to_string(); + value_iter = static_cast(map_iterator->_val_iterator.get()); + EXPECT_TRUE(value_iter->read_null_map_only()); + EXPECT_EQ(map_iterator->_key_iterator->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); +} + +TEST_F(ColumnReaderTest, MapFullProjectionStillRoutesPredicateSubPaths) { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), make_value_struct()); + map_iterator->set_column_name("m"); + + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"m"}; + + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"m", "KEYS"}; + predicate_access_paths.emplace_back(); + predicate_access_paths[1].data_access_path.path = {"m", "VALUES", "a"}; + + auto st = map_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); + + EXPECT_EQ(map_iterator->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iterator->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iterator->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* value_struct = static_cast(map_iterator->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); +} + +TEST_F(ColumnReaderTest, MetaOnlyAllPathsStillRoutePredicateSubPaths) { + { + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = + std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.set_column_name("a"); + + TColumnAccessPaths all_access_paths { + create_access_path({"a", ColumnIterator::ACCESS_OFFSET}), + create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"a", ColumnIterator::ACCESS_ALL})}; + + auto st = array_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); + EXPECT_FALSE(array_iterator.read_offset_only()); + EXPECT_EQ(array_iterator._item_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + } + + { + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), + std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), std::move(map_val_iter)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths all_access_paths { + create_access_path({"m", ColumnIterator::ACCESS_OFFSET}), + create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_access_path({"m", ColumnIterator::ACCESS_MAP_KEYS})}; + + auto st = map_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); + EXPECT_FALSE(map_iterator.read_offset_only()); + EXPECT_EQ(map_iterator._key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iterator._val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } +} + +TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPaths) { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), std::move(map_val_iter)); + map_iterator->set_column_name("item"); + + auto array_null_iter = std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(array_offsets_iter), + std::move(map_iterator), std::move(array_null_iter)); + array_iterator->set_column_name("col2"); + + auto struct_null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_col1 = std::make_unique(std::make_shared()); + sub_col1->set_column_name("col1"); + struct_sub_iters.emplace_back(std::move(sub_col1)); + struct_sub_iters.emplace_back(std::move(array_iterator)); + auto top_struct = std::make_unique(std::make_shared(), + std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + + TColumnAccessPaths access_paths; + access_paths.emplace_back(); + access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + TColumnAccessPaths predicate_access_paths = access_paths; + + auto st = top_struct->set_access_paths(access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set nested access paths: " << st.to_string(); + + EXPECT_EQ(top_struct->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(top_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); +} + +TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { + auto build_nested_iterator = []() { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), std::move(map_val_iter)); + map_iterator->set_column_name("item"); + + auto array_null_iter = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(array_offsets_iter), + std::move(map_iterator), std::move(array_null_iter)); + array_iterator->set_column_name("col2"); + + auto struct_null_iter = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_col1 = std::make_unique(std::make_shared()); + sub_col1->set_column_name("col1"); + struct_sub_iters.emplace_back(std::move(sub_col1)); + struct_sub_iters.emplace_back(std::move(array_iterator)); + auto top_struct = std::make_unique( + std::make_shared(), std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + return top_struct; + }; + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col1"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + EXPECT_EQ(top_struct->_read_requirement, ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(top_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "b"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + EXPECT_EQ(top_struct->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + } + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + EXPECT_EQ(top_struct->_read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(map_iter->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_FALSE(st.ok()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"wrong_root", "col2"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_FALSE(st.ok()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "wrong_item"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_FALSE(st.ok()); + } +} + +TEST_F(ColumnReaderTest, DeepNestedAccessPathsFiveLevels) { + auto make_item_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_p = std::make_unique(std::make_shared()); + sub_p->set_column_name("p"); + auto sub_q = std::make_unique(std::make_shared()); + sub_q->set_column_name("q"); + sub_iters.emplace_back(std::move(sub_p)); + sub_iters.emplace_back(std::move(sub_q)); + + auto item_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + item_struct->set_column_name("item"); + return item_struct; + }; + + auto make_value_struct = [make_item_struct]() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto array_offsets = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null = std::make_unique(std::make_shared()); + auto array_iter = std::make_unique( + std::make_shared(), std::move(array_offsets), make_item_struct(), + std::move(array_null)); + array_iter->set_column_name("arr"); + sub_iters.emplace_back(std::move(array_iter)); + + auto sub_z = std::make_unique(std::make_shared()); + sub_z->set_column_name("z"); + sub_iters.emplace_back(std::move(sub_z)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iter = std::make_unique( + std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), std::move(map_val_iter)); + map_iter->set_column_name("m"); + + auto struct_null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_x = std::make_unique(std::make_shared()); + sub_x->set_column_name("x"); + struct_sub_iters.emplace_back(std::move(sub_x)); + struct_sub_iters.emplace_back(std::move(map_iter)); + auto top_struct = std::make_unique(std::make_shared(), + std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "m", "VALUES", "arr", "*"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "m", "VALUES", "arr", "*", "q"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set deep access paths: " << st.to_string(); + + auto* map_ptr = static_cast(top_struct->_sub_column_iterators[1].get()); + EXPECT_EQ(map_ptr->_key_iterator->_read_requirement, ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(map_ptr->_val_iterator->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + + auto* value_struct = static_cast(map_ptr->_val_iterator.get()); + auto* array_iter = + static_cast(value_struct->_sub_column_iterators[0].get()); + auto* item_struct = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(item_struct->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_EQ(item_struct->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); +} + +TEST_F(ColumnReaderTest, NestedLazyOutputInLazyPredicatePhase) { + auto struct_null_iterator = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + StructFileColumnIterator struct_iterator(std::make_shared(), + std::move(struct_null_iterator), + std::move(struct_sub_iters)); + struct_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + struct_iterator.set_read_requirement_self(ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_FALSE(struct_iterator.has_lazy_read_target()); + EXPECT_FALSE(struct_iterator.need_to_read()); + struct_iterator._sub_column_iterators[0]->set_lazy_output_requirement(); + EXPECT_TRUE(struct_iterator.has_lazy_read_target()); + EXPECT_TRUE(struct_iterator.need_to_read()); + + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + array_iterator.set_read_requirement_self(ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_FALSE(array_iterator.has_lazy_read_target()); + EXPECT_FALSE(array_iterator.need_to_read()); + array_iterator._item_iterator->set_lazy_output_requirement(); + EXPECT_TRUE(array_iterator.has_lazy_read_target()); + EXPECT_TRUE(array_iterator.need_to_read()); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + map_iterator.set_read_requirement_self(ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_FALSE(map_iterator.has_lazy_read_target()); + EXPECT_FALSE(map_iterator.need_to_read()); + map_iterator._val_iterator->set_lazy_output_requirement(); + EXPECT_TRUE(map_iterator.has_lazy_read_target()); + EXPECT_TRUE(map_iterator.need_to_read()); +} + +TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { + auto build_nested_iterator = []() { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), std::move(map_val_iter)); + map_iterator->set_column_name("item"); + + auto array_null_iter = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(array_offsets_iter), + std::move(map_iterator), std::move(array_null_iter)); + array_iterator->set_column_name("col2"); + + auto struct_null_iter = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_col1 = std::make_unique(std::make_shared()); + sub_col1->set_column_name("col1"); + struct_sub_iters.emplace_back(std::move(sub_col1)); + struct_sub_iters.emplace_back(std::move(array_iterator)); + auto top_struct = std::make_unique( + std::make_shared(), std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + return top_struct; + }; + + auto assert_need_to_read = [](StructFileColumnIterator* top_struct) { + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + auto expect_scalar = [](ColumnIterator::ReadRequirement requirement, + ColumnIterator::ReadPhase mode) { + switch (mode) { + case ColumnIterator::ReadPhase::NORMAL: + return requirement != ColumnIterator::ReadRequirement::SKIP; + case ColumnIterator::ReadPhase::PREDICATE: + return requirement == ColumnIterator::ReadRequirement::PREDICATE; + case ColumnIterator::ReadPhase::LAZY: + return requirement == ColumnIterator::ReadRequirement::LAZY_OUTPUT; + default: + return false; + } + }; + auto expect_nested = [](ColumnIterator::ReadRequirement requirement, + ColumnIterator::ReadPhase mode) { + switch (mode) { + case ColumnIterator::ReadPhase::NORMAL: + return requirement != ColumnIterator::ReadRequirement::SKIP; + case ColumnIterator::ReadPhase::PREDICATE: + return requirement == ColumnIterator::ReadRequirement::PREDICATE; + default: + return false; + } + }; + + top_struct->set_read_phase(ColumnIterator::ReadPhase::NORMAL); + EXPECT_EQ(expect_nested(top_struct->read_requirement(), ColumnIterator::ReadPhase::NORMAL), + top_struct->need_to_read()); + EXPECT_EQ(expect_nested(array_iter->read_requirement(), ColumnIterator::ReadPhase::NORMAL), + array_iter->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->read_requirement(), ColumnIterator::ReadPhase::NORMAL), + map_iter->need_to_read()); + EXPECT_EQ( + expect_nested(value_struct->read_requirement(), ColumnIterator::ReadPhase::NORMAL), + value_struct->need_to_read()); + EXPECT_EQ(expect_scalar(map_iter->_key_iterator->read_requirement(), + ColumnIterator::ReadPhase::NORMAL), + map_iter->_key_iterator->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->_val_iterator->read_requirement(), + ColumnIterator::ReadPhase::NORMAL), + map_iter->_val_iterator->need_to_read()); + + top_struct->set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + EXPECT_EQ( + expect_nested(top_struct->read_requirement(), ColumnIterator::ReadPhase::PREDICATE), + top_struct->need_to_read()); + EXPECT_EQ( + expect_nested(array_iter->read_requirement(), ColumnIterator::ReadPhase::PREDICATE), + array_iter->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->read_requirement(), ColumnIterator::ReadPhase::PREDICATE), + map_iter->need_to_read()); + EXPECT_EQ(expect_nested(value_struct->read_requirement(), + ColumnIterator::ReadPhase::PREDICATE), + value_struct->need_to_read()); + EXPECT_EQ(expect_scalar(map_iter->_key_iterator->read_requirement(), + ColumnIterator::ReadPhase::PREDICATE), + map_iter->_key_iterator->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->_val_iterator->read_requirement(), + ColumnIterator::ReadPhase::PREDICATE), + map_iter->_val_iterator->need_to_read()); + + top_struct->set_read_phase(ColumnIterator::ReadPhase::LAZY); + EXPECT_EQ(top_struct->has_lazy_read_target(), top_struct->need_to_read()); + EXPECT_EQ(array_iter->has_lazy_read_target(), array_iter->need_to_read()); + EXPECT_EQ(map_iter->has_lazy_read_target(), map_iter->need_to_read()); + EXPECT_EQ(value_struct->has_lazy_read_target(), value_struct->need_to_read()); + EXPECT_EQ(expect_scalar(map_iter->_key_iterator->read_requirement(), + ColumnIterator::ReadPhase::LAZY), + map_iter->_key_iterator->need_to_read()); + EXPECT_EQ(map_iter->_val_iterator->has_lazy_read_target(), + map_iter->_val_iterator->need_to_read()); + }; + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + assert_need_to_read(top_struct.get()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + assert_need_to_read(top_struct.get()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + assert_need_to_read(top_struct.get()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_TRUE(st.ok()); + } } TEST_F(ColumnReaderTest, MultiAccessPaths) { @@ -202,20 +1986,318 @@ TEST_F(ColumnReaderTest, MultiAccessPaths) { auto st = iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); - ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_EQ(iterator->_read_requirement, ColumnIterator::ReadRequirement::LAZY_OUTPUT); - ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag, - ColumnIterator::ReadingFlag::SKIP_READING); - ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag, - ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_EQ(iterator->_sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + ASSERT_EQ(iterator->_sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); auto* array_iter = static_cast(iterator->_sub_column_iterators[1].get()); - ASSERT_EQ(array_iter->_item_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_EQ(array_iter->_item_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); auto* map_iter = static_cast(array_iter->_item_iterator.get()); - ASSERT_EQ(map_iter->_key_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); - ASSERT_EQ(map_iter->_val_iterator->_reading_flag, ColumnIterator::ReadingFlag::SKIP_READING); + ASSERT_EQ(map_iter->_key_iterator->_read_requirement, + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + ASSERT_EQ(map_iter->_val_iterator->_read_requirement, ColumnIterator::ReadRequirement::SKIP); +} + +TEST_F(ColumnReaderTest, StructNextBatchAndReadByRowidsUseSequentialChildReads) { + std::vector sub_column_iterators; + auto first_child = std::make_unique(); + auto* first_child_ptr = first_child.get(); + auto second_child = std::make_unique(); + auto* second_child_ptr = second_child.get(); + sub_column_iterators.emplace_back(std::move(first_child)); + sub_column_iterators.emplace_back(std::move(second_child)); + + StructFileColumnIterator struct_iterator(create_test_reader(), nullptr, + std::move(sub_column_iterators)); + + MutableColumnPtr dst = create_int_struct_column(2); + size_t rows = 3; + bool has_null = false; + auto st = struct_iterator.next_batch(&rows, dst, &has_null); + ASSERT_TRUE(st.ok()) << "struct next_batch failed: " << st.to_string(); + EXPECT_EQ(3, rows); + EXPECT_EQ(3, dst->size()); + EXPECT_THAT(first_child_ptr->next_batch_sizes, ::testing::ElementsAre(3)); + EXPECT_THAT(second_child_ptr->next_batch_sizes, ::testing::ElementsAre(3)); + + first_child_ptr->clear_tracking(); + second_child_ptr->clear_tracking(); + + const rowid_t rowids[] = {0, 1, 4, 5, 6}; + st = struct_iterator.read_by_rowids(rowids, std::size(rowids), dst); + ASSERT_TRUE(st.ok()) << "struct read_by_rowids failed: " << st.to_string(); + EXPECT_EQ(8, dst->size()); + EXPECT_THAT(first_child_ptr->seek_ordinals, ::testing::ElementsAre(0, 4)); + EXPECT_THAT(second_child_ptr->seek_ordinals, ::testing::ElementsAre(0, 4)); + EXPECT_THAT(first_child_ptr->next_batch_sizes, ::testing::ElementsAre(2, 3)); + EXPECT_THAT(second_child_ptr->next_batch_sizes, ::testing::ElementsAre(2, 3)); + EXPECT_TRUE(first_child_ptr->read_by_rowids_batches.empty()); + EXPECT_TRUE(second_child_ptr->read_by_rowids_batches.empty()); +} + +TEST_F(ColumnReaderTest, StructNullMapOnlyNextBatchSkipsSubColumns) { + auto null_iterator = std::make_unique(); + auto* null_iterator_ptr = null_iterator.get(); + std::vector sub_column_iterators; + auto child_iterator = std::make_unique(); + auto* child_iterator_ptr = child_iterator.get(); + child_iterator->set_column_name("field"); + sub_column_iterators.emplace_back(std::move(child_iterator)); + + StructFileColumnIterator struct_iterator(create_test_reader(true), std::move(null_iterator), + std::move(sub_column_iterators)); + struct_iterator.set_column_name("s"); + + TColumnAccessPaths null_path {create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + auto st = struct_iterator.set_access_paths(null_path, null_path); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + EXPECT_TRUE(struct_iterator.read_null_map_only()); + EXPECT_EQ(child_iterator_ptr->read_requirement(), ColumnIterator::ReadRequirement::SKIP); + + MutableColumnPtr dst = create_nullable_int_struct_column(1); + size_t rows = 2; + bool has_null = false; + st = struct_iterator.next_batch(&rows, dst, &has_null); + ASSERT_TRUE(st.ok()) << "struct null-map-only next_batch failed: " << st.to_string(); + EXPECT_TRUE(has_null); + EXPECT_EQ(2, dst->size()); + EXPECT_THAT(null_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(2)); + EXPECT_TRUE(child_iterator_ptr->next_batch_sizes.empty()); + + const auto& nullable_column = assert_cast(*dst); + EXPECT_EQ(2, nullable_column.get_null_map_column().size()); + const auto& nested_struct = assert_cast( + nullable_column.get_nested_column()); + EXPECT_EQ(2, nested_struct.get_column(0).size()); +} + +TEST_F(ColumnReaderTest, ArrayNullMapOnlyNextBatchAndReadByRowidsSkipItems) { + auto null_iterator = std::make_unique(); + auto* null_iterator_ptr = null_iterator.get(); + auto item_iterator = std::make_unique(); + auto* item_iterator_ptr = item_iterator.get(); + auto offset_iterator = create_tracking_offset_iterator(); + + ArrayFileColumnIterator array_iterator(create_test_reader(true), + std::move(offset_iterator.iterator), + std::move(item_iterator), std::move(null_iterator)); + array_iterator.set_column_name("a"); + + TColumnAccessPaths null_path {create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + auto st = array_iterator.set_access_paths(null_path, null_path); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + EXPECT_TRUE(array_iterator.read_null_map_only()); + EXPECT_EQ(item_iterator_ptr->read_requirement(), ColumnIterator::ReadRequirement::SKIP); + + MutableColumnPtr dst = create_nullable_int_array_column(); + size_t rows = 3; + bool has_null = false; + st = array_iterator.next_batch(&rows, dst, &has_null); + ASSERT_TRUE(st.ok()) << "array null-map-only next_batch failed: " << st.to_string(); + EXPECT_TRUE(has_null); + EXPECT_EQ(3, dst->size()); + EXPECT_THAT(null_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(3)); + EXPECT_TRUE(item_iterator_ptr->next_batch_sizes.empty()); + EXPECT_TRUE(offset_iterator.tracker->next_batch_sizes.empty()); + + null_iterator_ptr->clear_tracking(); + item_iterator_ptr->clear_tracking(); + + const rowid_t rowids[] = {1, 3}; + st = array_iterator.read_by_rowids(rowids, std::size(rowids), dst); + ASSERT_TRUE(st.ok()) << "array null-map-only read_by_rowids failed: " << st.to_string(); + EXPECT_EQ(5, dst->size()); + EXPECT_THAT(null_iterator_ptr->seek_ordinals, ::testing::ElementsAre(1, 3)); + EXPECT_THAT(null_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(1, 1)); + EXPECT_TRUE(item_iterator_ptr->next_batch_sizes.empty()); + EXPECT_TRUE(offset_iterator.tracker->next_batch_sizes.empty()); +} + +TEST_F(ColumnReaderTest, MapNullMapOnlyNextBatchAndReadByRowidsSkipKeysAndValues) { + auto null_iterator = std::make_unique(); + auto* null_iterator_ptr = null_iterator.get(); + auto key_iterator = std::make_unique(); + auto* key_iterator_ptr = key_iterator.get(); + auto value_iterator = std::make_unique(); + auto* value_iterator_ptr = value_iterator.get(); + auto offset_iterator = create_tracking_offset_iterator(); + + MapFileColumnIterator map_iterator(create_test_reader(true, 4), std::move(null_iterator), + std::move(offset_iterator.iterator), std::move(key_iterator), + std::move(value_iterator)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths null_path {create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + auto st = map_iterator.set_access_paths(null_path, null_path); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + EXPECT_TRUE(map_iterator.read_null_map_only()); + EXPECT_EQ(key_iterator_ptr->read_requirement(), ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(value_iterator_ptr->read_requirement(), ColumnIterator::ReadRequirement::SKIP); + + MutableColumnPtr dst = create_nullable_int_map_column(); + size_t rows = 3; + bool has_null = false; + st = map_iterator.next_batch(&rows, dst, &has_null); + ASSERT_TRUE(st.ok()) << "map null-map-only next_batch failed: " << st.to_string(); + EXPECT_TRUE(has_null); + EXPECT_EQ(3, dst->size()); + EXPECT_THAT(null_iterator_ptr->next_batch_sizes, ::testing::ElementsAre(3)); + EXPECT_TRUE(key_iterator_ptr->next_batch_sizes.empty()); + EXPECT_TRUE(value_iterator_ptr->next_batch_sizes.empty()); + EXPECT_TRUE(offset_iterator.tracker->next_batch_sizes.empty()); + + null_iterator_ptr->clear_tracking(); + key_iterator_ptr->clear_tracking(); + value_iterator_ptr->clear_tracking(); + + const rowid_t rowids[] = {1, 3}; + st = map_iterator.read_by_rowids(rowids, std::size(rowids), dst); + ASSERT_TRUE(st.ok()) << "map null-map-only read_by_rowids failed: " << st.to_string(); + EXPECT_EQ(5, dst->size()); + ASSERT_EQ(1, null_iterator_ptr->read_by_rowids_batches.size()); + EXPECT_THAT(null_iterator_ptr->read_by_rowids_batches[0], ::testing::ElementsAre(1, 3)); + EXPECT_TRUE(key_iterator_ptr->next_batch_sizes.empty()); + EXPECT_TRUE(value_iterator_ptr->next_batch_sizes.empty()); + EXPECT_TRUE(offset_iterator.tracker->next_batch_sizes.empty()); +} + +TEST_F(ColumnReaderTest, CollectPrefetchersHonorsNestedReadRequirements) { + auto null_iterator = std::make_unique(); + auto* null_iterator_ptr = null_iterator.get(); + std::vector sub_column_iterators; + auto predicate_child = std::make_unique(); + auto* predicate_child_ptr = predicate_child.get(); + auto lazy_child = std::make_unique(); + auto* lazy_child_ptr = lazy_child.get(); + sub_column_iterators.emplace_back(std::move(predicate_child)); + sub_column_iterators.emplace_back(std::move(lazy_child)); + + StructFileColumnIterator struct_iterator(create_test_reader(true), std::move(null_iterator), + std::move(sub_column_iterators)); + struct_iterator.set_read_requirement_self(ColumnIterator::ReadRequirement::PREDICATE); + predicate_child_ptr->set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + lazy_child_ptr->set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + struct_iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + std::map> prefetchers; + struct_iterator.collect_prefetchers(prefetchers, PrefetcherInitMethod::FROM_ROWIDS); + + EXPECT_THAT(null_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_THAT(predicate_child_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_TRUE(lazy_child_ptr->collect_methods.empty()); + EXPECT_THAT(prefetchers[PrefetcherInitMethod::FROM_ROWIDS], + ::testing::ElementsAre(null_iterator_ptr->prefetcher(), + predicate_child_ptr->prefetcher())); + + null_iterator_ptr->clear_tracking(); + predicate_child_ptr->clear_tracking(); + lazy_child_ptr->clear_tracking(); + prefetchers.clear(); + + struct_iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + struct_iterator.collect_prefetchers(prefetchers, PrefetcherInitMethod::FROM_ROWIDS); + + EXPECT_THAT(null_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_TRUE(predicate_child_ptr->collect_methods.empty()); + EXPECT_THAT(lazy_child_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); +} + +TEST_F(ColumnReaderTest, ArrayAndMapCollectPrefetchersUseAllDataBlocksForNestedData) { + { + auto null_iterator = std::make_unique(); + auto* null_iterator_ptr = null_iterator.get(); + auto item_iterator = std::make_unique(); + auto* item_iterator_ptr = item_iterator.get(); + auto offset_iterator = create_tracking_offset_iterator(); + auto* offset_iterator_ptr = offset_iterator.tracker; + + ArrayFileColumnIterator array_iterator(create_test_reader(true), + std::move(offset_iterator.iterator), + std::move(item_iterator), std::move(null_iterator)); + array_iterator.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + array_iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + std::map> prefetchers; + array_iterator.collect_prefetchers(prefetchers, PrefetcherInitMethod::FROM_ROWIDS); + + EXPECT_THAT(offset_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_THAT(null_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_THAT(item_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::ALL_DATA_BLOCKS)); + EXPECT_THAT(prefetchers[PrefetcherInitMethod::ALL_DATA_BLOCKS], + ::testing::ElementsAre(item_iterator_ptr->prefetcher())); + } + + { + auto null_iterator = std::make_unique(); + auto* null_iterator_ptr = null_iterator.get(); + auto key_iterator = std::make_unique(); + auto* key_iterator_ptr = key_iterator.get(); + auto value_iterator = std::make_unique(); + auto* value_iterator_ptr = value_iterator.get(); + auto offset_iterator = create_tracking_offset_iterator(); + auto* offset_iterator_ptr = offset_iterator.tracker; + + MapFileColumnIterator map_iterator(create_test_reader(true), std::move(null_iterator), + std::move(offset_iterator.iterator), + std::move(key_iterator), std::move(value_iterator)); + map_iterator.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + map_iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + std::map> prefetchers; + map_iterator.collect_prefetchers(prefetchers, PrefetcherInitMethod::FROM_ROWIDS); + + EXPECT_THAT(offset_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_THAT(null_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::FROM_ROWIDS)); + EXPECT_THAT(key_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::ALL_DATA_BLOCKS)); + EXPECT_THAT(value_iterator_ptr->collect_methods, + ::testing::ElementsAre(PrefetcherInitMethod::ALL_DATA_BLOCKS)); + EXPECT_THAT(prefetchers[PrefetcherInitMethod::ALL_DATA_BLOCKS], + ::testing::ElementsAre(key_iterator_ptr->prefetcher(), + value_iterator_ptr->prefetcher())); + } +} + +TEST_F(ColumnReaderTest, MapPredicateAccessAllWithOffsetKeepsKeysReadable) { + auto map_reader = create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_MAP); + auto key_iter = std::make_unique( + create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_STRING)); + key_iter->set_column_name("key"); + auto* key_ptr = key_iter.get(); + auto val_iter = std::make_unique( + create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_STRING)); + val_iter->set_column_name("value"); + auto* val_ptr = val_iter.get(); + auto offset_iterator = create_tracking_offset_iterator(); + + MapFileColumnIterator map_iter(map_reader, nullptr, std::move(offset_iterator.iterator), + std::move(key_iter), std::move(val_iter)); + map_iter.set_column_name("map_col"); + + TColumnAccessPaths access_paths {create_access_path( + {"map_col", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET})}; + auto st = map_iter.set_access_paths(access_paths, access_paths); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + + EXPECT_EQ(key_ptr->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_FALSE(key_ptr->read_offset_only()); + EXPECT_EQ(val_ptr->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_TRUE(val_ptr->read_offset_only()); } TEST_F(ColumnReaderTest, OffsetPeekUsesPageSentinelWhenNoRemaining) { @@ -285,7 +2367,7 @@ TEST_F(ColumnReaderTest, MapReadByRowidsSkipReadingResizesDestination) { MapFileColumnIterator map_iter(map_reader, std::move(null_iter), std::move(offsets_iter), std::move(key_iter), std::move(val_iter)); map_iter.set_column_name("map_col"); - map_iter.set_reading_flag(ColumnIterator::ReadingFlag::SKIP_READING); + map_iter.set_read_requirement(ColumnIterator::ReadRequirement::SKIP); // prepare an empty ColumnMap as destination auto keys = ColumnInt32::create(); @@ -329,14 +2411,14 @@ TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) { auto st = map_iter.set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); - // Key must be fully readable (NEED_TO_READ), NOT in OFFSET_ONLY mode. + // Key must be fully readable (LAZY_OUTPUT), NOT in OFFSET_ONLY mode. auto* key_ptr = static_cast(map_iter._key_iterator.get()); - ASSERT_EQ(key_ptr->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_EQ(key_ptr->_read_requirement, ColumnIterator::ReadRequirement::LAZY_OUTPUT); ASSERT_FALSE(key_ptr->read_offset_only()); // Value should be in OFFSET_ONLY mode since we only need string lengths. auto* val_ptr = static_cast(map_iter._val_iterator.get()); - ASSERT_EQ(val_ptr->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_EQ(val_ptr->_read_requirement, ColumnIterator::ReadRequirement::LAZY_OUTPUT); ASSERT_TRUE(val_ptr->read_offset_only()); } diff --git a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp new file mode 100644 index 00000000000000..b990201b49544c --- /dev/null +++ b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "storage/olap_common.h" +#include "storage/segment/column_reader.h" +#include "storage/tablet/tablet_schema.h" + +// Use #define private public to access SegmentIterator::_read_lazy_pruned_columns() +// and the small amount of state it consumes. This mirrors the existing +// segment_iterator_* white-box tests. +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wkeyword-macro" +#endif +#define private public +#include "storage/segment/segment_iterator.h" +#undef private +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +namespace doris::segment_v2 { +namespace { + +class TrackingLazyColumnIterator final : public ColumnIterator { +public: + Status seek_to_ordinal(ordinal_t ord) override { + seek_ordinals.push_back(ord); + return Status::OK(); + } + + Status read_by_rowids(const rowid_t* rowids, const size_t count, + MutableColumnPtr& dst) override { + read_phases.push_back(_read_phase); + read_rowids.assign(rowids, rowids + count); + ++read_by_rowids_count; + + auto& int_column = assert_cast&>(*dst); + for (size_t i = 0; i < count; ++i) { + int_column.insert_value(cast_set(rowids[i])); + } + return Status::OK(); + } + + void finalize_lazy_phase(MutableColumnPtr& dst) override { + finalize_phases.push_back(_read_phase); + ++finalize_count; + } + + ordinal_t get_current_ordinal() const override { return 0; } + + ReadPhase phase() const { return _read_phase; } + + std::vector seek_ordinals; + std::vector read_rowids; + std::vector read_phases; + std::vector finalize_phases; + int read_by_rowids_count = 0; + int finalize_count = 0; +}; + +TabletSchemaSPtr make_tablet_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + auto* col = schema_pb.add_column(); + col->set_unique_id(0); + col->set_name("c0"); + col->set_type("INT"); + col->set_is_key(true); + col->set_is_nullable(false); + + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + return tablet_schema; +} + +SchemaSPtr make_read_schema(const TabletSchemaSPtr& tablet_schema) { + std::vector read_column_ids(tablet_schema->num_columns()); + for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { + read_column_ids[cid] = cid; + } + return std::make_shared(tablet_schema->columns(), read_column_ids); +} + +Block make_int_block() { + Block block; + block.insert({ColumnInt32::create(), std::make_shared(), "c0"}); + return block; +} + +} // namespace + +class SegmentIteratorLazyPrunedTest : public ::testing::Test { +protected: + void SetUp() override { + _tablet_schema = make_tablet_schema(); + _read_schema = make_read_schema(_tablet_schema); + } + + std::unique_ptr make_iter(TrackingLazyColumnIterator** tracking_iter) { + auto iter = std::make_unique(nullptr, _read_schema); + iter->_opts.tablet_schema = _tablet_schema; + iter->_opts.stats = &_stats; + iter->_support_lazy_read_pruned_columns.insert(0); + iter->_column_iterators.resize(1); + + auto column_iter = std::make_unique(); + *tracking_iter = column_iter.get(); + iter->_column_iterators[0] = std::move(column_iter); + return iter; + } + + TabletSchemaSPtr _tablet_schema; + SchemaSPtr _read_schema; + OlapReaderStatistics _stats; +}; + +TEST_F(SegmentIteratorLazyPrunedTest, readsSelectedRowidsInLazyPhaseAndRestoresPhase) { + TrackingLazyColumnIterator* tracking_iter = nullptr; + auto iter = make_iter(&tracking_iter); + iter->_selected_size = 2; + iter->_block_rowids = {10, 20, 30, 40}; + iter->_sel_rowid_idx = {2, 0}; + + auto block = make_int_block(); + auto st = iter->_read_lazy_pruned_columns(&block); + ASSERT_TRUE(st.ok()) << st.to_string(); + + EXPECT_EQ(tracking_iter->read_by_rowids_count, 1); + EXPECT_EQ(tracking_iter->finalize_count, 1); + EXPECT_EQ(tracking_iter->read_rowids, (std::vector {30, 10})); + EXPECT_EQ(tracking_iter->read_phases, + (std::vector {ColumnIterator::ReadPhase::LAZY})); + EXPECT_EQ(tracking_iter->finalize_phases, + (std::vector {ColumnIterator::ReadPhase::LAZY})); + EXPECT_EQ(tracking_iter->phase(), ColumnIterator::ReadPhase::NORMAL); + + const auto& result = + assert_cast&>(*block.get_by_position(0).column); + ASSERT_EQ(result.size(), 2); + EXPECT_EQ(result.get_data()[0], 30); + EXPECT_EQ(result.get_data()[1], 10); +} + +TEST_F(SegmentIteratorLazyPrunedTest, emptySelectionStillFinalizesLazyPlaceholders) { + TrackingLazyColumnIterator* tracking_iter = nullptr; + auto iter = make_iter(&tracking_iter); + iter->_selected_size = 0; + + auto block = make_int_block(); + auto st = iter->_read_lazy_pruned_columns(&block); + ASSERT_TRUE(st.ok()) << st.to_string(); + + EXPECT_EQ(tracking_iter->read_by_rowids_count, 0); + EXPECT_EQ(tracking_iter->finalize_count, 1); + EXPECT_EQ(tracking_iter->finalize_phases, + (std::vector {ColumnIterator::ReadPhase::LAZY})); + EXPECT_EQ(tracking_iter->phase(), ColumnIterator::ReadPhase::NORMAL); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); +} + +} // namespace doris::segment_v2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java index 9e6807cc0b2905..9dcbfea57dd497 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java @@ -19,11 +19,11 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.processor.post.TopnFilterPushDownVisitor.PushDownContext; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.SortPhase; import org.apache.doris.nereids.trees.plans.algebra.TopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.types.DataType; /** * topN opt @@ -60,13 +60,20 @@ boolean checkTopN(TopN topN) { return false; } - Expression firstKey = topN.getOrderKeys().get(0).getExpr(); + DataType firstKeyType = topN.getOrderKeys().get(0).getExpr().getDataType(); - if (firstKey.getDataType().isFloatType() - || firstKey.getDataType().isDoubleType()) { - return false; - } - return true; + return isSupportedTopNRuntimeFilterType(firstKeyType); + } + + private boolean isSupportedTopNRuntimeFilterType(DataType dataType) { + return dataType.isBooleanType() + || dataType.isIntegralType() + || dataType.isDecimalLikeType() + || dataType.isStringLikeType() + || dataType.isDateLikeType() + || dataType.isTimeType() + || dataType.isIPType() + || dataType.isVarBinaryType(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java deleted file mode 100644 index 8f08699ec232d3..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MetaPathStriper.java +++ /dev/null @@ -1,300 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.analysis.ColumnAccessPathType; -import org.apache.doris.common.Pair; - -import com.google.common.collect.Multimap; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Strips redundant metadata-only (NULL/OFFSET) access paths using pure - * string-prefix comparison. Map-level wildcards must already have been - * expanded into {@code KEYS} + {@code VALUES} by the caller. - * - *

Single entry point: {@link #strip(int, Multimap, Multimap)}. - */ -public final class MetaPathStriper { - - private MetaPathStriper() {} - - /** - * Strip redundant metadata-only NULL/OFFSET paths using pure string-prefix - * comparison, keeping enough real paths for BE readers to avoid - * OFFSET_ONLY / NULL_MAP_ONLY modes that skip required child data. - * - *

Stripping is organised in two levels: - * - *

Level 1 — Same-prefix priority: when two paths share the same - * prefix and differ only in the final meta suffix, the higher-priority one - * eliminates the lower. - * 

{@code
-     *   Data  >  OFFSET  >  NULL
-     * }
- *
    - *
  • {@code Data} strips {@code OFFSET}: {@code [a]} strips {@code [a, OFFSET]}.
    • - *
  • {@code Data} strips {@code NULL}: {@code [a]} strips {@code [a, NULL]}.
    • - *
  • {@code OFFSET} strips {@code NULL}: {@code [a, OFFSET]} strips - * {@code [a, NULL]}.
    • - *
- * - *

Level 2 — Deeper-prefix coverage: when a covering path goes - * deeper into the type tree, its data reader already materialises the - * container, making a shallower meta-only path redundant. - *

    - *
  • Target suffix {@code OFFSET}, covered by deeper: - *
      - *
    • {@code Data}: {@code [a, *, field]} strips {@code [a, OFFSET]}.
      • - *
    • {@code OFFSET}: {@code [a, *, OFFSET]} strips {@code [a, OFFSET]}.
      • - *
    • {@code NULL}: {@code [a, *, NULL]} strips {@code [a, OFFSET]}.
      • - *
    - *
    • - *
  • Target suffix {@code NULL}, covered by deeper: - *
      - *
    • {@code Data}: {@code [a, b, c]} strips {@code [a, b, NULL]}.
      • - *
    • {@code OFFSET}: {@code [a, *, OFFSET]} strips {@code [a, NULL]}.
      • - *
    • {@code NULL}: {@code [a, *, NULL]} strips {@code [a, NULL]}.
      • - *
    - *
    • - *
- * - *

Pre-condition: map-level {@code *} wildcards must already have been - * expanded into {@code KEYS} + {@code VALUES} by the caller. This class - * uses pure string-prefix comparison and is type-unaware. - */ - public static void strip( - int slotId, - Multimap>> targetAccessPaths, - Multimap>> coveringAccessPaths) { - stripExactPrefixCoveredMetaPaths(slotId, targetAccessPaths, coveringAccessPaths); - stripNullBySamePrefixOffset(slotId, targetAccessPaths); - - stripMetaPathsByDeeperPrefix(slotId, AccessPathInfo.ACCESS_OFFSET, - targetAccessPaths, coveringAccessPaths); - stripMetaPathsByDeeperPrefix(slotId, AccessPathInfo.ACCESS_NULL, - targetAccessPaths, coveringAccessPaths); - } - - // ======================================================================== - // Path helpers - // ======================================================================== - - private static boolean isMetaPath(List path) { - if (path.isEmpty()) { - return false; - } - String lastComponent = path.get(path.size() - 1); - return AccessPathInfo.ACCESS_NULL.equals(lastComponent) - || AccessPathInfo.ACCESS_OFFSET.equals(lastComponent); - } - - private static List> collectPaths( - Collection>> a, - Collection>> b, boolean meta) { - List> result = new ArrayList<>(); - for (Pair> p : a) { - if (!p.second.isEmpty() && isMetaPath(p.second) == meta) { - result.add(p.second); - } - } - for (Pair> p : b) { - if (!p.second.isEmpty() && isMetaPath(p.second) == meta) { - result.add(p.second); - } - } - return result; - } - - private static boolean isPrefixCovered(List prefix, List coveringPath) { - if (coveringPath.isEmpty()) { - return true; - } - int minLen = Math.min(prefix.size(), coveringPath.size()); - for (int i = 0; i < minLen; i++) { - if (!prefix.get(i).equals(coveringPath.get(i))) { - return false; - } - } - return true; - } - - // ======================================================================== - // Level 1 — same-prefix priority - // ======================================================================== - - /** - * {@code [prefix]} strips {@code [prefix, OFFSET]} and {@code [prefix, NULL]}. - */ - private static void stripExactPrefixCoveredMetaPaths( - int slotId, - Multimap>> targetAccessPaths, - Multimap>> coveringAccessPaths) { - Collection>> targetPaths = targetAccessPaths.get(slotId); - if (targetPaths.isEmpty()) { - return; - } - - List> fullAccessPaths = collectPaths( - coveringAccessPaths.get(slotId), targetPaths, false); - - List>> pathsToRemove = new ArrayList<>(); - for (Pair> p : targetPaths) { - List path = p.second; - if (!isMetaPath(path)) { - continue; - } - List prefix = path.subList(0, path.size() - 1); - for (List fullAccessPath : fullAccessPaths) { - if (isPrefixCovered(prefix, fullAccessPath) - && prefix.size() >= fullAccessPath.size()) { - pathsToRemove.add(p); - break; - } - } - } - targetPaths.removeAll(pathsToRemove); - } - - /** - * {@code [prefix, OFFSET]} strips {@code [prefix, NULL]}. - */ - private static void stripNullBySamePrefixOffset( - int slotId, Multimap>> allAccessPaths) { - Collection>> slotPaths = allAccessPaths.get(slotId); - if (slotPaths.isEmpty()) { - return; - } - - List>> toRemove = new ArrayList<>(); - for (Pair> p : slotPaths) { - List path = p.second; - if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { - continue; - } - List prefix = path.subList(0, path.size() - 1); - for (Pair> q : slotPaths) { - List other = q.second; - if (other == path || other.isEmpty()) { - continue; - } - if (other.size() != path.size() - || !AccessPathInfo.ACCESS_OFFSET.equals(other.get(other.size() - 1))) { - continue; - } - List otherPrefix = other.subList(0, other.size() - 1); - if (isPrefixCovered(prefix, otherPrefix)) { - toRemove.add(p); - break; - } - } - } - slotPaths.removeAll(toRemove); - } - - // ======================================================================== - // Level 2 — deeper-prefix coverage - // ======================================================================== - - private static void stripMetaPathsByDeeperPrefix( - int slotId, String metaSuffix, - Multimap>> targetAccessPaths, - Multimap>> coveringAccessPaths) { - Collection>> targetPaths = targetAccessPaths.get(slotId); - if (targetPaths.isEmpty()) { - return; - } - Collection>> coveringPaths = - coveringAccessPaths.get(slotId); - - List> dataPaths = collectPaths(coveringPaths, targetPaths, false); - stripMetaByDeeperDataPaths(slotId, targetAccessPaths, dataPaths, metaSuffix); - - List> metaPaths = collectPaths(coveringPaths, targetPaths, true); - stripMetaByDeeperMetaPaths(slotId, metaSuffix, metaPaths, targetAccessPaths); - } - - /** - * Strip target meta paths covered by a data path. - */ - private static void stripMetaByDeeperDataPaths( - int slotId, - Multimap>> targetAccessPaths, - List> dataPaths, String metaSuffix) { - Collection>> targetPaths = - targetAccessPaths.get(slotId); - if (targetPaths.isEmpty() || dataPaths.isEmpty()) { - return; - } - - List>> toRemove = new ArrayList<>(); - for (Pair> p : new ArrayList<>(targetPaths)) { - List path = p.second; - if (path.isEmpty() || !metaSuffix.equals(path.get(path.size() - 1))) { - continue; - } - List prefix = path.subList(0, path.size() - 1); - for (List dataPath : dataPaths) { - if (isPrefixCovered(prefix, dataPath)) { - toRemove.add(p); - break; - } - } - } - targetPaths.removeAll(toRemove); - } - - /** - * Strip target meta paths covered by a strictly deeper meta path. - */ - private static void stripMetaByDeeperMetaPaths( - int slotId, String metaSuffix, - List> coveringPaths, - Multimap>> targetAccessPaths) { - Collection>> targetPaths = - targetAccessPaths.get(slotId); - if (targetPaths.isEmpty() || coveringPaths.isEmpty()) { - return; - } - - List>> toRemove = new ArrayList<>(); - for (Pair> p : targetPaths) { - List targetPath = p.second; - if (targetPath.isEmpty() || !metaSuffix.equals(targetPath.get(targetPath.size() - 1))) { - continue; - } - List targetPrefix = targetPath.subList(0, targetPath.size() - 1); - for (List coveringPath : coveringPaths) { - if (coveringPath == targetPath || coveringPath.isEmpty()) { - continue; - } - if (coveringPath.size() - 1 <= targetPath.size() - 1) { - continue; - } - if (isPrefixCovered(targetPrefix, coveringPath)) { - toRemove.add(p); - break; - } - } - } - targetPaths.removeAll(toRemove); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index 4518a77a3c8355..0095d11211e5f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -233,12 +233,17 @@ private static Map pruneDataType( for (Entry> kv : slotToAccessPaths.entrySet()) { Slot slot = kv.getKey(); List collectAccessPathResults = kv.getValue(); + boolean hasRegularAccessPath = collectAccessPathResults.stream() + .anyMatch(resultItem -> !isDataSkippingOnlyAccessPath(resultItem.getPath())); if (slot.getDataType() instanceof VariantType) { variantSlots.put(slot, slot.getDataType()); for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { List path = collectAccessPathResult.getPath(); TAccessPathType pathType = collectAccessPathResult.getType(); - allAccessPaths.put(slot.getExprId().asInt(), Pair.of(pathType, path)); + Pair> allPath = + normalizePredicateMetaPathForAllAccessPath( + slot, collectAccessPathResult, hasRegularAccessPath); + allAccessPaths.put(slot.getExprId().asInt(), allPath); if (collectAccessPathResult.isPredicate()) { predicateAccessPaths.put( slot.getExprId().asInt(), Pair.of(pathType, path) @@ -250,11 +255,14 @@ private static Map pruneDataType( for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { List path = collectAccessPathResult.getPath(); TAccessPathType pathType = collectAccessPathResult.getType(); + Pair> allPath = + normalizePredicateMetaPathForAllAccessPath( + slot, collectAccessPathResult, hasRegularAccessPath); DataTypeAccessTree allAccessTree = slotIdToAllAccessTree.computeIfAbsent( - slot, i -> DataTypeAccessTree.ofRoot(slot, pathType) + slot, i -> DataTypeAccessTree.ofRoot(slot, allPath.first) ); - allAccessTree.setAccessByPath(path, 0, pathType); - allAccessPaths.put(slot.getExprId().asInt(), Pair.of(pathType, path)); + allAccessTree.setAccessByPath(allPath.second, 0, allPath.first); + allAccessPaths.put(slot.getExprId().asInt(), allPath); if (collectAccessPathResult.isPredicate()) { DataTypeAccessTree predicateAccessTree = slotIdToPredicateAccessTree.computeIfAbsent( @@ -268,21 +276,21 @@ private static Map pruneDataType( } } - // phase 1.5: for slots with meta paths, expand map-star paths and strip - // redundant meta paths. Strip predicate first using the COMPLETE - // allAccessPaths as covering, then strip allAccessPaths self-covering. + // phase 1.5: for slots with meta paths, expand map-star paths before building final + // access path lists. Predicate NULL/OFFSET paths are kept in predicateAccessPaths. + // When regular data paths also exist, allAccessPaths uses the stripped data path for + // mixed-version safety with older BEs that only understand allAccessPaths, except + // map-star predicate metadata paths. Those paths must stay unstripped until this phase + // so map.*.OFFSET can become map.KEYS + map.VALUES.OFFSET instead of broad map.*. for (Entry kv : slotIdToAllAccessTree.entrySet()) { Slot slot = kv.getKey(); DataTypeAccessTree accessTree = kv.getValue(); if (!accessTree.hasOffsetPath() && !accessTree.hasNullPath()) { continue; } - int slotId = slot.getExprId().asInt(); // Expand both sets before stripping so covering is complete. expandMapStarPaths(slot, allAccessPaths); expandMapStarPaths(slot, predicateAccessPaths); - MetaPathStriper.strip(slotId, predicateAccessPaths, allAccessPaths); - MetaPathStriper.strip(slotId, allAccessPaths, allAccessPaths); } // second: build non-predicate access paths @@ -312,7 +320,7 @@ private static Map pruneDataType( buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); if (accessPathInfo != null) { - retainPredicatePathsInFinalAllAccessPaths( + addPredicatePathsToFinalAllAccessPaths( predicatePaths, accessPathInfo.getAllAccessPaths()); accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); } @@ -324,7 +332,7 @@ private static Map pruneDataType( buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); if (accessPathInfo != null) { - retainPredicatePathsInFinalAllAccessPaths( + addPredicatePathsToFinalAllAccessPaths( predicatePaths, accessPathInfo.getAllAccessPaths()); accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); } @@ -333,33 +341,86 @@ private static Map pruneDataType( return result; } + private static Pair> normalizePredicateMetaPathForAllAccessPath( + Slot slot, CollectAccessPathResult accessPathResult, boolean hasRegularAccessPath) { + List path = accessPathResult.getPath(); + TAccessPathType pathType = accessPathResult.getType(); + if (accessPathResult.isPredicate() && hasRegularAccessPath + && isDataSkippingOnlyAccessPath(path)) { + if (hasMapStarAccessPath(slot, path)) { + // Keep map-star metadata until expandMapStarPaths() turns it into precise + // KEYS/VALUES paths. Stripping here would broaden map value reads to map.*. + return Pair.of(pathType, path); + } + return Pair.of(TAccessPathType.DATA, stripDataSkippingSuffix(path)); + } + return Pair.of(pathType, path); + } + + private static boolean hasMapStarAccessPath(Slot slot, List path) { + List positions = new ArrayList<>(); + findMapStarPositions(path, slot.getDataType(), positions); + return !positions.isEmpty(); + } + /** - * Keep predicate access paths as a subset of final all access paths after NULL/OFFSET cleanup. - * Predicate paths are built from filter expressions first, but later all-path rewrites may drop - * metadata-only paths or collapse paths to whole-column access. Any predicate path not present - * in final all paths must be removed before sending access info to BE. + * Keep final allAccessPaths as a superset of non-metadata predicateAccessPaths. Predicate + * paths are collected from filter expressions first, but final all-path construction may later + * collapse ordinary paths to whole-column access. BE complex readers use allAccessPaths to + * decide which data sub-iterators can be pruned; any predicate data path missing from + * allAccessPaths can therefore make predicate reads disagree with pruning. * - *

Examples: - *

    - *
  • All paths {@code [s]}, predicate paths {@code [s.city.NULL]} becomes no predicate - * paths after parent NULL removal.
    • - *
  • All paths {@code [s.city.NULL, s.zip]}, predicate paths - * {@code [s.NULL, s.city.NULL]} becomes {@code [s.city.NULL]}.
    • - *
+ * Keep predicate NULL/OFFSET paths out of allAccessPaths for mixed-version safety. Older BEs + * decide current-level meta-only mode from allAccessPaths only, so sending both metadata and + * data paths there could make them skip required child data. Newer BEs still receive those + * metadata predicates through predicateAccessPaths. */ - private static void retainPredicatePathsInFinalAllAccessPaths( + private static void addPredicatePathsToFinalAllAccessPaths( List predicatePaths, List allPaths) { - if (predicatePaths.isEmpty()) { - return; + for (TColumnAccessPath predicatePath : predicatePaths) { + if (!isMetaPath(predicatePath) && !isCoveredByAllPath(predicatePath, allPaths)) { + allPaths.add(predicatePath); + } } + } - List toRemove = new ArrayList<>(); - for (TColumnAccessPath predicatePath : predicatePaths) { - if (!allPaths.contains(predicatePath)) { - toRemove.add(predicatePath); + private static boolean isCoveredByAllPath(TColumnAccessPath predicatePath, List allPaths) { + for (TColumnAccessPath allPath : allPaths) { + if (allPath.getType() == predicatePath.getType() + && isPrefixPath(getAccessPathList(allPath), getAccessPathList(predicatePath))) { + return true; } } - predicatePaths.removeAll(toRemove); + return false; + } + + private static boolean isPrefixPath(List prefix, List path) { + if (prefix.size() > path.size()) { + return false; + } + for (int i = 0; i < prefix.size(); ++i) { + if (!prefix.get(i).equals(path.get(i))) { + return false; + } + } + return true; + } + + private static boolean isMetaPath(TColumnAccessPath path) { + return isDataSkippingOnlyAccessPath(getAccessPathList(path)); + } + + private static boolean isDataSkippingOnlyAccessPath(List components) { + if (components.isEmpty()) { + return false; + } + String lastComponent = components.get(components.size() - 1); + return AccessPathInfo.ACCESS_NULL.equals(lastComponent) + || AccessPathInfo.ACCESS_OFFSET.equals(lastComponent); + } + + private static List stripDataSkippingSuffix(List components) { + return new ArrayList<>(components.subList(0, components.size() - 1)); } private static List buildColumnAccessPaths( diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 1bf84014bbc984..5a0b1112785ca5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5893,6 +5893,8 @@ public TQueryOptions toThrift() { tResult.setReadCsvEmptyLineAsNull(readCsvEmptyLineAsNull); tResult.setSerdeDialect(getSerdeDialect()); + tResult.setEnablePruneNestedColumn(enablePruneNestedColumns); + tResult.setEnableMatchWithoutInvertedIndex(enableMatchWithoutInvertedIndex); tResult.setEnableFallbackOnMissingInvertedIndex(enableFallbackOnMissingInvertedIndex); tResult.setEnableInvertedIndexSearcherCache(enableInvertedIndexSearcherCache); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java index f66a284121ca58..84868c93dd7226 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java @@ -42,6 +42,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.List; import java.util.Map; public class TopNRuntimeFilterTest extends SSBTestBase implements MemoPatternMatchSupported { @@ -125,6 +126,25 @@ public void testNotUseTopNRfOnWindow() { Assertions.assertFalse(checker.getCascadesContext().getTopnFilterContext().isTopnFilterSource(localTopN)); } + @Test + public void testNotUseTopNRfForUnsupportedComplexOrderKey() { + String sql = "select c_custkey from customer order by array(c_custkey) limit 5"; + PlanChecker checker = PlanChecker.from(connectContext).analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + List> localTopNs = plan.collectToList( + node -> node instanceof PhysicalTopN + && ((PhysicalTopN) node).getSortPhase().isLocal()); + Assertions.assertFalse(localTopNs.isEmpty(), plan.treeString()); + for (PhysicalTopN localTopN : localTopNs) { + Assertions.assertFalse( + checker.getCascadesContext().getTopnFilterContext().isTopnFilterSource(localTopN)); + } + } + @Test public void testProbeExprNullableThroughRightOuterJoin() { // topn node push down filter value to scan node. diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 7748e9c2b46e03..ad6fb4e1901312 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -171,31 +171,11 @@ public void testMap() throws Exception { public void testMapElementLengthWithMapValuesKeepsKeysPath() throws Exception { assertColumn("select length(map_col['a']), map_values(map_col)[1] from str_tbl", "map", - ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), + ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES"), path("map_col", "VALUES", "OFFSET")), ImmutableList.of() ); } - @Test - public void testStructRootArrayMixedAccessSuppressesOffsetPath() throws Exception { - assertAllAccessPathsContain( - "select cardinality(struct_element(s, 'arr')), " - + "struct_element(element_at(struct_element(s, 'arr'), 1), 'int_field') " - + "from nested_container_tbl", - ImmutableList.of(path("s", "arr", "*", "int_field")), - ImmutableList.of(path("s", "arr", "OFFSET"))); - } - - @Test - public void testStructRootMapMixedAccessKeepsKeysPath() throws Exception { - assertAllAccessPathsContain( - "select length(element_at(struct_element(s, 'm'), 'a')), " - + "element_at(map_values(struct_element(s, 'm')), 1) " - + "from nested_container_tbl", - ImmutableList.of(path("s", "m", "KEYS"), path("s", "m", "VALUES")), - ImmutableList.of(path("s", "m", "*", "OFFSET"), path("s", "m", "VALUES", "OFFSET"))); - } - @Test public void testCardinalityArrayElementKeepsOffsetPath() throws Exception { assertAllAccessPathsContain( @@ -204,53 +184,6 @@ public void testCardinalityArrayElementKeepsOffsetPath() throws Exception { ImmutableList.of(path("a", "*"))); } - @Test - public void testDeeperOffsetPathCoversShallowerOffsetPath() throws Exception { - // cardinality(a) on ARRAY> generates [a, OFFSET] - // cardinality(element_at(a, 1)) generates [a, *, OFFSET] - // [a, *, OFFSET] traverses into the outer array's items, so it must read - // through the outer array structure. The shallower [a, OFFSET] is therefore - // redundant and should be stripped. - assertAllAccessPathsContain( - "select cardinality(a), cardinality(element_at(a, 1)) from nested_array_tbl", - ImmutableList.of(path("a", "*", "OFFSET")), - ImmutableList.of(path("a", "OFFSET"))); - } - - @Test - public void testOffsetPathCoversNullPathWithSamePrefix() throws Exception { - assertAllAccessPathsContain( - "select cardinality(a), a is null from nested_array_tbl", - ImmutableList.of(path("a", "OFFSET")), - ImmutableList.of(path("a", "NULL"))); - } - - @Test - public void testDeeperNullPathCoversShallowerNullPath() throws Exception { - // a IS NULL on ARRAY> generates [a, NULL] - // element_at(a, 1) IS NULL generates [a, *, NULL] - // [a, *, NULL] traverses into the outer array's items, which requires - // reading the outer array's null bitmap. Therefore [a, NULL] is redundant - // and should be stripped. - assertAllAccessPathsContain( - "select a is null, element_at(a, 1) is null from nested_array_tbl", - ImmutableList.of(path("a", "*", "NULL")), - ImmutableList.of(path("a", "NULL"))); - } - - @Test - public void testDeeperOffsetPathCoversShallowerNullPathForArray() throws Exception { - // cardinality(element_at(a, 1)) on ARRAY> generates [a, *, OFFSET] - // a IS NULL generates [a, NULL] - // [a, *, OFFSET] reads the offset of inner-array elements, which requires - // reading through the outer array's null bitmap. Therefore [a, NULL] is - // redundant and should be stripped. - assertAllAccessPathsContain( - "select cardinality(element_at(a, 1)), a is null from nested_array_tbl", - ImmutableList.of(path("a", "*", "OFFSET")), - ImmutableList.of(path("a", "NULL"))); - } - @Test public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception { assertColumn("select cardinality(map_arr_col['a']) from map_array_tbl", @@ -259,114 +192,17 @@ public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception { ImmutableList.of()); } - @Test - public void testDeeperNullCoversValueOffsetForMapArray() throws Exception { - // cardinality(map_arr_col['a']) -> normalized to KEYS + VALUES.OFFSET - // element_at(map_arr_col['a'], 1) IS NULL -> *.*.NULL - // *.*.NULL goes deeper into the value-side array, so VALUES.OFFSET - // at the map value level is redundant. Without type-aware comparison - // * is not lexically equal to VALUES, so VALUES.OFFSET would survive - // and cause BE to skip the item iterator. - assertAllAccessPathsContain( - "select cardinality(map_arr_col['a']), " - + "element_at(map_arr_col['a'], 1) is null from map_array_tbl", - ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES", "*", "NULL")), - ImmutableList.of(path("map_arr_col", "VALUES", "OFFSET"))); - } - - @Test - public void testDataPathCoversNullPathWithMapAwareComparison() throws Exception { - // element_at(map_col, 'a') -> [map_col, KEYS] + [map_col, VALUES] - // element_at(map_values(map_col), 1) IS NULL -> [map_col, VALUES, NULL] - // Level 1 strips [map_col, VALUES, NULL] because [map_col, VALUES] covers it. - // - // NOT: map_values(map_col) IS NULL — visitMapValues special-cases a lone NULL - // suffix as a parent-map null check (isFunctionNullCheckPath), producing - // [map_col, NULL] instead of [map_col, VALUES, NULL]. - assertAllAccessPathsContain( - "select element_at(map_col, 'a') from str_tbl" - + " where element_at(map_values(map_col), 1) is null", - ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), - ImmutableList.of(path("map_col", "VALUES", "NULL"))); - } - - @Test - public void testDataPathCoversOffsetPathWithMapAwareComparison() throws Exception { - // element_at(map_col, 'a') -> [map_col, KEYS] + [map_col, VALUES] - // length(element_at(map_values(map_col), 1)) > 0 -> [map_col, VALUES, OFFSET] - // Level 2 strips [map_col, VALUES, OFFSET] because [map_col, VALUES] covers it. - // - // NOT: cardinality(map_values(map_col)) — visitCardinality unwraps MapValues - // and produces [map_col, OFFSET] instead of [map_col, VALUES, OFFSET]. - assertAllAccessPathsContain( - "select element_at(map_col, 'a') from str_tbl" - + " where length(element_at(map_values(map_col), 1)) > 0", - ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), - ImmutableList.of(path("map_col", "VALUES", "OFFSET"))); - } - - @Test - public void testMapValuesCoversStarNullPreservesKeysPath() throws Exception { - // After map-* normalization: - // element_at(map_col, 'a') -> [map_col, KEYS] + [map_col, VALUES] - // element_at(map_col, 'a') IS NULL -> [map_col, VALUES, NULL] + [map_col, KEYS] - // element_at(map_values(map_col), 1) -> [map_col, VALUES] - // Level 1 strips [map_col, VALUES, NULL] because [map_col, VALUES] covers it. - // The KEYS path is preserved for key lookup. - assertAllAccessPathsContain( - "select element_at(map_values(map_col), 1) from str_tbl" - + " where element_at(map_col, 'a') is null", - ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), - ImmutableList.of(path("map_col", "VALUES", "NULL"))); - } - - @Test - public void testSupplementalKeyPathShouldStripExistingKeyNullPath() throws Exception { - // After map-* normalization: - // projection: [map_arr_col, VALUES] - // filter: [map_arr_col, KEYS, NULL] - // OR [map_arr_col, *, *, NULL] → [map_arr_col, VALUES, *, NULL] - // + [map_arr_col, KEYS] - // Level 1 strips [map_arr_col, KEYS, NULL] because [map_arr_col, KEYS] now - // exists before the strip phase (produced by normalization, not supplemental). - // Level 2 strips [map_arr_col, VALUES, *, NULL] because [map_arr_col, VALUES] - // covers the value-side prefix. - assertAllAccessPathsContain( - "select element_at(map_values(map_arr_col), 1) from map_array_tbl" - + " where element_at(map_keys(map_arr_col), 1) is null" - + " or element_at(element_at(map_arr_col, 'a'), 1) is null", - ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES")), - ImmutableList.of(path("map_arr_col", "VALUES", "*", "NULL"), - path("map_arr_col", "KEYS", "NULL"))); - } - - @Test - public void testNestedMapElementLengthKeepsValueOffsetPath() throws Exception { - assertColumn("select length(element_at(element_at(s, 'm'), 'a')) from nested_container_tbl", - "struct>", - ImmutableList.of(path("s", "m", "KEYS"), path("s", "m", "VALUES", "OFFSET")), - ImmutableList.of()); - } - - @Test - public void testNestedMapElementIsNullKeepsValueIsNullPath() throws Exception { - assertColumn("select (element_at(element_at(s, 'm'), 'a')) is null from nested_container_tbl", - "struct>", - ImmutableList.of(path("s", "m", "KEYS"), path("s", "m", "VALUES", "NULL")), - ImmutableList.of()); - } - @Test public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { assertColumn("select struct_element(s, 'city') from tbl " + "where struct_element(s, 'city') is null", "struct", ImmutableList.of(path("s", "city")), - ImmutableList.of()); + ImmutableList.of(path("s", "city", "NULL"))); assertColumn("select cardinality(struct_element(s, 'data')), struct_element(s, 'data') from tbl", "struct>>>", - ImmutableList.of(path("s", "data")), + ImmutableList.of(path("s", "data"), path("s", "data", "OFFSET")), ImmutableList.of()); assertColumn("select cardinality(a), a from nested_array_tbl", @@ -376,13 +212,12 @@ public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { assertColumn("select cardinality(map_arr_col['a']), map_arr_col['a'] from map_array_tbl", "map>", - ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES")), + ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES"), path("map_arr_col", "VALUES", "OFFSET")), ImmutableList.of()); } @Test - public void testCardinalityMapElementOffsetCoveredByValueFieldAccess() throws Exception { - // [s, m, *, *, verified] strips [s, m, *, OFFSET] (pure string prefix). + public void testCardinalityMapElementOffsetPredicateStaysOutOfAllAccessPaths() throws Exception { Pair> result = collectComplexSlots( "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') " + "from map_array_value_tbl " @@ -393,14 +228,23 @@ public void testCardinalityMapElementOffsetCoveredByValueFieldAccess() throws Ex allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); } - Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified"))); - Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "VALUES", "OFFSET"))); - Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "VALUES", "OFFSET"))); - } - - @Test - public void testMapElementArrayNullPathCoveredByValueFieldAccess() throws Exception { - // [s, m, *, *, verified] strips [s, m, *, NULL] (pure string prefix). + Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "OFFSET")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "KEYS")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "OFFSET")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "KEYS")), + "predicateAccessPaths=" + predicateAccessPaths); + Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "VALUES", "OFFSET")), + "predicateAccessPaths=" + predicateAccessPaths); + } + + @Test + public void testMapElementArrayNullPredicateStaysOutOfAllAccessPaths() throws Exception { + // The map-star NULL path expands to precise KEYS/VALUES paths instead of broad s.m.*. Pair> result = collectComplexSlots( "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') " + "from map_array_value_tbl " @@ -411,9 +255,18 @@ public void testMapElementArrayNullPathCoveredByValueFieldAccess() throws Except allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); } - Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified"))); - Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "VALUES", "NULL"))); - Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "VALUES", "NULL"))); + Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "NULL")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "KEYS")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "NULL")), + "allAccessPaths=" + allAccessPaths); + Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "KEYS")), + "predicateAccessPaths=" + predicateAccessPaths); + Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "VALUES", "NULL")), + "predicateAccessPaths=" + predicateAccessPaths); } @Test @@ -797,11 +650,11 @@ public void testFilter() throws Throwable { ); // The IF expression itself is not collected as a null-only parent access here; the - // struct_element predicate still lets NCP prune the scan slot to the city field. - assertColumn("select 100 from tbl where if(id = 1, null, s) is not null or struct_element(s, 'city') = 'beijing'", - "struct", - ImmutableList.of(path("s", "city")), - ImmutableList.of(path("s", "city")) + // struct_element predicate is covered by the stripped parent path in allPaths. + assertColumn("select 100 from tbl where if(id = 1, null, s) is not null or element_at(s, 'city') = 'beijing'", + "struct>>>", + ImmutableList.of(path("s")), + ImmutableList.of(path("s", "NULL"), path("s", "city")) ); assertColumn("select 100 from tbl where struct_element(s, 'city') is not null", @@ -870,39 +723,28 @@ public void testMapKeysAndValuesFunctionNullCheckUseParentMapNullPath() throws E assertColumn("select map_keys(map_col) from str_tbl where map_keys(map_col) is null", "map", - ImmutableList.of(path("map_col", "KEYS")), - ImmutableList.of() + ImmutableList.of(path("map_col")), + ImmutableList.of(path("map_col", "NULL")) ); assertColumn("select map_values(map_col) from str_tbl where map_values(map_col) is null", "map", - ImmutableList.of(path("map_col", "VALUES")), - ImmutableList.of() + ImmutableList.of(path("map_col")), + ImmutableList.of(path("map_col", "NULL")) ); } @Test public void testProjectFilter() throws Throwable { - assertColumn("select s from tbl where element_at(s, 'city') is not null", - "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of() - ); - assertColumn("select s from tbl where struct_element(s, 'city') is null", - "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of() - ); - assertColumn("select element_at(s, 'data') from tbl where element_at(s, 'city') is not null", "struct>>>", - ImmutableList.of(path("s", "city", "NULL"), path("s", "data")), + ImmutableList.of(path("s", "city"), path("s", "data")), ImmutableList.of(path("s", "city", "NULL")) ); assertColumn("select element_at(s, 'data') from tbl where element_at(s, 'city') is not null and element_at(s, 'data') is not null", "struct>>>", - ImmutableList.of(path("s", "city", "NULL"), path("s", "data")), - ImmutableList.of(path("s", "city", "NULL")) + ImmutableList.of(path("s", "city"), path("s", "data")), + ImmutableList.of(path("s", "city", "NULL"), path("s", "data", "NULL")) ); } @@ -1556,7 +1398,6 @@ private void assertColumns(String sql, TreeSet actualPredicateAccessPaths = new TreeSet<>(slotDescriptor.getPredicateAccessPaths()); Assertions.assertEquals(expectPredicateAccessPathSet, actualPredicateAccessPaths); - Assertions.assertTrue(actualAllAccessPaths.containsAll(actualPredicateAccessPaths)); Map slotIdToDataTypes = new LinkedHashMap<>(); Consumer assertHasSameType = e -> { @@ -1625,25 +1466,28 @@ public void testStructIsNotNullPruning() throws Exception { @Test public void testStructIsNullMixedAccess() throws Exception { - // Parent NULL path must be stripped from allPaths when a child path is also required. - // Otherwise BE StructFileColumnIterator sees the parent NULL sub-path first, switches - // the whole struct iterator to NULL_MAP_ONLY, and skips the child iterator. - // predicateAccessPaths drops [s, NULL] too, keeping it a subset of allAccessPaths. - assertColumn("select struct_element(s, 'city') from tbl where s is null", - "struct", - ImmutableList.of(path("s", "city")), - ImmutableList.of()); + // Predicate metadata paths stay in predicatePaths. allPaths keeps the stripped data path + // shape when ordinary data paths also exist, so older BEs do not switch the mixed read to + // current-level metadata-only mode. + assertColumn("select element_at(s, 'city') from tbl where s is null", + "struct>>>", + ImmutableList.of(path("s")), + ImmutableList.of(path("s", "NULL"))); + + assertColumn("select s from tbl where element_at(s, 'city') is null", + "struct>>>", + ImmutableList.of(path("s")), + ImmutableList.of(path("s", "city", "NULL"))); // This shape is closer to the production bug: one predicate needs the parent // null map, another predicate needs a child null map, and the projection needs - // a different child data path. The parent [s.NULL] cannot remain in allPaths - // with [s.data], so it is also removed from predicate paths; [s.city.NULL] stays - // because it is still present in allPaths. - assertColumn("select struct_element(s, 'data') from tbl " - + "where s is null or struct_element(s, 'city') is null", + // a different child data path. allPaths strips predicate metadata to data paths and + // collapses to the whole struct for mixed-version safety. + assertColumn("select element_at(s, 'data') from tbl " + + "where s is null or element_at(s, 'city') is null", "struct>>>", - ImmutableList.of(path("s", "city", "NULL"), path("s", "data")), - ImmutableList.of(path("s", "city", "NULL"))); + ImmutableList.of(path("s")), + ImmutableList.of(path("s", "NULL"), path("s", "city", "NULL"))); } @Test @@ -1684,7 +1528,7 @@ public void testStringLengthPruning() throws Exception { assertColumn( "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from str_tbl", "struct", - ImmutableList.of(path("c_struct", "f3")), + ImmutableList.of(path("c_struct", "f3"), path("c_struct", "f3", "OFFSET")), ImmutableList.of()); } @@ -1848,9 +1692,19 @@ private SlotReference rewriteAndFindScanSlot(String sql, String columnName, } private void assertNoAccessPaths(SlotReference slot) { - Assertions.assertTrue(!slot.getAllAccessPaths().isPresent() || slot.getAllAccessPaths().get().isEmpty()); + String slotDebugInfo = String.format( + "slot=%s, name=%s, exprId=%s, qualifier=%s, dataType=%s, nullable=%s, " + + "subPath=%s, originalColumn=%s, allAccessPaths=%s, " + + "predicateAccessPaths=%s, displayAllAccessPaths=%s, " + + "displayPredicateAccessPaths=%s", + slot, slot.getName(), slot.getExprId(), slot.getQualifier(), slot.getDataType(), + slot.nullable(), slot.getSubPath(), slot.getOriginalColumn().map(Object::toString), + slot.getAllAccessPaths(), slot.getPredicateAccessPaths(), + slot.getDisplayAllAccessPaths(), slot.getDisplayPredicateAccessPaths()); + Assertions.assertTrue(!slot.getAllAccessPaths().isPresent() || slot.getAllAccessPaths().get().isEmpty(), + slotDebugInfo); Assertions.assertTrue(!slot.getPredicateAccessPaths().isPresent() - || slot.getPredicateAccessPaths().get().isEmpty()); + || slot.getPredicateAccessPaths().get().isEmpty(), slotDebugInfo); } private Pair> collectComplexSlots(String sql) throws Exception { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 9e6d34c7c39621..8a19feb4b7642c 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -511,6 +511,8 @@ struct TQueryOptions { // Fall back to RE2 when Hyperscan cannot compile a regular expression. 228: optional bool enable_hyperscan_fallback = true; + // Master uses 226 for this option; branch-4.2 already uses 226-228, so the id was moved. + 229: optional bool enable_prune_nested_column = false; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out index b3312aa670c066..60d6198b597288 100644 --- a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out +++ b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out @@ -1,86 +1,389 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- -1 {"city":"beijing", "data":[{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}], "value":1} -2 {"city":"shanghai", "data":[{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}], "value":2} -3 {"city":"guangzhou", "data":[{1:{"a":90, "b":60}, 2:{"a":110, "b":40}}], "value":3} -4 {"city":"shenzhen", "data":[{2:{"a":130, "b":20}, 1:{"a":150, "b":40}}], "value":4} -5 {"city":"hangzhou", "data":[{1:{"a":170, "b":80}, 2:{"a":190, "b":40}}], "value":5} -6 {"city":"nanjing", "data":[{2:{"a":210, "b":60}, 1:{"a":230, "b":40}}], "value":6} -7 {"city":"tianjin", "data":[{1:{"a":250, "b":20}, 2:{"a":270, "b":40}}], "value":7} -8 {"city":"chongqing", "data":[{2:{"a":290, "b":80}, 1:{"a":310, "b":40}}], "value":8} -9 {"city":"wuhan", "data":[{1:{"a":330, "b":60}, 2:{"a":350, "b":40}}], "value":9} -10 {"city":"xian", "data":[{2:{"a":370, "b":20}, 1:{"a":390, "b":40}}], "value":10} -11 {"city":"changsha", "data":[{1:{"a":410, "b":80}, 2:{"a":430, "b":40}}], "value":11} -12 {"city":"qingdao", "data":[{2:{"a":450, "b":60}, 1:{"a":470, "b":40}}], "value":12} -13 {"city":"dalian", "data":[{1:{"a":490, "b":20}, 2:{"a":510, "b":40}}], "value":13} +\N 300 +beijing 300 +chengdu 300 +guangzhou 300 +hangzhou 300 +nanjing 300 +shanghai 300 +shenzhen 300 +wuhan 300 +xian 300 -- !sql1 -- -1 [10] +1 [10, 5] + +-- !sql1_1 -- + +-- !sql1_2 -- -- !sql2 -- -1 beijing -2 shanghai +0 beijing +1 shanghai +2 shenzhen 3 guangzhou -4 shenzhen -5 hangzhou -6 nanjing -7 tianjin -8 chongqing -9 wuhan -10 xian -11 changsha -12 qingdao -13 dalian +4 hangzhou +5 chengdu +6 wuhan +7 xian +8 nanjing +9 \N +10 beijing +11 shanghai +12 shenzhen +13 guangzhou +14 hangzhou +15 chengdu +16 wuhan +17 xian +18 nanjing +19 \N + +-- !sql2_1 -- +100 beijing +101 shanghai +102 shenzhen +103 guangzhou +104 hangzhou +105 chengdu +106 wuhan +107 xian +108 nanjing +109 \N +110 beijing +111 shanghai +112 shenzhen +113 guangzhou +114 hangzhou +115 chengdu +116 wuhan +117 xian +118 nanjing +119 \N + +-- !sql2_2 -- +2999 \N +2998 nanjing +2997 xian +2996 wuhan +2995 chengdu +2994 hangzhou +2993 guangzhou +2992 shenzhen +2991 shanghai +2990 beijing +2989 \N +2988 nanjing +2987 xian +2986 wuhan +2985 chengdu +2984 hangzhou +2983 guangzhou +2982 shenzhen +2981 shanghai +2980 beijing -- !sql3 -- -1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}] -2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}] -3 [{1:{"a":90, "b":60}, 2:{"a":110, "b":40}}] -4 [{2:{"a":130, "b":20}, 1:{"a":150, "b":40}}] -5 [{1:{"a":170, "b":80}, 2:{"a":190, "b":40}}] -6 [{2:{"a":210, "b":60}, 1:{"a":230, "b":40}}] -7 [{1:{"a":250, "b":20}, 2:{"a":270, "b":40}}] -8 [{2:{"a":290, "b":80}, 1:{"a":310, "b":40}}] -9 [{1:{"a":330, "b":60}, 2:{"a":350, "b":40}}] -10 [{2:{"a":370, "b":20}, 1:{"a":390, "b":40}}] -11 [{1:{"a":410, "b":80}, 2:{"a":430, "b":40}}] -12 [{2:{"a":450, "b":60}, 1:{"a":470, "b":40}}] -13 [{1:{"a":490, "b":20}, 2:{"a":510, "b":40}}] +0 [{1:{"a":0, "b":0}, 2:{"a":20, "b":10}}, {1:{"a":0, "b":0}, 2:{"a":0, "b":0}}] +1 [{1:{"a":10, "b":11}, 2:{"a":30, "b":20}}, {2:{"a":5, "b":2.5}, 3:{"a":3, "b":1.5}}] +2 [{1:{"a":20, "b":22}, 2:{"a":40, "b":30}}, {3:{"a":10, "b":5}, 4:{"a":6, "b":3}}] +3 [{1:{"a":30, "b":33}, 2:{"a":50, "b":40}}, {1:{"a":15, "b":7.5}, 5:{"a":9, "b":4.5}}] +4 [{1:{"a":40, "b":44}, 2:{"a":60, "b":50}}, {2:{"a":20, "b":10}, 6:{"a":12, "b":6}}] +5 [{1:{"a":50, "b":50}, 2:{"a":70, "b":60}}, {3:{"a":25, "b":12.5}, 2:{"a":15, "b":7.5}}] +6 [{1:{"a":60, "b":61}, 2:{"a":80, "b":70}}, {1:{"a":30, "b":15}, 3:{"a":18, "b":9}}] +7 [{1:{"a":70, "b":72}, 2:{"a":90, "b":80}}, {2:{"a":35, "b":17.5}, 4:{"a":21, "b":10.5}}] +8 [{1:{"a":80, "b":83}, 2:{"a":100, "b":90}}, {3:{"a":40, "b":20}, 5:{"a":24, "b":12}}] +9 [{1:{"a":90, "b":94}, 2:{"a":110, "b":100}}, {1:{"a":45, "b":22.5}, 6:{"a":27, "b":13.5}}] +10 [{1:{"a":100, "b":100}, 2:{"a":120, "b":10}}, {2:{"a":30, "b":15}}] +11 [{1:{"a":110, "b":111}, 2:{"a":130, "b":20}}, {3:{"a":33, "b":16.5}}] +12 [{1:{"a":120, "b":122}, 2:{"a":140, "b":30}}, {1:{"a":60, "b":30}, 4:{"a":36, "b":18}}] +13 [{1:{"a":130, "b":133}, 2:{"a":150, "b":40}}, {2:{"a":65, "b":32.5}, 5:{"a":39, "b":19.5}}] +14 [{1:{"a":140, "b":144}, 2:{"a":160, "b":50}}, {3:{"a":70, "b":35}, 6:{"a":42, "b":21}}] +15 [{1:{"a":150, "b":150}, 2:{"a":170, "b":60}}, {1:{"a":75, "b":37.5}, 2:{"a":45, "b":22.5}}] +16 [{1:{"a":160, "b":161}, 2:{"a":180, "b":70}}, {2:{"a":80, "b":40}, 3:{"a":48, "b":24}}] +17 [{1:{"a":170, "b":172}, 2:{"a":190, "b":80}}, {3:{"a":85, "b":42.5}, 4:{"a":51, "b":25.5}}] +18 [{1:{"a":180, "b":183}, 2:{"a":200, "b":90}}, {1:{"a":90, "b":45}, 5:{"a":54, "b":27}}] +19 [{1:{"a":190, "b":194}, 2:{"a":210, "b":100}}, {2:{"a":95, "b":47.5}, 6:{"a":57, "b":28.5}}] + +-- !sql3_1 -- +200 [{1:{"a":2000, "b":2000}, 2:{"a":2020, "b":10}}, {3:{"a":1000, "b":500}, 2:{"a":600, "b":300}}] +201 [{1:{"a":2010, "b":2011}, 2:{"a":2030, "b":20}}, {1:{"a":1005, "b":502.5}, 3:{"a":603, "b":301.5}}] +202 [{1:{"a":2020, "b":2022}, 2:{"a":2040, "b":30}}, {2:{"a":1010, "b":505}, 4:{"a":606, "b":303}}] +203 [{1:{"a":2030, "b":2033}, 2:{"a":2050, "b":40}}, {3:{"a":1015, "b":507.5}, 5:{"a":609, "b":304.5}}] +204 [{1:{"a":2040, "b":2044}, 2:{"a":2060, "b":50}}, {1:{"a":1020, "b":510}, 6:{"a":612, "b":306}}] +205 [{1:{"a":2050, "b":2050}, 2:{"a":2070, "b":60}}, {2:{"a":615, "b":307.5}}] +206 [{1:{"a":2060, "b":2061}, 2:{"a":2080, "b":70}}, {3:{"a":618, "b":309}}] +207 [{1:{"a":2070, "b":2072}, 2:{"a":2090, "b":80}}, {1:{"a":1035, "b":517.5}, 4:{"a":621, "b":310.5}}] +208 [{1:{"a":2080, "b":2083}, 2:{"a":2100, "b":90}}, {2:{"a":1040, "b":520}, 5:{"a":624, "b":312}}] +209 [{1:{"a":2090, "b":2094}, 2:{"a":2110, "b":100}}, {3:{"a":1045, "b":522.5}, 6:{"a":627, "b":313.5}}] +210 [{1:{"a":2100, "b":2100}, 2:{"a":2120, "b":10}}, {1:{"a":1050, "b":525}, 2:{"a":630, "b":315}}] +211 [{1:{"a":2110, "b":2111}, 2:{"a":2130, "b":20}}, {2:{"a":1055, "b":527.5}, 3:{"a":633, "b":316.5}}] +212 [{1:{"a":2120, "b":2122}, 2:{"a":2140, "b":30}}, {3:{"a":1060, "b":530}, 4:{"a":636, "b":318}}] +213 [{1:{"a":2130, "b":2133}, 2:{"a":2150, "b":40}}, {1:{"a":1065, "b":532.5}, 5:{"a":639, "b":319.5}}] +214 [{1:{"a":2140, "b":2144}, 2:{"a":2160, "b":50}}, {2:{"a":1070, "b":535}, 6:{"a":642, "b":321}}] +215 [{1:{"a":2150, "b":2150}, 2:{"a":2170, "b":60}}, {3:{"a":1075, "b":537.5}, 2:{"a":645, "b":322.5}}] +216 [{1:{"a":2160, "b":2161}, 2:{"a":2180, "b":70}}, {1:{"a":1080, "b":540}, 3:{"a":648, "b":324}}] +217 [{1:{"a":2170, "b":2172}, 2:{"a":2190, "b":80}}, {2:{"a":1085, "b":542.5}, 4:{"a":651, "b":325.5}}] +218 [{1:{"a":2180, "b":2183}, 2:{"a":2200, "b":90}}, {3:{"a":1090, "b":545}, 5:{"a":654, "b":327}}] +219 [{1:{"a":2190, "b":2194}, 2:{"a":2210, "b":100}}, {1:{"a":1095, "b":547.5}, 6:{"a":657, "b":328.5}}] + +-- !sql3_2 -- +2999 [{1:{"a":29990, "b":29994}, 2:{"a":30010, "b":100}}, {3:{"a":14995, "b":7497.5}, 6:{"a":8997, "b":4498.5}}] +2998 [{1:{"a":29980, "b":29983}, 2:{"a":30000, "b":90}}, {2:{"a":14990, "b":7495}, 5:{"a":8994, "b":4497}}] +2997 [{1:{"a":29970, "b":29972}, 2:{"a":29990, "b":80}}, {1:{"a":14985, "b":7492.5}, 4:{"a":8991, "b":4495.5}}] +2996 [{1:{"a":29960, "b":29961}, 2:{"a":29980, "b":70}}, {3:{"a":8988, "b":4494}}] +2995 [{1:{"a":29950, "b":29950}, 2:{"a":29970, "b":60}}, {2:{"a":8985, "b":4492.5}}] +2994 [{1:{"a":29940, "b":29944}, 2:{"a":29960, "b":50}}, {1:{"a":14970, "b":7485}, 6:{"a":8982, "b":4491}}] +2993 [{1:{"a":29930, "b":29933}, 2:{"a":29950, "b":40}}, {3:{"a":14965, "b":7482.5}, 5:{"a":8979, "b":4489.5}}] +2992 [{1:{"a":29920, "b":29922}, 2:{"a":29940, "b":30}}, {2:{"a":14960, "b":7480}, 4:{"a":8976, "b":4488}}] +2991 [{1:{"a":29910, "b":29911}, 2:{"a":29930, "b":20}}, {1:{"a":14955, "b":7477.5}, 3:{"a":8973, "b":4486.5}}] +2990 [{1:{"a":29900, "b":29900}, 2:{"a":29920, "b":10}}, {3:{"a":14950, "b":7475}, 2:{"a":8970, "b":4485}}] +2989 [{1:{"a":29890, "b":29894}, 2:{"a":29910, "b":100}}, {2:{"a":14945, "b":7472.5}, 6:{"a":8967, "b":4483.5}}] +2988 [{1:{"a":29880, "b":29883}, 2:{"a":29900, "b":90}}, {1:{"a":14940, "b":7470}, 5:{"a":8964, "b":4482}}] +2987 [{1:{"a":29870, "b":29872}, 2:{"a":29890, "b":80}}, {3:{"a":14935, "b":7467.5}, 4:{"a":8961, "b":4480.5}}] +2986 [{1:{"a":29860, "b":29861}, 2:{"a":29880, "b":70}}, {2:{"a":14930, "b":7465}, 3:{"a":8958, "b":4479}}] +2985 [{1:{"a":29850, "b":29850}, 2:{"a":29870, "b":60}}, {1:{"a":14925, "b":7462.5}, 2:{"a":8955, "b":4477.5}}] +2984 [{1:{"a":29840, "b":29844}, 2:{"a":29860, "b":50}}, {3:{"a":14920, "b":7460}, 6:{"a":8952, "b":4476}}] +2983 [{1:{"a":29830, "b":29833}, 2:{"a":29850, "b":40}}, {2:{"a":14915, "b":7457.5}, 5:{"a":8949, "b":4474.5}}] +2982 [{1:{"a":29820, "b":29822}, 2:{"a":29840, "b":30}}, {1:{"a":14910, "b":7455}, 4:{"a":8946, "b":4473}}] +2981 [{1:{"a":29810, "b":29811}, 2:{"a":29830, "b":20}}, {3:{"a":8943, "b":4471.5}}] +2980 [{1:{"a":29800, "b":29800}, 2:{"a":29820, "b":10}}, {2:{"a":8940, "b":4470}}] -- !sql4 -- -1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}] -2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}] -3 [{1:{"a":90, "b":60}, 2:{"a":110, "b":40}}] -5 [{1:{"a":170, "b":80}, 2:{"a":190, "b":40}}] -7 [{1:{"a":250, "b":20}, 2:{"a":270, "b":40}}] -9 [{1:{"a":330, "b":60}, 2:{"a":350, "b":40}}] -11 [{1:{"a":410, "b":80}, 2:{"a":430, "b":40}}] -13 [{1:{"a":490, "b":20}, 2:{"a":510, "b":40}}] +3 [{1:{"a":30, "b":33}, 2:{"a":50, "b":40}}, {1:{"a":15, "b":7.5}, 5:{"a":9, "b":4.5}}] +13 [{1:{"a":130, "b":133}, 2:{"a":150, "b":40}}, {2:{"a":65, "b":32.5}, 5:{"a":39, "b":19.5}}] +23 [{1:{"a":230, "b":233}, 2:{"a":250, "b":40}}, {3:{"a":115, "b":57.5}, 5:{"a":69, "b":34.5}}] +33 [{1:{"a":330, "b":333}, 2:{"a":350, "b":40}}, {1:{"a":165, "b":82.5}, 5:{"a":99, "b":49.5}}] +43 [{1:{"a":430, "b":433}, 2:{"a":450, "b":40}}, {2:{"a":215, "b":107.5}, 5:{"a":129, "b":64.5}}] +53 [{1:{"a":530, "b":533}, 2:{"a":550, "b":40}}, {3:{"a":265, "b":132.5}, 5:{"a":159, "b":79.5}}] +63 [{1:{"a":630, "b":633}, 2:{"a":650, "b":40}}, {1:{"a":315, "b":157.5}, 5:{"a":189, "b":94.5}}] +73 [{1:{"a":730, "b":733}, 2:{"a":750, "b":40}}, {2:{"a":365, "b":182.5}, 5:{"a":219, "b":109.5}}] +83 [{1:{"a":830, "b":833}, 2:{"a":850, "b":40}}, {3:{"a":415, "b":207.5}, 5:{"a":249, "b":124.5}}] +93 [{1:{"a":930, "b":933}, 2:{"a":950, "b":40}}, {1:{"a":465, "b":232.5}, 5:{"a":279, "b":139.5}}] +103 [{1:{"a":1030, "b":1033}, 2:{"a":1050, "b":40}}, {2:{"a":515, "b":257.5}, 5:{"a":309, "b":154.5}}] +113 [{1:{"a":1130, "b":1133}, 2:{"a":1150, "b":40}}, {3:{"a":565, "b":282.5}, 5:{"a":339, "b":169.5}}] +123 [{1:{"a":1230, "b":1233}, 2:{"a":1250, "b":40}}, {1:{"a":615, "b":307.5}, 5:{"a":369, "b":184.5}}] +133 [{1:{"a":1330, "b":1333}, 2:{"a":1350, "b":40}}, {2:{"a":665, "b":332.5}, 5:{"a":399, "b":199.5}}] +143 [{1:{"a":1430, "b":1433}, 2:{"a":1450, "b":40}}, {3:{"a":715, "b":357.5}, 5:{"a":429, "b":214.5}}] +153 [{1:{"a":1530, "b":1533}, 2:{"a":1550, "b":40}}, {1:{"a":765, "b":382.5}, 5:{"a":459, "b":229.5}}] +163 [{1:{"a":1630, "b":1633}, 2:{"a":1650, "b":40}}, {2:{"a":815, "b":407.5}, 5:{"a":489, "b":244.5}}] +173 [{1:{"a":1730, "b":1733}, 2:{"a":1750, "b":40}}, {3:{"a":865, "b":432.5}, 5:{"a":519, "b":259.5}}] +183 [{1:{"a":1830, "b":1833}, 2:{"a":1850, "b":40}}, {1:{"a":915, "b":457.5}, 5:{"a":549, "b":274.5}}] +193 [{1:{"a":1930, "b":1933}, 2:{"a":1950, "b":40}}, {2:{"a":965, "b":482.5}, 5:{"a":579, "b":289.5}}] + +-- !sql4_1 -- +1003 [{1:{"a":10030, "b":10033}, 2:{"a":10050, "b":40}}, {2:{"a":5015, "b":2507.5}, 5:{"a":3009, "b":1504.5}}] +1013 [{1:{"a":10130, "b":10133}, 2:{"a":10150, "b":40}}, {3:{"a":5065, "b":2532.5}, 5:{"a":3039, "b":1519.5}}] +1023 [{1:{"a":10230, "b":10233}, 2:{"a":10250, "b":40}}, {1:{"a":5115, "b":2557.5}, 5:{"a":3069, "b":1534.5}}] +1033 [{1:{"a":10330, "b":10333}, 2:{"a":10350, "b":40}}, {2:{"a":5165, "b":2582.5}, 5:{"a":3099, "b":1549.5}}] +1043 [{1:{"a":10430, "b":10433}, 2:{"a":10450, "b":40}}, {3:{"a":5215, "b":2607.5}, 5:{"a":3129, "b":1564.5}}] +1053 [{1:{"a":10530, "b":10533}, 2:{"a":10550, "b":40}}, {1:{"a":5265, "b":2632.5}, 5:{"a":3159, "b":1579.5}}] +1063 [{1:{"a":10630, "b":10633}, 2:{"a":10650, "b":40}}, {2:{"a":5315, "b":2657.5}, 5:{"a":3189, "b":1594.5}}] +1073 [{1:{"a":10730, "b":10733}, 2:{"a":10750, "b":40}}, {3:{"a":5365, "b":2682.5}, 5:{"a":3219, "b":1609.5}}] +1083 [{1:{"a":10830, "b":10833}, 2:{"a":10850, "b":40}}, {1:{"a":5415, "b":2707.5}, 5:{"a":3249, "b":1624.5}}] +1093 [{1:{"a":10930, "b":10933}, 2:{"a":10950, "b":40}}, {2:{"a":5465, "b":2732.5}, 5:{"a":3279, "b":1639.5}}] +1103 [{1:{"a":11030, "b":11033}, 2:{"a":11050, "b":40}}, {3:{"a":5515, "b":2757.5}, 5:{"a":3309, "b":1654.5}}] +1113 [{1:{"a":11130, "b":11133}, 2:{"a":11150, "b":40}}, {1:{"a":5565, "b":2782.5}, 5:{"a":3339, "b":1669.5}}] +1123 [{1:{"a":11230, "b":11233}, 2:{"a":11250, "b":40}}, {2:{"a":5615, "b":2807.5}, 5:{"a":3369, "b":1684.5}}] +1133 [{1:{"a":11330, "b":11333}, 2:{"a":11350, "b":40}}, {3:{"a":5665, "b":2832.5}, 5:{"a":3399, "b":1699.5}}] +1143 [{1:{"a":11430, "b":11433}, 2:{"a":11450, "b":40}}, {1:{"a":5715, "b":2857.5}, 5:{"a":3429, "b":1714.5}}] +1153 [{1:{"a":11530, "b":11533}, 2:{"a":11550, "b":40}}, {2:{"a":5765, "b":2882.5}, 5:{"a":3459, "b":1729.5}}] +1163 [{1:{"a":11630, "b":11633}, 2:{"a":11650, "b":40}}, {3:{"a":5815, "b":2907.5}, 5:{"a":3489, "b":1744.5}}] +1173 [{1:{"a":11730, "b":11733}, 2:{"a":11750, "b":40}}, {1:{"a":5865, "b":2932.5}, 5:{"a":3519, "b":1759.5}}] +1183 [{1:{"a":11830, "b":11833}, 2:{"a":11850, "b":40}}, {2:{"a":5915, "b":2957.5}, 5:{"a":3549, "b":1774.5}}] +1193 [{1:{"a":11930, "b":11933}, 2:{"a":11950, "b":40}}, {3:{"a":5965, "b":2982.5}, 5:{"a":3579, "b":1789.5}}] + +-- !sql4_2 -- +2993 [{1:{"a":29930, "b":29933}, 2:{"a":29950, "b":40}}, {3:{"a":14965, "b":7482.5}, 5:{"a":8979, "b":4489.5}}] +2983 [{1:{"a":29830, "b":29833}, 2:{"a":29850, "b":40}}, {2:{"a":14915, "b":7457.5}, 5:{"a":8949, "b":4474.5}}] +2973 [{1:{"a":29730, "b":29733}, 2:{"a":29750, "b":40}}, {1:{"a":14865, "b":7432.5}, 5:{"a":8919, "b":4459.5}}] +2963 [{1:{"a":29630, "b":29633}, 2:{"a":29650, "b":40}}, {3:{"a":14815, "b":7407.5}, 5:{"a":8889, "b":4444.5}}] +2953 [{1:{"a":29530, "b":29533}, 2:{"a":29550, "b":40}}, {2:{"a":14765, "b":7382.5}, 5:{"a":8859, "b":4429.5}}] +2943 [{1:{"a":29430, "b":29433}, 2:{"a":29450, "b":40}}, {1:{"a":14715, "b":7357.5}, 5:{"a":8829, "b":4414.5}}] +2933 [{1:{"a":29330, "b":29333}, 2:{"a":29350, "b":40}}, {3:{"a":14665, "b":7332.5}, 5:{"a":8799, "b":4399.5}}] +2923 [{1:{"a":29230, "b":29233}, 2:{"a":29250, "b":40}}, {2:{"a":14615, "b":7307.5}, 5:{"a":8769, "b":4384.5}}] +2913 [{1:{"a":29130, "b":29133}, 2:{"a":29150, "b":40}}, {1:{"a":14565, "b":7282.5}, 5:{"a":8739, "b":4369.5}}] +2903 [{1:{"a":29030, "b":29033}, 2:{"a":29050, "b":40}}, {3:{"a":14515, "b":7257.5}, 5:{"a":8709, "b":4354.5}}] +2893 [{1:{"a":28930, "b":28933}, 2:{"a":28950, "b":40}}, {2:{"a":14465, "b":7232.5}, 5:{"a":8679, "b":4339.5}}] +2883 [{1:{"a":28830, "b":28833}, 2:{"a":28850, "b":40}}, {1:{"a":14415, "b":7207.5}, 5:{"a":8649, "b":4324.5}}] +2873 [{1:{"a":28730, "b":28733}, 2:{"a":28750, "b":40}}, {3:{"a":14365, "b":7182.5}, 5:{"a":8619, "b":4309.5}}] +2863 [{1:{"a":28630, "b":28633}, 2:{"a":28650, "b":40}}, {2:{"a":14315, "b":7157.5}, 5:{"a":8589, "b":4294.5}}] +2853 [{1:{"a":28530, "b":28533}, 2:{"a":28550, "b":40}}, {1:{"a":14265, "b":7132.5}, 5:{"a":8559, "b":4279.5}}] +2843 [{1:{"a":28430, "b":28433}, 2:{"a":28450, "b":40}}, {3:{"a":14215, "b":7107.5}, 5:{"a":8529, "b":4264.5}}] +2833 [{1:{"a":28330, "b":28333}, 2:{"a":28350, "b":40}}, {2:{"a":14165, "b":7082.5}, 5:{"a":8499, "b":4249.5}}] +2823 [{1:{"a":28230, "b":28233}, 2:{"a":28250, "b":40}}, {1:{"a":14115, "b":7057.5}, 5:{"a":8469, "b":4234.5}}] +2813 [{1:{"a":28130, "b":28133}, 2:{"a":28150, "b":40}}, {3:{"a":14065, "b":7032.5}, 5:{"a":8439, "b":4219.5}}] +2803 [{1:{"a":28030, "b":28033}, 2:{"a":28050, "b":40}}, {2:{"a":14015, "b":7007.5}, 5:{"a":8409, "b":4204.5}}] -- !sql5 -- -1 beijing -2 shanghai 3 guangzhou -5 hangzhou -7 tianjin -9 wuhan -11 changsha -13 dalian +13 guangzhou +23 guangzhou +33 guangzhou +43 guangzhou +53 guangzhou +63 guangzhou +73 guangzhou +83 guangzhou +93 guangzhou +103 guangzhou +113 guangzhou +123 guangzhou +133 guangzhou +143 guangzhou +153 guangzhou +163 guangzhou +173 guangzhou +183 guangzhou +193 guangzhou -- !sql5_1 -- -61 +1003 guangzhou +1013 guangzhou +1023 guangzhou +1033 guangzhou +1043 guangzhou +1053 guangzhou +1063 guangzhou +1073 guangzhou +1083 guangzhou +1093 guangzhou +1103 guangzhou +1113 guangzhou +1123 guangzhou +1133 guangzhou +1143 guangzhou +1153 guangzhou +1163 guangzhou +1173 guangzhou +1183 guangzhou +1193 guangzhou -- !sql5_2 -- +2993 guangzhou +2983 guangzhou +2973 guangzhou +2963 guangzhou +2953 guangzhou +2943 guangzhou +2933 guangzhou +2923 guangzhou +2913 guangzhou +2903 guangzhou +2893 guangzhou +2883 guangzhou +2873 guangzhou +2863 guangzhou +2853 guangzhou +2843 guangzhou +2833 guangzhou +2823 guangzhou +2813 guangzhou +2803 guangzhou + +-- !sql5_3 -- 61 +-- !sql5_4 -- +61 + +-- !sql5_5 -- +9 {"city":null, "data":[{1:{"a":90, "b":94}, 2:{"a":110, "b":100}}, {1:{"a":45, "b":22.5}, 6:{"a":27, "b":13.5}}], "value":9} +19 {"city":null, "data":[{1:{"a":190, "b":194}, 2:{"a":210, "b":100}}, {2:{"a":95, "b":47.5}, 6:{"a":57, "b":28.5}}], "value":19} +29 {"city":null, "data":[{1:{"a":290, "b":294}, 2:{"a":310, "b":100}}, {3:{"a":145, "b":72.5}, 6:{"a":87, "b":43.5}}], "value":29} +39 {"city":null, "data":[{1:{"a":390, "b":394}, 2:{"a":410, "b":100}}, {1:{"a":195, "b":97.5}, 6:{"a":117, "b":58.5}}], "value":39} +49 {"city":null, "data":[{1:{"a":490, "b":494}, 2:{"a":510, "b":100}}, {2:{"a":245, "b":122.5}, 6:{"a":147, "b":73.5}}], "value":49} +59 {"city":null, "data":[{1:{"a":590, "b":594}, 2:{"a":610, "b":100}}, {3:{"a":295, "b":147.5}, 6:{"a":177, "b":88.5}}], "value":59} +69 {"city":null, "data":[{1:{"a":690, "b":694}, 2:{"a":710, "b":100}}, {1:{"a":345, "b":172.5}, 6:{"a":207, "b":103.5}}], "value":69} +79 {"city":null, "data":[{1:{"a":790, "b":794}, 2:{"a":810, "b":100}}, {2:{"a":395, "b":197.5}, 6:{"a":237, "b":118.5}}], "value":79} +89 {"city":null, "data":[{1:{"a":890, "b":894}, 2:{"a":910, "b":100}}, {3:{"a":445, "b":222.5}, 6:{"a":267, "b":133.5}}], "value":89} +99 {"city":null, "data":[{1:{"a":990, "b":994}, 2:{"a":1010, "b":100}}, {1:{"a":495, "b":247.5}, 6:{"a":297, "b":148.5}}], "value":99} +109 {"city":null, "data":[{1:{"a":1090, "b":1094}, 2:{"a":1110, "b":100}}, {2:{"a":545, "b":272.5}, 6:{"a":327, "b":163.5}}], "value":109} +119 {"city":null, "data":[{1:{"a":1190, "b":1194}, 2:{"a":1210, "b":100}}, {3:{"a":595, "b":297.5}, 6:{"a":357, "b":178.5}}], "value":119} +129 {"city":null, "data":[{1:{"a":1290, "b":1294}, 2:{"a":1310, "b":100}}, {1:{"a":645, "b":322.5}, 6:{"a":387, "b":193.5}}], "value":129} +139 {"city":null, "data":[{1:{"a":1390, "b":1394}, 2:{"a":1410, "b":100}}, {2:{"a":695, "b":347.5}, 6:{"a":417, "b":208.5}}], "value":139} +149 {"city":null, "data":[{1:{"a":1490, "b":1494}, 2:{"a":1510, "b":100}}, {3:{"a":745, "b":372.5}, 6:{"a":447, "b":223.5}}], "value":149} +159 {"city":null, "data":[{1:{"a":1590, "b":1594}, 2:{"a":1610, "b":100}}, {1:{"a":795, "b":397.5}, 6:{"a":477, "b":238.5}}], "value":159} +169 {"city":null, "data":[{1:{"a":1690, "b":1694}, 2:{"a":1710, "b":100}}, {2:{"a":845, "b":422.5}, 6:{"a":507, "b":253.5}}], "value":169} +179 {"city":null, "data":[{1:{"a":1790, "b":1794}, 2:{"a":1810, "b":100}}, {3:{"a":895, "b":447.5}, 6:{"a":537, "b":268.5}}], "value":179} +189 {"city":null, "data":[{1:{"a":1890, "b":1894}, 2:{"a":1910, "b":100}}, {1:{"a":945, "b":472.5}, 6:{"a":567, "b":283.5}}], "value":189} +199 {"city":null, "data":[{1:{"a":1990, "b":1994}, 2:{"a":2010, "b":100}}, {2:{"a":995, "b":497.5}, 6:{"a":597, "b":298.5}}], "value":199} + -- !sql6 -- -2 +5 12.5 +15 \N +25 \N +35 87.5 +45 \N +55 \N +65 162.5 +75 \N +85 \N +95 237.5 +105 \N +115 \N +125 312.5 +135 \N +145 \N +155 387.5 +165 \N +175 \N +185 462.5 +195 \N + +-- !sql6_1 -- +1005 \N +1015 \N +1025 2562.5 +1035 \N +1045 \N +1055 2637.5 +1065 \N +1075 \N +1085 2712.5 +1095 \N +1105 \N +1115 2787.5 +1125 \N +1135 \N +1145 2862.5 +1155 \N +1165 \N +1175 2937.5 +1185 \N +1195 \N + +-- !sql6_2 -- +2995 \N +2985 \N +2975 7437.5 +2965 \N +2955 \N +2945 7362.5 +2935 \N +2925 \N +2915 7287.5 +2905 \N +2895 \N +2885 7212.5 +2875 \N +2865 \N +2855 7137.5 +2845 \N +2835 \N +2825 7062.5 +2815 \N +2805 \N -- !sql7 -- +2 + +-- !sql8 -- 0.41 0.99 --- !sql8 -- +-- !sql9 -- \N added_z diff --git a/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out index 8b2d33fd6e3e32..e4dc236aa24916 100644 --- a/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out +++ b/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out @@ -1,7 +1,19 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !struct_root_arr_mixed -- 1 2 10 +2 0 \N +3 1 30 -- !struct_root_map_mixed -- 1 1 x +2 6 longer +3 \N only-b + +-- !struct_root_arr_predicate_mixed -- +1 hello +3 empty + +-- !struct_root_map_predicate_mixed -- +1 x +2 longer diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy index 4e70c26819f482..be802dc52ee9bc 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy @@ -15,7 +15,11 @@ // specific language governing permissions and limitations // under the License. +import org.apache.doris.regression.action.ProfileAction + suite("test_pruned_columns") { + sql "set batch_size = 32;" + sql "set enable_prune_nested_column = true" sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns`""" sql """ CREATE TABLE `tbl_test_pruned_columns` ( @@ -23,59 +27,171 @@ suite("test_pruned_columns") { `s` struct>>, value:int> NULL ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY RANDOM BUCKETS AUTO + DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ( "replication_allocation" = "tag.location.default: 1" ); """ sql """ - insert into `tbl_test_pruned_columns` values - (1, named_struct('city', 'beijing', 'data', array(map(1, named_struct('a', 10, 'b', 20.0), 2, named_struct('a', 30, 'b', 40))), 'value', 1)), - (2, named_struct('city', 'shanghai', 'data', array(map(2, named_struct('a', 50, 'b', 40.0), 1, named_struct('a', 70, 'b', 80))), 'value', 2)), - (3, named_struct('city', 'guangzhou', 'data', array(map(1, named_struct('a', 90, 'b', 60.0), 2, named_struct('a', 110, 'b', 40))), 'value', 3)), - (4, named_struct('city', 'shenzhen', 'data', array(map(2, named_struct('a', 130, 'b', 20.0), 1, named_struct('a', 150, 'b', 40))), 'value', 4)), - (5, named_struct('city', 'hangzhou', 'data', array(map(1, named_struct('a', 170, 'b', 80.0), 2, named_struct('a', 190, 'b', 40))), 'value', 5)), - (6, named_struct('city', 'nanjing', 'data', array(map(2, named_struct('a', 210, 'b', 60.0), 1, named_struct('a', 230, 'b', 40))), 'value', 6)), - (7, named_struct('city', 'tianjin', 'data', array(map(1, named_struct('a', 250, 'b', 20.0), 2, named_struct('a', 270, 'b', 40))), 'value', 7)), - (8, named_struct('city', 'chongqing', 'data', array(map(2, named_struct('a', 290, 'b', 80.0), 1, named_struct('a', 310, 'b', 40))), 'value', 8)), - (9, named_struct('city', 'wuhan', 'data', array(map(1, named_struct('a', 330, 'b', 60.0), 2, named_struct('a', 350, 'b', 40))), 'value', 9)), - (10, named_struct('city', 'xian', 'data', array(map(2, named_struct('a', 370, 'b', 20.0), 1, named_struct('a', 390, 'b', 40))), 'value', 10)), - (11, named_struct('city', 'changsha', 'data', array(map(1, named_struct('a', 410, 'b', 80.0), 2, named_struct('a', 430, 'b', 40))), 'value', 11)), - (12, named_struct('city', 'qingdao', 'data', array(map(2, named_struct('a', 450, 'b', 60.0), 1, named_struct('a', 470, 'b', 40))), 'value', 12)), - (13, named_struct('city', 'dalian', 'data', array(map(1, named_struct('a', 490, 'b', 20.0), 2, named_struct('a', 510, 'b', 40))), 'value', 13)); + insert into `tbl_test_pruned_columns` + select + number as id, + named_struct( + 'city', + case (number % 10) + when 0 then 'beijing' + when 1 then 'shanghai' + when 2 then 'shenzhen' + when 3 then 'guangzhou' + when 4 then 'hangzhou' + when 5 then 'chengdu' + when 6 then 'wuhan' + when 7 then 'xian' + when 8 then 'nanjing' + else null + end, + 'data', + array( + map( + 1, named_struct('a', number * 10, 'b', (number * 10 + number % 5) * 1.0), + 2, named_struct('a', number * 10 + 20, 'b', (number % 10 + 1) * 10.0) + ), + map( + (number % 3 + 1), named_struct('a', number * 5, 'b', number * 2.5), + (number % 5 + 2), named_struct('a', number * 3, 'b', number * 1.5) + ) + ), + 'value', + number + ) as s + from numbers("number" = "3000"); """ qt_sql """ - select * from `tbl_test_pruned_columns` order by 1; + select element_at(s, 'city'), count() from `tbl_test_pruned_columns` group by element_at(s, 'city') order by 1, 2; """ qt_sql1 """ - select b.id, array_map(x -> element_at(map_values(x)[1], 'a'), element_at(s, 'data')) from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id order by 1; + select + b.id + , array_map(x -> element_at(map_values(x)[1], 'a') + , element_at(s, 'data')) + from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id + order by 1, 2 limit 0, 20; + """ + + qt_sql1_1 """ + select + b.id + , array_map(x -> element_at(map_values(x)[1], 'a') + , element_at(s, 'data')) + from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id + order by 1, 2 limit 100, 20; + """ + + qt_sql1_2 """ + select + b.id + , array_map(x -> element_at(map_values(x)[1], 'a') + , element_at(s, 'data')) + from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id + order by 1 desc, 2 limit 100, 20; """ qt_sql2 """ - select id, element_at(s, 'city') from `tbl_test_pruned_columns` order by 1; + select id, element_at(s, 'city') from `tbl_test_pruned_columns` order by 1 limit 0, 20; + """ + + qt_sql2_1 """ + select id, element_at(s, 'city') from `tbl_test_pruned_columns` order by 1 limit 100, 20; + """ + + qt_sql2_2 """ + select id, element_at(s, 'city') from `tbl_test_pruned_columns` order by 1 desc limit 0, 20; """ qt_sql3 """ - select id, element_at(s, 'data') from `tbl_test_pruned_columns` order by 1; + select id, element_at(s, 'data') from `tbl_test_pruned_columns` order by 1 limit 0, 20; + """ + + qt_sql3_1 """ + select id, element_at(s, 'data') from `tbl_test_pruned_columns` order by 1 limit 200, 20; + """ + + qt_sql3_2 """ + select id, element_at(s, 'data') from `tbl_test_pruned_columns` order by 1 desc limit 0, 20; """ qt_sql4 """ - select id, element_at(s, 'data') from `tbl_test_pruned_columns` where element_at(element_at(s, 'data')[1][2], 'b') = 40 order by 1; + select + id + , element_at(s, 'data') + from `tbl_test_pruned_columns` + where element_at(element_at(s, 'data')[1][2], 'b') = 40 + order by 1 limit 0, 20; + """ + + qt_sql4_1 """ + select + id + , element_at(s, 'data') + from `tbl_test_pruned_columns` + where element_at(element_at(s, 'data')[1][2], 'b') = 40 + order by 1 limit 100, 20; + """ + + qt_sql4_2 """ + select + id + , element_at(s, 'data') + from `tbl_test_pruned_columns` + where element_at(element_at(s, 'data')[1][2], 'b') = 40 + order by 1 desc limit 0, 20; """ qt_sql5 """ - select id, element_at(s, 'city') from `tbl_test_pruned_columns` where element_at(element_at(s, 'data')[1][2], 'b') = 40 order by 1; + select + id + , element_at(s, 'city') + from `tbl_test_pruned_columns` + where element_at(element_at(s, 'data')[1][2], 'b') = 40 + order by 1, 2 limit 0, 20; """ qt_sql5_1 """ - select /*+ set enable_prune_nested_column = 1; */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); + select + id + , element_at(s, 'city') + from `tbl_test_pruned_columns` + where element_at(element_at(s, 'data')[1][2], 'b') = 40 + order by 1, 2 limit 100, 20; """ qt_sql5_2 """ - select /*+ set enable_prune_nested_column = 0; */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); + select + id + , element_at(s, 'city') + from `tbl_test_pruned_columns` + where element_at(element_at(s, 'data')[1][2], 'b') = 40 + order by 1 desc, 2 limit 0, 20; + """ + + qt_sql5_3 """ + select /*+ SET_VAR(enable_prune_nested_column=true) */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); + """ + + qt_sql5_4 """ + select /*+ SET_VAR(enable_prune_nested_column=false) */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); + """ + + qt_sql5_5 """ + select + id + , s + from `tbl_test_pruned_columns` + where element_at(s, 'city') is null + order by 1 limit 0, 20; """ sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns_map`""" @@ -98,10 +214,100 @@ suite("test_pruned_columns") { """ qt_sql6 """ - select count(element_at(dynamic_attributes['theme_preference'], 'confidence_score')) from `tbl_test_pruned_columns_map`; + select + id + , element_at(element_at(s, 'data')[2][3], 'b') + from `tbl_test_pruned_columns` + where element_at(s, 'city') = 'chengdu' + order by 1, 2 limit 0, 20; + """ + + sql "set enable_profile = true" + sql "set profile_level = 2" + sql "set enable_common_expr_pushdown = true" + + def lazyPrunedToken = "lazy_pruned_column_recovery_" + UUID.randomUUID().toString() + sql """ + select + "${lazyPrunedToken}" + , id + , element_at(s, 'data') + from `tbl_test_pruned_columns` + where element_at(s, 'city') = 'chengdu' + order by 1 limit 0, 20; + """ + + def profileAction = new ProfileAction(context) + def profileCompletionStateName = "Profile Completion State" + def profileCompletionStateComplete = "COMPLETE" + def lazyPrunedCounterName = "LazyReadPrunedTime" + def lazyPrunedProfile = "" + def lazyPrunedProfileState = "" + for (int attempt = 0; attempt < 60; attempt++) { + for (def profileItem : profileAction.getProfileList()) { + if (profileItem["Sql Statement"].toString().contains(lazyPrunedToken)) { + lazyPrunedProfileState = profileItem[profileCompletionStateName]?.toString() + def currentProfile = profileAction.getProfile(profileItem["Profile ID"].toString()) + if (currentProfile != null && !currentProfile.isEmpty()) { + lazyPrunedProfile = currentProfile + } + break + } + } + if (lazyPrunedProfileState == profileCompletionStateComplete + && lazyPrunedProfile.contains(lazyPrunedCounterName)) { + break + } + Thread.sleep(500) + } + assertTrue(lazyPrunedProfile != null && !lazyPrunedProfile.isEmpty(), + "profile not found for ${lazyPrunedToken}") + assertTrue(lazyPrunedProfileState == profileCompletionStateComplete, + "profile is not complete for ${lazyPrunedToken}, state: ${lazyPrunedProfileState}") + logger.info("${lazyPrunedToken} profile: ${lazyPrunedProfile}") + + def lazyPrunedTimer = (lazyPrunedProfile =~ /${lazyPrunedCounterName}:\s*([0-9.]+)(ns|us|ms|s)/) + boolean foundLazyPrunedTimer = false + boolean nonZeroLazyPrunedTimer = false + while (lazyPrunedTimer.find()) { + foundLazyPrunedTimer = true + if ((lazyPrunedTimer.group(1) as BigDecimal) > 0) { + nonZeroLazyPrunedTimer = true + break + } + } + assertTrue(foundLazyPrunedTimer, + "LazyReadPrunedTime not found in profile for ${lazyPrunedToken}") + assertTrue(nonZeroLazyPrunedTimer, + "LazyReadPrunedTime is zero in profile for ${lazyPrunedToken}: ${lazyPrunedProfile}") + + qt_sql6_1 """ + select + id + , element_at(element_at(s, 'data')[2][3], 'b') + from `tbl_test_pruned_columns` + where element_at(s, 'city') = 'chengdu' + order by 1, 2 limit 100, 20; """ + qt_sql6_2 """ + select + id + , element_at(element_at(s, 'data')[2][3], 'b') + from `tbl_test_pruned_columns` + where element_at(s, 'city') = 'chengdu' + order by 1 desc, 2 limit 0, 20; + """ + + sql "set enable_profile = false" + sql "unset variable profile_level" + sql "set enable_common_expr_pushdown = false" + qt_sql7 """ + select count(element_at(dynamic_attributes['theme_preference'], 'confidence_score')) from `tbl_test_pruned_columns_map`; + """ + + qt_sql8 """ select element_at(dynamic_attributes['theme_preference'], 'confidence_score') from `tbl_test_pruned_columns_map` order by id; """ @@ -113,12 +319,12 @@ suite("test_pruned_columns") { `s_info` STRUCT, `arr_s` ARRAY>, `map_s` MAP> - ) - UNIQUE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 4 + ) + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 4 PROPERTIES ( "replication_num" = "1", - "light_schema_change" = "true" + "light_schema_change" = "true" ); """ sql """ @@ -134,7 +340,7 @@ suite("test_pruned_columns") { INSERT INTO nested_sc_tbl VALUES (3, struct(30.5, 'v3', 888), array(struct(500, 600, 'added_z'), struct(501, 601, 'added_z_2')), map('k3', struct(3, 3.3))); """ - qt_sql8 """ + qt_sql9 """ select element_at(element_at(arr_s, 1), 'z') as inner_z FROM nested_sc_tbl ORDER BY id; """ -} \ No newline at end of file +} diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy index 327925754e62d7..cb4baa6bdf74cb 100644 --- a/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy +++ b/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy @@ -30,21 +30,36 @@ suite("nested_container_offset_pruning") { PROPERTIES ("replication_allocation" = "tag.location.default: 1") """ sql """ - INSERT INTO nested_container_offset_pruning_tbl VALUES ( - 1, - named_struct( - 'arr', array( - named_struct('str_field', 'hello', 'int_field', 10), - named_struct('str_field', 'world', 'int_field', 20) - ), - 'm', {'a': 'x', 'b': 'y'} + INSERT INTO nested_container_offset_pruning_tbl VALUES + ( + 1, + named_struct( + 'arr', array( + named_struct('str_field', 'hello', 'int_field', 10), + named_struct('str_field', 'world', 'int_field', 20) + ), + 'm', {'a': 'x', 'b': 'y'} + ) + ), + ( + 2, + named_struct( + 'arr', array(), + 'm', {'a': 'longer', 'c': ''} + ) + ), + ( + 3, + named_struct( + 'arr', array(named_struct('str_field', 'empty', 'int_field', 30)), + 'm', {'b': 'only-b'} + ) ) - ) """ - // cardinality(s.arr) only needs array offsets, but element_at(...).int_field also needs - // array item data. The redundant s.arr.OFFSET path must be removed even though the root slot - // itself is STRUCT. + // cardinality(s.arr) needs array offsets, and element_at(...).int_field also needs array item + // data. Keep both paths: the BE consumes the current array-level metadata at the array + // iterator without forwarding it to the item iterator. order_qt_struct_root_arr_mixed """ SELECT id, cardinality(struct_element(s, 'arr')), @@ -52,9 +67,8 @@ suite("nested_container_offset_pruning") { FROM nested_container_offset_pruning_tbl ORDER BY id """ - // Same issue for nested maps: length(element_at(s.m, 'a')) needs the key lookup path, - // while map_values(s.m)[1] needs full value data. Dedup must therefore keep KEYS + VALUES - // and drop only the redundant value-side OFFSET path under the nested map container. + // Same issue for nested maps: length(element_at(s.m, 'a')) needs the key lookup path and + // value-string offsets, while map_values(s.m)[1] needs full value data. order_qt_struct_root_map_mixed """ SELECT id, length(element_at(struct_element(s, 'm'), 'a')), @@ -62,6 +76,26 @@ suite("nested_container_offset_pruning") { FROM nested_container_offset_pruning_tbl ORDER BY id """ + // Predicate on array offsets + output from array item should keep both current-level metadata + // and child data paths across FE access-path construction and BE iterator routing. + order_qt_struct_root_arr_predicate_mixed """ + SELECT id, + element_at(element_at(element_at(s, 'arr'), 1), 'str_field') + FROM nested_container_offset_pruning_tbl + WHERE cardinality(element_at(s, 'arr')) >= 1 + ORDER BY id + """ + + // Predicate length(map['a']) needs KEYS + VALUES.OFFSET, while projecting map['a'] still needs + // the full value data for the same map branch. + order_qt_struct_root_map_predicate_mixed """ + SELECT id, + element_at(element_at(s, 'm'), 'a') + FROM nested_container_offset_pruning_tbl + WHERE length(element_at(element_at(s, 'm'), 'a')) >= 1 + ORDER BY id + """ + explain { sql """ SELECT cardinality(struct_element(s, 'arr')), @@ -69,7 +103,7 @@ suite("nested_container_offset_pruning") { FROM nested_container_offset_pruning_tbl """ contains "s.arr.*.int_field" - notContains "s.arr.OFFSET" + contains "s.arr.OFFSET" } explain { @@ -80,6 +114,26 @@ suite("nested_container_offset_pruning") { """ contains "s.m.KEYS" contains "s.m.VALUES" - notContains "OFFSET" + contains "s.m.VALUES.OFFSET" + } + + explain { + sql """ + SELECT element_at(element_at(element_at(s, 'arr'), 1), 'str_field') + FROM nested_container_offset_pruning_tbl + WHERE cardinality(element_at(s, 'arr')) >= 1 + """ + contains "s.arr.*.str_field" + contains "s.arr.OFFSET" + } + + explain { + sql """ + SELECT element_at(element_at(s, 'm'), 'a') + FROM nested_container_offset_pruning_tbl + WHERE length(element_at(element_at(s, 'm'), 'a')) >= 1 + """ + contains "all access paths: [s.m.KEYS, s.m.VALUES, s.m.VALUES.OFFSET]" + contains "predicate access paths: [s.m.KEYS, s.m.VALUES.OFFSET]" } } diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy index 7cba3bc1c40a89..654ac283faf735 100644 --- a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy +++ b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy @@ -23,9 +23,8 @@ // The EXPLAIN plan should show: // nested columns: : all access paths: [.NULL] // -// When the same column is also accessed for data (e.g., projected or used in -// struct_element), the NULL-only path must be stripped from allAccessPaths and -// predicateAccessPaths unless the same path is still present in allAccessPaths. +// When the same column is also accessed for data (e.g., projected or used in element_at), +// allAccessPaths keep the data path and predicateAccessPaths keep the predicate metadata path. suite("null_column_pruning") { sql """ DROP TABLE IF EXISTS ncp_tbl """ @@ -123,8 +122,7 @@ suite("null_column_pruning") { sql "select id, arr_col from ncp_tbl where arr_col is null" contains "nested columns" contains "all access paths: [arr_col]" - notContains "arr_col.NULL" - notContains "predicate access paths:" + contains "predicate access paths: [arr_col.NULL]" } order_qt_array_full_access_strips_null """ @@ -145,8 +143,7 @@ suite("null_column_pruning") { sql "select id, map_col from ncp_tbl where map_col is null" contains "nested columns" contains "all access paths: [map_col]" - notContains "map_col.NULL" - notContains "predicate access paths:" + contains "predicate access paths: [map_col.NULL]" } order_qt_map_full_access_strips_null """ @@ -186,44 +183,38 @@ suite("null_column_pruning") { order_qt_10 "select int_col from ncp_tbl where int_col is null"; // ─── Mixed: struct IS NULL + partial field access ─────────────────────────── - // struct_col IS NULL in WHERE + struct_element in SELECT → child data is also needed. - // The parent struct_col.NULL path must NOT stay in allAccessPaths with child paths. - // BE StructFileColumnIterator treats a leading NULL sub-path as NULL_MAP_ONLY; if - // allAccessPaths were [struct_col.NULL, struct_col.city], BE would skip the city - // child iterator and default-fill the projected value. The normal nullable struct - // read materializes the parent null map together with child data, and - // predicateAccessPaths is filtered so it remains a subset of allAccessPaths. + // struct_col IS NULL in WHERE + element_at in SELECT needs struct data for projection, while + // the predicate keeps the parent null map separately. explain { sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null" contains "nested columns" - contains "all access paths: [struct_col.city]" - notContains "predicate access paths:" + contains "all access paths: [struct_col]" + contains "predicate access paths: [struct_col.NULL]" } order_qt_11 "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null"; - // This query verifies the real correctness risk: one branch needs the parent null - // map, another branch needs a child null map, and the projection needs another - // child data path. Keeping struct_col.NULL in allAccessPaths would put BE in - // NULL_MAP_ONLY mode for the whole struct and return the default zip value instead - // of reading the zip child column. + // This query verifies the real correctness risk: predicate paths need both parent and child + // null maps, while allAccessPaths keeps the struct data path for projection. explain { sql "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null" contains "nested columns" - contains "all access paths: [struct_col.city.NULL, struct_col.zip]" - contains "predicate access paths: [struct_col.city.NULL]" + contains "all access paths: [struct_col]" + contains "predicate access paths:" + contains "struct_col.NULL" + contains "struct_col.city.NULL" } order_qt_parent_null_with_child_data "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null"; // ─── Non-optimizable: struct IS NULL + full struct projected ──────────────── - // Full struct access covers its own null flag, so [struct_col.NULL] is stripped - // from allAccessPaths but kept in predicateAccessPaths. + // Full struct access stays in allAccessPaths. The predicate keeps [struct_col.NULL] + // in predicateAccessPaths. explain { sql "select struct_col from ncp_tbl where struct_col is null" contains "nested columns" contains "all access paths: [struct_col]" - notContains "predicate access paths:" + contains "predicate access paths: [struct_col.NULL]" } order_qt_12 "select struct_col from ncp_tbl where struct_col is null"; @@ -231,13 +222,12 @@ suite("null_column_pruning") { // ─── Nested struct field IS NULL ──────────────────────────────────────────── // struct_element(struct_col, 'city') IS NULL should produce a null-flag-only // predicate path [struct_col.city.NULL] while the projection reads city data. - // [struct_col.city.NULL] is stripped from allAccessPaths because [struct_col.city] - // covers the same prefix (full city data includes its null flag). + // [struct_col.city.NULL] remains in predicateAccessPaths beside the projected city data path. explain { sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null" contains "nested columns" - contains "all access paths: [struct_col.city]" - notContains "predicate access paths:" + contains "struct_col.city" + contains "predicate access paths: [struct_col.city.NULL]" } order_qt_13 "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null"; @@ -400,26 +390,25 @@ suite("null_column_pruning") { order_qt_24 "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is not null"; // ─── Mixed: map_keys IS NULL + map_keys projected ────────────────────────── - // Projection needs key data, while the predicate checks whether the parent map - // is NULL. The parent NULL path must not stay in either access path list, so BE - // does not switch the whole map iterator to NULL_MAP_ONLY and skip the keys child. + // Projection needs map data, while the predicate checks whether the parent map is NULL. The + // parent NULL path stays in predicateAccessPaths. explain { sql "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null" contains "nested columns" - contains "all access paths: [map_col.KEYS]" - notContains "predicate access paths:" + contains "all access paths: [map_col]" + contains "predicate access paths: [map_col.NULL]" } order_qt_25 "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null"; // ─── Mixed: map_values IS NULL + map_values projected ────────────────────── - // Projection needs value data, while the predicate checks whether the parent + // Projection needs map data, while the predicate checks whether the parent // map is NULL. A NULL value element does not make map_values(map_col) NULL. explain { sql "select map_values(map_col) from ncp_tbl where map_values(map_col) is null" contains "nested columns" - contains "all access paths: [map_col.VALUES]" - notContains "predicate access paths:" + contains "all access paths: [map_col]" + contains "predicate access paths: [map_col.NULL]" } order_qt_26 "select map_values(map_col) from ncp_tbl where map_values(map_col) is null"; @@ -526,13 +515,13 @@ suite("null_column_pruning") { order_qt_33 "select 1 from ncp_tbl_nn where id is null"; // ─── length(str_col) = 0 OR str_col IS NULL ──────────────────────────────── - // length(str_col) already uses the OFFSET path, and BE can derive null-ness - // from that layout, so the extra NULL-only path is redundant. + // The OR predicate needs both pieces of metadata: length(str_col) uses OFFSET and + // str_col IS NULL uses NULL. Keep both paths in the predicate metadata set. explain { sql "select 1 from ncp_tbl where length(str_col) = 0 or str_col is null" contains "nested columns" contains "str_col.OFFSET" - notContains "str_col.NULL" + contains "str_col.NULL" } order_qt_34 "select 1 from ncp_tbl where length(str_col) = 0 or str_col is null"; diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy index afb83a401bd1d9..aca98d9f817ad4 100644 --- a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy +++ b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy @@ -76,7 +76,7 @@ suite("string_length_column_pruning") { sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id" contains "nested columns" contains "str_col.OFFSET" - notContains "str_col.NULL" + contains "str_col.NULL" notContains "all access paths: [id]" } sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id" @@ -146,10 +146,12 @@ suite("string_length_column_pruning") { notContains "type=bigint" } sql "select sum(cardinality(arr_col)) from slcp_str_tbl" - // arr_col also accessed via element_at → full element data needed, OFFSET suppressed. + // arr_col is also accessed via element_at, so full element data is needed. Keep OFFSET + // as well because cardinality(arr_col) still needs array offset metadata. explain { sql "select cardinality(arr_col), arr_col[1] from slcp_str_tbl" - notContains "OFFSET" + contains "arr_col.*" + contains "arr_col.OFFSET" notContains "type=bigint" } @@ -262,29 +264,35 @@ suite("string_length_column_pruning") { notContains "type=bigint" } - // value also accessed directly (arr[0]) → full VALUES needed, OFFSET suppressed + // Value is also accessed directly (arr[0]), so full VALUES element data is needed. Keep + // VALUES.OFFSET as well because cardinality(map_arr_col['a']) still needs array offset metadata. explain { sql "select cardinality(map_arr_col['a']), map_arr_col['b'][0] from slcp_str_tbl" - notContains "OFFSET" + contains "map_arr_col.KEYS" + contains "map_arr_col.VALUES.*" + contains "map_arr_col.VALUES.OFFSET" notContains "type=bigint" } - // [map_arr_struct_col, VALUES, *, verified] strips [map_arr_struct_col, VALUES, OFFSET]. + // The struct field is also accessed directly, so keep only the required verified field data. + // VALUES.OFFSET is still needed by cardinality(map_arr_struct_col['a']). explain { sql "select cardinality(map_arr_struct_col['a']), map_arr_struct_col['a'][1].verified from slcp_str_tbl" contains "nested columns" contains "map_arr_struct_col.KEYS" contains "map_arr_struct_col.VALUES.*.verified" - notContains "map_arr_struct_col.VALUES.OFFSET" + contains "map_arr_struct_col.VALUES.OFFSET" notContains "type=bigint" } - // [map_arr_col, VALUES] strips [map_arr_col, VALUES, OFFSET]. + // Returning map_arr_col['a'] needs full VALUES data. Keep VALUES.OFFSET too because + // cardinality(map_arr_col['a']) also reads the selected value-array offsets. explain { sql "select id, cardinality(map_arr_col['a']), map_arr_col['a'] from slcp_str_tbl" contains "nested columns" - contains "all access paths: [map_arr_col.KEYS, map_arr_col.VALUES]" - notContains "map_arr_col.VALUES.OFFSET" + contains "map_arr_col.KEYS" + contains "map_arr_col.VALUES" + contains "map_arr_col.VALUES.OFFSET" notContains "predicate access paths:" notContains "type=bigint" } @@ -294,14 +302,13 @@ suite("string_length_column_pruning") { order by id """ - // [map_arr_struct_col, VALUES, *, verified] strips [map_arr_struct_col, VALUES, OFFSET]. - // KEYS (data path) remains in predicateAccessPaths. + // Keep the value-array offset in predicate paths: predicate evaluation needs it for + // cardinality(), and lazy materialization still needs the verified field after filtering. explain { sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where cardinality(map_arr_struct_col['a']) > 0" contains "nested columns" - contains "all access paths: [map_arr_struct_col.KEYS, map_arr_struct_col.VALUES.*.verified]" - contains "predicate access paths: [map_arr_struct_col.KEYS]" - notContains "map_arr_struct_col.VALUES.OFFSET" + contains "all access paths: [map_arr_struct_col.KEYS, map_arr_struct_col.VALUES.*.verified, map_arr_struct_col.VALUES.OFFSET]" + contains "predicate access paths: [map_arr_struct_col.KEYS, map_arr_struct_col.VALUES.OFFSET]" notContains "type=bigint" } @@ -311,15 +318,13 @@ suite("string_length_column_pruning") { order by 1 """ - // [map_arr_struct_col, VALUES, *, verified] strips [map_arr_struct_col, VALUES, NULL]. - // KEYS (data path) remains in predicateAccessPaths. + // Predicate keeps the value-array NULL path so IS NULL can read the value-array null map + // during predicate evaluation, while allAccessPaths keeps the data paths used by projection. explain { sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where map_arr_struct_col['a'] is null" contains "nested columns" - contains "map_arr_struct_col.KEYS" - contains "map_arr_struct_col.VALUES.*.verified" - contains "predicate access paths: [map_arr_struct_col.KEYS]" - notContains "map_arr_struct_col.VALUES.NULL" + contains "all access paths: [map_arr_struct_col.KEYS, map_arr_struct_col.VALUES.*.verified, map_arr_struct_col.VALUES.NULL]" + contains "predicate access paths: [map_arr_struct_col.KEYS, map_arr_struct_col.VALUES.NULL]" } // ─── Non-optimizable cases ────────────────────────────────────────────────── @@ -434,10 +439,13 @@ suite("string_length_column_pruning") { notContains "bigint" } - // length(map_col['a']) + direct map access → OFFSET suppressed, full VALUES needed + // length(map_col['a']) + direct map value access still needs full VALUES data. + // Keep VALUES.OFFSET as the length() access on map value still depends on the map value offsets. explain { sql "select length(map_col['a']), map_col['b'] from slcp_str_tbl" - notContains "OFFSET" + contains "map_col.KEYS" + contains "map_col.VALUES" + contains "map_col.VALUES.OFFSET" notContains "bigint" } @@ -521,7 +529,9 @@ suite("string_length_column_pruning") { FROM slcp_struct_root_tbl """ contains "s.arr.*.int_field" - notContains "s.arr.OFFSET" + // cardinality(element_at(s, 'arr')) still needs the nested array offsets, + // while the selected element field needs the int_field data. + contains "s.arr.OFFSET" } explain { @@ -532,12 +542,15 @@ suite("string_length_column_pruning") { """ contains "s.m.KEYS" contains "s.m.VALUES" - notContains "OFFSET" + // The length() on a map value needs value-string offsets even though the query + // also directly reads map values. + contains "s.m.VALUES.OFFSET" } // ─── Map element_at + map_values mixed access ───────────────────────────────── - // [map_col, VALUES] strips [map_col, VALUES, OFFSET]. KEYS is kept for element_at lookup. + // Full VALUES data is still needed for map_values(), while the length() access on a + // map value keeps VALUES.OFFSET. KEYS is kept for element_at lookup. order_qt_map_element_with_map_values """ select length(map_col['a']), map_values(map_col)[1] from slcp_str_tbl """ @@ -547,14 +560,17 @@ suite("string_length_column_pruning") { contains "nested columns" contains "KEYS" contains "VALUES" - notContains "OFFSET" + contains "map_col.VALUES.OFFSET" notContains "bigint" } - // [map_col, VALUES] strips [map_col, VALUES, OFFSET]. + // Full VALUES data is needed for map_col['a']; length(map_values(map_col)[1]) keeps + // VALUES.OFFSET for value-string offsets. explain { sql "select length(map_values(map_col)[1]), map_col['a'] from slcp_str_tbl" - notContains "OFFSET" + contains "map_col.KEYS" + contains "map_col.VALUES" + contains "map_col.VALUES.OFFSET" notContains "bigint" } } From 27460072524b055620e13dbe9fd3d744e67ff0a5 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 9 Sep 2026 15:54:22 +0800 Subject: [PATCH 3/4] branch-4.2 [fix](nereids) Disambiguate NULL/OFFSET metadata from same-named nested fields (#65805) Problem Summary: Nested-column pruning makes Doris read only the referenced parts of a struct/map/array column. Two kinds of special meta access paths serve that purpose: a terminal NULL component means the query only needs the null flag (IS [NOT] NULL), and a terminal OFFSET component means only offsets are needed (length/cardinality-style functions). The BE satisfies them in NULL_MAP_ONLY / OFFSET_ONLY meta-read modes that skip the payload data entirely. While the optimization itself is sound, the way meta paths were expressed on the wire and handled on both sides had several correctness holes: * Wire ambiguity. Every access path was sent as a DATA-typed list of string components, and NULL/OFFSET lived in the same list as real child names, so a struct field literally named OFFSET or NULL could be swallowed as metadata and its data silently pruned away, and there was no FE/BE version contract to evolve the encoding during rolling upgrades. * FE null-map correctness. The old logic keyed on expression/slot nullability, so a physical NOT NULL dimension column of a LEFT JOIN received a [col, NULL] path and the BE aborted trying to read a null map from a NOT NULL column. * FE pruning bookkeeping. NULL/OFFSET were detected by string suffix, so real fields with those names collided inside the pruning logic too. * BE routing/read-mode logic. A terminal NULL/OFFSET was treated as current-level metadata at every container, meta-only modes could be entered even when a sibling DATA path still required payload reads, and Map descendant routing depended on physical key/value child column names while silently ignoring unknown selectors. What this PR does: * Versioned, type-selected access paths: adds an optional version to TColumnAccessPath (thrift + protobuf + FE/BE descriptor conversion; TCOLUMN_ACCESS_PATH_VERSION_LEGACY = 0, TCOLUMN_ACCESS_PATH_VERSION_TYPED = 1). In the typed format the path type is authoritative. New BEs still decode the legacy all-DATA encoding from old FEs, so the supported rolling upgrade is: upgrade all BEs first, then let FEs send typed paths. * FE: marks NULL/OFFSET collector contexts as META; checks the physical column's nullability instead of slot nullability; falls back to plain data reads for variant sub-columns; keeps the exact meta path when a sibling data path is also read; and keys pruning on the path type instead of string suffixes. * BE: validates path versions and type-selected payloads; partitions current data / current metadata / descendant routing with explicit per-container ownership; derives NULL_MAP_ONLY / OFFSET_ONLY only when no current or predicate data path requires payload; keeps sibling META paths explicit; routes Map children by the logical KEYS/VALUES selectors; rejects unrecognized Map selectors instead of silently pruning everything. * Tests: BE ColumnReaderTest coverage for typed-vs-legacy encoding, metadata-vs-same-named-data disambiguation, meta-read-mode legality, sibling-data-path handling and Map selector validation; SlotDescriptor protobuf round-trip of versions and payloads; FE PruneNestedColumnTest cases; the nereids_rules_p0/column_pruning regression suites, plus the new left_join_not_null_column suite guarding the outer-join null-map crash. branch-4.2 adaptations: - branch-4.2 keeps the FE access-path model as thrift types instead of master's ColumnAccessPath/DescriptorToThriftConverter, so the typed version is stamped in NestedColumnPruning.buildColumnAccessPaths (the branch's single producer) and preserved by SlotTypeReplacer.replaceIcebergAccessPathToId. The master-only DescriptorToThriftConverter/DescriptorToThriftConverterTest files do not exist on this branch and were not added; the equivalent coverage lives in PruneNestedColumnTest, whose path()/metaPath() helpers now carry the typed version. - Ported FE code was mapped onto this branch's names (TAccessPathType, TColumnAccessPath, isFunctionNullCheckPath) and keeps the segment-wise access path comparator introduced by the branch's nested-column-pruning port. - Unit tests for lambda access paths and Map lazy-read-by-rowids that only exist on master (they come from PRs outside this chain) were not added, and the regression suites lambda_null_pruning / map_contains_arg_pruning do not exist on branch-4.2. Co-authored-by: Hu Shenggang --- be/src/runtime/descriptors.cpp | 6 + be/src/storage/segment/column_reader.cpp | 787 ++++++++------- be/src/storage/segment/column_reader.h | 58 +- be/test/runtime/descriptor_test.cpp | 59 +- .../storage/segment/column_reader_test.cpp | 912 +++++++++++++++--- .../AccessPathExpressionCollector.java | 114 ++- .../rewrite/AccessPathPlanCollector.java | 17 +- .../rules/rewrite/NestedColumnPruning.java | 137 ++- .../rules/rewrite/SlotTypeReplacer.java | 12 + .../rules/rewrite/PruneNestedColumnTest.java | 515 ++++++++-- .../rewrite/VariantPruningLogicTest.java | 2 + gensrc/proto/descriptors.proto | 3 + gensrc/thrift/Descriptors.thrift | 7 + .../left_join_not_null_column.out | 4 + .../left_join_not_null_column.groovy | 71 ++ .../column_pruning/null_column_pruning.groovy | 27 +- 16 files changed, 2045 insertions(+), 686 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/column_pruning/left_join_not_null_column.out create mode 100644 regression-test/suites/nereids_rules_p0/column_pruning/left_join_not_null_column.groovy diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index edd1f8b99d3cc2..91756d5cec595b 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -105,6 +105,9 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc) auto convert_to_thrift_column_access_path = [](const PColumnAccessPath& pb_path) { TColumnAccessPath thrift_path; thrift_path.type = (TAccessPathType::type)pb_path.type(); + if (pb_path.has_version()) { + thrift_path.__set_version(pb_path.version()); + } if (pb_path.has_data_access_path()) { thrift_path.__isset.data_access_path = true; for (int i = 0; i < pb_path.data_access_path().path_size(); ++i) { @@ -161,6 +164,9 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const { doris::PColumnAccessPath* pb_path) { pb_path->Clear(); pb_path->set_type((PAccessPathType)thrift_path.type); // 使用 reinterpret_cast 进行类型转换 + if (thrift_path.__isset.version) { + pb_path->set_version(thrift_path.version); + } if (thrift_path.__isset.data_access_path) { auto* pb_data = pb_path->mutable_data_access_path(); pb_data->Clear(); diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 7a993044f53099..696cb5d3694d8f 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -18,12 +18,14 @@ #include "storage/segment/column_reader.h" #include +#include #include #include #include #include #include +#include #include #include #include @@ -91,25 +93,260 @@ inline bool read_as_string(PrimitiveType type) { type == PrimitiveType::TYPE_BITMAP || type == PrimitiveType::TYPE_FIXED_LENGTH_OBJECT; } -bool is_current_level_meta_access_path(const TColumnAccessPath& path) { - if (path.data_access_path.path.size() != 1) { - return false; - } - const auto& component = path.data_access_path.path[0]; +bool is_meta_access_path_component(const std::string& component) { return StringCaseEqual()(component, ColumnIterator::ACCESS_OFFSET) || StringCaseEqual()(component, ColumnIterator::ACCESS_NULL); } -bool is_current_level_data_access_path(const TColumnAccessPath& path, - const std::string& column_name) { - return path.data_access_path.path.size() == 1 && - StringCaseEqual()(path.data_access_path.path[0], column_name); +bool uses_legacy_access_path_encoding(const TColumnAccessPath& path) { + return !path.__isset.version || + path.version == g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY; } -void remove_current_level_meta_access_paths(TColumnAccessPaths& paths) { - auto removed = std::ranges::remove_if(paths, is_current_level_meta_access_path); - paths.erase(removed.begin(), removed.end()); -} +namespace { + +// Nested access paths are processed one container level at a time: +// +// 1. Each Map/Array/Struct iterator's set_access_paths() calls _prepare_nested_access_paths(). For +// the all-path and predicate-path channels independently, _split_access_paths() validates the +// wire encoding and type-selected payload, removes the current iterator name, and separates +// requests consumed by the current iterator from paths that still address a data descendant. A +// current DATA request requires all data children. Struct owns current-level NULL metadata; +// Map and Array own NULL and OFFSET metadata. A supported metadata-only request can stop before +// descendant routing and mark every data child SKIP. +// 2. This router interprets only the first remaining component and routes the path according to +// the container topology: +// - Struct components already name fields. Select the paths for each field without rewriting. +// - Array `*` names its only item. Retarget `*` to the item iterator name. +// - Map `KEYS` and `VALUES` directly name its logical children. Map `*` creates a complete DATA +// path for `KEYS` and routes any trailing components through `VALUES`. +// 3. The container forwards the routed all-path and predicate-path channels to each selected +// child's set_access_paths(), then finalizes that child's PREDICATE/LAZY_OUTPUT/SKIP +// requirement. The child repeats the same flow, which handles arbitrary Map/Array/Struct +// nesting. +// +// At this point versions have been validated, legacy paths have DATA type, and every payload +// selected by type is non-empty. Routing never interprets or rewrites an unselected compatibility +// payload. +class DescendantAccessPathRouter final { +public: + DescendantAccessPathRouter() = delete; + + // Preserve the two set_access_paths() input channels. all_paths originates from the + // all-access-path superset, while predicate_paths separately records predicate-phase paths. + // Routing may omit all_paths when the parent already requires complete child data. + struct ChildAccessPaths { + TColumnAccessPaths all_paths; + TColumnAccessPaths predicate_paths; + + bool empty() const { return all_paths.empty() && predicate_paths.empty(); } + }; + + struct MapChildAccessPaths { + ChildAccessPaths key; + ChildAccessPaths value; + }; + + // Map children use the logical KEYS/VALUES selectors as their access-path names, independent + // of physical child column names. Expand a wildcard to complete keys and route its trailing + // qualifiers to values. + static Result route_map_paths_to_children( + TColumnAccessPaths all_paths, TColumnAccessPaths predicate_paths) { + MapChildAccessPaths child_paths; + auto status = distribute_map_paths(std::move(all_paths), child_paths.key.all_paths, + child_paths.value.all_paths); + if (!status.ok()) { + return ResultError(std::move(status)); + } + status = distribute_map_paths(std::move(predicate_paths), child_paths.key.predicate_paths, + child_paths.value.predicate_paths); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return child_paths; + } + + // Array has one data child. Retarget its logical wildcard selector to the item iterator name. + static Result route_array_paths_to_item(TColumnAccessPaths all_paths, + TColumnAccessPaths predicate_paths, + const std::string& item_name) { + ChildAccessPaths child_paths {.all_paths = std::move(all_paths), + .predicate_paths = std::move(predicate_paths)}; + auto retarget_wildcard_paths_to_item = [&](TColumnAccessPaths& paths) -> Status { + for (auto& path : paths) { + const bool is_wildcard = DORIS_TRY(selected_payload_head_matches( + path, ColumnIterator::ACCESS_ALL, PathHeadMatchMode::EXACT)); + if (is_wildcard) { + RETURN_IF_ERROR(replace_selected_payload_head(path, item_name)); + } + } + return Status::OK(); + }; + + auto status = retarget_wildcard_paths_to_item(child_paths.all_paths); + if (!status.ok()) { + return ResultError(std::move(status)); + } + status = retarget_wildcard_paths_to_item(child_paths.predicate_paths); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return child_paths; + } + + // Struct selectors already use child field names. Select the paths for one child without + // rewriting them, so that the child can validate and strip its own name. + static Result select_struct_paths_for_child( + const TColumnAccessPaths& all_paths, const TColumnAccessPaths& predicate_paths, + const std::string& child_name, bool include_all_paths) { + ChildAccessPaths child_paths; + auto select_matching_paths = [&](const TColumnAccessPaths& source_paths, + TColumnAccessPaths& child_paths) -> Status { + for (const auto& path : source_paths) { + const bool matches_child = DORIS_TRY(selected_payload_head_matches( + path, child_name, PathHeadMatchMode::CASE_INSENSITIVE)); + if (matches_child) { + child_paths.emplace_back(path); + } + } + return Status::OK(); + }; + + if (include_all_paths) { + auto status = select_matching_paths(all_paths, child_paths.all_paths); + if (!status.ok()) { + return ResultError(std::move(status)); + } + } + auto status = select_matching_paths(predicate_paths, child_paths.predicate_paths); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return child_paths; + } + +private: + enum class PathHeadMatchMode { EXACT, CASE_INSENSITIVE }; + enum class MapSelector { WILDCARD, KEYS, VALUES }; + + // TColumnAccessPath may carry both payload fields after compatibility forwarding. Selected + // means data_access_path for DATA and meta_access_path for META. Visit only that payload and + // leave the other payload untouched. + template + static Status visit_selected_payload(AccessPath& access_path, Visitor&& visitor) { + switch (access_path.type) { + case TAccessPathType::DATA: + if (!access_path.__isset.data_access_path) { + return Status::InternalError( + "Invalid DATA access path: data_access_path payload is not set"); + } + return std::forward(visitor)(access_path.data_access_path); + case TAccessPathType::META: + if (!access_path.__isset.meta_access_path) { + return Status::InternalError( + "Invalid META access path: meta_access_path payload is not set"); + } + return std::forward(visitor)(access_path.meta_access_path); + default: + return Status::InternalError("Invalid access path type: {}", + static_cast(access_path.type)); + } + } + + static Result selected_payload_head_matches(const TColumnAccessPath& access_path, + const std::string& expected_head, + PathHeadMatchMode match_mode) { + bool matches = false; + auto status = visit_selected_payload(access_path, [&](const auto& payload) { + DORIS_CHECK(!payload.path.empty()); + matches = match_mode == PathHeadMatchMode::EXACT + ? payload.path.front() == expected_head + : StringCaseEqual()(payload.path.front(), expected_head); + return Status::OK(); + }); + if (!status.ok()) { + return ResultError(std::move(status)); + } + return matches; + } + + static Status replace_selected_payload_head(TColumnAccessPath& access_path, + const std::string& child_name) { + return visit_selected_payload(access_path, [&](auto& payload) { + DORIS_CHECK(!payload.path.empty()); + payload.path.front() = child_name; + return Status::OK(); + }); + } + + // Map `*` applies trailing qualifiers only to values, while locating entries still requires + // complete KEYS as DATA. Construct a fresh key path because the source may be META, and + // preserve its version so child routing keeps the same legacy/typed encoding. + static TColumnAccessPath make_full_data_path_for_map_keys( + const TColumnAccessPath& version_source) { + TColumnAccessPath child_path; + child_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_path; + data_path.__set_path({ColumnIterator::ACCESS_MAP_KEYS}); + child_path.__set_data_access_path(data_path); + if (version_source.__isset.version) { + child_path.__set_version(version_source.version); + } + return child_path; + } + + static Result classify_map_selector(const TColumnAccessPath& access_path) { + std::optional selector; + auto status = visit_selected_payload(access_path, [&](const auto& payload) { + DORIS_CHECK(!payload.path.empty()); + const auto& head = payload.path.front(); + if (head == ColumnIterator::ACCESS_ALL) { + selector = MapSelector::WILDCARD; + } else if (head == ColumnIterator::ACCESS_MAP_KEYS) { + selector = MapSelector::KEYS; + } else if (head == ColumnIterator::ACCESS_MAP_VALUES) { + selector = MapSelector::VALUES; + } else { + return Status::InternalError( + "Invalid map access path selector '{}': expected '*', 'KEYS', or 'VALUES'", + head); + } + return Status::OK(); + }); + if (!status.ok()) { + return ResultError(std::move(status)); + } + DORIS_CHECK(selector.has_value()); + return *selector; + } + + static Status distribute_map_paths(TColumnAccessPaths source_paths, + TColumnAccessPaths& key_paths, + TColumnAccessPaths& value_paths) { + for (auto& path : source_paths) { + const auto selector = DORIS_TRY(classify_map_selector(path)); + switch (selector) { + case MapSelector::WILDCARD: + // A wildcard needs complete keys for runtime lookup, while any remaining + // qualifiers apply only to the value. The value keeps its type and version. + key_paths.emplace_back(make_full_data_path_for_map_keys(path)); + RETURN_IF_ERROR( + replace_selected_payload_head(path, ColumnIterator::ACCESS_MAP_VALUES)); + value_paths.emplace_back(std::move(path)); + break; + case MapSelector::KEYS: + key_paths.emplace_back(std::move(path)); + break; + case MapSelector::VALUES: + value_paths.emplace_back(std::move(path)); + break; + } + } + return Status::OK(); + } +}; + +} // namespace Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, const io::FileReaderSPtr& file_reader, @@ -850,13 +1087,11 @@ Status ColumnReader::new_map_iterator(ColumnIteratorUPtr* iterator, &key_iterator, tablet_column && tablet_column->get_subtype_count() > 1 ? &tablet_column->get_sub_column(0) : nullptr)); - key_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : ""); ColumnIteratorUPtr val_iterator; RETURN_IF_ERROR(_sub_readers[1]->new_iterator( &val_iterator, tablet_column && tablet_column->get_subtype_count() > 1 ? &tablet_column->get_sub_column(1) : nullptr)); - val_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(1).name() : ""); ColumnIteratorUPtr offsets_iterator; RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator, nullptr)); auto* file_iter = static_cast(offsets_iterator.release()); @@ -929,44 +1164,122 @@ void ColumnIterator::_recovery_from_place_holder_column(MutableColumnPtr& dst) { } } -Result ColumnIterator::_get_sub_access_paths( - TColumnAccessPaths sub_access_paths, bool is_predicate) { - // Access paths passed to a complex iterator always start with the current - // column name. Strip that component and return the remaining child-relative - // paths to the caller. For example, when this iterator is for column `s`, - // path `s.a.b` is converted to `a.b` and then dispatched to child `a`. - // - // If stripping the current column consumes the whole path, the current - // iterator itself is requested rather than one of its children. Mark the - // current iterator according to the path source: predicate paths must be read - // in the predicate phase, while all/output paths become lazy output targets. - // Empty or mismatched paths indicate an FE/BE access-path contract violation. - for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) { - TColumnAccessPath& name_path = *it; - if (name_path.data_access_path.path.empty()) { +Result ColumnIterator::_split_access_paths( + TColumnAccessPaths access_paths) const { + AccessPathSplit split; + for (auto& path : access_paths) { + const bool uses_legacy_encoding = uses_legacy_access_path_encoding(path); + if (!uses_legacy_encoding && + path.version != g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED) { + return ResultError( + Status::InternalError("Unsupported access path version: {}", path.version)); + } + + std::vector* components = nullptr; + if (uses_legacy_encoding) { + if (path.type != TAccessPathType::DATA) { + return ResultError(Status::InternalError("Invalid legacy access path type: {}", + static_cast(path.type))); + } + if (!path.__isset.data_access_path) { + return ResultError(Status::InternalError( + "Invalid legacy access path: data_access_path payload is not set")); + } + components = &path.data_access_path.path; + } else { + switch (path.type) { + case TAccessPathType::DATA: + if (!path.__isset.data_access_path) { + return ResultError(Status::InternalError( + "Invalid DATA access path: data_access_path payload is not set")); + } + components = &path.data_access_path.path; + break; + case TAccessPathType::META: + if (!path.__isset.meta_access_path) { + return ResultError(Status::InternalError( + "Invalid META access path: meta_access_path payload is not set")); + } + components = &path.meta_access_path.path; + break; + default: + return ResultError(Status::InternalError("Invalid access path type: {}", + static_cast(path.type))); + } + } + + if (components->empty()) { return ResultError(Status::InternalError( "Invalid access path for column '{}': path is empty", _column_name)); } - if (!StringCaseEqual()(name_path.data_access_path.path[0], _column_name)) { + if (!StringCaseEqual()((*components)[0], _column_name)) { return ResultError(Status::InternalError( R"(Invalid access path for column: expected name "{}", got "{}")", _column_name, - name_path.data_access_path.path[0])); + (*components)[0])); } - name_path.data_access_path.path.erase(name_path.data_access_path.path.begin()); - if (!name_path.data_access_path.path.empty()) { - ++it; - } else { - if (is_predicate) { - set_read_requirement(ReadRequirement::PREDICATE); - } else { - set_lazy_output_requirement(); + components->erase(components->begin()); + if (components->empty()) { + split.reads_current_data = true; + continue; + } + + const bool is_current_level_meta = + components->size() == 1 && is_meta_access_path_component((*components)[0]) && + (path.type == TAccessPathType::META || uses_legacy_encoding); + if (is_current_level_meta) { + if (StringCaseEqual()((*components)[0], ACCESS_OFFSET)) { + split.current_meta_mode = MetaReadMode::OFFSET_ONLY; + } else if (split.current_meta_mode == MetaReadMode::DEFAULT) { + split.current_meta_mode = MetaReadMode::NULL_MAP_ONLY; } - it = sub_access_paths.erase(it); + continue; + } + + split.descendant_paths.emplace_back(std::move(path)); + } + return split; +} + +Result ColumnIterator::_prepare_nested_access_paths( + const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths, NestedMetaSupport meta_support, + const std::function& set_all_data_descendants_read_requirement) { + const auto requirement_before = _read_requirement; + if (!predicate_access_paths.empty()) { + set_read_requirement_self(ReadRequirement::PREDICATE); + } + + NestedAccessPathPlan plan; + auto all_split = _split_access_paths(all_access_paths); + if (!all_split.has_value()) { + return ResultError(std::move(all_split).error()); + } + plan.all = std::move(all_split).value(); + + auto predicate_split = _split_access_paths(predicate_access_paths); + if (!predicate_split.has_value()) { + return ResultError(std::move(predicate_split).error()); + } + plan.predicate = std::move(predicate_split).value(); + if (plan.all.reads_current_data) { + set_lazy_output_requirement(); + } + if (plan.predicate.reads_current_data) { + set_read_requirement(ReadRequirement::PREDICATE); + } + + if (!plan.predicate.has_descendant_paths()) { + RETURN_IF_ERROR_RESULT(_check_and_set_meta_read_mode(requirement_before, plan.all)); + plan.skip_data_descendants = + read_null_map_only() || + (meta_support == NestedMetaSupport::NULL_MAP_AND_OFFSET && read_offset_only()); + if (plan.skip_data_descendants) { + set_all_data_descendants_read_requirement(ReadRequirement::SKIP); } } - return sub_access_paths; + return plan; } ///====================== MapFileColumnIterator ============================//// @@ -982,6 +1295,10 @@ MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr reade if (_map_reader->is_nullable()) { _null_iterator = std::move(null_iterator); } + // Access paths identify map children by logical selectors rather than storage/schema child + // names. These names are consumed by each child's _split_access_paths(). + _key_iterator->set_column_name(ACCESS_MAP_KEYS); + _val_iterator->set_column_name(ACCESS_MAP_VALUES); } Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) { @@ -1420,120 +1737,26 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc return Status::OK(); } - const auto requirement_before_access_path = _read_requirement; - if (!predicate_access_paths.empty()) { - set_read_requirement_self(ReadRequirement::PREDICATE); - DLOG(INFO) << "Map column iterator set sub-column " << _column_name << " to PREDICATE"; - } - - const bool has_current_level_data_path = - std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { - return is_current_level_data_access_path(path, _column_name); - }); - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = - DORIS_TRY(_get_sub_access_paths(predicate_access_paths, true)); - if (has_current_level_data_path) { - remove_current_level_meta_access_paths(sub_all_access_paths); - } - const bool has_current_level_predicate_meta_path = - std::ranges::any_of(sub_predicate_access_paths, is_current_level_meta_access_path); - // Current-level predicate metadata paths are consumed by this map iterator and must not be - // forwarded to key/value children. The FE keeps all_access_paths as a superset of predicate - // paths, so meta-only mode is still decided from sub_all_access_paths below. - remove_current_level_meta_access_paths(sub_predicate_access_paths); - - if (sub_predicate_access_paths.empty() && _read_requirement == ReadRequirement::PREDICATE && - !has_current_level_predicate_meta_path) { - // if no sub-column in predicate_access_paths, but current column is PREDICATE, - // then we should set key/value iterator to PREDICATE too. - _key_iterator->set_read_requirement(ReadRequirement::PREDICATE); - _val_iterator->set_read_requirement(ReadRequirement::PREDICATE); - } - - if (sub_predicate_access_paths.empty()) { - // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY). Only skip key/value - // iterators when no predicate sub-path needs them in the predicate phase. - _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); - if (read_offset_only()) { - _key_iterator->set_read_requirement(ReadRequirement::SKIP); - _val_iterator->set_read_requirement(ReadRequirement::SKIP); - DLOG(INFO) << "Map column iterator set column " << _column_name - << " to OFFSET_ONLY meta read mode, key/value columns set to SKIP"; - return Status::OK(); - } - if (read_null_map_only()) { - _key_iterator->set_read_requirement(ReadRequirement::SKIP); - _val_iterator->set_read_requirement(ReadRequirement::SKIP); - DLOG(INFO) << "Map column iterator set column " << _column_name - << " to NULL_MAP_ONLY meta read mode, key/value columns set to SKIP"; - return Status::OK(); - } + auto plan = DORIS_TRY(_prepare_nested_access_paths( + all_access_paths, predicate_access_paths, NestedMetaSupport::NULL_MAP_AND_OFFSET, + [this](ReadRequirement requirement) { + _key_iterator->set_read_requirement(requirement); + _val_iterator->set_read_requirement(requirement); + })); + if (plan.skip_data_descendants) { + return Status::OK(); } - // A current-level data path is consumed by _get_sub_access_paths() and leaves - // sub_all_access_paths empty after marking key/value as lazy-read targets. Predicate - // sub-paths still have to be routed to child iterators for the predicate phase. - if (sub_all_access_paths.empty() && sub_predicate_access_paths.empty()) { + if (!plan.all.has_descendant_paths() && !plan.predicate.has_descendant_paths()) { return Status::OK(); } - TColumnAccessPaths key_all_access_paths; - TColumnAccessPaths val_all_access_paths; - TColumnAccessPaths key_predicate_access_paths; - TColumnAccessPaths val_predicate_access_paths; - - for (auto paths : sub_all_access_paths) { - if (paths.data_access_path.path[0] == ACCESS_ALL) { - // ACCESS_ALL means element_at(map, key) style access: the key column must be - // fully read so that the runtime can match the requested key, while any sub-path - // qualifiers (e.g. OFFSET) apply only to the value column. - // For key: create a path with just the column name (= full data access). - TColumnAccessPath key_path; - key_path.__set_type(paths.type); - TDataAccessPath key_data_path; - key_data_path.__set_path({_key_iterator->column_name()}); - key_path.__set_data_access_path(key_data_path); - key_all_access_paths.emplace_back(std::move(key_path)); - // For value: pass the full sub-path so qualifiers like OFFSET propagate. - paths.data_access_path.path[0] = _val_iterator->column_name(); - val_all_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == ACCESS_MAP_KEYS) { - paths.data_access_path.path[0] = _key_iterator->column_name(); - key_all_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == ACCESS_MAP_VALUES) { - paths.data_access_path.path[0] = _val_iterator->column_name(); - val_all_access_paths.emplace_back(paths); - } - } - for (auto paths : sub_predicate_access_paths) { - if (paths.data_access_path.path[0] == ACCESS_ALL) { - // Same logic as above: key needs full data, value gets the sub-path. - TColumnAccessPath key_path; - key_path.__set_type(paths.type); - TDataAccessPath key_data_path; - key_data_path.__set_path({_key_iterator->column_name()}); - key_path.__set_data_access_path(key_data_path); - key_predicate_access_paths.emplace_back(std::move(key_path)); - paths.data_access_path.path[0] = _val_iterator->column_name(); - val_predicate_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == ACCESS_MAP_KEYS) { - paths.data_access_path.path[0] = _key_iterator->column_name(); - key_predicate_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == ACCESS_MAP_VALUES) { - paths.data_access_path.path[0] = _val_iterator->column_name(); - val_predicate_access_paths.emplace_back(paths); - } - } - - const auto need_read_keys = - !key_all_access_paths.empty() || !key_predicate_access_paths.empty(); - const auto need_read_values = - !val_all_access_paths.empty() || !val_predicate_access_paths.empty(); - - if (need_read_keys) { - RETURN_IF_ERROR( - _key_iterator->set_access_paths(key_all_access_paths, key_predicate_access_paths)); + auto child_paths = DORIS_TRY(DescendantAccessPathRouter::route_map_paths_to_children( + std::move(plan.all.descendant_paths), std::move(plan.predicate.descendant_paths))); + + if (!child_paths.key.empty()) { + RETURN_IF_ERROR(_key_iterator->set_access_paths(child_paths.key.all_paths, + child_paths.key.predicate_paths)); // Apply LAZY_OUTPUT after child predicate paths have been handled. Read requirements are // monotonic, so a predicate-only child already promoted to PREDICATE will not // be downgraded, while a non-predicate child becomes a lazy materialization target. @@ -1543,9 +1766,9 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc DLOG(INFO) << "Map column iterator set key column to SKIP"; } - if (need_read_values) { - RETURN_IF_ERROR( - _val_iterator->set_access_paths(val_all_access_paths, val_predicate_access_paths)); + if (!child_paths.value.empty()) { + RETURN_IF_ERROR(_val_iterator->set_access_paths(child_paths.value.all_paths, + child_paths.value.predicate_paths)); // Same as keys: predicate-only value paths stay PREDICATE because this // post-processing update cannot lower a stronger child requirement. _val_iterator->set_read_requirement_self(ReadRequirement::LAZY_OUTPUT); @@ -1805,70 +2028,30 @@ Status StructFileColumnIterator::set_access_paths( return Status::OK(); } - const auto requirement_before_access_path = _read_requirement; - if (!predicate_access_paths.empty()) { - set_read_requirement_self(ReadRequirement::PREDICATE); - DLOG(INFO) << "Struct column iterator set sub-column " << _column_name << " to PREDICATE"; - } - - const bool has_current_level_data_path = - std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { - return is_current_level_data_access_path(path, _column_name); - }); - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = - DORIS_TRY(_get_sub_access_paths(predicate_access_paths, true)); - if (has_current_level_data_path) { - remove_current_level_meta_access_paths(sub_all_access_paths); - } - const bool has_current_level_predicate_meta_path = - std::ranges::any_of(sub_predicate_access_paths, is_current_level_meta_access_path); - // Current-level predicate metadata paths are consumed by this struct iterator and must not be - // forwarded to child fields. The FE keeps all_access_paths as a superset of predicate paths, so - // NULL_MAP_ONLY is still decided from sub_all_access_paths below. - remove_current_level_meta_access_paths(sub_predicate_access_paths); - - if (sub_predicate_access_paths.empty()) { - // Check for NULL_MAP_ONLY mode: only read null map, skip all sub-columns. - // Do not take this early return when predicate child paths must still be read. - _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); - if (read_null_map_only()) { - for (auto& sub_iterator : _sub_column_iterators) { - sub_iterator->set_read_requirement(ReadRequirement::SKIP); - } - DLOG(INFO) << "Struct column iterator set column " << _column_name - << " to NULL_MAP_ONLY meta read mode, all sub-columns set to SKIP"; - return Status::OK(); - } - } + auto plan = DORIS_TRY(_prepare_nested_access_paths( + all_access_paths, predicate_access_paths, NestedMetaSupport::NULL_MAP, + [this](ReadRequirement requirement) { + for (auto& sub_iterator : _sub_column_iterators) { + sub_iterator->set_read_requirement(requirement); + } + })); - const auto no_sub_column_to_skip = sub_all_access_paths.empty(); - const auto no_predicate_sub_column = sub_predicate_access_paths.empty(); + if (plan.skip_data_descendants) { + DLOG(INFO) << "Struct column iterator set column " << _column_name + << " to NULL_MAP_ONLY meta read mode, all sub-columns set to SKIP"; + return Status::OK(); + } + const bool reads_all_sub_columns = plan.all.reads_current_data; + const bool include_all_paths = !reads_all_sub_columns; for (auto& sub_iterator : _sub_column_iterators) { const auto name = sub_iterator->column_name(); - TColumnAccessPaths sub_all_access_paths_of_this; - if (!no_sub_column_to_skip) { - for (const auto& paths : sub_all_access_paths) { - if (paths.data_access_path.path[0] == name) { - sub_all_access_paths_of_this.emplace_back(paths); - } - } - } - - TColumnAccessPaths sub_predicate_access_paths_of_this; - if (!no_predicate_sub_column) { - for (const auto& paths : sub_predicate_access_paths) { - if (StringCaseEqual()(paths.data_access_path.path[0], name)) { - sub_predicate_access_paths_of_this.emplace_back(paths); - } - } - } + auto paths = DORIS_TRY(DescendantAccessPathRouter::select_struct_paths_for_child( + plan.all.descendant_paths, plan.predicate.descendant_paths, name, + include_all_paths)); - // Predicate-only child paths still need to be routed to the child iterator - // even when the child is not requested by ordinary projection access paths. - const bool need_to_read = no_sub_column_to_skip || !sub_all_access_paths_of_this.empty() || - !sub_predicate_access_paths_of_this.empty(); + // Predicate paths must still reach the child even when no non-predicate path selects it. + const bool need_to_read = reads_all_sub_columns || !paths.empty(); if (!need_to_read) { set_read_requirement_self(ReadRequirement::SKIP); sub_iterator->set_read_requirement(ReadRequirement::SKIP); @@ -1876,15 +2059,7 @@ Status StructFileColumnIterator::set_access_paths( continue; } - if (no_predicate_sub_column && _read_requirement == ReadRequirement::PREDICATE && - !has_current_level_predicate_meta_path) { - // if no sub-column in predicate_access_paths, but current column is PREDICATE, - // then we should set sub iterator to PREDICATE too. - sub_iterator->set_read_requirement(ReadRequirement::PREDICATE); - } - - RETURN_IF_ERROR(sub_iterator->set_access_paths(sub_all_access_paths_of_this, - sub_predicate_access_paths_of_this)); + RETURN_IF_ERROR(sub_iterator->set_access_paths(paths.all_paths, paths.predicate_paths)); // Set LAZY_OUTPUT after routing child predicate paths. If the child was needed only for // predicate evaluation, set_access_paths() has already promoted it to // PREDICATE and this monotonic update will not downgrade it. Otherwise, this @@ -2261,87 +2436,25 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a return Status::OK(); } - const auto requirement_before_access_path = _read_requirement; - if (!predicate_access_paths.empty()) { - set_read_requirement_self(ReadRequirement::PREDICATE); - DLOG(INFO) << "Array column iterator set sub-column " << _column_name << " to PREDICATE"; - } - - const bool has_current_level_data_path = - std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { - return is_current_level_data_access_path(path, _column_name); - }); - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = - DORIS_TRY(_get_sub_access_paths(predicate_access_paths, true)); - if (has_current_level_data_path) { - // A current-level data path already reads the array offsets while materializing the array. - // Do not let a redundant current-level OFFSET/NULL path switch this iterator into a - // meta-only mode that would skip item data. - remove_current_level_meta_access_paths(sub_all_access_paths); - } - const bool has_current_level_predicate_meta_path = - std::ranges::any_of(sub_predicate_access_paths, is_current_level_meta_access_path); - // Current-level predicate metadata paths are consumed by this array iterator and must not be - // forwarded to the item iterator. The FE keeps all_access_paths as a superset of predicate - // paths, so meta-only mode is still decided from sub_all_access_paths below. - auto removed = - std::ranges::remove_if(sub_predicate_access_paths, is_current_level_meta_access_path); - sub_predicate_access_paths.erase(removed.begin(), removed.end()); - - if (sub_predicate_access_paths.empty()) { - // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY). Only skip the item - // iterator when no predicate sub-path needs it in the predicate phase. - _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); - if (read_offset_only()) { - _item_iterator->set_read_requirement(ReadRequirement::SKIP); - DLOG(INFO) << "Array column iterator set column " << _column_name - << " to OFFSET_ONLY meta read mode, item column set to SKIP"; - return Status::OK(); - } - if (read_null_map_only()) { - _item_iterator->set_read_requirement(ReadRequirement::SKIP); - DLOG(INFO) << "Array column iterator set column " << _column_name - << " to NULL_MAP_ONLY meta read mode, item column set to SKIP"; - return Status::OK(); - } - } - // OFFSET/NULL at the current array level is consumed by this iterator. After deciding that - // the array is not in a meta-only mode, do not forward those paths to the item iterator. - remove_current_level_meta_access_paths(sub_all_access_paths); - - const auto no_sub_column_to_skip = sub_all_access_paths.empty(); - const auto no_predicate_sub_column = sub_predicate_access_paths.empty(); + auto plan = DORIS_TRY(_prepare_nested_access_paths( + all_access_paths, predicate_access_paths, NestedMetaSupport::NULL_MAP_AND_OFFSET, + [this](ReadRequirement requirement) { + _item_iterator->set_read_requirement(requirement); + })); - if (!no_sub_column_to_skip) { - for (auto& path : sub_all_access_paths) { - if (path.data_access_path.path[0] == ACCESS_ALL) { - path.data_access_path.path[0] = _item_iterator->column_name(); - } - } + if (plan.skip_data_descendants) { + return Status::OK(); } - if (no_predicate_sub_column) { - // Current-level predicate meta paths (OFFSET/NULL) are consumed by the array itself and - // removed before forwarding paths to the item iterator. If they are the only predicate - // paths, the item iterator may still be needed later for lazy materialization, but it must - // not be promoted to PREDICATE. Only propagate the predicate requirement when the - // parent predicate really applies to the item/whole value instead of array metadata only. - if (_read_requirement == ReadRequirement::PREDICATE && - !has_current_level_predicate_meta_path) { - _item_iterator->set_read_requirement(ReadRequirement::PREDICATE); - } - } else { - for (auto& path : sub_predicate_access_paths) { - if (path.data_access_path.path[0] == ACCESS_ALL) { - path.data_access_path.path[0] = _item_iterator->column_name(); - } - } - } + const bool has_all_descendant = plan.all.has_descendant_paths(); + const bool has_predicate_descendant = plan.predicate.has_descendant_paths(); + auto child_paths = DORIS_TRY(DescendantAccessPathRouter::route_array_paths_to_item( + std::move(plan.all.descendant_paths), std::move(plan.predicate.descendant_paths), + _item_iterator->column_name())); - if (!no_sub_column_to_skip || !no_predicate_sub_column) { - RETURN_IF_ERROR( - _item_iterator->set_access_paths(sub_all_access_paths, sub_predicate_access_paths)); + if (has_all_descendant || has_predicate_descendant) { + RETURN_IF_ERROR(_item_iterator->set_access_paths(child_paths.all_paths, + child_paths.predicate_paths)); // Predicate-only item paths stay PREDICATE because this update runs after // child set_access_paths() and read requirements are monotonic. Non-predicate item paths are // marked as lazy materialization targets. @@ -2370,26 +2483,7 @@ Status StringFileColumnIterator::init(const ColumnIteratorOptions& opts) { Status StringFileColumnIterator::set_access_paths( const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { - if (all_access_paths.empty() && predicate_access_paths.empty()) { - return Status::OK(); - } - - const auto requirement_before_access_path = _read_requirement; - if (!predicate_access_paths.empty()) { - set_read_requirement(ReadRequirement::PREDICATE); - } - - const bool has_current_level_data_path = - std::ranges::any_of(all_access_paths, [this](const TColumnAccessPath& path) { - return is_current_level_data_access_path(path, _column_name); - }); - // Strip the column name from path[0] before checking for meta-only modes. - // Raw paths look like ["col_name", "OFFSET"] or ["col_name", "NULL"]. - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - if (has_current_level_data_path) { - remove_current_level_meta_access_paths(sub_all_access_paths); - } - _check_and_set_meta_read_mode(requirement_before_access_path, sub_all_access_paths); + RETURN_IF_ERROR(FileColumnIterator::set_access_paths(all_access_paths, predicate_access_paths)); // OFFSET_ONLY mode is fundamentally incompatible with CHAR columns: // CHAR is stored padded to its declared length (see // OlapColumnDataConvertorChar::clone_and_padding), so the per-row length @@ -2425,10 +2519,32 @@ Status StringFileColumnIterator::set_access_paths( //////////////////////////////////////////////////////////////////////////////// +Status FileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths) { + if (all_access_paths.empty() && predicate_access_paths.empty()) { + return Status::OK(); + } + + const auto requirement_before_access_path = _read_requirement; + if (!predicate_access_paths.empty()) { + set_read_requirement(ReadRequirement::PREDICATE); + } + + auto all_split = DORIS_TRY(_split_access_paths(all_access_paths)); + auto predicate_split = DORIS_TRY(_split_access_paths(predicate_access_paths)); + if (all_split.reads_current_data) { + set_lazy_output_requirement(); + } + if (predicate_split.reads_current_data) { + set_read_requirement(ReadRequirement::PREDICATE); + } + return _check_and_set_meta_read_mode(requirement_before_access_path, all_split); +} + FileColumnIterator::FileColumnIterator(std::shared_ptr reader) : _reader(reader) {} -void ColumnIterator::_check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, - const TColumnAccessPaths& sub_all_access_paths) { +Status ColumnIterator::_check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, + const AccessPathSplit& all_access_paths) { _meta_read_mode = MetaReadMode::DEFAULT; if (requirement_before_access_path != ReadRequirement::NORMAL && requirement_before_access_path != ReadRequirement::SKIP) { @@ -2436,32 +2552,15 @@ void ColumnIterator::_check_and_set_meta_read_mode(ReadRequirement requirement_b // to materialize data. In that case a later predicate NULL/OFFSET path is only // an additional predicate requirement and must not downgrade the read to // meta-only. - return; + return Status::OK(); } - bool has_offset_path = false; - bool has_null_path = false; - for (const auto& path : sub_all_access_paths) { - if (!is_current_level_meta_access_path(path)) { - _meta_read_mode = MetaReadMode::DEFAULT; - return; - } - const auto& component = path.data_access_path.path[0]; - if (StringCaseEqual()(component, ACCESS_OFFSET)) { - has_offset_path = true; - } else { - has_null_path = true; - } - } - if (has_offset_path) { - // OFFSET_ONLY skips actual child/string data, but nullable complex iterators still - // materialize the current-level null map. So OFFSET covers OFFSET+NULL metadata. - _meta_read_mode = MetaReadMode::OFFSET_ONLY; - } else if (has_null_path) { - _meta_read_mode = MetaReadMode::NULL_MAP_ONLY; - } else { - _meta_read_mode = MetaReadMode::DEFAULT; + if (all_access_paths.reads_current_data || all_access_paths.has_descendant_paths()) { + return Status::OK(); } + + _meta_read_mode = all_access_paths.current_meta_mode; + return Status::OK(); } Status FileColumnIterator::init(const ColumnIteratorOptions& opts) { diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index e7dd0c2869aaf7..a3bd46e5f701fb 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -24,6 +24,7 @@ #include // for size_t #include // for uint32_t +#include #include #include // for unique_ptr #include @@ -492,18 +493,48 @@ class ColumnIterator { } protected: + struct AccessPathSplit { + TColumnAccessPaths descendant_paths; + bool reads_current_data = false; + MetaReadMode current_meta_mode = MetaReadMode::DEFAULT; + + bool has_descendant_paths() const { return !descendant_paths.empty(); } + }; + + // Nested columns share the same current-level access-path planning, while their data-child + // topology and descendant routing remain container-specific. + struct NestedAccessPathPlan { + AccessPathSplit all; + AccessPathSplit predicate; + bool skip_data_descendants = false; + }; + + // At their current level, Struct supports null-map metadata. Map and Array additionally + // support offsets. + enum class NestedMetaSupport { NULL_MAP, NULL_MAP_AND_OFFSET }; + void _convert_to_place_holder_column(MutableColumnPtr& dst, size_t count); void _recovery_from_place_holder_column(MutableColumnPtr& dst); - // Derive current-level meta-only read mode from access paths. Meta-only is valid only when - // this iterator had no data-read requirement before applying the current paths, and every - // visible path at this level is NULL/OFFSET metadata. - void _check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, - const TColumnAccessPaths& sub_all_access_paths); - - Result _get_sub_access_paths(TColumnAccessPaths access_paths, - bool is_predicate = false); + // Derive current-level meta-only read mode from an explicit access-path split. Meta-only is + // valid only when this iterator had no data-read requirement before applying the current paths, + // no current DATA path exists, and no path must be routed to a descendant iterator. + Status _check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, + const AccessPathSplit& all_access_paths); + + // Apply the common current-level access-path state transitions and select a supported + // parent-owned meta-only mode. When that mode skips data descendants, synchronously invoke the + // callback once with SKIP before returning the routing plan. The callback is never retained. + Result _prepare_nested_access_paths( + const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths, NestedMetaSupport meta_support, + const std::function& set_all_data_descendants_read_requirement); + + // Normalize the wire encoding, strip this iterator's column name, and explicitly partition + // paths consumed by this iterator from paths that must be routed to descendants. This helper is + // intentionally side-effect free; callers apply DATA/predicate read requirements explicitly. + Result _split_access_paths(TColumnAccessPaths access_paths) const; ColumnIteratorOptions _opts; ReadRequirement _read_requirement {ReadRequirement::NORMAL}; @@ -534,6 +565,9 @@ class FileColumnIterator : public ColumnIterator { Status read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) override; + Status set_access_paths(const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths) override; + ordinal_t get_current_ordinal() const override { return _current_ordinal; } // get row ranges by zone map @@ -610,10 +644,10 @@ class EmptyFileColumnIterator final : public ColumnIterator { ordinal_t get_current_ordinal() const override { return 0; } }; -// StringFileColumnIterator extends FileColumnIterator with meta-only reading -// support for string/binary column types. When the OFFSET path is detected in -// set_access_paths, it sets only_read_offsets on the ColumnIteratorOptions so -// that the BinaryPlainPageDecoder skips chars memcpy and only fills offsets. +// StringFileColumnIterator extends FileColumnIterator's NULL metadata support with OFFSET-only +// reading for string/binary column types. When the OFFSET path is detected in set_access_paths, it +// sets only_read_offsets on the ColumnIteratorOptions so that the BinaryPlainPageDecoder skips +// chars memcpy and only fills offsets. class StringFileColumnIterator final : public FileColumnIterator { public: explicit StringFileColumnIterator(std::shared_ptr reader); diff --git a/be/test/runtime/descriptor_test.cpp b/be/test/runtime/descriptor_test.cpp index 20c2b1000c8303..0a665cfc34c343 100644 --- a/be/test/runtime/descriptor_test.cpp +++ b/be/test/runtime/descriptor_test.cpp @@ -15,8 +15,10 @@ // specific language governing permissions and limitations // under the License. +#include #include #include +#include #include #include "common/exception.h" @@ -196,4 +198,59 @@ TEST_F(SlotDescriptorTest, DebugString) { EXPECT_TRUE(debug_str2.find("is_virtual=true") != std::string::npos); } -} // namespace doris \ No newline at end of file +TEST_F(SlotDescriptorTest, AccessPathsPreservedThroughProtobuf) { + TColumnAccessPath data_path; + data_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + data_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_payload; + data_payload.__set_path({"s", "field"}); + data_path.__set_data_access_path(data_payload); + + TColumnAccessPath meta_path; + meta_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + meta_path.__set_type(TAccessPathType::META); + TMetaAccessPath meta_payload; + meta_payload.__set_path({"s", "field", "NULL"}); + meta_path.__set_meta_access_path(meta_payload); + + TColumnAccessPath legacy_path; + legacy_path.__set_type(TAccessPathType::DATA); + data_payload.__set_path({"s", "legacy", "NULL"}); + legacy_path.__set_data_access_path(data_payload); + + TColumnAccessPath explicit_legacy_path; + explicit_legacy_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + explicit_legacy_path.__set_type(TAccessPathType::DATA); + data_payload.__set_path({"s", "legacy", "VALUES"}); + explicit_legacy_path.__set_data_access_path(data_payload); + + TSlotDescriptor thrift_descriptor = create_basic_slot_descriptor(); + thrift_descriptor.__set_all_access_paths( + {data_path, meta_path, legacy_path, explicit_legacy_path}); + thrift_descriptor.__set_predicate_access_paths({meta_path}); + + SlotDescriptor original(thrift_descriptor); + PSlotDescriptor protobuf_descriptor; + original.to_protobuf(&protobuf_descriptor); + ASSERT_EQ(protobuf_descriptor.all_access_paths_size(), 4); + EXPECT_TRUE(protobuf_descriptor.all_access_paths(0).has_version()); + EXPECT_EQ(protobuf_descriptor.all_access_paths(0).version(), + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + EXPECT_TRUE(protobuf_descriptor.all_access_paths(1).has_version()); + EXPECT_FALSE(protobuf_descriptor.all_access_paths(2).has_version()); + EXPECT_TRUE(protobuf_descriptor.all_access_paths(3).has_version()); + EXPECT_EQ(protobuf_descriptor.all_access_paths(3).version(), + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + SlotDescriptor round_trip(protobuf_descriptor); + + EXPECT_EQ(round_trip.all_access_paths(), thrift_descriptor.all_access_paths); + EXPECT_EQ(round_trip.predicate_access_paths(), thrift_descriptor.predicate_access_paths); + EXPECT_FALSE(round_trip.all_access_paths()[1].__isset.data_access_path); + EXPECT_FALSE(round_trip.predicate_access_paths()[0].__isset.data_access_path); + EXPECT_FALSE(round_trip.all_access_paths()[2].__isset.version); + EXPECT_TRUE(round_trip.all_access_paths()[3].__isset.version); + EXPECT_EQ(round_trip.all_access_paths()[3].version, + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); +} + +} // namespace doris diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index 2ca69b87aa63ce..6bcdd148f8f766 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -16,6 +16,7 @@ // under the License. #include "storage/segment/column_reader.h" +#include #include #include #include @@ -73,14 +74,16 @@ class TestColumnIterator final : public ColumnIterator { _read_requirement = requirement; } - Result get_sub_access_paths(const TColumnAccessPaths& access_paths, - bool is_predicate = false) { - return _get_sub_access_paths(access_paths, is_predicate); + using ColumnIterator::AccessPathSplit; + + Result split_access_paths(const TColumnAccessPaths& access_paths) const { + return _split_access_paths(access_paths); } - void check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, - const TColumnAccessPaths& access_paths) { - _check_and_set_meta_read_mode(requirement_before_access_path, access_paths); + Status check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path, + const TColumnAccessPaths& access_paths) { + auto split = DORIS_TRY(_split_access_paths(access_paths)); + return _check_and_set_meta_read_mode(requirement_before_access_path, split); } void convert_to_place_holder_column(MutableColumnPtr& dst, size_t count) { @@ -88,14 +91,35 @@ class TestColumnIterator final : public ColumnIterator { } }; -TColumnAccessPath create_access_path(std::vector path) { +TColumnAccessPath create_data_access_path(std::vector path) { TColumnAccessPath access_path; + access_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + access_path.__set_type(TAccessPathType::DATA); TDataAccessPath data_access_path; data_access_path.__set_path(std::move(path)); access_path.__set_data_access_path(std::move(data_access_path)); return access_path; } +TColumnAccessPath create_legacy_data_access_path(std::vector path) { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_access_path; + data_access_path.__set_path(std::move(path)); + access_path.__set_data_access_path(std::move(data_access_path)); + return access_path; +} + +TColumnAccessPath create_meta_access_path(std::vector path) { + TColumnAccessPath access_path; + access_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + access_path.__set_type(TAccessPathType::META); + TMetaAccessPath meta_access_path; + meta_access_path.__set_path(std::move(path)); + access_path.__set_meta_access_path(std::move(meta_access_path)); + return access_path; +} + std::shared_ptr create_test_reader( bool is_nullable = false, uint64_t num_rows = 0, FieldType field_type = FieldType::OLAP_FIELD_TYPE_INT) { @@ -148,6 +172,13 @@ class TrackingColumnIterator final : public ColumnIterator { ordinal_t get_current_ordinal() const override { return _current_ordinal; } + Status set_access_paths(const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths) override { + routed_all_access_paths = all_access_paths; + routed_predicate_access_paths = predicate_access_paths; + return ColumnIterator::set_access_paths(all_access_paths, predicate_access_paths); + } + void collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) override { @@ -164,12 +195,16 @@ class TrackingColumnIterator final : public ColumnIterator { next_batch_sizes.clear(); read_by_rowids_batches.clear(); collect_methods.clear(); + routed_all_access_paths.clear(); + routed_predicate_access_paths.clear(); } std::vector seek_ordinals; std::vector next_batch_sizes; std::vector> read_by_rowids_batches; std::vector collect_methods; + TColumnAccessPaths routed_all_access_paths; + TColumnAccessPaths routed_predicate_access_paths; private: void record_collect_method(PrefetcherInitMethod init_method) { @@ -416,9 +451,9 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { // Only reading sub_col_1 // sub_col_2 should be set to SKIP - all_access_paths[0].data_access_path.path = {"self", "sub_col_1"}; + all_access_paths[0] = create_data_access_path({"self", "sub_col_1"}); - predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"}; + predicate_access_paths[0] = create_data_access_path({"self", "sub_col_1"}); st = iterator->set_access_paths(all_access_paths, predicate_access_paths); // invalid name leads to error @@ -436,7 +471,7 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { ColumnIterator::ReadRequirement::SKIP); // Reading all sub columns - all_access_paths[0].data_access_path.path = {"self"}; + all_access_paths[0] = create_data_access_path({"self"}); iterator = create_struct_iterator(); iterator->set_column_name("self"); st = iterator->set_access_paths(all_access_paths, predicate_access_paths); @@ -512,35 +547,377 @@ TEST_F(ColumnReaderTest, MetaReadModePrefersOffsetOverNull) { auto assert_meta_read_mode = [](TColumnAccessPaths access_paths, bool offset_only, bool null_map_only) { TestColumnIterator iterator; - iterator.check_and_set_meta_read_mode(ColumnIterator::ReadRequirement::NORMAL, - access_paths); + iterator.set_column_name("self"); + auto st = iterator.check_and_set_meta_read_mode(ColumnIterator::ReadRequirement::NORMAL, + access_paths); + ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(iterator.read_offset_only(), offset_only); EXPECT_EQ(iterator.read_null_map_only(), null_map_only); }; assert_meta_read_mode(TColumnAccessPaths {}, false, false); - assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_OFFSET})}, - true, false); - assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_NULL})}, - false, true); - assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_OFFSET}), - create_access_path({ColumnIterator::ACCESS_NULL})}, - true, false); - assert_meta_read_mode(TColumnAccessPaths {create_access_path({"child"})}, false, false); - assert_meta_read_mode(TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_OFFSET}), - create_access_path({"child"})}, - false, false); + assert_meta_read_mode( + TColumnAccessPaths {create_meta_access_path({"self", ColumnIterator::ACCESS_OFFSET})}, + true, false); + assert_meta_read_mode( + TColumnAccessPaths {create_meta_access_path({"self", ColumnIterator::ACCESS_NULL})}, + false, true); + assert_meta_read_mode( + TColumnAccessPaths {create_meta_access_path({"self", ColumnIterator::ACCESS_OFFSET}), + create_meta_access_path({"self", ColumnIterator::ACCESS_NULL})}, + true, false); + assert_meta_read_mode(TColumnAccessPaths {create_data_access_path({"self", "child"})}, false, + false); + assert_meta_read_mode( + TColumnAccessPaths {create_data_access_path({"self", ColumnIterator::ACCESS_OFFSET})}, + false, false); + assert_meta_read_mode( + TColumnAccessPaths {create_data_access_path({"self", ColumnIterator::ACCESS_NULL})}, + false, false); + assert_meta_read_mode( + TColumnAccessPaths {create_meta_access_path({"self", ColumnIterator::ACCESS_OFFSET}), + create_data_access_path({"self", "child"})}, + false, false); { TestColumnIterator iterator; - iterator.check_and_set_meta_read_mode( + iterator.set_column_name("self"); + auto st = iterator.check_and_set_meta_read_mode( ColumnIterator::ReadRequirement::LAZY_OUTPUT, - TColumnAccessPaths {create_access_path({ColumnIterator::ACCESS_NULL})}); + TColumnAccessPaths { + create_meta_access_path({"self", ColumnIterator::ACCESS_NULL})}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_FALSE(iterator.read_null_map_only()); + EXPECT_FALSE(iterator.read_offset_only()); + } +} + +TEST_F(ColumnReaderTest, TypedAccessPathRequiresMatchingPayload) { + FileColumnIterator iterator(create_test_reader(true)); + iterator.set_column_name("c"); + + TColumnAccessPath unknown_type_path; + unknown_type_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + TDataAccessPath data_access_path; + data_access_path.__set_path({"c"}); + unknown_type_path.__set_data_access_path(data_access_path); + auto st = iterator.set_access_paths({unknown_type_path}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("Invalid access path type")); + + TColumnAccessPath missing_meta_payload; + missing_meta_payload.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + missing_meta_payload.__set_type(TAccessPathType::META); + missing_meta_payload.__set_data_access_path(data_access_path); + st = iterator.set_access_paths({missing_meta_payload}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("meta_access_path payload is not set")); + + TColumnAccessPath missing_data_payload; + missing_data_payload.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + missing_data_payload.__set_type(TAccessPathType::DATA); + TMetaAccessPath meta_access_path; + meta_access_path.__set_path({"c", ColumnIterator::ACCESS_NULL}); + missing_data_payload.__set_meta_access_path(meta_access_path); + st = iterator.set_access_paths({missing_data_payload}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("data_access_path payload is not set")); + + TColumnAccessPath missing_legacy_payload; + missing_legacy_payload.__set_type(TAccessPathType::DATA); + st = iterator.set_access_paths({missing_legacy_payload}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), + ::testing::HasSubstr( + "Invalid legacy access path: data_access_path payload is not set")); + + for (const bool explicit_legacy_version : {false, true}) { + TColumnAccessPath invalid_legacy_type; + invalid_legacy_type.__set_type(TAccessPathType::META); + invalid_legacy_type.__set_data_access_path(data_access_path); + invalid_legacy_type.__set_meta_access_path(meta_access_path); + if (explicit_legacy_version) { + invalid_legacy_type.__set_version( + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + } + st = iterator.set_access_paths({invalid_legacy_type}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("Invalid legacy access path type")); + } + + FileColumnIterator data_precedence_iterator(create_test_reader(true)); + data_precedence_iterator.set_column_name("c"); + auto compatible_data_path = create_data_access_path({"c"}); + compatible_data_path.__set_meta_access_path(meta_access_path); + st = data_precedence_iterator.set_access_paths({compatible_data_path}, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_FALSE(data_precedence_iterator.read_null_map_only()); + EXPECT_FALSE(data_precedence_iterator.read_offset_only()); + + auto compatible_meta_path = create_meta_access_path({"c", ColumnIterator::ACCESS_NULL}); + compatible_meta_path.__set_data_access_path(data_access_path); + st = iterator.set_access_paths({compatible_meta_path}, {compatible_meta_path}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(iterator.read_null_map_only()); + + FileColumnIterator wrong_root_iterator(create_test_reader(true)); + wrong_root_iterator.set_column_name("c"); + auto wrong_root_path = create_meta_access_path({"other", ColumnIterator::ACCESS_NULL}); + st = wrong_root_iterator.set_access_paths({}, {wrong_root_path}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("expected name \"c\", got \"other\"")); + + auto unsupported_version_path = create_data_access_path({"c"}); + unsupported_version_path.__set_version( + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED + 1); + st = iterator.set_access_paths({unsupported_version_path}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("Unsupported access path version")); +} + +TEST_F(ColumnReaderTest, AccessPathVersionControlsLegacyMetaEncoding) { + for (const bool explicit_legacy_version : {false, true}) { + SCOPED_TRACE(explicit_legacy_version ? "explicit-version-0" : "missing-version"); + FileColumnIterator iterator(create_test_reader(true)); + iterator.set_column_name("c"); + auto legacy_null_path = create_legacy_data_access_path({"c", ColumnIterator::ACCESS_NULL}); + if (explicit_legacy_version) { + legacy_null_path.__set_version( + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + } + auto st = iterator.set_access_paths({legacy_null_path}, {legacy_null_path}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(iterator.read_null_map_only()); + } + + { + FileColumnIterator iterator(create_test_reader(true)); + iterator.set_column_name("c"); + auto typed_data_path = create_data_access_path({"c", ColumnIterator::ACCESS_NULL}); + auto st = iterator.set_access_paths({typed_data_path}, {typed_data_path}); + ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_FALSE(iterator.read_null_map_only()); EXPECT_FALSE(iterator.read_offset_only()); } } +TEST_F(ColumnReaderTest, LegacyDataSpecialComponentsRemainDataSelectors) { + auto make_map_iterator = []() { + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + auto key_iterator = std::make_unique(create_test_reader()); + auto value_iterator = std::make_unique(create_test_reader()); + auto map_iterator = std::make_unique( + create_test_reader(), nullptr, std::move(offsets_iterator), std::move(key_iterator), + std::move(value_iterator)); + map_iterator->set_column_name("m"); + return map_iterator; + }; + + struct SelectorCase { + const char* component; + bool explicit_legacy_version; + ColumnIterator::ReadRequirement key_requirement; + ColumnIterator::ReadRequirement value_requirement; + }; + for (const auto& test_case : {SelectorCase {ColumnIterator::ACCESS_MAP_KEYS, false, + ColumnIterator::ReadRequirement::LAZY_OUTPUT, + ColumnIterator::ReadRequirement::SKIP}, + SelectorCase {ColumnIterator::ACCESS_MAP_VALUES, true, + ColumnIterator::ReadRequirement::SKIP, + ColumnIterator::ReadRequirement::LAZY_OUTPUT}}) { + SCOPED_TRACE(test_case.component); + auto map_iterator = make_map_iterator(); + auto path = create_legacy_data_access_path({"m", test_case.component}); + if (test_case.explicit_legacy_version) { + path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + } + + auto st = map_iterator->set_access_paths({path}, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_FALSE(map_iterator->read_null_map_only()); + EXPECT_FALSE(map_iterator->read_offset_only()); + EXPECT_EQ(map_iterator->_key_iterator->read_requirement(), test_case.key_requirement); + EXPECT_EQ(map_iterator->_val_iterator->read_requirement(), test_case.value_requirement); + } + + // `*` is also a legacy DATA selector. It keeps keys readable while OFFSET is promoted only + // after the remaining path reaches the value iterator. + auto map_iterator = make_map_iterator(); + auto offset_path = create_legacy_data_access_path( + {"m", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET}); + offset_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + auto st = map_iterator->set_access_paths({offset_path}, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_EQ(map_iterator->_key_iterator->read_requirement(), + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_TRUE(map_iterator->_val_iterator->read_offset_only()); +} + +TEST_F(ColumnReaderTest, TypedMetaPathsRouteThroughScalarAndComplexIterators) { + { + FileColumnIterator scalar_iterator(create_test_reader(true)); + scalar_iterator.set_column_name("i"); + TColumnAccessPaths null_path {create_meta_access_path({"i", ColumnIterator::ACCESS_NULL})}; + auto st = scalar_iterator.set_access_paths(null_path, null_path); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(scalar_iterator.read_null_map_only()); + } + + { + auto item_iterator = std::make_unique(create_test_reader()); + item_iterator->set_column_name("item"); + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + ArrayFileColumnIterator array_iterator(create_test_reader(), std::move(offsets_iterator), + std::move(item_iterator), nullptr); + array_iterator.set_column_name("a"); + + TColumnAccessPaths offset_path { + create_meta_access_path({"a", ColumnIterator::ACCESS_OFFSET})}; + auto st = array_iterator.set_access_paths(offset_path, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(array_iterator.read_offset_only()); + EXPECT_EQ(array_iterator._item_iterator->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); + } + + { + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + auto key_iterator = std::make_unique(create_test_reader()); + auto value_iterator = std::make_unique(create_test_reader()); + MapFileColumnIterator map_iterator(create_test_reader(), nullptr, + std::move(offsets_iterator), std::move(key_iterator), + std::move(value_iterator)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths offset_path { + create_meta_access_path({"m", ColumnIterator::ACCESS_OFFSET})}; + auto st = map_iterator.set_access_paths(offset_path, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(map_iterator.read_offset_only()); + EXPECT_EQ(map_iterator._key_iterator->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); + EXPECT_EQ(map_iterator._val_iterator->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); + } + + { + std::vector sub_iterators; + auto string_iterator = std::make_unique(create_test_reader()); + string_iterator->set_column_name("text"); + sub_iterators.emplace_back(std::move(string_iterator)); + StructFileColumnIterator struct_iterator(create_test_reader(), nullptr, + std::move(sub_iterators)); + struct_iterator.set_column_name("s"); + + TColumnAccessPaths offset_path { + create_meta_access_path({"s", "text", ColumnIterator::ACCESS_OFFSET})}; + auto st = struct_iterator.set_access_paths(offset_path, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + auto* text_iterator = static_cast( + struct_iterator._sub_column_iterators[0].get()); + EXPECT_TRUE(text_iterator->read_offset_only()); + } + + { + auto item_iterator = std::make_unique(create_test_reader()); + item_iterator->set_column_name("item"); + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + ArrayFileColumnIterator array_iterator(create_test_reader(), std::move(offsets_iterator), + std::move(item_iterator), nullptr); + array_iterator.set_column_name("a"); + + TColumnAccessPaths offset_path {create_meta_access_path( + {"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET})}; + auto st = array_iterator.set_access_paths(offset_path, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + auto* typed_item_iterator = + static_cast(array_iterator._item_iterator.get()); + EXPECT_TRUE(typed_item_iterator->read_offset_only()); + } + + { + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + auto key_iterator = std::make_unique(create_test_reader()); + auto value_iterator = std::make_unique(create_test_reader()); + MapFileColumnIterator map_iterator(create_test_reader(), nullptr, + std::move(offsets_iterator), std::move(key_iterator), + std::move(value_iterator)); + map_iterator.set_column_name("m"); + + TColumnAccessPaths offset_path {create_meta_access_path( + {"m", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET})}; + auto st = map_iterator.set_access_paths(offset_path, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + auto* typed_key_iterator = + static_cast(map_iterator._key_iterator.get()); + auto* typed_value_iterator = + static_cast(map_iterator._val_iterator.get()); + EXPECT_FALSE(typed_key_iterator->read_offset_only()); + EXPECT_EQ(typed_key_iterator->read_requirement(), + ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_TRUE(typed_value_iterator->read_offset_only()); + } +} + +TEST_F(ColumnReaderTest, TypedDataFieldsNamedMetaComponentsAreNotTreatedAsMetaPaths) { + auto make_struct_iterator = [](const std::string& field_name) { + std::vector sub_iterators; + auto field_iterator = std::make_unique(create_test_reader()); + field_iterator->set_column_name(field_name); + sub_iterators.emplace_back(std::move(field_iterator)); + auto struct_iterator = std::make_unique( + create_test_reader(), nullptr, std::move(sub_iterators)); + struct_iterator->set_column_name("s"); + return struct_iterator; + }; + + for (const std::string field_name : + {ColumnIterator::ACCESS_NULL, ColumnIterator::ACCESS_OFFSET}) { + SCOPED_TRACE(field_name); + auto struct_iterator = make_struct_iterator(field_name); + TColumnAccessPaths data_path {create_data_access_path({"s", field_name})}; + auto st = struct_iterator->set_access_paths(data_path, data_path); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_FALSE(struct_iterator->read_null_map_only()); + EXPECT_FALSE(struct_iterator->read_offset_only()); + ASSERT_EQ(struct_iterator->_sub_column_iterators.size(), 1); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->read_requirement(), + ColumnIterator::ReadRequirement::PREDICATE); + } + + auto struct_iterator = make_struct_iterator(ColumnIterator::ACCESS_NULL); + TColumnAccessPaths meta_path {create_meta_access_path({"s", ColumnIterator::ACCESS_NULL})}; + auto st = struct_iterator->set_access_paths(meta_path, meta_path); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(struct_iterator->read_null_map_only()); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); +} + +TEST_F(ColumnReaderTest, LegacyStructMetaComponentsRemainSentinels) { + auto null_iterator = std::make_unique(create_test_reader()); + std::vector sub_iterators; + auto field_iterator = std::make_unique(create_test_reader()); + field_iterator->set_column_name(ColumnIterator::ACCESS_NULL); + sub_iterators.emplace_back(std::move(field_iterator)); + StructFileColumnIterator struct_iterator(create_test_reader(), std::move(null_iterator), + std::move(sub_iterators)); + struct_iterator.set_column_name("s"); + + auto legacy_path = create_legacy_data_access_path({"s", ColumnIterator::ACCESS_NULL}); + legacy_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + auto st = struct_iterator.set_access_paths({legacy_path}, {legacy_path}); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(struct_iterator.read_null_map_only()); + EXPECT_EQ(struct_iterator._sub_column_iterators[0]->read_requirement(), + ColumnIterator::ReadRequirement::SKIP); +} + TEST_F(ColumnReaderTest, PlaceHolderLifecycleInLazyMode) { TestColumnIterator iterator; iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); @@ -734,23 +1111,270 @@ TEST_F(ColumnReaderTest, FinalizeLazyModeOnNestedStruct) { EXPECT_EQ(0, nested_after->size()); } -TEST_F(ColumnReaderTest, GetSubAccessPathsSetsPredicateFlag) { +TEST_F(ColumnReaderTest, SplitAccessPathsClassifiesCurrentAndDescendantPaths) { TestColumnIterator iterator; iterator.set_column_name("self"); + iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); - TColumnAccessPaths access_paths; - access_paths.emplace_back(); - access_paths[0].data_access_path.path = {"self"}; + { + auto split = TEST_TRY(iterator.split_access_paths( + TColumnAccessPaths {create_data_access_path({"self"})})); + EXPECT_TRUE(split.reads_current_data); + EXPECT_EQ(split.current_meta_mode, ColumnIterator::MetaReadMode::DEFAULT); + EXPECT_TRUE(split.descendant_paths.empty()); + } - iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); - auto sub_paths = TEST_TRY(iterator.get_sub_access_paths(access_paths)); - EXPECT_TRUE(sub_paths.empty()); - EXPECT_EQ(iterator._read_requirement, ColumnIterator::ReadRequirement::LAZY_OUTPUT); + { + auto split = TEST_TRY(iterator.split_access_paths(TColumnAccessPaths { + create_meta_access_path({"self", ColumnIterator::ACCESS_NULL}), + create_meta_access_path({"self", ColumnIterator::ACCESS_OFFSET})})); + EXPECT_FALSE(split.reads_current_data); + EXPECT_EQ(split.current_meta_mode, ColumnIterator::MetaReadMode::OFFSET_ONLY); + EXPECT_TRUE(split.descendant_paths.empty()); + } - iterator.force_set_read_requirement(ColumnIterator::ReadRequirement::NORMAL); - sub_paths = TEST_TRY(iterator.get_sub_access_paths(access_paths, true)); - EXPECT_TRUE(sub_paths.empty()); - EXPECT_EQ(iterator._read_requirement, ColumnIterator::ReadRequirement::PREDICATE); + { + auto split = TEST_TRY(iterator.split_access_paths(TColumnAccessPaths { + create_data_access_path({"self", "child"}), + create_meta_access_path({"self", "child", ColumnIterator::ACCESS_NULL})})); + EXPECT_FALSE(split.reads_current_data); + EXPECT_EQ(split.current_meta_mode, ColumnIterator::MetaReadMode::DEFAULT); + ASSERT_EQ(split.descendant_paths.size(), 2); + EXPECT_EQ(split.descendant_paths[0].type, TAccessPathType::DATA); + EXPECT_EQ(split.descendant_paths[0].data_access_path.path, + (std::vector {"child"})); + EXPECT_EQ(split.descendant_paths[1].type, TAccessPathType::META); + EXPECT_EQ(split.descendant_paths[1].meta_access_path.path, + (std::vector {"child", ColumnIterator::ACCESS_NULL})); + } + + { + auto legacy_null = create_legacy_data_access_path({"self", ColumnIterator::ACCESS_NULL}); + auto legacy_offset = + create_legacy_data_access_path({"self", ColumnIterator::ACCESS_OFFSET}); + legacy_offset.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + auto split = TEST_TRY(iterator.split_access_paths( + TColumnAccessPaths {std::move(legacy_null), std::move(legacy_offset)})); + EXPECT_FALSE(split.reads_current_data); + EXPECT_EQ(split.current_meta_mode, ColumnIterator::MetaReadMode::OFFSET_ONLY); + EXPECT_TRUE(split.descendant_paths.empty()); + } + + { + auto legacy_keys = + create_legacy_data_access_path({"self", ColumnIterator::ACCESS_MAP_KEYS}); + auto split = TEST_TRY(iterator.split_access_paths( + TColumnAccessPaths {create_data_access_path({"self", ColumnIterator::ACCESS_NULL}), + std::move(legacy_keys)})); + EXPECT_FALSE(split.reads_current_data); + EXPECT_EQ(split.current_meta_mode, ColumnIterator::MetaReadMode::DEFAULT); + ASSERT_EQ(split.descendant_paths.size(), 2); + EXPECT_EQ(split.descendant_paths[0].data_access_path.path, + (std::vector {ColumnIterator::ACCESS_NULL})); + EXPECT_EQ(split.descendant_paths[1].data_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_KEYS})); + } + + // Splitting is a pure classification step. Callers explicitly apply lazy/predicate state. + EXPECT_EQ(iterator._read_requirement, ColumnIterator::ReadRequirement::NORMAL); +} + +TEST_F(ColumnReaderTest, MapRejectsUnrecognizedDescendantSelectors) { + auto make_map_iterator = []() { + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + auto map_iterator = std::make_unique( + create_test_reader(), nullptr, std::move(offsets_iterator), + std::make_unique(), + std::make_unique()); + map_iterator->set_column_name("m"); + return map_iterator; + }; + + std::vector invalid_paths; + invalid_paths.emplace_back(create_data_access_path({"m", "UNKNOWN"})); + invalid_paths.emplace_back( + create_meta_access_path({"m", "UNKNOWN", ColumnIterator::ACCESS_NULL})); + invalid_paths.emplace_back(create_legacy_data_access_path({"m", "UNKNOWN"})); + + for (const auto& path : invalid_paths) { + auto map_iterator = make_map_iterator(); + auto st = map_iterator->set_access_paths({path}, {}); + EXPECT_FALSE(st.ok()); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("Invalid map access path selector")); + EXPECT_THAT(st.to_string(), ::testing::HasSubstr("UNKNOWN")); + } + + auto path = create_meta_access_path( + {"m", ColumnIterator::ACCESS_MAP_VALUES, ColumnIterator::ACCESS_OFFSET}); + TDataAccessPath unselected_data_payload; + unselected_data_payload.__set_path({"m", "UNKNOWN"}); + path.__set_data_access_path(std::move(unselected_data_payload)); + auto map_iterator = make_map_iterator(); + auto st = map_iterator->set_access_paths({path}, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); +} + +TEST_F(ColumnReaderTest, MapChildPathRoutingUsesLogicalSelectorsAndPreservesVersion) { + auto key_iterator = std::make_unique(); + key_iterator->set_column_name("physical_key"); + auto* key = key_iterator.get(); + auto value_iterator = std::make_unique(); + value_iterator->set_column_name("physical_value"); + auto* value = value_iterator.get(); + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + MapFileColumnIterator map_iterator(create_test_reader(), nullptr, std::move(offsets_iterator), + std::move(key_iterator), std::move(value_iterator)); + map_iterator.set_column_name("m"); + EXPECT_EQ(key->column_name(), ColumnIterator::ACCESS_MAP_KEYS); + EXPECT_EQ(value->column_name(), ColumnIterator::ACCESS_MAP_VALUES); + + auto path = create_meta_access_path( + {"m", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET}); + TDataAccessPath unselected_data_payload; + unselected_data_payload.__set_path(path.meta_access_path.path); + path.__set_data_access_path(std::move(unselected_data_payload)); + auto legacy_path = create_data_access_path( + {"m", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET}); + legacy_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + + auto st = map_iterator.set_access_paths({path}, {legacy_path}); + ASSERT_TRUE(st.ok()) << st.to_string(); + + ASSERT_EQ(key->routed_all_access_paths.size(), 1); + const auto& key_path = key->routed_all_access_paths[0]; + EXPECT_EQ(key_path.type, TAccessPathType::DATA); + ASSERT_TRUE(key_path.__isset.version); + EXPECT_EQ(key_path.version, g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + EXPECT_EQ(key_path.data_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_KEYS})); + ASSERT_EQ(key->routed_predicate_access_paths.size(), 1); + const auto& legacy_key_path = key->routed_predicate_access_paths[0]; + EXPECT_EQ(legacy_key_path.type, TAccessPathType::DATA); + ASSERT_TRUE(legacy_key_path.__isset.version); + EXPECT_EQ(legacy_key_path.version, g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + EXPECT_EQ(legacy_key_path.data_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_KEYS})); + + ASSERT_EQ(value->routed_all_access_paths.size(), 1); + const auto& value_path = value->routed_all_access_paths[0]; + EXPECT_EQ(value_path.type, TAccessPathType::META); + ASSERT_TRUE(value_path.__isset.version); + EXPECT_EQ(value_path.version, g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + EXPECT_EQ(value_path.meta_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_VALUES, + ColumnIterator::ACCESS_OFFSET})); + EXPECT_EQ(value_path.data_access_path.path, + (std::vector {"m", ColumnIterator::ACCESS_ALL, + ColumnIterator::ACCESS_OFFSET})); + ASSERT_EQ(value->routed_predicate_access_paths.size(), 1); + const auto& legacy_value_path = value->routed_predicate_access_paths[0]; + EXPECT_EQ(legacy_value_path.type, TAccessPathType::DATA); + ASSERT_TRUE(legacy_value_path.__isset.version); + EXPECT_EQ(legacy_value_path.version, + g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY); + EXPECT_EQ(legacy_value_path.data_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_VALUES, + ColumnIterator::ACCESS_OFFSET})); +} + +TEST_F(ColumnReaderTest, NestedMapWildcardRoutingUsesLogicalSelectorsWithoutPhysicalNames) { + auto inner_key_iterator = std::make_unique(); + auto* inner_key = inner_key_iterator.get(); + auto inner_value_iterator = std::make_unique(); + auto* inner_value = inner_value_iterator.get(); + auto inner_offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + auto inner_map_iterator = std::make_unique( + create_test_reader(), nullptr, std::move(inner_offsets_iterator), + std::move(inner_key_iterator), std::move(inner_value_iterator)); + + auto outer_key_iterator = std::make_unique(); + auto* outer_key = outer_key_iterator.get(); + auto outer_offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + MapFileColumnIterator outer_map_iterator( + create_test_reader(), nullptr, std::move(outer_offsets_iterator), + std::move(outer_key_iterator), std::move(inner_map_iterator)); + outer_map_iterator.set_column_name("m"); + + auto path = + create_meta_access_path({"m", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_ALL, + ColumnIterator::ACCESS_OFFSET}); + auto st = outer_map_iterator.set_access_paths({path}, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + + ASSERT_EQ(outer_key->routed_all_access_paths.size(), 1); + EXPECT_EQ(outer_key->routed_all_access_paths[0].data_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_KEYS})); + ASSERT_EQ(inner_key->routed_all_access_paths.size(), 1); + EXPECT_EQ(inner_key->routed_all_access_paths[0].data_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_KEYS})); + ASSERT_EQ(inner_value->routed_all_access_paths.size(), 1); + EXPECT_EQ(inner_value->routed_all_access_paths[0].meta_access_path.path, + (std::vector {ColumnIterator::ACCESS_MAP_VALUES, + ColumnIterator::ACCESS_OFFSET})); +} + +TEST_F(ColumnReaderTest, ArrayItemPathRoutingRewritesOnlySelectedPayload) { + auto item_iterator = std::make_unique(); + item_iterator->set_column_name("item"); + auto* item = item_iterator.get(); + auto offsets_iterator = std::make_unique( + std::make_unique(create_test_reader())); + ArrayFileColumnIterator array_iterator(create_test_reader(), std::move(offsets_iterator), + std::move(item_iterator), nullptr); + array_iterator.set_column_name("a"); + + auto path = create_meta_access_path( + {"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET}); + TDataAccessPath unselected_data_payload; + unselected_data_payload.__set_path(path.meta_access_path.path); + path.__set_data_access_path(std::move(unselected_data_payload)); + + auto st = array_iterator.set_access_paths({path}, {}); + ASSERT_TRUE(st.ok()) << st.to_string(); + + ASSERT_EQ(item->routed_all_access_paths.size(), 1); + const auto& item_path = item->routed_all_access_paths[0]; + EXPECT_EQ(item_path.type, TAccessPathType::META); + ASSERT_TRUE(item_path.__isset.version); + EXPECT_EQ(item_path.version, g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + EXPECT_EQ(item_path.meta_access_path.path, + (std::vector {"item", ColumnIterator::ACCESS_OFFSET})); + EXPECT_EQ(item_path.data_access_path.path, + (std::vector {"a", ColumnIterator::ACCESS_ALL, + ColumnIterator::ACCESS_OFFSET})); +} + +TEST_F(ColumnReaderTest, StructCurrentMetaDoesNotRouteToDataFieldWithSameName) { + std::vector sub_iterators; + auto null_field_iterator = std::make_unique(); + null_field_iterator->set_column_name(ColumnIterator::ACCESS_NULL); + auto* null_field = null_field_iterator.get(); + sub_iterators.emplace_back(std::move(null_field_iterator)); + auto data_field_iterator = std::make_unique(); + data_field_iterator->set_column_name("data"); + auto* data_field = data_field_iterator.get(); + sub_iterators.emplace_back(std::move(data_field_iterator)); + + StructFileColumnIterator struct_iterator(create_test_reader(), nullptr, + std::move(sub_iterators)); + struct_iterator.set_column_name("s"); + TColumnAccessPaths all_access_paths { + create_meta_access_path({"s", ColumnIterator::ACCESS_NULL}), + create_data_access_path({"s", "data"})}; + TColumnAccessPaths predicate_access_paths {create_data_access_path({"s", "data"})}; + + auto st = struct_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(null_field->routed_all_access_paths.empty()); + EXPECT_TRUE(null_field->routed_predicate_access_paths.empty()); + EXPECT_EQ(null_field->read_requirement(), ColumnIterator::ReadRequirement::SKIP); + ASSERT_EQ(data_field->routed_all_access_paths.size(), 1); + ASSERT_EQ(data_field->routed_predicate_access_paths.size(), 1); + EXPECT_EQ(data_field->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); } TEST_F(ColumnReaderTest, NestedIteratorsPropagateReadPhase) { @@ -810,7 +1434,7 @@ TEST_F(ColumnReaderTest, AccessPathsPropagatePredicateToChildren) { TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"s"}; + all_access_paths[0] = create_data_access_path({"s"}); TColumnAccessPaths predicate_access_paths = all_access_paths; auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); @@ -832,7 +1456,7 @@ TEST_F(ColumnReaderTest, AccessPathsPropagatePredicateToChildren) { array_iterator.set_column_name("a"); TColumnAccessPaths array_access_paths; array_access_paths.emplace_back(); - array_access_paths[0].data_access_path.path = {"a"}; + array_access_paths[0] = create_data_access_path({"a"}); TColumnAccessPaths array_predicate_paths = array_access_paths; st = array_iterator.set_access_paths(array_access_paths, array_predicate_paths); ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); @@ -851,7 +1475,7 @@ TEST_F(ColumnReaderTest, AccessPathsPropagatePredicateToChildren) { map_iterator.set_column_name("m"); TColumnAccessPaths map_access_paths; map_access_paths.emplace_back(); - map_access_paths[0].data_access_path.path = {"m"}; + map_access_paths[0] = create_data_access_path({"m"}); TColumnAccessPaths map_predicate_paths = map_access_paths; st = map_iterator.set_access_paths(map_access_paths, map_predicate_paths); ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); @@ -876,8 +1500,8 @@ TEST_F(ColumnReaderTest, StructPredicateOnlyChildPathStillRoutesToChild) { std::move(sub_iters)); struct_iterator.set_column_name("s"); - TColumnAccessPaths all_access_paths {create_access_path({"s", "a"})}; - TColumnAccessPaths predicate_access_paths {create_access_path({"s", "b"})}; + TColumnAccessPaths all_access_paths {create_data_access_path({"s", "a"})}; + TColumnAccessPaths predicate_access_paths {create_data_access_path({"s", "b"})}; auto st = struct_iterator.set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); @@ -894,7 +1518,7 @@ TEST_F(ColumnReaderTest, StructPredicateOnlyChildPathStillRoutesToChild) { EXPECT_EQ(struct_iterator._sub_column_iterators[1]->column_name(), "b"); } -TEST_F(ColumnReaderTest, CurrentLevelPredicateNullPathUsesMetaOnlyMode) { +TEST_F(ColumnReaderTest, LegacyCurrentLevelPredicateNullPathUsesMetaOnlyMode) { auto make_struct_iterator = []() { auto null_iter = std::make_unique(std::make_shared()); std::vector sub_iters; @@ -914,9 +1538,9 @@ TEST_F(ColumnReaderTest, CurrentLevelPredicateNullPathUsesMetaOnlyMode) { { auto struct_iterator = make_struct_iterator(); TColumnAccessPaths all_access_paths { - create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"s", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"s", ColumnIterator::ACCESS_NULL})}; auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); @@ -934,9 +1558,10 @@ TEST_F(ColumnReaderTest, CurrentLevelPredicateNullPathUsesMetaOnlyMode) { { auto struct_iterator = make_struct_iterator(); TColumnAccessPaths all_access_paths { - create_access_path({"s"}), create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"s"}), + create_legacy_data_access_path({"s", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"s", ColumnIterator::ACCESS_NULL})}; auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); @@ -961,9 +1586,9 @@ TEST_F(ColumnReaderTest, CurrentLevelPredicateNullPathUsesMetaOnlyMode) { array_iterator.set_column_name("a"); TColumnAccessPaths all_access_paths { - create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"a", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"a", ColumnIterator::ACCESS_NULL})}; auto st = array_iterator.set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); @@ -984,9 +1609,9 @@ TEST_F(ColumnReaderTest, CurrentLevelPredicateNullPathUsesMetaOnlyMode) { map_iterator.set_column_name("m"); TColumnAccessPaths all_access_paths { - create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"m", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + create_legacy_data_access_path({"m", ColumnIterator::ACCESS_NULL})}; auto st = map_iterator.set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); @@ -1017,10 +1642,10 @@ TEST_F(ColumnReaderTest, StructPredicateMetaPathDoesNotOverrideExistingDataNeed) auto struct_iterator = make_struct_iterator(); TColumnAccessPaths all_access_paths { - create_access_path({"s"}), - create_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + create_data_access_path({"s"}), + create_meta_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + create_meta_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); @@ -1033,7 +1658,7 @@ TEST_F(ColumnReaderTest, StructPredicateMetaPathDoesNotOverrideExistingDataNeed) ColumnIterator::ReadRequirement::LAZY_OUTPUT); struct_iterator = make_struct_iterator(); - all_access_paths = {create_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + all_access_paths = {create_meta_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; predicate_access_paths = all_access_paths; st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set predicate-only struct access paths: " << st.to_string(); @@ -1044,6 +1669,39 @@ TEST_F(ColumnReaderTest, StructPredicateMetaPathDoesNotOverrideExistingDataNeed) ColumnIterator::ReadRequirement::SKIP); } +TEST_F(ColumnReaderTest, StructSiblingDataPathDoesNotDisablePredicateMetaOnlyRead) { + auto city_iterator = std::make_unique( + create_test_reader(true, 0, FieldType::OLAP_FIELD_TYPE_STRING)); + city_iterator->set_column_name("city"); + auto* city_iterator_ptr = city_iterator.get(); + + auto data_iterator = std::make_unique(create_test_reader()); + data_iterator->set_column_name("data"); + auto* data_iterator_ptr = data_iterator.get(); + + std::vector sub_iterators; + sub_iterators.emplace_back(std::move(city_iterator)); + sub_iterators.emplace_back(std::move(data_iterator)); + StructFileColumnIterator struct_iterator( + create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_STRUCT), nullptr, + std::move(sub_iterators)); + struct_iterator.set_column_name("s"); + + TColumnAccessPaths all_access_paths { + create_data_access_path({"s", "data"}), + create_meta_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths { + create_meta_access_path({"s", "city", ColumnIterator::ACCESS_NULL})}; + + auto st = struct_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << st.to_string(); + + EXPECT_EQ(struct_iterator.read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(data_iterator_ptr->read_requirement(), ColumnIterator::ReadRequirement::LAZY_OUTPUT); + EXPECT_TRUE(city_iterator_ptr->read_null_map_only()); + EXPECT_EQ(city_iterator_ptr->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); +} + TEST_F(ColumnReaderTest, ArrayPredicateMetaPathDoesNotOverrideExistingDataNeed) { auto make_array_iterator = []() { auto item_iter = @@ -1060,11 +1718,11 @@ TEST_F(ColumnReaderTest, ArrayPredicateMetaPathDoesNotOverrideExistingDataNeed) }; auto array_iterator = make_array_iterator(); - TColumnAccessPaths all_access_paths { - create_access_path({"a"}), - create_access_path({"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; - TColumnAccessPaths predicate_access_paths { - create_access_path({"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths all_access_paths {create_data_access_path({"a"}), + create_meta_access_path({"a", ColumnIterator::ACCESS_ALL, + ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths {create_meta_access_path( + {"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; auto st = array_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); @@ -1073,8 +1731,8 @@ TEST_F(ColumnReaderTest, ArrayPredicateMetaPathDoesNotOverrideExistingDataNeed) EXPECT_EQ(item_iter->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); array_iterator = make_array_iterator(); - all_access_paths = { - create_access_path({"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; + all_access_paths = {create_meta_access_path( + {"a", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_NULL})}; predicate_access_paths = all_access_paths; st = array_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set predicate-only array access paths: " << st.to_string(); @@ -1088,10 +1746,8 @@ TEST_F(ColumnReaderTest, MapPredicateMetaPathDoesNotOverrideExistingDataNeed) { auto offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto key_iter = std::make_unique(std::make_shared()); - key_iter->set_column_name("key"); auto value_iter = std::make_unique(std::make_shared()); - value_iter->set_column_name("value"); auto map_iterator = std::make_unique( std::make_shared(), std::move(null_iter), std::move(offsets_iter), std::move(key_iter), std::move(value_iter)); @@ -1100,10 +1756,11 @@ TEST_F(ColumnReaderTest, MapPredicateMetaPathDoesNotOverrideExistingDataNeed) { }; auto map_iterator = make_map_iterator(); - TColumnAccessPaths all_access_paths {create_access_path({"m"}), - create_access_path({"m", ColumnIterator::ACCESS_MAP_VALUES, - ColumnIterator::ACCESS_NULL})}; - TColumnAccessPaths predicate_access_paths {create_access_path( + TColumnAccessPaths all_access_paths { + create_data_access_path({"m"}), + create_meta_access_path( + {"m", ColumnIterator::ACCESS_MAP_VALUES, ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths {create_meta_access_path( {"m", ColumnIterator::ACCESS_MAP_VALUES, ColumnIterator::ACCESS_NULL})}; auto st = map_iterator->set_access_paths(all_access_paths, predicate_access_paths); @@ -1115,7 +1772,7 @@ TEST_F(ColumnReaderTest, MapPredicateMetaPathDoesNotOverrideExistingDataNeed) { EXPECT_EQ(value_iter->read_requirement(), ColumnIterator::ReadRequirement::PREDICATE); map_iterator = make_map_iterator(); - all_access_paths = {create_access_path( + all_access_paths = {create_meta_access_path( {"m", ColumnIterator::ACCESS_MAP_VALUES, ColumnIterator::ACCESS_NULL})}; predicate_access_paths = all_access_paths; st = map_iterator->set_access_paths(all_access_paths, predicate_access_paths); @@ -1139,7 +1796,6 @@ TEST_F(ColumnReaderTest, MapFullProjectionStillRoutesPredicateSubPaths) { auto value_struct = std::make_unique( std::make_shared(), std::move(null_iter), std::move(sub_iters)); - value_struct->set_column_name("value"); return value_struct; }; @@ -1147,7 +1803,6 @@ TEST_F(ColumnReaderTest, MapFullProjectionStillRoutesPredicateSubPaths) { auto map_offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto map_key_iter = std::make_unique(std::make_shared()); - map_key_iter->set_column_name("key"); auto map_iterator = std::make_unique( std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), std::move(map_key_iter), make_value_struct()); @@ -1155,13 +1810,13 @@ TEST_F(ColumnReaderTest, MapFullProjectionStillRoutesPredicateSubPaths) { TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"m"}; + all_access_paths[0] = create_data_access_path({"m"}); TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"m", "KEYS"}; + predicate_access_paths[0] = create_data_access_path({"m", "KEYS"}); predicate_access_paths.emplace_back(); - predicate_access_paths[1].data_access_path.path = {"m", "VALUES", "a"}; + predicate_access_paths[1] = create_data_access_path({"m", "VALUES", "a"}); auto st = map_iterator->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); @@ -1180,6 +1835,33 @@ TEST_F(ColumnReaderTest, MapFullProjectionStillRoutesPredicateSubPaths) { } TEST_F(ColumnReaderTest, MetaOnlyAllPathsStillRoutePredicateSubPaths) { + { + auto struct_null_iter = + std::make_unique(std::make_shared()); + std::vector sub_iters; + auto selected_iter = std::make_unique(std::make_shared()); + selected_iter->set_column_name("selected"); + auto skipped_iter = std::make_unique(std::make_shared()); + skipped_iter->set_column_name("skipped"); + sub_iters.emplace_back(std::move(selected_iter)); + sub_iters.emplace_back(std::move(skipped_iter)); + StructFileColumnIterator struct_iterator(std::make_shared(), + std::move(struct_null_iter), std::move(sub_iters)); + struct_iterator.set_column_name("s"); + + TColumnAccessPaths all_access_paths { + create_meta_access_path({"s", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths predicate_access_paths {create_data_access_path({"s", "selected"})}; + + auto st = struct_iterator.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + EXPECT_FALSE(struct_iterator.read_null_map_only()); + EXPECT_EQ(struct_iterator._sub_column_iterators[0]->_read_requirement, + ColumnIterator::ReadRequirement::PREDICATE); + EXPECT_EQ(struct_iterator._sub_column_iterators[1]->_read_requirement, + ColumnIterator::ReadRequirement::SKIP); + } + { auto array_item_iterator = std::make_unique(std::make_shared()); @@ -1193,10 +1875,10 @@ TEST_F(ColumnReaderTest, MetaOnlyAllPathsStillRoutePredicateSubPaths) { array_iterator.set_column_name("a"); TColumnAccessPaths all_access_paths { - create_access_path({"a", ColumnIterator::ACCESS_OFFSET}), - create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + create_meta_access_path({"a", ColumnIterator::ACCESS_OFFSET}), + create_meta_access_path({"a", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"a", ColumnIterator::ACCESS_ALL})}; + create_data_access_path({"a", ColumnIterator::ACCESS_ALL})}; auto st = array_iterator.set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); @@ -1217,10 +1899,10 @@ TEST_F(ColumnReaderTest, MetaOnlyAllPathsStillRoutePredicateSubPaths) { map_iterator.set_column_name("m"); TColumnAccessPaths all_access_paths { - create_access_path({"m", ColumnIterator::ACCESS_OFFSET}), - create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + create_meta_access_path({"m", ColumnIterator::ACCESS_OFFSET}), + create_meta_access_path({"m", ColumnIterator::ACCESS_NULL})}; TColumnAccessPaths predicate_access_paths { - create_access_path({"m", ColumnIterator::ACCESS_MAP_KEYS})}; + create_data_access_path({"m", ColumnIterator::ACCESS_MAP_KEYS})}; auto st = map_iterator.set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); @@ -1245,7 +1927,6 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPaths) { auto value_struct = std::make_unique( std::make_shared(), std::move(null_iter), std::move(sub_iters)); - value_struct->set_column_name("value"); return value_struct; }; @@ -1253,7 +1934,6 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPaths) { auto map_offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto map_key_iter = std::make_unique(std::make_shared()); - map_key_iter->set_column_name("key"); auto map_val_iter = make_value_struct(); auto map_iterator = std::make_unique( std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), @@ -1281,7 +1961,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPaths) { TColumnAccessPaths access_paths; access_paths.emplace_back(); - access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES", "a"}); TColumnAccessPaths predicate_access_paths = access_paths; auto st = top_struct->set_access_paths(access_paths, predicate_access_paths); @@ -1321,7 +2001,6 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto value_struct = std::make_unique( std::make_shared(), std::move(null_iter), std::move(sub_iters)); - value_struct->set_column_name("value"); return value_struct; }; @@ -1329,7 +2008,6 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto map_offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto map_key_iter = std::make_unique(std::make_shared()); - map_key_iter->set_column_name("key"); auto map_val_iter = make_value_struct(); auto map_iterator = std::make_unique( std::make_shared(), std::move(map_null_iter), @@ -1363,7 +2041,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col1"}; + all_access_paths[0] = create_data_access_path({"root", "col1"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1380,10 +2058,10 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "KEYS"}); TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "b"}; + predicate_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES", "b"}); auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); EXPECT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); @@ -1393,7 +2071,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2"}; + all_access_paths[0] = create_data_access_path({"root", "col2"}); TColumnAccessPaths predicate_access_paths = all_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1423,7 +2101,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { TColumnAccessPaths all_access_paths; TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + predicate_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES", "a"}); auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); EXPECT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); @@ -1432,7 +2110,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "KEYS"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1451,7 +2129,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1476,7 +2154,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1501,7 +2179,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES", "a"}); TColumnAccessPaths predicate_access_paths = all_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1530,10 +2208,10 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*"}); TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + predicate_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES"}); auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); @@ -1551,7 +2229,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {}; + all_access_paths[0] = create_data_access_path({}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1562,7 +2240,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"wrong_root", "col2"}; + all_access_paths[0] = create_data_access_path({"wrong_root", "col2"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1573,7 +2251,7 @@ TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "wrong_item"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "wrong_item"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1616,7 +2294,6 @@ TEST_F(ColumnReaderTest, DeepNestedAccessPathsFiveLevels) { auto value_struct = std::make_unique( std::make_shared(), std::move(null_iter), std::move(sub_iters)); - value_struct->set_column_name("value"); return value_struct; }; @@ -1624,7 +2301,6 @@ TEST_F(ColumnReaderTest, DeepNestedAccessPathsFiveLevels) { auto map_offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto map_key_iter = std::make_unique(std::make_shared()); - map_key_iter->set_column_name("key"); auto map_val_iter = make_value_struct(); auto map_iter = std::make_unique( std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), @@ -1644,10 +2320,10 @@ TEST_F(ColumnReaderTest, DeepNestedAccessPathsFiveLevels) { TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "m", "VALUES", "arr", "*"}; + all_access_paths[0] = create_data_access_path({"root", "m", "VALUES", "arr", "*"}); TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"root", "m", "VALUES", "arr", "*", "q"}; + predicate_access_paths[0] = create_data_access_path({"root", "m", "VALUES", "arr", "*", "q"}); auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set deep access paths: " << st.to_string(); @@ -1733,7 +2409,6 @@ TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { auto value_struct = std::make_unique( std::make_shared(), std::move(null_iter), std::move(sub_iters)); - value_struct->set_column_name("value"); return value_struct; }; @@ -1741,7 +2416,6 @@ TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { auto map_offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto map_key_iter = std::make_unique(std::make_shared()); - map_key_iter->set_column_name("key"); auto map_val_iter = make_value_struct(); auto map_iterator = std::make_unique( std::make_shared(), std::move(map_null_iter), @@ -1853,7 +2527,7 @@ TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES", "a"}); TColumnAccessPaths predicate_access_paths = all_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1865,7 +2539,7 @@ TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "KEYS"}); TColumnAccessPaths predicate_access_paths; auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); @@ -1877,10 +2551,10 @@ TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*"}); TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + predicate_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES"}); auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); @@ -1891,10 +2565,10 @@ TEST_F(ColumnReaderTest, NestedReadPhaseLazyOutputMatrix) { auto top_struct = build_nested_iterator(); TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + all_access_paths[0] = create_data_access_path({"root", "col2", "*", "VALUES"}); TColumnAccessPaths predicate_access_paths; predicate_access_paths.emplace_back(); - predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + predicate_access_paths[0] = create_data_access_path({"root", "col2", "*", "KEYS"}); auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); EXPECT_TRUE(st.ok()); @@ -1978,7 +2652,7 @@ TEST_F(ColumnReaderTest, MultiAccessPaths) { // all access paths: // self.sub_col_2.*.KEYS // predicates paths empty - all_access_paths[0].data_access_path.path = {"self", "sub_col_2", "*", "KEYS"}; + all_access_paths[0] = create_data_access_path({"self", "sub_col_2", "*", "KEYS"}); TColumnAccessPaths predicate_access_paths; @@ -2054,7 +2728,7 @@ TEST_F(ColumnReaderTest, StructNullMapOnlyNextBatchSkipsSubColumns) { std::move(sub_column_iterators)); struct_iterator.set_column_name("s"); - TColumnAccessPaths null_path {create_access_path({"s", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths null_path {create_meta_access_path({"s", ColumnIterator::ACCESS_NULL})}; auto st = struct_iterator.set_access_paths(null_path, null_path); ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); EXPECT_TRUE(struct_iterator.read_null_map_only()); @@ -2089,7 +2763,7 @@ TEST_F(ColumnReaderTest, ArrayNullMapOnlyNextBatchAndReadByRowidsSkipItems) { std::move(item_iterator), std::move(null_iterator)); array_iterator.set_column_name("a"); - TColumnAccessPaths null_path {create_access_path({"a", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths null_path {create_meta_access_path({"a", ColumnIterator::ACCESS_NULL})}; auto st = array_iterator.set_access_paths(null_path, null_path); ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); EXPECT_TRUE(array_iterator.read_null_map_only()); @@ -2133,7 +2807,7 @@ TEST_F(ColumnReaderTest, MapNullMapOnlyNextBatchAndReadByRowidsSkipKeysAndValues std::move(value_iterator)); map_iterator.set_column_name("m"); - TColumnAccessPaths null_path {create_access_path({"m", ColumnIterator::ACCESS_NULL})}; + TColumnAccessPaths null_path {create_meta_access_path({"m", ColumnIterator::ACCESS_NULL})}; auto st = map_iterator.set_access_paths(null_path, null_path); ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); EXPECT_TRUE(map_iterator.read_null_map_only()); @@ -2277,11 +2951,9 @@ TEST_F(ColumnReaderTest, MapPredicateAccessAllWithOffsetKeepsKeysReadable) { auto map_reader = create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_MAP); auto key_iter = std::make_unique( create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_STRING)); - key_iter->set_column_name("key"); auto* key_ptr = key_iter.get(); auto val_iter = std::make_unique( create_test_reader(false, 0, FieldType::OLAP_FIELD_TYPE_STRING)); - val_iter->set_column_name("value"); auto* val_ptr = val_iter.get(); auto offset_iterator = create_tracking_offset_iterator(); @@ -2289,7 +2961,7 @@ TEST_F(ColumnReaderTest, MapPredicateAccessAllWithOffsetKeepsKeysReadable) { std::move(key_iter), std::move(val_iter)); map_iter.set_column_name("map_col"); - TColumnAccessPaths access_paths {create_access_path( + TColumnAccessPaths access_paths {create_meta_access_path( {"map_col", ColumnIterator::ACCESS_ALL, ColumnIterator::ACCESS_OFFSET})}; auto st = map_iter.set_access_paths(access_paths, access_paths); ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); @@ -2394,9 +3066,7 @@ TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) { auto offsets_iter = std::make_unique( std::make_unique(std::make_shared())); auto key_iter = std::make_unique(std::make_shared()); - key_iter->set_column_name("key"); auto val_iter = std::make_unique(std::make_shared()); - val_iter->set_column_name("value"); MapFileColumnIterator map_iter(map_reader, std::move(null_iter), std::move(offsets_iter), std::move(key_iter), std::move(val_iter)); @@ -2405,7 +3075,7 @@ TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) { // path: [map_col, *, OFFSET] — simulates length(map_col['c_phone']) TColumnAccessPaths all_access_paths; all_access_paths.emplace_back(); - all_access_paths[0].data_access_path.path = {"map_col", "*", "OFFSET"}; + all_access_paths[0] = create_meta_access_path({"map_col", "*", "OFFSET"}); TColumnAccessPaths predicate_access_paths; auto st = map_iter.set_access_paths(all_access_paths, predicate_access_paths); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java index a325fab81455ed..a814617fbecccb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.analysis.AccessPathInfo; +import org.apache.doris.catalog.Column; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectorContext; import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree; @@ -123,17 +124,17 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con if (slotReference.hasSubColPath()) { path.addAll(slotReference.getSubPath()); } - // Strip NULL suffix for variant sub-column access — null-flag-only optimization - // does not apply to variant sub-column data layout. List builderPath = context.accessPathBuilder.getPathList(); - if (builderPath.size() > 1 - && AccessPathInfo.ACCESS_NULL.equals(builderPath.get(builderPath.size() - 1))) { + TAccessPathType pathType = context.type; + if (pathType == TAccessPathType.META) { + // Variant readers do not support metadata-only access paths. Drop the synthetic + // NULL/OFFSET component and read the referenced variant path as ordinary data. builderPath = new ArrayList<>(builderPath.subList(0, builderPath.size() - 1)); + pathType = TAccessPathType.DATA; } path.addAll(builderPath); int slotId = slotReference.getExprId().asInt(); - slotToAccessPaths.put(slotId, new CollectAccessPathResult( - path, context.bottomFilter, TAccessPathType.DATA)); + slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, pathType)); return null; } if (dataType instanceof VariantType) { @@ -146,20 +147,37 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con return null; } if (dataType instanceof NestedColumnPrunable) { + // A META path ending in NULL directly on the slot means "read the slot's null map". + // Check the physical column's nullability (via getOriginalColumn), NOT the slot's + // nullability which may be synthetic (e.g. from outer join). If the physical column + // has no null map or is unknown, suppress this path. + // (Field-level null paths like [s, field, NULL] were already validated upstream.) + if (context.type == TAccessPathType.META + && isFunctionNullCheckPath(context.accessPathBuilder.accessPath) + && !hasPhysicalNullMap(slotReference)) { + return null; + } context.accessPathBuilder.addPrefix(slotReference.getName().toLowerCase()); ImmutableList path = Utils.fastToImmutableList(context.accessPathBuilder.accessPath); int slotId = slotReference.getExprId().asInt(); slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type)); + return null; } if (dataType.isStringLikeType()) { int slotId = slotReference.getExprId().asInt(); if (!context.accessPathBuilder.isEmpty()) { // Accessed via an offset-only function (e.g. length()) or null-check (IS NULL). // Builder already has "OFFSET"/"NULL" at the tail; add the column name as prefix. + // For META NULL paths, suppress when the physical column has no null map. + if (context.type == TAccessPathType.META + && isFunctionNullCheckPath(context.accessPathBuilder.accessPath) + && !hasPhysicalNullMap(slotReference)) { + return null; + } context.accessPathBuilder.addPrefix(slotReference.getName()); ImmutableList path = ImmutableList.copyOf(context.accessPathBuilder.accessPath); slotToAccessPaths.put(slotId, - new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + new CollectAccessPathResult(path, context.bottomFilter, context.type)); } else { // Direct access to the string column → record a DATA path so that any // concurrent offset-only path for the same slot is suppressed. @@ -172,17 +190,23 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con // For any other nullable column type (e.g. INT, BIGINT) accessed via IS NULL / IS NOT NULL: // record the [col_name, NULL] path so NestedColumnPruning can emit null-only access paths. // Skip NestedColumnPrunable types (already handled above) and string types (handled above). + // Check getOriginalColumn() rather than slotReference.nullable(): the latter may be + // inflated by outer join, while the former reflects the physical column's null map. + // Only NULL paths need the null-map check; OFFSET paths don't depend on nullability. if (!(dataType instanceof NestedColumnPrunable) && !dataType.isStringLikeType() - && !context.accessPathBuilder.isEmpty() && slotReference.nullable()) { + && isFunctionNullCheckPath(context.accessPathBuilder.accessPath) + && hasPhysicalNullMap(slotReference)) { context.accessPathBuilder.addPrefix(slotReference.getName()); ImmutableList path = ImmutableList.copyOf(context.accessPathBuilder.accessPath); int slotId = slotReference.getExprId().asInt(); slotToAccessPaths.put(slotId, - new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + new CollectAccessPathResult(path, context.bottomFilter, context.type)); } // For any other nullable column type accessed directly (not via IS NULL / length / etc.): - // record a [col_name] full-access path so that when the column is also used via IS NULL, - // stripNullSuffixPaths correctly suppresses the null-only optimization. + // record a [col_name] full-access path. When the same column also has a META NULL + // path (e.g. from IS NULL), both paths are sent to BE. The presence of a DATA path + // signals that full column data is needed, preventing the BE from entering + // NULL_MAP_ONLY mode which would read only the null bitmap. if (!(dataType instanceof NestedColumnPrunable) && !dataType.isStringLikeType() && !(dataType instanceof VariantType) && context.accessPathBuilder.isEmpty() && slotReference.nullable()) { @@ -222,6 +246,7 @@ public Void visitLength(Length length, CollectorContext context) { CollectorContext offsetContext = new CollectorContext(context.statementContext, context.bottomFilter); offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_OFFSET); + offsetContext.setType(TAccessPathType.META); return arg.accept(this, offsetContext); } // fall through to default (recurse into children with fresh contexts) @@ -240,6 +265,7 @@ public Void visitMapSize(MapSize mapSize, CollectorContext context) { CollectorContext offsetContext = new CollectorContext(context.statementContext, context.bottomFilter); offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_OFFSET); + offsetContext.setType(TAccessPathType.META); return arg.accept(this, offsetContext); } return visit(mapSize, context); @@ -259,6 +285,7 @@ public Void visitCardinality(Cardinality cardinality, CollectorContext context) CollectorContext offsetContext = new CollectorContext(context.statementContext, context.bottomFilter); offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_OFFSET); + offsetContext.setType(TAccessPathType.META); // cardinality(map_keys(m)) == cardinality(m) == cardinality(map_values(m)): // all three count map entries, so emit the same [map_col, OFFSET] path. Expression effectiveArg = (arg instanceof MapKeys || arg instanceof MapValues) @@ -298,9 +325,10 @@ public Void visitCast(Cast cast, CollectorContext context) { DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA); List replacePath = new ArrayList<>(context.accessPathBuilder.getPathList()); - if (originTree.replacePathByAnotherTree(castTree, replacePath, 0)) { + if (originTree.replacePathByAnotherTree(castTree, replacePath, 0, context.type)) { CollectorContext castContext = new CollectorContext(context.statementContext, context.bottomFilter); castContext.accessPathBuilder.accessPath.addAll(replacePath); + castContext.setType(context.type); return continueCollectAccessPath(cast.child(), castContext); } } @@ -340,6 +368,23 @@ public Void visitElementAt(ElementAt elementAt, CollectorContext context) { Expression fieldName = arguments.get(1); DataType fieldType = fieldName.getDataType(); if (fieldName.isLiteral() && (fieldType.isIntegerLikeType() || fieldType.isStringLikeType())) { + // Only emit META [s, field, NULL] when the selected field itself is nullable. + if (context.type == TAccessPathType.META + && isFunctionNullCheckPath(context.accessPathBuilder.getPathList())) { + StructField field = resolveStructField( + (StructType) first.getDataType(), fieldName); + if (field == null || !field.isNullable()) { + // Non-nullable leaf: no META NULL path for the field is needed. + // However the field must still appear in the type so pruneDataType + // preserves it — the filter expression still references + // element_at(s, 'f') IS NULL and won't be rewritten to s IS NULL. + // Fall through with a fresh DATA context to emit [s, f] DATA. + context = new CollectorContext( + context.statementContext, context.bottomFilter); + } + // Nullable leaf: fall through to add field prefix → META [s, field, NULL] + } + if (fieldType.isIntegerLikeType()) { int fieldIndex = ((Number) ((Literal) fieldName).getValue()).intValue(); List fields = ((StructType) first.getDataType()).getFields(); @@ -371,6 +416,7 @@ public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) { = new CollectorContext(context.statementContext, context.bottomFilter); removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size())); removeStarContext.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS); + removeStarContext.setType(context.type); return continueCollectAccessPath(mapKeys.getArgument(0), removeStarContext); } context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS); @@ -390,6 +436,7 @@ public Void visitMapValues(MapValues mapValues, CollectorContext context) { = new CollectorContext(context.statementContext, context.bottomFilter); removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size())); removeStarContext.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_VALUES); + removeStarContext.setType(context.type); return continueCollectAccessPath(mapValues.getArgument(0), removeStarContext); } context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_VALUES); @@ -592,11 +639,36 @@ public Void visitIsNull(IsNull isNull, CollectorContext context) { CollectorContext nullContext = new CollectorContext(context.statementContext, context.bottomFilter); nullContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_NULL); + nullContext.setType(TAccessPathType.META); return continueCollectAccessPath(arg, nullContext); } return visit(isNull, context); } + /** + * Resolve the StructField selected by a constant int/string literal. + * Returns null when the selector is not a recognized literal or index is out of bounds. + */ + // VisibleForTesting + static StructField resolveStructField(StructType structType, Expression fieldExpr) { + if (!fieldExpr.isLiteral()) { + return null; + } + if (fieldExpr.getDataType().isIntegerLikeType()) { + int index = ((Number) ((Literal) fieldExpr).getValue()).intValue(); + List fields = structType.getFields(); + if (index >= 1 && index <= fields.size()) { + return fields.get(index - 1); + } + return null; + } + if (fieldExpr.getDataType().isStringLikeType()) { + String name = ((Literal) fieldExpr).getStringValue().toLowerCase(); + return structType.getField(name); + } + return null; + } + @Override public Void visitIf(If ifExpr, CollectorContext context) { if (isFunctionNullCheckPath(context.accessPathBuilder.accessPath)) { @@ -751,15 +823,29 @@ public boolean equals(Object o) { return false; } CollectAccessPathResult that = (CollectAccessPathResult) o; - return isPredicate == that.isPredicate && Objects.equals(path, that.path); + return isPredicate == that.isPredicate + && type == that.type + && Objects.equals(path, that.path); } @Override public int hashCode() { - return path.hashCode(); + return Objects.hash(path, isPredicate, type); } } + /** + * Check whether the physical column backing a SlotReference has a null map on disk. + * Uses the catalog Column's isAllowNull, not the slot's nullable() which may be + * inflated by outer join (via withNullable(true)). Returns false when the slot has + * no physical column (conservative: assume no null map). + */ + private static boolean hasPhysicalNullMap(SlotReference slot) { + return slot.getOriginalColumn() + .map(Column::isAllowNull) + .orElse(false); + } + // if the map type is changed, we can not prune the type, because the map type need distinct the keys, // e.g. select map_values(cast(map(3.0, 1, 3.1, 2) as map)); // the result is [2] because the keys: 3.0 and 3.1 will cast to 3 and the second entry remained. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java index adb5e5eebe12d9..6427e5c5e82917 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java @@ -371,7 +371,8 @@ public Void visitLogicalFileScan(LogicalFileScan fileScan, StatementContext cont } Collection accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt()); if (!accessPaths.isEmpty()) { - scanSlotToAccessPaths.put(slot, normalizeDataSkippingOnlyAccessPaths(accessPaths)); + scanSlotToAccessPaths.put( + slot, normalizeDataSkippingOnlyAccessPaths(accessPaths)); } } return null; @@ -385,7 +386,8 @@ public Void visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, StatementCon } Collection accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt()); if (!accessPaths.isEmpty()) { - scanSlotToAccessPaths.put(slot, normalizeDataSkippingOnlyAccessPaths(accessPaths)); + scanSlotToAccessPaths.put( + slot, normalizeDataSkippingOnlyAccessPaths(accessPaths)); } } return null; @@ -418,7 +420,7 @@ static List normalizeDataSkippingOnlyAccessPaths( List normalizedAccessPaths = new ArrayList<>(); for (CollectAccessPathResult accessPath : accessPaths) { List path = accessPath.getPath(); - if (isDataSkippingOnlyAccessPath(path) && path.size() > 1) { + if (path.size() > 1 && accessPath.getType() == TAccessPathType.META) { // NULL/OFFSET suffixes are OLAP segment-reader-only optimizations. External // table and TVF readers use access paths as real nested field paths, so read // the referenced column/sub-column normally instead of sending a pseudo field. @@ -432,13 +434,4 @@ static List normalizeDataSkippingOnlyAccessPaths( } return normalizedAccessPaths; } - - private static boolean isDataSkippingOnlyAccessPath(List path) { - if (path.isEmpty()) { - return false; - } - String lastComponent = path.get(path.size() - 1); - return AccessPathInfo.ACCESS_NULL.equals(lastComponent) - || AccessPathInfo.ACCESS_OFFSET.equals(lastComponent); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index 0095d11211e5f8..5086325d66a0c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.VariantType; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.DescriptorsConstants; import org.apache.doris.thrift.TAccessPathType; import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TDataAccessPath; @@ -215,14 +216,9 @@ private static Map pruneDataType( Map slotIdToPredicateAccessTree = new LinkedHashMap<>(); Map variantSlots = new LinkedHashMap<>(); - // Segment-wise comparison preserves the distinction between the key "a.b" and path a/b. - Comparator>> pathComparator = (left, right) -> { - int comparison = comparePathSegments(left.second, right.second); - if (comparison != 0) { - return comparison; - } - return left.first.compareTo(right.first); - }; + Comparator>> pathComparator = Comparator + .comparing((Pair> path) -> path.first) + .thenComparing(path -> path.second, NestedColumnPruning::comparePathComponents); Multimap>> allAccessPaths = TreeMultimap.create( Comparator.naturalOrder(), pathComparator); @@ -233,16 +229,12 @@ private static Map pruneDataType( for (Entry> kv : slotToAccessPaths.entrySet()) { Slot slot = kv.getKey(); List collectAccessPathResults = kv.getValue(); - boolean hasRegularAccessPath = collectAccessPathResults.stream() - .anyMatch(resultItem -> !isDataSkippingOnlyAccessPath(resultItem.getPath())); if (slot.getDataType() instanceof VariantType) { variantSlots.put(slot, slot.getDataType()); for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { List path = collectAccessPathResult.getPath(); TAccessPathType pathType = collectAccessPathResult.getType(); - Pair> allPath = - normalizePredicateMetaPathForAllAccessPath( - slot, collectAccessPathResult, hasRegularAccessPath); + Pair> allPath = Pair.of(pathType, path); allAccessPaths.put(slot.getExprId().asInt(), allPath); if (collectAccessPathResult.isPredicate()) { predicateAccessPaths.put( @@ -255,9 +247,7 @@ private static Map pruneDataType( for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { List path = collectAccessPathResult.getPath(); TAccessPathType pathType = collectAccessPathResult.getType(); - Pair> allPath = - normalizePredicateMetaPathForAllAccessPath( - slot, collectAccessPathResult, hasRegularAccessPath); + Pair> allPath = Pair.of(pathType, path); DataTypeAccessTree allAccessTree = slotIdToAllAccessTree.computeIfAbsent( slot, i -> DataTypeAccessTree.ofRoot(slot, allPath.first) ); @@ -277,18 +267,15 @@ private static Map pruneDataType( } // phase 1.5: for slots with meta paths, expand map-star paths before building final - // access path lists. Predicate NULL/OFFSET paths are kept in predicateAccessPaths. - // When regular data paths also exist, allAccessPaths uses the stripped data path for - // mixed-version safety with older BEs that only understand allAccessPaths, except - // map-star predicate metadata paths. Those paths must stay unstripped until this phase - // so map.*.OFFSET can become map.KEYS + map.VALUES.OFFSET instead of broad map.*. + // access path lists. This turns map.*.OFFSET into precise map.KEYS + + // map.VALUES.OFFSET paths instead of broad map.* access. for (Entry kv : slotIdToAllAccessTree.entrySet()) { Slot slot = kv.getKey(); DataTypeAccessTree accessTree = kv.getValue(); if (!accessTree.hasOffsetPath() && !accessTree.hasNullPath()) { continue; } - // Expand both sets before stripping so covering is complete. + // Expand both sets so all-path and predicate-path routing stay consistent. expandMapStarPaths(slot, allAccessPaths); expandMapStarPaths(slot, predicateAccessPaths); } @@ -313,7 +300,7 @@ private static Map pruneDataType( new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); } - // third: build predicate access path (strip already done in phase 1.5) + // third: build predicate access paths for (Entry kv : slotIdToPredicateAccessTree.entrySet()) { Slot slot = kv.getKey(); List predicatePaths = @@ -341,26 +328,15 @@ private static Map pruneDataType( return result; } - private static Pair> normalizePredicateMetaPathForAllAccessPath( - Slot slot, CollectAccessPathResult accessPathResult, boolean hasRegularAccessPath) { - List path = accessPathResult.getPath(); - TAccessPathType pathType = accessPathResult.getType(); - if (accessPathResult.isPredicate() && hasRegularAccessPath - && isDataSkippingOnlyAccessPath(path)) { - if (hasMapStarAccessPath(slot, path)) { - // Keep map-star metadata until expandMapStarPaths() turns it into precise - // KEYS/VALUES paths. Stripping here would broaden map value reads to map.*. - return Pair.of(pathType, path); + private static int comparePathComponents(List left, List right) { + int commonSize = Math.min(left.size(), right.size()); + for (int i = 0; i < commonSize; i++) { + int result = left.get(i).compareTo(right.get(i)); + if (result != 0) { + return result; } - return Pair.of(TAccessPathType.DATA, stripDataSkippingSuffix(path)); } - return Pair.of(pathType, path); - } - - private static boolean hasMapStarAccessPath(Slot slot, List path) { - List positions = new ArrayList<>(); - findMapStarPositions(path, slot.getDataType(), positions); - return !positions.isEmpty(); + return Integer.compare(left.size(), right.size()); } /** @@ -370,10 +346,8 @@ private static boolean hasMapStarAccessPath(Slot slot, List path) { * decide which data sub-iterators can be pruned; any predicate data path missing from * allAccessPaths can therefore make predicate reads disagree with pruning. * - * Keep predicate NULL/OFFSET paths out of allAccessPaths for mixed-version safety. Older BEs - * decide current-level meta-only mode from allAccessPaths only, so sending both metadata and - * data paths there could make them skip required child data. Newer BEs still receive those - * metadata predicates through predicateAccessPaths. + * Metadata predicate paths are already retained in allAccessPaths while the access trees are + * built, so only missing data paths need to be added here. */ private static void addPredicatePathsToFinalAllAccessPaths( List predicatePaths, List allPaths) { @@ -407,20 +381,7 @@ private static boolean isPrefixPath(List prefix, List path) { } private static boolean isMetaPath(TColumnAccessPath path) { - return isDataSkippingOnlyAccessPath(getAccessPathList(path)); - } - - private static boolean isDataSkippingOnlyAccessPath(List components) { - if (components.isEmpty()) { - return false; - } - String lastComponent = components.get(components.size() - 1); - return AccessPathInfo.ACCESS_NULL.equals(lastComponent) - || AccessPathInfo.ACCESS_OFFSET.equals(lastComponent); - } - - private static List stripDataSkippingSuffix(List components) { - return new ArrayList<>(components.subList(0, components.size() - 1)); + return path.getType() == TAccessPathType.META; } private static List buildColumnAccessPaths( @@ -437,12 +398,14 @@ private static List buildColumnAccessPaths( dataAccessPath.setPath(new ArrayList<>(pathInfo.second)); TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA); accessPath.setDataAccessPath(dataAccessPath); + accessPath.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); paths.add(accessPath); } else { TMetaAccessPath dataAccessPath = new TMetaAccessPath(); dataAccessPath.setPath(new ArrayList<>(pathInfo.second)); TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META); accessPath.setMetaAccessPath(dataAccessPath); + accessPath.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); paths.add(accessPath); } // only retain access the whole root @@ -464,22 +427,12 @@ private static List buildColumnAccessPaths( } else { accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName))); } + accessPath.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); return new ArrayList<>(ImmutableList.of(accessPath)); } return paths; } - private static int comparePathSegments(List left, List right) { - int commonLength = Math.min(left.size(), right.size()); - for (int index = 0; index < commonLength; index++) { - int comparison = left.get(index).compareTo(right.get(index)); - if (comparison != 0) { - return comparison; - } - } - return Integer.compare(left.size(), right.size()); - } - private static boolean shouldSkipAccessInfo( Slot slot, DataType prunedDataType, List allPaths) { if (!prunedDataType.equals(slot.getDataType())) { @@ -650,10 +603,23 @@ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast } /** replacePathByAnotherTree */ - public boolean replacePathByAnotherTree(DataTypeAccessTree cast, List path, int index) { + public boolean replacePathByAnotherTree( + DataTypeAccessTree cast, List path, int index, TAccessPathType pathType) { if (index >= path.size()) { return true; } + if (pathType == TAccessPathType.META && index == path.size() - 1) { + String metaPath = path.get(index); + if (!metaPath.equals(AccessPathInfo.ACCESS_NULL) + && !metaPath.equals(AccessPathInfo.ACCESS_OFFSET)) { + throw new AnalysisException("unsupported metadata access path: " + metaPath); + } + // A type-changing cast can change both the physical OFFSET representation and + // nullness (for example, a failed string-to-number cast). Only translate the + // metadata suffix when the source and target node types are identical; otherwise + // visitCast falls back to reading the original value data before evaluating cast. + return type.equals(cast.type); + } if (cast.type instanceof StructType) { List fields = ((StructType) cast.type).getFields(); for (int i = 0; i < fields.size(); i++) { @@ -662,17 +628,17 @@ public boolean replacePathByAnotherTree(DataTypeAccessTree cast, List pa String originFieldName = ((StructType) type).getFields().get(i).getName(); path.set(index, originFieldName); return children.get(originFieldName).replacePathByAnotherTree( - cast.children.get(castFieldName), path, index + 1 + cast.children.get(castFieldName), path, index + 1, pathType ); } } } else if (cast.type instanceof ArrayType) { return children.values().iterator().next().replacePathByAnotherTree( - cast.children.values().iterator().next(), path, index + 1); + cast.children.values().iterator().next(), path, index + 1, pathType); } else if (cast.type instanceof MapType) { String fieldName = path.get(index); return children.get(AccessPathInfo.ACCESS_MAP_VALUES).replacePathByAnotherTree( - cast.children.get(fieldName), path, index + 1 + cast.children.get(fieldName), path, index + 1, pathType ); } return false; @@ -689,10 +655,11 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType this.pathType = TAccessPathType.DATA; } - // NULL path component: the column is accessed only via IS NULL / IS NOT NULL. + // Terminal NULL component: the column is accessed only via IS NULL / IS NOT NULL. // Mark null-check-only and return without setting accessAll or accessPartialChild, // so that parent nodes can distinguish "null-only leaf" from "has real sub-access". - if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_NULL)) { + if (pathType == TAccessPathType.META && accessIndex == path.size() - 1 + && path.get(accessIndex).equals(AccessPathInfo.ACCESS_NULL)) { hasNullPath = true; return; } @@ -710,8 +677,9 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType } return; } else if (this.type.isArrayType()) { - if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_OFFSET)) { - // length(array_col) — only the offset array is needed, not element data. + if (pathType == TAccessPathType.META && accessIndex == path.size() - 1 + && path.get(accessIndex).equals(AccessPathInfo.ACCESS_OFFSET)) { + // cardinality(array_col) — only the offset array is needed, not element data. hasOffsetPath = true; return; } @@ -723,8 +691,9 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType return; } else if (this.type.isMapType()) { String fieldName = path.get(accessIndex); - if (fieldName.equals(AccessPathInfo.ACCESS_OFFSET)) { - // length(map_col) — only the offset array is needed, not key/value data. + if (pathType == TAccessPathType.META && accessIndex == path.size() - 1 + && fieldName.equals(AccessPathInfo.ACCESS_OFFSET)) { + // cardinality(map_col) — only the offset array is needed, not key/value data. hasOffsetPath = true; return; } @@ -761,7 +730,8 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType } else if (type.isStringLikeType()) { // String leaf accessed via the offset array (e.g. path ends in "offset"). // Mark offset-only so pruneDataType() can return BigIntType instead of full data. - if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_OFFSET)) { + if (pathType == TAccessPathType.META && accessIndex == path.size() - 1 + && path.get(accessIndex).equals(AccessPathInfo.ACCESS_OFFSET)) { hasOffsetPath = true; return; // do NOT set accessAll — offset-only is distinguishable from full access } @@ -954,7 +924,8 @@ private static List>> expandOnePath( } result.add(Pair.of(type, allValues)); - // KEYS-terminating path for each position + // KEYS-terminating path for each position. Map lookup still reads key data even when + // the values path only reads metadata. for (int i = 0; i < n; i++) { int keysPos = positions.get(i); List keysPath = new ArrayList<>(); @@ -966,7 +937,7 @@ private static List>> expandOnePath( keysPath.add(component); } keysPath.add(AccessPathInfo.ACCESS_MAP_KEYS); - result.add(Pair.of(type, keysPath)); + result.add(Pair.of(TAccessPathType.DATA, keysPath)); } return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java index c9cf0dfc259210..e4ca2a096007a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.MoreFieldsThread; +import org.apache.doris.thrift.DescriptorsConstants; import org.apache.doris.thrift.TAccessPathType; import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TDataAccessPath; @@ -635,6 +636,7 @@ private List replaceIcebergAccessPathToId( ); TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.DATA); newAccessPath.data_access_path = new TDataAccessPath(icebergColumnAccessPath); + copyAccessPathVersion(accessPath, newAccessPath); replacedAccessPaths.add(newAccessPath); } else { icebergColumnAccessPath.addAll(accessPath.meta_access_path.path); @@ -643,6 +645,7 @@ private List replaceIcebergAccessPathToId( ); TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.META); newAccessPath.meta_access_path = new TMetaAccessPath(icebergColumnAccessPath); + copyAccessPathVersion(accessPath, newAccessPath); replacedAccessPaths.add(newAccessPath); } } @@ -688,6 +691,15 @@ private void replaceIcebergAccessPathToId(List originPath, int index, Da } } + /** Keep the FE/BE access-path wire contract when a path is rebuilt with Iceberg field ids. */ + private static void copyAccessPathVersion(TColumnAccessPath source, TColumnAccessPath target) { + if (source.isSetVersion()) { + target.setVersion(source.getVersion()); + } else { + target.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + } + } + private void tryRecordReplaceSlots(Plan plan, Object checkObj, Set shouldReplaceSlots) { if (checkObj instanceof SupportPruneNestedColumn && ((SupportPruneNestedColumn) checkObj).supportPruneNestedColumn()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index ad6fb4e1901312..3922522b47f584 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ArrayItemReference; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -41,8 +42,10 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.NestedColumnPrunable; import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.VariantType; @@ -50,19 +53,23 @@ import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; +import org.apache.doris.thrift.DescriptorsConstants; import org.apache.doris.thrift.TAccessPathType; import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TDataAccessPath; import org.apache.doris.thrift.TMetaAccessPath; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -139,6 +146,39 @@ public void createTable() throws Exception { + " >\n" + ") properties ('replication_num'='1')"); + createTable("create table meta_name_tbl(\n" + + " id int,\n" + + " s struct<`NULL`: string, `OFFSET`: string>\n" + + ") properties ('replication_num'='1')"); + + // Nested struct for verifying multi-level META NULL path generation. + // All fields are nullable by default (no NOT NULL in DDL). + createTable("create table nested_struct_tbl(\n" + + " id int,\n" + + " s struct<`outer`: struct<`a`: string, `inner_f`: string>>\n" + + ") properties ('replication_num'='1')"); + + // Tables for outer-join nullability test: verifying that synthetic nullability + // from outer join does NOT cause META NULL paths on physically NOT NULL columns. + createTable("create table driving_tbl(\n" + + " id int\n" + + ") properties ('replication_num'='1')"); + + createTable("create table not_null_struct_tbl(\n" + + " id int,\n" + + " s struct not null\n" + + ") properties ('replication_num'='1')"); + + // Table for verifying that NOT NULL struct FIELD is preserved in pruned type + // when a sibling field is also accessed. + // Doris DDL does not support NOT NULL on individual struct fields, so the + // NOT NULL branch is tested via testNotNullFieldPreservedInAccessPaths + // which constructs the StructType programmatically. + createTable("create table nullable_struct_tbl_two_fields(\n" + + " id int,\n" + + " s struct\n" + + ") properties ('replication_num'='1')"); + connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name()); connectContext.getSessionVariable().enableNereidsTimeout = false; } @@ -171,7 +211,10 @@ public void testMap() throws Exception { public void testMapElementLengthWithMapValuesKeepsKeysPath() throws Exception { assertColumn("select length(map_col['a']), map_values(map_col)[1] from str_tbl", "map", - ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES"), path("map_col", "VALUES", "OFFSET")), + ImmutableList.of( + path("map_col", "KEYS"), + path("map_col", "VALUES"), + metaPath("map_col", "VALUES", "OFFSET")), ImmutableList.of() ); } @@ -180,7 +223,7 @@ public void testMapElementLengthWithMapValuesKeepsKeysPath() throws Exception { public void testCardinalityArrayElementKeepsOffsetPath() throws Exception { assertAllAccessPathsContain( "select cardinality(element_at(a, 1)) from nested_array_tbl", - ImmutableList.of(path("a", "*", "OFFSET")), + ImmutableList.of(metaPath("a", "*", "OFFSET")), ImmutableList.of(path("a", "*"))); } @@ -188,21 +231,21 @@ public void testCardinalityArrayElementKeepsOffsetPath() throws Exception { public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception { assertColumn("select cardinality(map_arr_col['a']) from map_array_tbl", "map>", - ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES", "OFFSET")), + ImmutableList.of(path("map_arr_col", "KEYS"), metaPath("map_arr_col", "VALUES", "OFFSET")), ImmutableList.of()); } @Test - public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { + public void testFullFieldAccessKeepsExactMetadataPath() throws Exception { assertColumn("select struct_element(s, 'city') from tbl " + "where struct_element(s, 'city') is null", "struct", - ImmutableList.of(path("s", "city")), - ImmutableList.of(path("s", "city", "NULL"))); + ImmutableList.of(path("s", "city"), metaPath("s", "city", "NULL")), + ImmutableList.of(metaPath("s", "city", "NULL"))); assertColumn("select cardinality(struct_element(s, 'data')), struct_element(s, 'data') from tbl", "struct>>>", - ImmutableList.of(path("s", "data"), path("s", "data", "OFFSET")), + ImmutableList.of(path("s", "data"), metaPath("s", "data", "OFFSET")), ImmutableList.of()); assertColumn("select cardinality(a), a from nested_array_tbl", @@ -212,7 +255,10 @@ public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { assertColumn("select cardinality(map_arr_col['a']), map_arr_col['a'] from map_array_tbl", "map>", - ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES"), path("map_arr_col", "VALUES", "OFFSET")), + ImmutableList.of( + path("map_arr_col", "KEYS"), + path("map_arr_col", "VALUES"), + metaPath("map_arr_col", "VALUES", "OFFSET")), ImmutableList.of()); } @@ -228,17 +274,17 @@ public void testCardinalityMapElementOffsetPredicateStaysOutOfAllAccessPaths() t allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); } - Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "OFFSET")), + Assertions.assertFalse(allAccessPaths.contains(metaPath("s", "m", "*", "OFFSET")), "allAccessPaths=" + allAccessPaths); Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "KEYS")), "allAccessPaths=" + allAccessPaths); Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified")), "allAccessPaths=" + allAccessPaths); - Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "OFFSET")), + Assertions.assertTrue(allAccessPaths.contains(metaPath("s", "m", "VALUES", "OFFSET")), "allAccessPaths=" + allAccessPaths); Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "KEYS")), "predicateAccessPaths=" + predicateAccessPaths); - Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "VALUES", "OFFSET")), + Assertions.assertTrue(predicateAccessPaths.contains(metaPath("s", "m", "VALUES", "OFFSET")), "predicateAccessPaths=" + predicateAccessPaths); } @@ -255,17 +301,17 @@ public void testMapElementArrayNullPredicateStaysOutOfAllAccessPaths() throws Ex allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); } - Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "NULL")), + Assertions.assertFalse(allAccessPaths.contains(metaPath("s", "m", "*", "NULL")), "allAccessPaths=" + allAccessPaths); Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "KEYS")), "allAccessPaths=" + allAccessPaths); Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "*", "verified")), "allAccessPaths=" + allAccessPaths); - Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "VALUES", "NULL")), + Assertions.assertTrue(allAccessPaths.contains(metaPath("s", "m", "VALUES", "NULL")), "allAccessPaths=" + allAccessPaths); Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "KEYS")), "predicateAccessPaths=" + predicateAccessPaths); - Assertions.assertTrue(predicateAccessPaths.contains(path("s", "m", "VALUES", "NULL")), + Assertions.assertTrue(predicateAccessPaths.contains(metaPath("s", "m", "VALUES", "NULL")), "predicateAccessPaths=" + predicateAccessPaths); } @@ -350,6 +396,18 @@ public void testVariantPredicateAccessPath() throws Exception { ); } + @Test + public void testVariantRootNullCheckFallsBackToData() { + SlotReference slot = rewriteAndFindScanSlot( + "select 1 from variant_tbl where v is null", "v", false); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("v"))), + new TreeSet<>(slot.getAllAccessPaths().get())); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("v"))), + new TreeSet<>(slot.getPredicateAccessPaths().get())); + } + @Test public void testVariantProjectAndPredicateAccessPaths() throws Exception { assertVariantSubColumnSlots("select v['a'] from variant_tbl where v['b']['c'] = 1", @@ -490,6 +548,25 @@ public void testPruneCast() throws Exception { ImmutableList.of() ); + assertColumn("select length(element_at(cast(c_struct as struct), 'v')) from str_tbl", + "struct", + ImmutableList.of(metaPath("c_struct", "f3", "OFFSET")), + ImmutableList.of() + ); + + assertColumn("select length(element_at(cast(c_struct as struct), 'k')) from str_tbl", + "struct", + ImmutableList.of(path("c_struct")), + ImmutableList.of() + ); + + assertColumn("select 1 from str_tbl " + + "where element_at(cast(c_struct as struct), 'v') is null", + "struct", + ImmutableList.of(path("c_struct")), + ImmutableList.of(path("c_struct")) + ); + assertColumns("select element_at(s, 'city') from (select * from tbl union all select * from tbl2)t", ImmutableList.of( Triple.of( @@ -645,63 +722,67 @@ public void testProject() throws Exception { public void testFilter() throws Throwable { assertColumn("select 100 from tbl where s is not null", "struct>>>", - ImmutableList.of(path("s", "NULL")), - ImmutableList.of(path("s", "NULL")) + ImmutableList.of(metaPath("s", "NULL")), + ImmutableList.of(metaPath("s", "NULL")) ); - // The IF expression itself is not collected as a null-only parent access here; the - // struct_element predicate is covered by the stripped parent path in allPaths. + // The IF expression contributes a parent metadata path, while the struct_element + // predicate keeps its independent data path. assertColumn("select 100 from tbl where if(id = 1, null, s) is not null or element_at(s, 'city') = 'beijing'", - "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of(path("s", "NULL"), path("s", "city")) + "struct", + ImmutableList.of(metaPath("s", "NULL"), path("s", "city")), + ImmutableList.of(metaPath("s", "NULL"), path("s", "city")) ); assertColumn("select 100 from tbl where struct_element(s, 'city') is not null", "struct", - ImmutableList.of(path("s", "city", "NULL")), - ImmutableList.of(path("s", "city", "NULL")) + ImmutableList.of(metaPath("s", "city", "NULL")), + ImmutableList.of(metaPath("s", "city", "NULL")) ); assertColumn("select 100 from tbl where struct_element(s, 'data') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "NULL")), - ImmutableList.of(path("s", "data", "NULL")) + ImmutableList.of(metaPath("s", "data", "NULL")), + ImmutableList.of(metaPath("s", "data", "NULL")) ); assertColumn("select 100 from tbl where element_at(s, 'data')[1] is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "NULL")), - ImmutableList.of(path("s", "data", "*", "NULL")) + ImmutableList.of(metaPath("s", "data", "*", "NULL")), + ImmutableList.of(metaPath("s", "data", "*", "NULL")) ); assertColumn("select 100 from tbl where map_keys(element_at(s, 'data')[1]) is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "NULL")), - ImmutableList.of(path("s", "data", "*", "NULL")) + ImmutableList.of(metaPath("s", "data", "*", "NULL")), + ImmutableList.of(metaPath("s", "data", "*", "NULL")) ); assertColumn("select 100 from tbl where map_values(element_at(s, 'data')[1]) is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "NULL")), - ImmutableList.of(path("s", "data", "*", "NULL")) + ImmutableList.of(metaPath("s", "data", "*", "NULL")), + ImmutableList.of(metaPath("s", "data", "*", "NULL")) ); assertColumn("select 100 from tbl where element_at(map_values(element_at(s, 'data')[1])[1], 'a') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "VALUES", "a", "NULL")), - ImmutableList.of(path("s", "data", "*", "VALUES", "a", "NULL")) + ImmutableList.of(metaPath("s", "data", "*", "VALUES", "a", "NULL")), + ImmutableList.of(metaPath("s", "data", "*", "VALUES", "a", "NULL")) ); assertColumn("select 100 from tbl where element_at(s, 'data')[1][1] is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "NULL")), - ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "NULL")) + ImmutableList.of(path("s", "data", "*", "KEYS"), metaPath("s", "data", "*", "VALUES", "NULL")), + ImmutableList.of(path("s", "data", "*", "KEYS"), metaPath("s", "data", "*", "VALUES", "NULL")) ); assertColumn("select 100 from tbl where element_at(element_at(s, 'data')[1][1], 'a') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "a", "NULL")), - ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "a", "NULL")) + ImmutableList.of(path("s", "data", "*", "KEYS"), + metaPath("s", "data", "*", "VALUES", "a", "NULL")), + ImmutableList.of(path("s", "data", "*", "KEYS"), + metaPath("s", "data", "*", "VALUES", "a", "NULL")) ); assertColumn("select 100 from tbl where element_at(element_at(s, 'data')[1][1], 'b') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "b", "NULL")), - ImmutableList.of(path("s", "data", "*", "KEYS"), path("s", "data", "*", "VALUES", "b", "NULL")) + ImmutableList.of(path("s", "data", "*", "KEYS"), + metaPath("s", "data", "*", "VALUES", "b", "NULL")), + ImmutableList.of(path("s", "data", "*", "KEYS"), + metaPath("s", "data", "*", "VALUES", "b", "NULL")) ); } @@ -712,24 +793,24 @@ public void testMapKeysAndValuesFunctionNullCheckUseParentMapNullPath() throws E // read the parent map null map, not the KEYS/VALUES child null maps. assertColumn("select 100 from str_tbl where map_keys(map_col) is null", "map", - ImmutableList.of(path("map_col", "NULL")), - ImmutableList.of(path("map_col", "NULL")) + ImmutableList.of(metaPath("map_col", "NULL")), + ImmutableList.of(metaPath("map_col", "NULL")) ); assertColumn("select 100 from str_tbl where map_values(map_col) is null", "map", - ImmutableList.of(path("map_col", "NULL")), - ImmutableList.of(path("map_col", "NULL")) + ImmutableList.of(metaPath("map_col", "NULL")), + ImmutableList.of(metaPath("map_col", "NULL")) ); assertColumn("select map_keys(map_col) from str_tbl where map_keys(map_col) is null", "map", - ImmutableList.of(path("map_col")), - ImmutableList.of(path("map_col", "NULL")) + ImmutableList.of(path("map_col", "KEYS"), metaPath("map_col", "NULL")), + ImmutableList.of(metaPath("map_col", "NULL")) ); assertColumn("select map_values(map_col) from str_tbl where map_values(map_col) is null", "map", - ImmutableList.of(path("map_col")), - ImmutableList.of(path("map_col", "NULL")) + ImmutableList.of(path("map_col", "VALUES"), metaPath("map_col", "NULL")), + ImmutableList.of(metaPath("map_col", "NULL")) ); } @@ -737,14 +818,19 @@ public void testMapKeysAndValuesFunctionNullCheckUseParentMapNullPath() throws E public void testProjectFilter() throws Throwable { assertColumn("select element_at(s, 'data') from tbl where element_at(s, 'city') is not null", "struct>>>", - ImmutableList.of(path("s", "city"), path("s", "data")), - ImmutableList.of(path("s", "city", "NULL")) + ImmutableList.of( + path("s", "data"), + metaPath("s", "city", "NULL")), + ImmutableList.of(metaPath("s", "city", "NULL")) ); assertColumn("select element_at(s, 'data') from tbl where element_at(s, 'city') is not null and element_at(s, 'data') is not null", "struct>>>", - ImmutableList.of(path("s", "city"), path("s", "data")), - ImmutableList.of(path("s", "city", "NULL"), path("s", "data", "NULL")) + ImmutableList.of( + path("s", "data"), + metaPath("s", "city", "NULL"), + metaPath("s", "data", "NULL")), + ImmutableList.of(metaPath("s", "city", "NULL"), metaPath("s", "data", "NULL")) ); } @@ -1150,6 +1236,51 @@ public void testDataTypeAccessTreeKeepsVariantTerminalPath() { ((StructType) prunedType).getFields().get(0).getDataType()); } + @Test + public void testDataPathNamedLikeMetadataComponent() { + StructType structType = new StructType(ImmutableList.of( + new StructField("NULL", StringType.INSTANCE, true, ""), + new StructField("OFFSET", StringType.INSTANCE, true, ""))); + SlotReference slot = new SlotReference("s", structType); + + DataTypeAccessTree nullFieldTree = DataTypeAccessTree.ofRoot(slot, TAccessPathType.DATA); + nullFieldTree.setAccessByPath(ImmutableList.of("s", "NULL"), 0, TAccessPathType.DATA); + Assertions.assertEquals("STRUCT<`null`:TEXT>", nullFieldTree.pruneDataType().get().toSql()); + + DataTypeAccessTree offsetFieldTree = DataTypeAccessTree.ofRoot(slot, TAccessPathType.DATA); + offsetFieldTree.setAccessByPath(ImmutableList.of("s", "OFFSET"), 0, TAccessPathType.DATA); + Assertions.assertEquals("STRUCT", offsetFieldTree.pruneDataType().get().toSql()); + + DataTypeAccessTree nullMetadataTree = DataTypeAccessTree.ofRoot(slot, TAccessPathType.META); + nullMetadataTree.setAccessByPath( + ImmutableList.of("s", "NULL", "NULL"), 0, TAccessPathType.META); + Assertions.assertEquals("STRUCT<`null`:TEXT>", nullMetadataTree.pruneDataType().get().toSql()); + + DataTypeAccessTree offsetMetadataTree = DataTypeAccessTree.ofRoot(slot, TAccessPathType.META); + offsetMetadataTree.setAccessByPath( + ImmutableList.of("s", "OFFSET", "OFFSET"), 0, TAccessPathType.META); + Assertions.assertEquals("STRUCT", offsetMetadataTree.pruneDataType().get().toSql()); + + CollectAccessPathResult dataPath = new CollectAccessPathResult( + ImmutableList.of("s", "NULL"), false, TAccessPathType.DATA); + CollectAccessPathResult metadataPath = new CollectAccessPathResult( + ImmutableList.of("s", "NULL"), false, TAccessPathType.META); + Assertions.assertNotEquals(dataPath, metadataPath); + } + + @Test + public void testMetadataPathBelowSameNamedStructField() throws Exception { + assertColumn("select 1 from meta_name_tbl where element_at(s, 'NULL') is null", + "struct", + ImmutableList.of(metaPath("s", "null", "NULL")), + ImmutableList.of(metaPath("s", "null", "NULL"))); + + assertColumn("select length(element_at(s, 'OFFSET')) from meta_name_tbl", + "struct", + ImmutableList.of(metaPath("s", "offset", "OFFSET")), + ImmutableList.of()); + } + @Test public void testWithVariant() throws Exception { connectContext.getSessionVariable().enableDecimal256 = true; @@ -1451,8 +1582,8 @@ public void testStructIsNullPruning() throws Exception { // struct column IS NULL → null-only access, emit [s, NULL] path, type stays struct assertColumn("select 1 from tbl where s is null", "struct>>>", - ImmutableList.of(path("s", "NULL")), - ImmutableList.of(path("s", "NULL"))); + ImmutableList.of(metaPath("s", "NULL")), + ImmutableList.of(metaPath("s", "NULL"))); } @Test @@ -1460,34 +1591,80 @@ public void testStructIsNotNullPruning() throws Exception { // struct column IS NOT NULL → same null-only access pattern assertColumn("select 1 from tbl where s is not null", "struct>>>", - ImmutableList.of(path("s", "NULL")), - ImmutableList.of(path("s", "NULL"))); + ImmutableList.of(metaPath("s", "NULL")), + ImmutableList.of(metaPath("s", "NULL"))); + } + + @Test + public void testResolveStructFieldNullable() { + StructType type = new StructType(ImmutableList.of( + new StructField("not_null_f", StringType.INSTANCE, false, ""), + new StructField("nullable_f", StringType.INSTANCE, true, "") + )); + // String-like literal: select by name + StructField notNullField = AccessPathExpressionCollector.resolveStructField( + type, new org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral("not_null_f")); + Assertions.assertNotNull(notNullField); + Assertions.assertFalse(notNullField.isNullable()); + + StructField nullableField = AccessPathExpressionCollector.resolveStructField( + type, new org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral("nullable_f")); + Assertions.assertNotNull(nullableField); + Assertions.assertTrue(nullableField.isNullable()); + + // Integer-like literal: select by 1-based index + StructField fieldByIndex = AccessPathExpressionCollector.resolveStructField( + type, new org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral(1)); + Assertions.assertNotNull(fieldByIndex); + Assertions.assertEquals("not_null_f", fieldByIndex.getName()); + Assertions.assertFalse(fieldByIndex.isNullable()); + + // Out-of-bounds index returns null + StructField outOfBounds = AccessPathExpressionCollector.resolveStructField( + type, new org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral(99)); + Assertions.assertNull(outOfBounds); + + // Non-literal returns null + StructField nonLiteral = AccessPathExpressionCollector.resolveStructField( + type, NullLiteral.INSTANCE); + Assertions.assertNull(nonLiteral); + } + + @Test + public void testScalarIsNullProducesMetaPath() { + SlotReference slot = rewriteAndFindScanSlot("select 1 from tbl where id is null", "id", false); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(metaPath("id", "NULL"))), + new TreeSet<>(slot.getAllAccessPaths().get())); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(metaPath("id", "NULL"))), + new TreeSet<>(slot.getPredicateAccessPaths().get())); } @Test public void testStructIsNullMixedAccess() throws Exception { - // Predicate metadata paths stay in predicatePaths. allPaths keeps the stripped data path - // shape when ordinary data paths also exist, so older BEs do not switch the mixed read to - // current-level metadata-only mode. + // Predicate metadata paths stay typed in allPaths even when another child needs data. assertColumn("select element_at(s, 'city') from tbl where s is null", - "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of(path("s", "NULL"))); + "struct", + ImmutableList.of(path("s", "city"), metaPath("s", "NULL")), + ImmutableList.of(metaPath("s", "NULL"))); assertColumn("select s from tbl where element_at(s, 'city') is null", "struct>>>", ImmutableList.of(path("s")), - ImmutableList.of(path("s", "city", "NULL"))); + ImmutableList.of(metaPath("s", "city", "NULL"))); // This shape is closer to the production bug: one predicate needs the parent // null map, another predicate needs a child null map, and the projection needs - // a different child data path. allPaths strips predicate metadata to data paths and - // collapses to the whole struct for mixed-version safety. + // a different child data path. Keep all three requirements independent. assertColumn("select element_at(s, 'data') from tbl " + "where s is null or element_at(s, 'city') is null", "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of(path("s", "NULL"), path("s", "city", "NULL"))); + ImmutableList.of( + path("s", "data"), + metaPath("s", "NULL"), + metaPath("s", "city", "NULL")), + ImmutableList.of(metaPath("s", "NULL"), metaPath("s", "city", "NULL"))); } @Test @@ -1497,7 +1674,7 @@ public void testStringLengthPruning() throws Exception { "select length(str_col) from str_tbl", "str_col", true, - ImmutableList.of(path("str_col", "OFFSET"))); + ImmutableList.of(metaPath("str_col", "OFFSET"))); // ── Case 2: length(str_col) + direct projection of str_col ─ suppressed ───── assertStringColumn( @@ -1513,13 +1690,13 @@ public void testStringLengthPruning() throws Exception { false, ImmutableList.of()); - // ── Case 4: length applied to a struct field ─ struct pruned to bigint field ─ + // ── Case 4: length applied to a struct field ─ metadata-only field read ──── // c_struct has {f1:int, f3:string}; only f3 accessed offset-only → - // pruned type is struct, access path is DATA(["c_struct","f3","offset"]) + // pruned type is struct, access path is META(["c_struct","f3","OFFSET"]) assertColumn( "select length(struct_element(c_struct, 'f3')) from str_tbl", "struct", - ImmutableList.of(path("c_struct", "f3", "OFFSET")), + ImmutableList.of(metaPath("c_struct", "f3", "OFFSET")), ImmutableList.of()); // ── Case 5: length(struct field) + direct read of same field ─ suppressed ─── @@ -1528,29 +1705,47 @@ public void testStringLengthPruning() throws Exception { assertColumn( "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from str_tbl", "struct", - ImmutableList.of(path("c_struct", "f3"), path("c_struct", "f3", "OFFSET")), + ImmutableList.of(path("c_struct", "f3"), metaPath("c_struct", "f3", "OFFSET")), + ImmutableList.of()); + + assertColumn( + "select length(map_keys(map_col)[1]) from str_tbl", + "map", + ImmutableList.of(metaPath("map_col", "KEYS", "OFFSET")), + ImmutableList.of()); + + assertColumn( + "select length(map_values(map_col)[1]) from str_tbl", + "map", + ImmutableList.of(metaPath("map_col", "VALUES", "OFFSET")), ImmutableList.of()); } @Test - public void testNonOlapDataSkippingOnlyAccessPathFallback() { + public void testNonOlapMetadataAccessPathFallback() { List normalizedAccessPaths = AccessPathPlanCollector.normalizeDataSkippingOnlyAccessPaths(ImmutableList.of( + new CollectAccessPathResult( + ImmutableList.of("s", "city", "NULL"), true, TAccessPathType.META), new CollectAccessPathResult( ImmutableList.of("s", "city", "NULL"), true, TAccessPathType.DATA), new CollectAccessPathResult( - ImmutableList.of("array_column", "OFFSET"), false, TAccessPathType.DATA), + ImmutableList.of("s", "NULL"), false, TAccessPathType.DATA), + new CollectAccessPathResult( + ImmutableList.of("s", "OFFSET"), false, TAccessPathType.DATA), new CollectAccessPathResult( ImmutableList.of("s", "city"), false, TAccessPathType.DATA))); - Assertions.assertEquals(3, normalizedAccessPaths.size()); + Assertions.assertEquals(5, normalizedAccessPaths.size()); Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(0).getPath()); Assertions.assertTrue(normalizedAccessPaths.get(0).isPredicate()); Assertions.assertEquals(TAccessPathType.DATA, normalizedAccessPaths.get(0).getType()); - Assertions.assertEquals(ImmutableList.of("array_column"), normalizedAccessPaths.get(1).getPath()); - Assertions.assertFalse(normalizedAccessPaths.get(1).isPredicate()); + Assertions.assertEquals(ImmutableList.of("s", "city", "NULL"), normalizedAccessPaths.get(1).getPath()); + Assertions.assertTrue(normalizedAccessPaths.get(1).isPredicate()); Assertions.assertEquals(TAccessPathType.DATA, normalizedAccessPaths.get(1).getType()); - Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(2).getPath()); + Assertions.assertEquals(ImmutableList.of("s", "NULL"), normalizedAccessPaths.get(2).getPath()); + Assertions.assertEquals(ImmutableList.of("s", "OFFSET"), normalizedAccessPaths.get(3).getPath()); + Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(4).getPath()); } @Test @@ -1558,10 +1753,10 @@ public void testMvRewritePlanFragmentSkipsNullOnlyAccessPath() { SlotReference normalSlot = rewriteAndFindScanSlot( "select 1 from str_tbl where str_col is not null", "str_col", false); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))), + new TreeSet<>(ImmutableList.of(metaPath("str_col", "NULL"))), new TreeSet<>(normalSlot.getAllAccessPaths().get())); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))), + new TreeSet<>(ImmutableList.of(metaPath("str_col", "NULL"))), new TreeSet<>(normalSlot.getPredicateAccessPaths().get())); // MV fragment: IS NULL degrades to full column read via default visitor. @@ -1573,10 +1768,10 @@ public void testMvRewritePlanFragmentSkipsNullOnlyAccessPath() { SlotReference nestedNormalSlot = rewriteAndFindScanSlot( "select 1 from tbl where struct_element(s, 'city') is not null", "s", false); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))), + new TreeSet<>(ImmutableList.of(metaPath("s", "city", "NULL"))), new TreeSet<>(nestedNormalSlot.getAllAccessPaths().get())); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))), + new TreeSet<>(ImmutableList.of(metaPath("s", "city", "NULL"))), new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get())); // MV fragment: IS NULL degrades to element_at via default visitor, @@ -1596,10 +1791,10 @@ public void testMvRewritePlanFragmentSkipsOffsetOnlyAccessPath() { SlotReference normalSlot = rewriteAndFindScanSlot( "select 1 from str_tbl where length(str_col) > 0", "str_col", false); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("str_col", "OFFSET"))), + new TreeSet<>(ImmutableList.of(metaPath("str_col", "OFFSET"))), new TreeSet<>(normalSlot.getAllAccessPaths().get())); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("str_col", "OFFSET"))), + new TreeSet<>(ImmutableList.of(metaPath("str_col", "OFFSET"))), new TreeSet<>(normalSlot.getPredicateAccessPaths().get())); SlotReference fragmentSlot = rewriteAndFindScanSlot( @@ -1610,10 +1805,10 @@ public void testMvRewritePlanFragmentSkipsOffsetOnlyAccessPath() { "select 1 from str_tbl where length(struct_element(c_struct, 'f3')) > 0", "c_struct", false); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))), + new TreeSet<>(ImmutableList.of(metaPath("c_struct", "f3", "OFFSET"))), new TreeSet<>(nestedNormalSlot.getAllAccessPaths().get())); Assertions.assertEquals( - new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))), + new TreeSet<>(ImmutableList.of(metaPath("c_struct", "f3", "OFFSET"))), new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get())); // MV fragment: length() degrades to element_at via default visitor, @@ -1729,6 +1924,7 @@ private Pair> collectComplexSlots(String sql) private TColumnAccessPath path(String... path) { TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA); accessPath.data_access_path = new TDataAccessPath(ImmutableList.copyOf(path)); + accessPath.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); return accessPath; } @@ -1754,6 +1950,7 @@ private interface CheckedRunnable { private TColumnAccessPath metaPath(String... path) { TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META); accessPath.meta_access_path = new TMetaAccessPath(ImmutableList.copyOf(path)); + accessPath.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); return accessPath; } @@ -1803,4 +2000,152 @@ private void assertVariantSubColumnSlotCount(String sql, List expectedSu Assertions.assertEquals(expectedCount, actualCount); } + + /** + * Verify that synthetic nullability from outer join does NOT cause META NULL paths + * on physically NOT NULL columns. When a NOT NULL struct sits on the nullable side + * of a LEFT JOIN, the slot's {@code nullable()} returns true (from outer join + * semantics), but {@code getOriginalColumn().isAllowNull()} returns false + * (physical column has no null map). The fix in AccessPathExpressionCollector + * should suppress the {@code [s, NULL]} META path in this case. + */ + @Test + public void testNotNullStructOnOuterJoinNullableSide() throws Exception { + // driving_tbl LEFT JOIN not_null_struct_tbl: + // not_null_struct_tbl.s is NOT NULL in the schema, but after LEFT JOIN the + // slot becomes nullable (right side of LEFT JOIN → withNullable(true)). + // element_at(s, 'f') IS NULL in WHERE: + // - s.nullable() = true (synthetic, from outer join) + // - s.getOriginalColumn().isAllowNull() = false (physical, no null map) + // Expected: [s, f] DATA is present (field is read for IS NULL evaluation), + // [s, NULL] META must NOT be present (no physical null map). + assertAllAccessPathsContain( + "select driving_tbl.id from driving_tbl" + + " left join not_null_struct_tbl" + + " on driving_tbl.id = not_null_struct_tbl.id" + + " where element_at(not_null_struct_tbl.s, 'f') is null", + // expect-contain: field is read (DATA path) + ImmutableList.of(path("s", "f")), + // expect-NOT-contain: struct-level NULL path must be suppressed + ImmutableList.of(metaPath("s", "NULL"))); + } + + /** + * Verifies that a NOT NULL struct field is preserved in the pruned type when a + * sibling field is accessed via SELECT and the IS NULL check emits struct-level + * META NULL. Before the fix, the early return at visitElementAt dropped the NOT NULL + * field's path, causing pruneDataType to remove it from the struct type. The filter + * expression still referenced element_at(s, 'f') IS NULL and could not be rebuilt. + */ + @Test + public void testNullableFieldPreservedWithSiblingProjection() throws Exception { + // s STRUCT NULL (both fields nullable by default) + // SELECT element_at(s, 'g') → [s, g] DATA + // WHERE element_at(s, 'f') IS NULL → [s, NULL] META + [s, f, NULL] META + // Both fields preserved in pruned type. + assertColumn( + "select element_at(s, 'g') from nullable_struct_tbl_two_fields" + + " where element_at(s, 'f') is null", + "struct", + ImmutableList.of( + path("s", "g"), + metaPath("s", "f", "NULL")), + ImmutableList.of( + metaPath("s", "f", "NULL"))); + } + + /** + * Tests the NOT NULL struct field branch in visitElementAt line 375-384, + * complementing {@link #testNotNullStructOnOuterJoinNullableSide()} and + * {@link #testNullableFieldPreservedWithSiblingProjection()}. + * + *

Relationship with other tests

+ *
    + *
  • {@code testNotNullStructOnOuterJoinNullableSide}: the struct itself is + * NOT NULL → no struct null map → [s, NULL] META must be suppressed.
    • + *
  • {@code testNullableFieldPreservedWithSiblingProjection}: the struct IS + * nullable AND the field IS nullable → [s, NULL] META + [s, f, NULL] META + * both emitted, field preserved via the META path.
    • + *
  • This test: the struct IS nullable BUT the field is NOT NULL → + * [s, NULL] META is emitted for the struct null map, but [s, f, NULL] + * META is NOT emitted (no field-level null map). The fix must still emit + * [s, f] DATA so pruneDataType preserves the field in the struct type — + * the filter expression {@code element_at(s, 'f') IS NULL} still + * references 'f' and won't be rewritten to {@code s IS NULL}.
    • + *
+ * + *

Why manual construction instead of SQL

+ * Doris DDL does not support {@code NOT NULL} on individual struct fields + * ({@code struct} triggers a syntax error). This test + * therefore constructs the {@link StructType} with a nullable=false field + * programmatically and calls {@link AccessPathExpressionCollector} directly. + * Because the {@link SlotReference} lacks an {@code originalColumn}, + * {@code hasPhysicalNullMap} returns false, so [s, NULL] META is suppressed + * by the slot-level guard — that path is covered by the SQL-based + * {@code testNullableFieldPreservedWithSiblingProjection}. + */ + @Test + public void testNotNullFieldPreservedInAccessPaths() { + // Scenario from Review 2: + // Project(element_at(s, 'g')) + // Filter(element_at(s, 'f') IS NULL) + // Scan(s STRUCT NULL) + // + // Before fix: visitElementAt emitted [s, NULL] META then returned null, + // dropping field 'f'. pruneDataType removed 'f' from the struct type + // because no path referenced it. The filter still referenced + // element_at(s, 'f') IS NULL → rebuild failed. + // + // After fix: visitElementAt emits [s, NULL] META, then falls through + // with a fresh DATA context to emit [s, f] DATA, preserving 'f'. + + // s STRUCT + StructType structType = new StructType(ImmutableList.of( + new StructField("f", IntegerType.INSTANCE, false, ""), // NOT NULL + new StructField("g", IntegerType.INSTANCE, true, ""))); // nullable + SlotReference slot = new SlotReference("s", structType, true); // struct is nullable + + // SELECT element_at(s, 'g') → collector emits [s, g] DATA + ElementAt selectG = new ElementAt(slot, new org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral("g")); + Multimap paths1 = ArrayListMultimap.create(); + new AccessPathExpressionCollector( + connectContext.getStatementContext(), paths1, false, false) + .collect(selectG); + + // WHERE element_at(s, 'f') IS NULL → should emit [s, NULL] META + [s, f] DATA + ElementAt whereF = new ElementAt(slot, new org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral("f")); + IsNull isNull = new IsNull(whereF); + Multimap paths2 = ArrayListMultimap.create(); + new AccessPathExpressionCollector( + connectContext.getStatementContext(), paths2, true, false) + .collect(isNull); + + // Merge results + TreeSet allPaths = new TreeSet<>( + Comparator.comparing(CollectAccessPathResult::toString)); + allPaths.addAll(paths1.get(slot.getExprId().asInt())); + allPaths.addAll(paths2.get(slot.getExprId().asInt())); + + // [s, g] DATA must exist (from SELECT) + Assertions.assertTrue(allPaths.contains( + new CollectAccessPathResult( + ImmutableList.of("s", "g"), false, TAccessPathType.DATA)), + "expected [s,g] DATA in: " + allPaths); + // [s, f] DATA must exist (NOT NULL field preserved — the fix) + // isPredicate=true because it comes from the WHERE clause. + Assertions.assertTrue(allPaths.contains( + new CollectAccessPathResult( + ImmutableList.of("s", "f"), true, TAccessPathType.DATA)), + "expected [s,f] DATA (NOT NULL field preserved) in: " + allPaths); + // NOTE: [s, NULL] META is not asserted here because the manually constructed + // SlotReference has no originalColumn, so hasPhysicalNullMap returns false and + // the META NULL path is suppressed at visitSlotReference. The [s, NULL] META + // behavior is covered by testNullableFieldPreservedWithSiblingProjection. + // + // [s, f, NULL] META must NOT exist (f is NOT NULL, no field-level null map) + Assertions.assertFalse(allPaths.contains( + new CollectAccessPathResult( + ImmutableList.of("s", "f", "NULL"), true, TAccessPathType.META)), + "f is NOT NULL, expected NO [s,f,NULL] META in: " + allPaths); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java index 2b17d366bbda96..62cac47d7fb5d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; +import org.apache.doris.thrift.DescriptorsConstants; import org.apache.doris.thrift.TAccessPathType; import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TDataAccessPath; @@ -302,6 +303,7 @@ private void assertAllAccessPathsContain( private TColumnAccessPath path(String... path) { TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA); accessPath.data_access_path = new TDataAccessPath(ImmutableList.copyOf(path)); + accessPath.setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); return accessPath; } diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index f62204ac6f6094..b3080b5cbb6c9e 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -64,6 +64,9 @@ message PColumnAccessPath { required PAccessPathType type = 1; optional PDataAccessPath data_access_path = 2; optional PMetaAccessPath meta_access_path = 3; + // 0/absent is the legacy all-DATA encoding with special path components. Version 1 uses + // typed payloads and may carry meta_access_path. + optional int32 version = 4; } message PSlotDescriptor { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 3bce52b2f87fed..3472a8a659058c 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -64,10 +64,17 @@ struct TMetaAccessPath { 1: required list path } +const i32 TCOLUMN_ACCESS_PATH_VERSION_LEGACY = 0 +const i32 TCOLUMN_ACCESS_PATH_VERSION_TYPED = 1 + struct TColumnAccessPath { 1: required TAccessPathType type 2: optional TDataAccessPath data_access_path 3: optional TMetaAccessPath meta_access_path + // The version is absent for legacy senders. Legacy paths all have DATA type and encode + // KEYS/VALUES/* selectors plus NULL/OFFSET metadata in data_access_path. Starting from the + // typed version, type selects the authoritative payload and META may use meta_access_path. + 4: optional i32 version } struct TColumn { diff --git a/regression-test/data/nereids_rules_p0/column_pruning/left_join_not_null_column.out b/regression-test/data/nereids_rules_p0/column_pruning/left_join_not_null_column.out new file mode 100644 index 00000000000000..8a98c58828acaa --- /dev/null +++ b/regression-test/data/nereids_rules_p0/column_pruning/left_join_not_null_column.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !left_join_not_null_column -- +3 6 60 600.00 1 + diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/left_join_not_null_column.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/left_join_not_null_column.groovy new file mode 100644 index 00000000000000..e7e9aa1b06de81 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/left_join_not_null_column.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("left_join_not_null_column") { + sql "set enable_prune_nested_column = true" + + sql "drop table if exists left_join_not_null_dim" + sql "drop table if exists left_join_not_null_fact" + + sql """ + create table left_join_not_null_dim ( + id int not null, + segment varchar(32) not null + ) + unique key(id) + distributed by hash(id) buckets 1 + properties ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + sql """ + create table left_join_not_null_fact ( + fact_id int not null, + dim_id int not null, + quantity int not null, + amount decimal(10, 2) not null + ) + unique key(fact_id) + distributed by hash(fact_id) buckets 1 + properties ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + sql "insert into left_join_not_null_dim values (1, 'segment-a'), (2, 'segment-b')" + sql "insert into left_join_not_null_fact values (1, 1, 10, 100.00), (2, 2, 20, 200.00), (3, 3, 30, 300.00)" + + explain { + sql """ + select count(*), sum(f.fact_id), sum(f.quantity), sum(f.amount), + sum(case when d.segment is null then 1 else 0 end) + from left_join_not_null_fact f + left join left_join_not_null_dim d on f.dim_id = d.id + """ + notContains "segment.NULL" + } + + qt_left_join_not_null_column """ + select count(*), sum(f.fact_id), sum(f.quantity), sum(f.amount), + sum(case when d.segment is null then 1 else 0 end) + from left_join_not_null_fact f + left join left_join_not_null_dim d on f.dim_id = d.id + """ +} diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy index 654ac283faf735..8c1b3883669548 100644 --- a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy +++ b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy @@ -18,7 +18,7 @@ // Regression tests for the IS NULL / IS NOT NULL column pruning optimization. // // When IS NULL (or IS NOT NULL) is the *only* use of a nullable column, the FE -// should emit a DATA access path with a "NULL" component so that the BE can +// should emit a META access path with a "NULL" component so that the BE can // satisfy the query by reading only the null flag instead of the full column data. // The EXPLAIN plan should show: // nested columns: : all access paths: [.NULL] @@ -183,23 +183,23 @@ suite("null_column_pruning") { order_qt_10 "select int_col from ncp_tbl where int_col is null"; // ─── Mixed: struct IS NULL + partial field access ─────────────────────────── - // struct_col IS NULL in WHERE + element_at in SELECT needs struct data for projection, while - // the predicate keeps the parent null map separately. + // struct_col IS NULL in WHERE + element_at in SELECT needs only the projected field data, + // while the predicate keeps the parent null map separately. explain { sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null" contains "nested columns" - contains "all access paths: [struct_col]" + contains "all access paths: [struct_col.city, struct_col.NULL]" contains "predicate access paths: [struct_col.NULL]" } order_qt_11 "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null"; // This query verifies the real correctness risk: predicate paths need both parent and child - // null maps, while allAccessPaths keeps the struct data path for projection. + // null maps, while allAccessPaths keeps only the projected field data path. explain { sql "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null" contains "nested columns" - contains "all access paths: [struct_col]" + contains "all access paths: [struct_col.zip, struct_col.NULL, struct_col.city.NULL]" contains "predicate access paths:" contains "struct_col.NULL" contains "struct_col.city.NULL" @@ -220,9 +220,8 @@ suite("null_column_pruning") { order_qt_12 "select struct_col from ncp_tbl where struct_col is null"; // ─── Nested struct field IS NULL ──────────────────────────────────────────── - // struct_element(struct_col, 'city') IS NULL should produce a null-flag-only - // predicate path [struct_col.city.NULL] while the projection reads city data. - // [struct_col.city.NULL] remains in predicateAccessPaths beside the projected city data path. + // struct_element(struct_col, 'city') IS NULL needs both the parent Struct null map and the + // selected field null map, while the projection reads city data. explain { sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null" contains "nested columns" @@ -390,24 +389,24 @@ suite("null_column_pruning") { order_qt_24 "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is not null"; // ─── Mixed: map_keys IS NULL + map_keys projected ────────────────────────── - // Projection needs map data, while the predicate checks whether the parent map is NULL. The - // parent NULL path stays in predicateAccessPaths. + // Projection needs only map keys, while the predicate checks whether the parent map is NULL. + // Keep the parent NULL path independently in both path sets. explain { sql "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null" contains "nested columns" - contains "all access paths: [map_col]" + contains "all access paths: [map_col.KEYS, map_col.NULL]" contains "predicate access paths: [map_col.NULL]" } order_qt_25 "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null"; // ─── Mixed: map_values IS NULL + map_values projected ────────────────────── - // Projection needs map data, while the predicate checks whether the parent + // Projection needs only map values, while the predicate checks whether the parent // map is NULL. A NULL value element does not make map_values(map_col) NULL. explain { sql "select map_values(map_col) from ncp_tbl where map_values(map_col) is null" contains "nested columns" - contains "all access paths: [map_col]" + contains "all access paths: [map_col.VALUES, map_col.NULL]" contains "predicate access paths: [map_col.NULL]" } From 63572a9fa58bafc0e95756d72a153d0a866aab5c Mon Sep 17 00:00:00 2001 From: englefly Date: Tue, 22 Sep 2026 15:03:30 +0800 Subject: [PATCH 4/4] branch-4.2 [fix](be) Review follow-ups for typed NULL/OFFSET access paths (#68314) Addresses the four review comments on #68314: * column_reader.cpp: FileColumnIterator::read_by_rowids() inserted `count` placeholders into the nested column before the rowid loop and appended `total_read_count` more at the end, so the nested column ended up longer than the null map (an internally inconsistent ColumnNullable). Keep only the insertion after the real read count is known, matching master. * segment_iterator_lazy_pruned_test.cpp: build `_schema_block_id_map` in the fixture the same way `_vec_init_lazy_materialization()` does, so the direct `_read_lazy_pruned_columns()` calls no longer index an empty vector. * column_reader_test.cpp: the skipped-read destination must start truly empty; assert the append semantics explicitly with a second read as well. * PruneNestedColumnTest: keep the verified `[s, f]` DATA expectation of testNotNullStructOnOuterJoinNullableSide (the collected set for that query is exactly that one path) and add testNotNullStructIsNullOnOuterJoinNullableSide, which is the shape that actually puts the root `[s, NULL]` path in front of the hasPhysicalNullMap guard, so the suppression the PR introduces is really covered. Test: * FE: ./run-fe-ut.sh --run org.apache.doris.nereids.rules.rewrite.PruneNestedColumnTest -> 64 tests, 0 failures; mvn checkstyle:check -> 0 violations. * BE: BUILD_TYPE_UT=Release ./run-be-ut.sh --run --filter='ColumnReaderTest.NullMapOnlyReadBySparseRowidsAcrossPages:ColumnReaderTest.MapReadByRowidsSkipReadingAppendsPlaceholders:SegmentIteratorLazyPrunedTest.*' -> 4 tests, 0 failures. --- be/src/storage/segment/column_reader.cpp | 6 ++-- .../storage/segment/column_reader_test.cpp | 9 +++-- .../segment_iterator_lazy_pruned_test.cpp | 7 ++++ .../rules/rewrite/PruneNestedColumnTest.java | 34 +++++++++++++++++-- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 696cb5d3694d8f..2b88b42d6f5f0d 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -2803,8 +2803,10 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co const size_t base_size = null_map_data.size(); null_map_data.resize(base_size + count); - nullable_col.get_nested_column().insert_many_defaults(count); - + // The nested column is filled exactly once, below, with the number of rows actually + // read. Filling `count` placeholders here as well would append them twice and leave + // the nested column longer than the null map, i.e. an internally inconsistent + // ColumnNullable (see the insert_many_defaults(total_read_count) at the end). size_t remaining = count; size_t total_read_count = 0; size_t nrows_to_read = 0; diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index 6bcdd148f8f766..d19158e1a06056 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -3027,7 +3027,7 @@ TEST_F(ColumnReaderTest, OffsetCalculateOffsetsUsesPageSentinelForLastOffset) { ASSERT_EQ(static_cast(105), data[2]); } -TEST_F(ColumnReaderTest, MapReadByRowidsSkipReadingResizesDestination) { +TEST_F(ColumnReaderTest, MapReadByRowidsSkipReadingAppendsPlaceholders) { // create a basic map iterator with dummy readers/iterators auto map_reader = std::make_shared(); auto null_iter = std::make_unique(std::make_shared()); @@ -3045,7 +3045,6 @@ TEST_F(ColumnReaderTest, MapReadByRowidsSkipReadingResizesDestination) { auto keys = ColumnInt32::create(); auto values = ColumnInt32::create(); auto offsets = ColumnArray::ColumnOffsets::create(); - offsets->get_data().push_back(0); auto column_map = ColumnMap::create(std::move(keys), std::move(values), std::move(offsets)); MutableColumnPtr dst = std::move(column_map); @@ -3055,6 +3054,12 @@ TEST_F(ColumnReaderTest, MapReadByRowidsSkipReadingResizesDestination) { ASSERT_TRUE(st.ok()) << "read_by_rowids failed: " << st.to_string(); ASSERT_EQ(count, dst->size()); + + // A skipped read appends `count` placeholder rows; it must not treat `count` as the + // absolute destination size, so reading again appends another `count` rows. + st = map_iter.read_by_rowids(rowids, count, dst); + ASSERT_TRUE(st.ok()) << "second read_by_rowids failed: " << st.to_string(); + ASSERT_EQ(2 * count, dst->size()); } TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) { // Regression test: when the access path is [map_col, *, OFFSET] diff --git a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp index b990201b49544c..d405a255b77830 100644 --- a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp +++ b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp @@ -125,6 +125,13 @@ class SegmentIteratorLazyPrunedTest : public ::testing::Test { auto iter = std::make_unique(nullptr, _read_schema); iter->_opts.tablet_schema = _tablet_schema; iter->_opts.stats = &_stats; + // Production fills this map in _vec_init_lazy_materialization(); the tests below call + // _read_lazy_pruned_columns() directly, so build the same column-id -> block-position + // map for the one-column read schema (column id 0 lives at block position 0). + iter->_schema_block_id_map.resize(_read_schema->columns().size(), -1); + for (int i = 0; i < _read_schema->num_column_ids(); i++) { + iter->_schema_block_id_map[_read_schema->column_id(i)] = i; + } iter->_support_lazy_read_pruned_columns.insert(0); iter->_column_iterators.resize(1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 3922522b47f584..5371040f1b53f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -2002,12 +2002,18 @@ private void assertVariantSubColumnSlotCount(String sql, List expectedSu } /** - * Verify that synthetic nullability from outer join does NOT cause META NULL paths - * on physically NOT NULL columns. When a NOT NULL struct sits on the nullable side + * Verify that synthetic nullability from outer join does NOT cause a root META NULL path + * on a physically NOT NULL column. When a NOT NULL struct sits on the nullable side * of a LEFT JOIN, the slot's {@code nullable()} returns true (from outer join * semantics), but {@code getOriginalColumn().isAllowNull()} returns false * (physical column has no null map). The fix in AccessPathExpressionCollector * should suppress the {@code [s, NULL]} META path in this case. + * + *

Because this filter sits above the join rather than directly on the scan, the path + * collected for the field is a data read ({@code [s, f]} DATA): the IS NULL check is evaluated + * on the join output, where the field value has to be materialized. The root-level + * {@code [s, NULL]} META path is the one that must not appear, and + * {@link #testNotNullStructIsNullOnOuterJoinNullableSide()} covers that path directly. */ @Test public void testNotNullStructOnOuterJoinNullableSide() throws Exception { @@ -2030,6 +2036,30 @@ public void testNotNullStructOnOuterJoinNullableSide() throws Exception { ImmutableList.of(metaPath("s", "NULL"))); } + /** + * The direct root-level case of the same guard: {@code s IS NULL} on the nullable side of a + * LEFT JOIN, where {@code s} is physically NOT NULL. A direct null check on the struct is what + * puts the one-component {@code [NULL]} path in front of {@code visitSlotReference}, so this is + * the case that exercises the {@code hasPhysicalNullMap} check: the path must be dropped, leaving + * the struct unpruned (read as a whole) instead of sending the BE a NULL_MAP_ONLY read it + * cannot satisfy. It is kept out of a WHERE clause on purpose — a null-rejecting filter there + * lets the optimizer rewrite the LEFT JOIN into an anti join, which removes the struct from the + * plan entirely and would make the assertion vacuous. + */ + @Test + public void testNotNullStructIsNullOnOuterJoinNullableSide() throws Exception { + Pair> result = collectComplexSlots( + "select not_null_struct_tbl.s is null from driving_tbl" + + " left join not_null_struct_tbl" + + " on driving_tbl.id = not_null_struct_tbl.id"); + Assertions.assertEquals(1, result.second.size(), + "[s, NULL] suppression must be observable on the struct slot"); + List allAccessPaths = result.second.get(0).getAllAccessPaths(); + Assertions.assertFalse(allAccessPaths.contains(metaPath("s", "NULL")), + "the physical struct column has no null map, so no [s, NULL] META path is allowed: " + + allAccessPaths); + } + /** * Verifies that a NOT NULL struct field is preserved in the pruned type when a * sibling field is accessed via SELECT and the IS NULL check emits struct-level