From 4a39e80c2d345413ab6fa4384a13548ef5d3c54d Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Tue, 22 Sep 2026 23:58:55 +0800 Subject: [PATCH 1/2] [core] Rebuild file index over the correct column after schema evolution FileIndexProcessor computes projectedIndexCols as the index columns' positions in the file schema, and creates the index writer with fileSchema.project(projectedIndexCols). The reader, however, was built with ReadBuilder.withProjection(projectedIndexCols), which resolves the indices against the current table schema. Once a schema change shifts column positions (for example dropping a middle column and adding another), the file-schema index no longer matches the table-schema index, so the reader returns a different column and the file index is rebuilt over it. A later query on the indexed column then silently prunes files that actually match. Read the columns with withReadType(fileSchema.project(projectedIndexCols)), the same file-schema projection the writer uses, so the reader and writer stay in the file's own column space. rewrite_file_index reads a single data file whose committed schema is that file schema, and file indexes are probed at query time with predicates devolved to the file schema, so the per-file index must be built in the file's column and type space. The name-mapping half of schema evolution was added with the rewrite_file_index procedure in #6562, but the reader projection was never aligned to it. --- .../paimon/index/FileIndexProcessor.java | 7 +- .../paimon/index/FileIndexProcessorTest.java | 77 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java b/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java index de6560881655..3030ac3bb20a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java @@ -136,9 +136,14 @@ public DataFileMeta process(BinaryRow partition, int bucket, ManifestEntry manif fileIndexOptions, schemaInfo.colNameMapping); if (dataFileIndexWriter != null) { + // projectedIndexCols index into the file schema. withProjection would re-interpret + // them against the current table schema, so a schema change that shifts columns (drop + // a middle column, add another) would read the wrong column and rebuild the index over + // it. Read with the same file-schema projection the writer above uses. + RowType indexReadType = schemaInfo.fileSchema.project(schemaInfo.projectedIndexCols); try (RecordReader reader = table.newReadBuilder() - .withProjection(schemaInfo.projectedIndexCols) + .withReadType(indexReadType) .newRead() .createReader( DataSplit.builder() diff --git a/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java b/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java index fa6f51e34d81..a22c04da1951 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; import org.apache.paimon.fileindex.FileIndexFormat; +import org.apache.paimon.fileindex.FileIndexReader; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; @@ -46,6 +47,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -174,4 +176,79 @@ public void testProcessReadsTheSchemasOfTheTableBranch() throws Exception { DataFileMeta processed = processor.process(entry.partition(), entry.bucket(), entry); assertThat(processed.extraFiles()).isNotEmpty(); } + + @Test + public void testRebuildsIndexOnCorrectColumnAfterColumnDropAndAdd() throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path warehouse = new Path(tempDir.toString()); + Map options = new HashMap<>(); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.FILE_FORMAT.key(), "parquet"); + options.put(CoreOptions.FILE_INDEX + ".bloom-filter.columns", "v"); + // v is the third field, so it sits at index 2 in the file schema. + RowType rowType = + RowType.of( + new DataType[] {DataTypes.INT(), DataTypes.INT(), DataTypes.INT()}, + new String[] {"k", "a", "v"}); + + Identifier identifier = Identifier.create("mydb", "t"); + try (FileSystemCatalog catalog = new FileSystemCatalog(fileIO, warehouse)) { + catalog.createDatabase("mydb", false); + catalog.createTable( + identifier, + new Schema( + rowType.getFields(), + Collections.emptyList(), + Collections.singletonList("k"), + options, + ""), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + + String commitUser = UUID.randomUUID().toString(); + try (TableWriteImpl write = table.newWrite(commitUser); + TableCommitImpl commit = table.newCommit(commitUser)) { + write.write(GenericRow.of(1, 10, 100)); + commit.commit(1, write.prepareCommit(false, 1)); + } + + // Drop the middle column and add another: v keeps file-schema index 2, but the + // current table schema now has a different column (w) at index 2. + table.schemaManager().commitChanges(SchemaChange.dropColumn("a")); + table.schemaManager().commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); + FileStoreTable evolved = (FileStoreTable) catalog.getTable(identifier); + + List entries = evolved.store().newScan().plan().files(); + assertThat(entries).isNotEmpty(); + ManifestEntry entry = entries.get(0); + assertThat(entry.file().schemaId()).isEqualTo(0L); + + FileIndexProcessor processor = new FileIndexProcessor(evolved); + DataFileMeta processed = processor.process(entry.partition(), entry.bucket(), entry); + assertThat(processed.extraFiles()).isNotEmpty(); + + String indexFile = + processed.extraFiles().stream() + .filter(name -> name.endsWith(DataFilePathFactory.INDEX_PATH_SUFFIX)) + .findFirst() + .orElseThrow(() -> new AssertionError("no file index was written")); + Path indexPath = + new Path( + evolved.store() + .pathFactory() + .bucketPath(entry.partition(), entry.bucket()), + indexFile); + // The bloom filter for v must still consider the written value 100 present. Before + // the fix the reader projected the current-schema column at index 2 (w, absent from + // this file), so the index was rebuilt over nulls and 100 was reported as missing. + try (FileIndexFormat.Reader reader = + FileIndexFormat.createReader(fileIO.newInputStream(indexPath), rowType)) { + Set vReaders = reader.readColumnIndex("v"); + assertThat(vReaders).isNotEmpty(); + for (FileIndexReader vReader : vReaders) { + assertThat(vReader.visitEqual(null, 100).remain()).isTrue(); + } + } + } + } } From c6e4764c89c2829c9e7f95c3738373482ad6b1e0 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Wed, 23 Sep 2026 10:04:52 +0800 Subject: [PATCH 2/2] [core] Assert the rebuilt file index with a deterministic bitmap index Switch the schema-evolution regression test from a bloom filter to a bitmap index. A bloom-filter positive is probabilistic, so remain() == true does not strictly prove the value was indexed from the correct column - a false positive could let the old wrong-column behavior pass. A bitmap index returns an exact empty result for an absent value, so the test now reads the (embedded) index and asserts the indexed value is present while an absent value is not. --- .../paimon/index/FileIndexProcessorTest.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java b/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java index a22c04da1951..19a5c899e540 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/FileIndexProcessorTest.java @@ -26,11 +26,13 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.fileindex.FileIndexFormat; import org.apache.paimon.fileindex.FileIndexReader; +import org.apache.paimon.fs.ByteArraySeekableStream; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFilePathFactory; import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; @@ -184,7 +186,10 @@ public void testRebuildsIndexOnCorrectColumnAfterColumnDropAndAdd() throws Excep Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.FILE_FORMAT.key(), "parquet"); - options.put(CoreOptions.FILE_INDEX + ".bloom-filter.columns", "v"); + // A bitmap index is exact: querying an absent value returns an empty result, + // so the regression assertion below is deterministic (a bloom filter's + // probabilistic false positives could let the wrong-column behavior pass). + options.put(CoreOptions.FILE_INDEX + ".bitmap.columns", "v"); // v is the third field, so it sits at index 2 in the file schema. RowType rowType = RowType.of( @@ -225,28 +230,22 @@ public void testRebuildsIndexOnCorrectColumnAfterColumnDropAndAdd() throws Excep FileIndexProcessor processor = new FileIndexProcessor(evolved); DataFileMeta processed = processor.process(entry.partition(), entry.bucket(), entry); - assertThat(processed.extraFiles()).isNotEmpty(); - - String indexFile = - processed.extraFiles().stream() - .filter(name -> name.endsWith(DataFilePathFactory.INDEX_PATH_SUFFIX)) - .findFirst() - .orElseThrow(() -> new AssertionError("no file index was written")); - Path indexPath = - new Path( - evolved.store() - .pathFactory() - .bucketPath(entry.partition(), entry.bucket()), - indexFile); - // The bloom filter for v must still consider the written value 100 present. Before - // the fix the reader projected the current-schema column at index 2 (w, absent from - // this file), so the index was rebuilt over nulls and 100 was reported as missing. + // The small single-row bitmap index is embedded in the manifest, not a side file. + byte[] embedded = processed.embeddedIndex(); + assertThat(embedded).isNotEmpty(); + + // The bitmap index for v must still contain the written value 100. Before the fix + // the reader projected the current-schema column at index 2 (w, absent from this + // file), so the index was rebuilt over nulls and 100 would be absent. + FieldRef vRef = new FieldRef(0, "v", DataTypes.INT()); try (FileIndexFormat.Reader reader = - FileIndexFormat.createReader(fileIO.newInputStream(indexPath), rowType)) { + FileIndexFormat.createReader(new ByteArraySeekableStream(embedded), rowType)) { Set vReaders = reader.readColumnIndex("v"); assertThat(vReaders).isNotEmpty(); for (FileIndexReader vReader : vReaders) { - assertThat(vReader.visitEqual(null, 100).remain()).isTrue(); + // Exact: 100 was indexed from v, 999 never was. + assertThat(vReader.visitEqual(vRef, 100).remain()).isTrue(); + assertThat(vReader.visitEqual(vRef, 999).remain()).isFalse(); } } }