feat: add BTree global index core components

src/paimon/common/global_index/btree/btree_defs.h

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace paimon {
+struct BtreeDefs {
+    BtreeDefs() = delete;
+    ~BtreeDefs() = delete;
+    static inline const char kIdentifier[] = "btree";
+    /// "btree-index.compression" - The compression algorithm to use for BTreeIndex.
+    /// Default value is "none".
+    static inline const char kBtreeIndexCompression[] = "btree-index.compression";
+    /// "btree-index.compression-level" - The compression level of the compression algorithm.
+    /// Default value is 1.
+    static inline const char kBtreeIndexCompressionLevel[] = "btree-index.compression-level";
+    /// "btree-index.block-size" - The block size to use for BTreeIndex.
+    /// Default value is 64 KB.
+    static inline const char kBtreeIndexBlockSize[] = "btree-index.block-size";
+    /// "btree-index.cache-size" - The cache size to use for BTreeIndex.
+    /// Default value is 128 MB.
+    static inline const char kBtreeIndexCacheSize[] = "btree-index.cache-size";
+    /// "btree-index.high-priority-pool-ratio" - The high priority pool ratio to use for BTreeIndex.
+    /// Default value is 0.1.
+    static inline const char kBtreeIndexHighPriorityPoolRatio[] =
+        "btree-index.high-priority-pool-ratio";
+
+    /// "btree-index.read-buffer-size" - Optional. Specifies the read buffer size for the B-tree
+    /// index. This setting can be tuned based on query patterns:
+    ///   - For range queries (e.g., `VisitLessThan`, `VisitGreaterOrEqual`), increasing the buffer
+    ///   size (e.g., to 1MB) may improve I/O bandwidth and sequential read performance.
+    ///   - For point queries (e.g., `VisitEqual`), buffering can introduce negative effects due to
+    ///   read amplification; it is recommended to leave this option unset.
+    ///
+    /// If specified, read block with `BufferedInputStream`.
+    static inline const char kBtreeIndexReadBufferSize[] = "btree-index.read-buffer-size";
+
+    static inline const char kDefaultBtreeIndexBlockSize[] = "64KB";
+    static inline const char kDefaultBtreeIndexCompression[] = "none";
+    static inline const char kDefaultBtreeIndexCacheSize[] = "128MB";
+    static inline const int32_t kDefaultBtreeIndexCompressionLevel = 1;
+    static inline const double kDefaultBtreeIndexHighPriorityPoolRatio = 0.1;
+};
+}  // namespace paimon

src/paimon/common/global_index/btree/btree_file_footer.cpp

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/global_index/btree/btree_file_footer.h"
+
+#include <fmt/format.h>
+
+namespace paimon {
+
+Result<std::shared_ptr<BTreeFileFooter>> BTreeFileFooter::Read(MemorySliceInput* input) {
+    // read version and verify magic number
+    PAIMON_RETURN_NOT_OK(input->SetPosition(kEncodingLength - 8));
+
+    int32_t version = input->ReadInt();
+    int32_t magic_number = input->ReadInt();
+    if (magic_number != kMagicNumber) {
+        return Status::Invalid(
+            fmt::format("File is not a btree index file (expected magic number {:#x}, got {:#x})",
+                        kMagicNumber, magic_number));
+    }
+
+    PAIMON_RETURN_NOT_OK(input->SetPosition(0));
+
+    // read bloom filter and index handles
+    auto offset = input->ReadLong();
+    auto size = input->ReadInt();
+    auto expected_entries = input->ReadLong();
+    std::optional<BloomFilterHandle> bloom_filter_handle =
+        BloomFilterHandle(offset, size, expected_entries);
+    if (bloom_filter_handle->Offset() == 0 && bloom_filter_handle->Size() == 0 &&
+        bloom_filter_handle->ExpectedEntries() == 0) {
+        bloom_filter_handle = std::nullopt;
+    }
+
+    offset = input->ReadLong();
+    size = input->ReadInt();
+    BlockHandle index_block_handle(offset, size);
+
+    offset = input->ReadLong();
+    size = input->ReadInt();
+    std::optional<BlockHandle> null_bitmap_handle = BlockHandle(offset, size);
+    if (null_bitmap_handle->Offset() == 0 && null_bitmap_handle->Size() == 0) {
+        null_bitmap_handle = std::nullopt;
+    }
+
+    return std::make_shared<BTreeFileFooter>(version, bloom_filter_handle, index_block_handle,
+                                             null_bitmap_handle);
+}
+
+MemorySlice BTreeFileFooter::Write(const std::shared_ptr<BTreeFileFooter>& footer,
+                                   MemoryPool* pool) {
+    MemorySliceOutput output(kEncodingLength, pool);
+    return BTreeFileFooter::Write(footer, &output);
+}
+
+MemorySlice BTreeFileFooter::Write(const std::shared_ptr<BTreeFileFooter>& footer,
+                                   MemorySliceOutput* output) {
+    // write bloom filter handle
+    const auto& bloom_filter_handle = footer->GetBloomFilterHandle();
+    if (!bloom_filter_handle.has_value()) {
+        output->WriteValue(static_cast<int64_t>(0));
+        output->WriteValue(static_cast<int32_t>(0));
+        output->WriteValue(static_cast<int64_t>(0));
+    } else {
+        output->WriteValue(bloom_filter_handle->Offset());
+        output->WriteValue(bloom_filter_handle->Size());
+        output->WriteValue(bloom_filter_handle->ExpectedEntries());
+    }
+
+    // write index block handle
+    const auto& index_block_handle = footer->GetIndexBlockHandle();
+    output->WriteValue(index_block_handle.Offset());
+    output->WriteValue(index_block_handle.Size());
+
+    // write null bitmap handle
+    const auto& null_bitmap_handle = footer->GetNullBitmapHandle();
+    if (!null_bitmap_handle.has_value()) {
+        output->WriteValue(static_cast<int64_t>(0));
+        output->WriteValue(static_cast<int32_t>(0));
+    } else {
+        output->WriteValue(null_bitmap_handle->Offset());
+        output->WriteValue(null_bitmap_handle->Size());
+    }
+
+    // write version and magic number
+    output->WriteValue(footer->GetVersion());
+    output->WriteValue(kMagicNumber);
+
+    return output->ToSlice();
+}
+
+}  // namespace paimon

src/paimon/common/global_index/btree/btree_file_footer.h

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <optional>
+
+#include "paimon/common/memory/memory_slice_input.h"
+#include "paimon/common/memory/memory_slice_output.h"
+#include "paimon/common/sst/block_handle.h"
+#include "paimon/common/sst/bloom_filter_handle.h"
+
+namespace paimon {
+/// The Footer for BTree file.
+class BTreeFileFooter {
+ public:
+    static Result<std::shared_ptr<BTreeFileFooter>> Read(MemorySliceInput* input);
+    static MemorySlice Write(const std::shared_ptr<BTreeFileFooter>& footer, MemoryPool* pool);
+    static MemorySlice Write(const std::shared_ptr<BTreeFileFooter>& footer,
+                             MemorySliceOutput* output);
+
+ public:
+    BTreeFileFooter(const std::optional<BloomFilterHandle>& bloom_filter_handle,
+                    const BlockHandle& index_block_handle,
+                    const std::optional<BlockHandle>& null_bitmap_handle)
+        : BTreeFileFooter(kCurrentVersion, bloom_filter_handle, index_block_handle,
+                          null_bitmap_handle) {}
+
+    BTreeFileFooter(int32_t version, const std::optional<BloomFilterHandle>& bloom_filter_handle,
+                    const BlockHandle& index_block_handle,
+                    const std::optional<BlockHandle>& null_bitmap_handle)
+        : version_(version),
+          bloom_filter_handle_(bloom_filter_handle),
+          index_block_handle_(index_block_handle),
+          null_bitmap_handle_(null_bitmap_handle) {}
+
+    int32_t GetVersion() const {
+        return version_;
+    }
+
+    const std::optional<BloomFilterHandle>& GetBloomFilterHandle() const {
+        return bloom_filter_handle_;
+    }
+
+    const BlockHandle& GetIndexBlockHandle() const {
+        return index_block_handle_;
+    }
+
+    const std::optional<BlockHandle>& GetNullBitmapHandle() const {
+        return null_bitmap_handle_;
+    }
+
+ public:
+    static constexpr int32_t kMagicNumber = 0x50425449;
+    static constexpr int32_t kCurrentVersion = 1;
+    static constexpr int32_t kEncodingLength = 52;
+
+ private:
+    int32_t version_;
+    std::optional<BloomFilterHandle> bloom_filter_handle_;
+    BlockHandle index_block_handle_;
+    std::optional<BlockHandle> null_bitmap_handle_;
+};
+
+}  // namespace paimon