Move protobuf and flatbuffer definitions into their owning crates

.cargo/config.toml

@@ -4,8 +4,8 @@ rustflags = [
 ]
 
 [alias]
-xtask = "run -p xtask --"
 vx = "run -p vortex-tui --"
+xtask = "run -p xtask --"
 
 [publish]
 # NOTE(aduffy): we run into frequent issues auto-releasing our workspace

.github/actions/setup-flatc/action.yml

@@ -1,16 +1,85 @@
 name: "Setup flatc"
-description: "Download and install flatc binary"
+description: "Install the pinned flatc, from the official release binary or from source"
 inputs:
   flatc_version:
     description: "Version of the flatc binary"
     default: "25.12.19"
 runs:
   using: "composite"
   steps:
-    - name: Download flatc
-      id: download-flatc
+    # The generated bindings are not source-compatible across flatc releases (24.x names a
+    # `size` field `size_`, for example), so the version has to match exactly. Distro packages
+    # lag well behind it, hence the release binary, or a source build where none is published.
+    - name: Install flatc
       shell: bash
       run: |
-        wget -O /tmp/flatc.zip "https://github.com/google/flatbuffers/releases/download/v${{ inputs.flatc_version }}/Linux.flatc.binary.clang++-18.zip"
-        unzip /tmp/flatc.zip flatc
-        mv flatc /usr/local/bin/
+        set -euo pipefail
+
+        VERSION="${{ inputs.flatc_version }}"
+        if [ "$(flatc --version 2>/dev/null)" = "flatc version ${VERSION}" ]; then
+          echo "flatc ${VERSION} is already installed"
+          exit 0
+        fi
+
+        ARCH=$(uname -m)
+        OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+        # Official binaries exist for x86_64 glibc Linux, macOS and Windows only.
+        ASSET=""
+        case "$OS" in
+          linux)
+            if [ "$ARCH" = "x86_64" ] && ! ldd /bin/sh 2>&1 | grep -qi musl; then
+              ASSET="Linux.flatc.binary.clang++-18.zip"
+            fi
+            ;;
+          darwin)
+            case "$ARCH" in
+              arm64)  ASSET="Mac.flatc.binary.zip" ;;
+              x86_64) ASSET="MacIntel.flatc.binary.zip" ;;
+            esac
+            ;;
+          *) ASSET="Windows.flatc.binary.zip" ;;
+        esac
+
+        # This sometimes runs in a container where the user is root and there is no sudo.
+        if command -v sudo &>/dev/null
+        then
+          CMD=sudo
+        else
+          CMD=
+        fi
+
+        RELEASE="https://github.com/google/flatbuffers/releases/download/v${VERSION}"
+        if [ -n "$ASSET" ]; then
+          curl -fsSL -o /tmp/flatc.zip "${RELEASE}/${ASSET}"
+          unzip -o /tmp/flatc.zip -d /tmp/flatc
+          rm -f /tmp/flatc.zip
+        else
+          echo "No official flatc binary for ${OS}/${ARCH}; building ${VERSION} from source"
+          git clone --depth 1 --branch "v${VERSION}" \
+            https://github.com/google/flatbuffers.git /tmp/flatbuffers
+          cmake -S /tmp/flatbuffers -B /tmp/flatbuffers/build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DFLATBUFFERS_BUILD_TESTS=OFF \
+            -DFLATBUFFERS_BUILD_FLATLIB=OFF \
+            -DFLATBUFFERS_BUILD_FLATHASH=OFF \
+            -DFLATBUFFERS_INSTALL=OFF
+          cmake --build /tmp/flatbuffers/build --target flatc \
+            --parallel "$(nproc 2>/dev/null || echo 4)"
+          mkdir -p /tmp/flatc
+          mv /tmp/flatbuffers/build/flatc /tmp/flatc/flatc
+          rm -rf /tmp/flatbuffers
+        fi
+
+        if [ "$OS" = "linux" ] || [ "$OS" = "darwin" ]; then
+          $CMD mv /tmp/flatc/flatc /usr/local/bin/
+        else
+          # Windows runners do not have /usr/local/bin on PATH for non-bash steps.
+          mkdir -p "$HOME/.local/bin"
+          mv /tmp/flatc/flatc.exe "$HOME/.local/bin/"
+          cygpath -w "$HOME/.local/bin" >> "$GITHUB_PATH"
+          export PATH="$HOME/.local/bin:$PATH"
+        fi
+
+        rm -rf /tmp/flatc
+        flatc --version

.github/actions/setup-prebuild/action.yml

@@ -170,3 +170,7 @@ runs:
         components: ${{ inputs.components }}
         targets: ${{ inputs.targets }}
         enable-sccache: "false"
+
+    # Pins the version; the prebuild AMIs bake in a flatc that may lag it.
+    - name: Install flatc (for FlatBuffers code generation)
+      uses: ./.github/actions/setup-flatc

.github/actions/setup-rust/action.yml

@@ -83,3 +83,6 @@ runs:
     - name: Install Protoc (for lance-encoding build step)
       if: runner.os != 'Windows'
       uses: ./.github/actions/setup-protoc
+
+    - name: Install flatc (for FlatBuffers code generation)
+      uses: ./.github/actions/setup-flatc

.github/workflows/ci.yml

@@ -809,12 +809,9 @@ jobs:
       - uses: ./.github/actions/setup-prebuild
         with:
           enable-sccache: "true"
+      - uses: ./.github/actions/setup-flatc
       - name: Install nightly for cbindgen macro expansion
         run: rustup toolchain install $NIGHTLY_TOOLCHAIN
-      - name: "regenerate all .fbs/.proto Rust code"
-        run: |
-          cargo run --profile ci -p xtask -- generate-fbs
-          cargo run --profile ci -p xtask -- generate-proto
       - name: "regenerate the edition records"
         run: |
           cargo run --profile ci -p xtask -- generate-editions
@@ -833,16 +830,27 @@ jobs:
           git status --porcelain
           test -z "$(git status --porcelain)"
 
-      - name: "Checkout develop flatbuffers"
-        working-directory: vortex-flatbuffers/
+      # Schemas live under `<crate>/flatbuffers/` but resolve includes against those dirs
+      # collectively, so flatten each revision into one tree for flatc to compare.
+      - name: "Collect flatbuffer schemas from this revision and from develop"
         run: |
-          cp -R flatbuffers flatbuffers.HEAD
           git fetch origin develop --depth 1
-          git checkout origin/develop -- flatbuffers
+          collect() {
+            git ls-tree -r --name-only "$1" \
+              | grep -E '(^|/)flatbuffers/.*\.fbs$' \
+              | while read -r path; do
+                  rel="${path#*/flatbuffers/}"
+                  mkdir -p "$2/$(dirname "$rel")"
+                  git show "$1:$path" > "$2/$rel"
+                done
+          }
+          collect HEAD "$RUNNER_TEMP/fbs.head"
+          collect origin/develop "$RUNNER_TEMP/fbs.develop"
       - name: "Verify flatbuffer back-compat"
-        working-directory: vortex-flatbuffers/
+        working-directory: ${{ runner.temp }}
         run: |
-          find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{}
+          find fbs.develop/ -type f -name "*.fbs" | sed 's|^fbs.develop/||' \
+            | xargs -I{} -n1 flatc -I fbs.head --conform-includes fbs.develop --conform fbs.develop/{} fbs.head/{}
 
   ffi-c-test:
     name: "C API test build"

.github/workflows/codspeed.yml

@@ -56,7 +56,7 @@ jobs:
           - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
           - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }
           - { shard: 7, name: "Encodings 4 & layout", packages: "vortex-sparse vortex-zigzag vortex-zstd vortex-layout" }
-          - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks vortex-row" }
+          - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-btrblocks vortex-row" }
           - { shard: 9, name: "Tensor & spatial", packages: "vortex-tensor vortex-spatial" }
     name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})"
     timeout-minutes: 30

.github/workflows/musl.yml

@@ -54,6 +54,9 @@ jobs:
 
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
 
+      # Alpine only packages flatc versions we cannot use, so this builds the pinned one.
+      - uses: ./.github/actions/setup-flatc
+
       - name: Install nextest
         shell: bash
         # Prebuilt static musl nextest binary; building it from source would

.github/workflows/rust-instrumented.yml

@@ -100,9 +100,9 @@ jobs:
             --llvm-path "${LLVM_TOOLS_BIN}" \
             --threads $(nproc) \
             --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
-            --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
-            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
-            --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
+            --ignore 'home/*' --ignore 'xtask/*' --ignore 'vortex-build/*' --ignore 'target/*' --ignore 'vortex-error/*' \
+            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' \
+            --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
             --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' \
             --ignore benchmarks/* --ignore 'vortex-test/*' \
             -o ${{ env.GRCOV_OUTPUT_FILE }}

AGENTS.md

@@ -29,6 +29,11 @@ documentation in `docs/`, and benchmark tooling in `vortex-bench/` and `benchmar
   and the OpenDAL-backed services (`cos://`, `oss://`). Every binding resolves URLs through it.
 - `vortex-scan`, `vortex-session`, `vortex-datafusion`, and `vortex-duckdb` contain scan
   and execution integrations.
+- FlatBuffers (`.fbs`) and Protocol Buffers (`.proto`) schemas live in the crate that owns the
+  types they describe (`vortex-array`, `vortex-layout`, `vortex-file`, `vortex-ipc`), and are
+  compiled into `OUT_DIR` by that crate's `build.rs` via `vortex-build`. Generated code is never
+  checked in, and a schema that includes another crate's declares that crate with `depends_on`.
+  Building therefore requires `flatc` on `PATH` (or `FLATC` set); `protoc` is not needed.
 - `vortex-python` contains Python bindings. RST-flavored project docs live in `docs/`.
 
 ## Scoped Guidance

CONTRIBUTING.md

@@ -36,6 +36,18 @@ The contribution process is outlined below:
 
 ## Development Workflows
 
+### Build prerequisites
+
+Bindings for the `.fbs` and `.proto` schemas are generated at build time into `OUT_DIR` by the
+`build.rs` of the crate that owns each schema, and are never checked in.
+
+FlatBuffers generation shells out to the [`flatc`](https://github.com/google/flatbuffers/releases)
+compiler, so building any Vortex crate requires it on `PATH`, or its location in the `FLATC`
+environment variable. CI pins version `25.12.19`; other recent versions work, but may produce
+cosmetically different generated code.
+
+Protocol Buffers generation parses schemas in pure Rust, so `protoc` is not required.
+
 The repository uses [`uv`](https://docs.astral.sh/uv/) to manage its Python workspace. From the
 repository root, create or update the development environment with: