feat: Floe binary row serialization format

.codecov.yaml

@@ -16,6 +16,8 @@ flag_management:
       carryforward: true
     - name: arrow-extension
       carryforward: true
+    - name: flow-php-extension
+      carryforward: true
     - name: telemetry-tests
       carryforward: true
 

.github/workflows/job-flow-php-extension.yml

@@ -0,0 +1,196 @@
+name: Flow PHP Extension
+
+on:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: false
+
+jobs:
+  build:
+    name: Build and Test Extension
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest']
+        php: ['8.3', '8.4', '8.5']
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup PHP Environment
+        uses: ./.github/actions/setup-php-env
+        with:
+          php-version: ${{ matrix.php }}
+          dependencies: locked
+          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, zstd'
+          tools: 'composer:v2, php-config'
+          coverage: 'pcov'
+          install-dependencies: 'false'
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          toolchain: stable
+
+      - name: Cache cargo registry and build
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            src/extension/flow-php-ext/target
+          key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-php${{ matrix.php }}-cargo-
+
+      - name: Install build dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential clang libclang-dev
+
+      - name: Install build dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install llvm
+          echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV"
+          echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV"
+
+      - name: Set LIBCLANG_PATH (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          LIBCLANG_SO=$(find /usr/lib \
+            \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \
+            -not -path '*/clang/*/lib/*' \
+            2>/dev/null | head -1)
+          if [ -z "$LIBCLANG_SO" ]; then
+            echo "Could not locate libclang.so under /usr/lib" >&2
+            exit 1
+          fi
+          echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV"
+          echo "Detected libclang at: $LIBCLANG_SO"
+
+      - name: Build extension
+        working-directory: src/extension/flow-php-ext
+        run: make build
+
+      - name: Install Composer Dependencies
+        uses: ./.github/actions/composer-install
+        with:
+          php-version: ${{ matrix.php }}
+          dependencies: locked
+
+      - name: Run PHPT tests
+        working-directory: src/extension/flow-php-ext
+        run: make test
+
+      - name: Install extension and verify
+        working-directory: src/extension/flow-php-ext
+        run: |
+          EXT_DIR=$(php -r 'echo ini_get("extension_dir");')
+          sudo cp ext/modules/flow_php.so "$EXT_DIR/"
+          echo "extension=flow_php.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')"
+          php -m | grep flow_php
+
+      - name: Run ETL Unit Tests with Flow PHP Extension
+        run: just test --testsuite=etl-unit ${{ (matrix.php == '8.3' && matrix.os == 'ubuntu-latest') && '--coverage-clover=./var/phpunit/coverage/clover/coverage.xml' || '' }}
+
+      - name: Run ETL Integration Tests with Flow PHP Extension
+        run: just test --testsuite=etl-integration
+
+      - name: Upload to Codecov
+        if: ${{ !cancelled() && matrix.php == '8.3' && matrix.os == 'ubuntu-latest' }}
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: ./var/phpunit/coverage/clover
+          flags: flow-php-extension
+
+  pie-install-test:
+    name: Test PIE Installation
+    runs-on: ${{ matrix.os }}
+    env:
+      RUSTUP_TOOLCHAIN: stable
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest']
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup PHP Environment
+        uses: ./.github/actions/setup-php-env
+        with:
+          php-version: '8.3'
+          dependencies: locked
+          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
+          tools: 'composer:v2, phpize, php-config'
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          toolchain: stable
+
+      - name: Install build dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential autoconf automake libtool clang libclang-dev
+
+      - name: Install build dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install autoconf automake libtool llvm
+          echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV"
+          # See the build job: keep clang and libclang from the same llvm so
+          # bindgen does not mix brew's libclang with Xcode's clang headers.
+          echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV"
+
+      - name: Set LIBCLANG_PATH (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          # Find the canonical libclang.so shared library, excluding the
+          # Clang runtime support libraries under llvm-*/lib/clang/*/lib/.
+          LIBCLANG_SO=$(find /usr/lib \
+            \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \
+            -not -path '*/clang/*/lib/*' \
+            2>/dev/null | head -1)
+          if [ -z "$LIBCLANG_SO" ]; then
+            echo "Could not locate libclang.so under /usr/lib" >&2
+            exit 1
+          fi
+          echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV"
+          echo "Detected libclang at: $LIBCLANG_SO"
+
+      - name: Install PIE
+        run: |
+          sudo curl -L -o /usr/local/bin/pie https://github.com/php/pie/releases/latest/download/pie.phar
+          sudo chmod +x /usr/local/bin/pie
+          pie --version
+
+      - name: Prepare local package for PIE installation
+        working-directory: src/extension/flow-php-ext
+        run: |
+          jq '. + {"version": "0.0.9999"}' composer.json > composer.tmp.json
+          mv composer.tmp.json composer.json
+
+      - name: Install extension via PIE (from local)
+        run: |
+          sudo pie repository:remove packagist.org
+          sudo pie repository:add path ${{ github.workspace }}/src/extension/flow-php-ext
+          sudo env "PATH=$PATH" "LIBCLANG_PATH=$LIBCLANG_PATH" ${CLANG_PATH:+"CLANG_PATH=$CLANG_PATH"} "RUSTUP_TOOLCHAIN=$RUSTUP_TOOLCHAIN" "CARGO_HOME=$CARGO_HOME" "RUSTUP_HOME=$RUSTUP_HOME" pie install flow-php/flow-php-ext:0.0.9999@dev
+
+      - name: Enable extension
+        run: |
+          php -m | grep -q flow_php || echo "extension=flow_php" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')"
+
+      - name: Verify extension is loaded
+        run: |
+          php -m | grep flow_php

.github/workflows/monorepo-split.yml

@@ -128,6 +128,8 @@ jobs:
             split_repository: 'pg-query-ext'
           - local_path: 'src/extension/arrow-ext'
             split_repository: 'arrow-ext'
+          - local_path: 'src/extension/flow-php-ext'
+            split_repository: 'flow-php-ext'
 
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

.github/workflows/test-suite.yml

@@ -60,5 +60,10 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  flow-php-extension:
+    uses: ./.github/workflows/job-flow-php-extension.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   mutation-tests:
     uses: ./.github/workflows/job-mutation-tests.yml

.nix/pkgs/flow-php/package.nix

@@ -7,11 +7,13 @@
     php-zstd,
     php-pg-query-ext,
     php-arrow-ext,
+    php-flow-php-ext,
     with-pcov ? true,
     with-xdebug ? false,
     with-blackfire ? false,
     with-pg-query-ext ? false,
     with-arrow-ext ? false,
+    with-flow-php-ext ? false,
     with-grpc ? false,
     with-protobuf ? true
 }:
@@ -38,6 +40,7 @@ let
         ++ (if with-blackfire then [blackfire] else [])
         ++ (if with-pg-query-ext then [(php-pg-query-ext.override { inherit php; })] else [])
         ++ (if with-arrow-ext then [(php-arrow-ext.override { inherit php; })] else [])
+        ++ (if with-flow-php-ext then [(php-flow-php-ext.override { inherit php; })] else [])
         ++ (if with-grpc then [grpc] else [])
         ++ (if with-protobuf then [
             (protobuf.overrideAttrs (old: {

.nix/pkgs/php-flow-php-ext/package.nix

@@ -0,0 +1,66 @@
+{
+  php,
+  lib,
+  stdenv,
+  rustPlatform,
+  clang,
+  llvmPackages,
+  flow-php-ext-version ? "dev",
+}:
+
+let
+  extSrc = builtins.path {
+    path = ../../../src/extension/flow-php-ext;
+    name = "flow-php-ext-src";
+    filter = path: type:
+      let baseName = baseNameOf path;
+      in !(
+        baseName == "target" ||
+        baseName == "vendor" ||
+        baseName == "ext" ||
+        baseName == ".gitignore" ||
+        baseName == ".gitattributes" ||
+        baseName == "composer.json" ||
+        baseName == "composer.lock"
+      );
+  };
+  pkg = rustPlatform.buildRustPackage {
+    pname = "php-flow-php-ext";
+    version = flow-php-ext-version;
+
+    src = extSrc;
+
+    cargoLock.lockFile = ../../../src/extension/flow-php-ext/Cargo.lock;
+
+    nativeBuildInputs = [
+      clang
+      llvmPackages.libclang
+      php.unwrapped
+      php.unwrapped.dev
+    ];
+
+    env = {
+      LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
+      PHP_CONFIG = "${php.unwrapped.dev}/bin/php-config";
+      PHP = "${php.unwrapped}/bin/php";
+      FLOW_PHP_EXT_VERSION = flow-php-ext-version;
+    };
+
+    installPhase = let
+      targetDir = "target/${stdenv.hostPlatform.rust.rustcTargetSpec}/release";
+    in ''
+      runHook preInstall
+      mkdir -p $out/lib/php/extensions
+      cp ${targetDir}/libflow_php${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/php/extensions/flow_php.so
+      runHook postInstall
+    '';
+
+    doCheck = false;
+
+    meta = with lib; {
+      description = "Flow PHP native extension (Rust) - Floe frame-body encoder/decoder for DataFrame Rows";
+      license = licenses.mit;
+    };
+  };
+in
+pkg // { extensionName = "flow_php"; }

Justfile

@@ -48,15 +48,15 @@ lint-links:
 # Audit GitHub Actions workflows (actionlint static checks + zizmor security audit).
 # Workflows that live under src/ are split out to their own repos, so they are not
 # auto-discovered with the root .github/workflows and must be listed explicitly:
-#   - the arrow-ext release workflow (custom, full audit);
+#   - the extension release workflows (custom, full audit);
 #   - every per-package subtree-split readonly.yaml (added by hand per package, so all
 #     are audited to catch drift; dangerous-triggers is exempted for them in
 #     .github/zizmor.yml — they only run `gh pr close`, never check out PR code).
 lint-actions:
     #!/usr/bin/env bash
     set -uo pipefail
     rc=0
-    extra_workflows=(src/extension/arrow-ext/.github/workflows/release.yml)
+    extra_workflows=(src/extension/arrow-ext/.github/workflows/release.yml src/extension/flow-php-ext/.github/workflows/release.yml)
     while IFS= read -r workflow; do
         extra_workflows+=("$workflow")
     done < <(find src -path '*/.github/workflows/readonly.yaml' | sort)

bin/docs.php

@@ -56,6 +56,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
     {
         $paths = [
             __DIR__ . '/../src/core/etl/src/Flow/ETL/DSL/functions.php',
+            __DIR__ . '/../src/core/etl/src/Flow/Floe/DSL/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php',

composer.json

@@ -200,7 +200,8 @@
                 "src/lib/telemetry/src/Flow",
                 "src/lib/types/src/Flow",
                 "src/tools/documentation/src/Flow",
-                "src/extension/arrow-ext/php/Flow"
+                "src/extension/arrow-ext/php/Flow",
+                "src/extension/flow-php-ext/php/Flow"
             ],
             "Flow\\Doctrine\\Bulk\\": [
                 "src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk"
@@ -245,6 +246,7 @@
             "src/bridge/telemetry/otlp/src/Flow/Bridge/Telemetry/OTLP/DSL/functions.php",
             "src/cli/src/Flow/CLI/DSL/functions.php",
             "src/core/etl/src/Flow/ETL/DSL/functions.php",
+            "src/core/etl/src/Flow/Floe/DSL/functions.php",
             "src/functions.php",
             "src/lib/array-dot/src/Flow/ArrayDot/array_dot.php",
             "src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php",