From 31c4aa8f414d29cd00e9787c39ff6c1cd542be43 Mon Sep 17 00:00:00 2001
From: Matthew Ball <mgball@uci.edu>
Date: Sat, 12 Sep 2026 21:41:54 -0700
Subject: [PATCH 1/4] feat(engine): columnar Arrow wire execution for scan and
 filter

Add an opt-in columnar execution path so a scan and its downstream
filters exchange Apache Arrow batches instead of row Tuples, skipping
per-row boxing and Iceberg materialization on pipelined edges.

- ColumnarFrame data payload carries Arrow IPC bytes across the wire,
  gated by COLUMNAR_WIRE and NetworkOutputBuffer.columnarWire
- SpecializedFilterOpExec gains a vectorized processBatchMultiPort and a
  native-Arrow processColumnarBatch (numeric SIMD fast path)
- CSVScanSourceOpExec.produceColumnarBatch parses CSV straight into
Arrow
- OutputManager emits columnar batches and still materializes surviving
  rows to result storage for the GUI result panel
- DataProcessor/DPThread/EndChannelHandler drain the columnar path,
  falling back to the row path on multi-receiver edges
- ArrowUtils: serialize/deserialize roots and tuples, row selection
- tests: vectorized filter correctness, Arrow IPC round-trip, columnar
  repro (row == columnar), scan/filter and end-to-end A/B benches

In-engine A/B (scan 6M lineitem -> filter -> count, single worker):
100.4s row -> 16.5s columnar (~6.1x).

Also adds reference docs: Amber vs Spark/Flink speed analysis and the
8.1-8.4 optimization explainers.
---
 .../messaginglayer/InputManager.scala         |   5 +
 .../messaginglayer/OutputManager.scala        |  40 +++-
 .../pythonworker/PythonProxyClient.scala      |   3 +
 .../partitioners/Partitioner.scala            |  16 +-
 .../engine/architecture/worker/DPThread.scala |   3 +-
 .../architecture/worker/DataProcessor.scala   |  70 ++++--
 .../promisehandlers/EndChannelHandler.scala   |  25 ++-
 .../common/ambermessage/DataPayload.scala     |   8 +-
 .../common/ambermessage/WorkflowMessage.scala |   5 +-
 .../amber/engine/e2e/ColumnarReproSpec.scala  | 111 ++++++++++
 .../e2e/ColumnarWorkflowBenchSpec.scala       | 135 ++++++++++++
 .../executor/ColumnarOperatorExecutor.scala   |  33 +++
 .../core/executor/OperatorExecutor.scala      |   6 +
 .../executor/SourceOperatorExecutor.scala     |   8 +
 .../amber/core/tuple/ColumnarBatch.scala      |  76 +++++++
 .../apache/texera/amber/util/ArrowUtils.scala |  83 +++++++-
 .../amber/util/ArrowIpcRoundTripSpec.scala    |  69 ++++++
 .../filter/SpecializedFilterOpExec.scala      | 199 +++++++++++++++++-
 .../source/scan/csv/CSVScanSourceOpExec.scala |  39 ++++
 .../filter/ArrowColumnarFilterBenchSpec.scala | 106 ++++++++++
 .../filter/ColumnarScanFilterBenchSpec.scala  | Bin 0 -> 5614 bytes
 .../VectorizedFilterCorrectnessSpec.scala     | 183 ++++++++++++++++
 todo.md                                       |   4 +
 23 files changed, 1199 insertions(+), 28 deletions(-)
 create mode 100644 amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarReproSpec.scala
 create mode 100644 amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarWorkflowBenchSpec.scala
 create mode 100644 common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
 create mode 100644 common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/ColumnarBatch.scala
 create mode 100644 common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala
 create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ArrowColumnarFilterBenchSpec.scala
 create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ColumnarScanFilterBenchSpec.scala
 create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
 create mode 100644 todo.md

diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/InputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/InputManager.scala
index 4e297829431..e5c978ba6e4 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/InputManager.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/InputManager.scala
@@ -152,4 +152,9 @@ class InputManager(
     inputBatch = batch
     currentInputIdx = -1
   }
+
+  // Marks the current batch fully consumed (hasUnfinishedInput becomes false).
+  def skipToEnd(): Unit = {
+    currentInputIdx = if (inputBatch == null) -1 else inputBatch.length - 1
+  }
 }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
index b594cc8ab03..6b5cb7f5200 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
@@ -37,7 +37,9 @@ import org.apache.texera.amber.engine.architecture.worker.managers.{
   PortStorageWriterTerminateSignal
 }
 import org.apache.texera.amber.engine.common.AmberLogging
-import org.apache.texera.amber.util.VirtualIdentityUtils
+import org.apache.texera.amber.engine.common.ambermessage.ColumnarFrame
+import org.apache.texera.amber.util.{ArrowUtils, VirtualIdentityUtils}
+import org.apache.arrow.vector.VectorSchemaRoot
 
 import java.net.URI
 import scala.collection.mutable
@@ -197,6 +199,42 @@ class OutputManager(
     saveStateToStorageIfNeeded(state)
   }
 
+  // Columnar emit is safe only when each output link has a single receiver
+  // (e.g. one-to-one); hash/range shuffles would need per-key column splitting.
+  def canEmitColumnar: Boolean =
+    networkOutputBuffers.nonEmpty && networkOutputBuffers.groupBy(_._1._1).forall(_._2.size == 1)
+
+  // Send a whole Arrow batch downstream as a ColumnarFrame (no per-tuple path).
+  def emitColumnarBatch(root: VectorSchemaRoot): Unit = {
+    val bytes = ArrowUtils.serializeRoot(root)
+    val rowCount = root.getRowCount
+    val texeraSchema = ArrowUtils.toTexeraSchema(root.getSchema)
+    networkOutputBuffers.keys.foreach {
+      case (_, receiver) =>
+        outputGateway.sendTo(receiver, ColumnarFrame(bytes, rowCount, texeraSchema))
+    }
+    // Materialize to result storage for the GUI result panel, matching the row
+    // path's saveTupleToStorageIfNeeded. Only decodes when a port needs storage.
+    if (outputPortResultWriterThreads.nonEmpty) {
+      var i = 0
+      while (i < rowCount) {
+        saveTupleToStorageIfNeeded(ArrowUtils.getTexeraTuple(i, root))
+        i += 1
+      }
+    }
+  }
+
+  // Columnar batches pending emit, drained one-per-step by the DP loop so that
+  // backpressure applies between batches (unlike a synchronous emit).
+  private var columnarOutputIter: Iterator[VectorSchemaRoot] = Iterator.empty
+  def setColumnarOutput(it: Iterator[VectorSchemaRoot]): Unit = columnarOutputIter = it
+  def hasUnfinishedColumnarOutput: Boolean = columnarOutputIter.hasNext
+  def emitOneColumnarBatch(): Unit = {
+    val root = columnarOutputIter.next()
+    emitColumnarBatch(root)
+    root.close()
+  }
+
   def addPort(portId: PortIdentity, schema: Schema, storageURIBaseOption: Option[URI]): Unit = {
     // each port can only be added and initialized once.
     if (this.ports.contains(portId)) {
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
index 144c3ac57a3..bc04a274d38 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
@@ -124,6 +124,9 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
     dataPayload match {
       case DataFrame(frame) =>
         writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
+      case ColumnarFrame(bytes, _, _) =>
+        // Columnar wire reaching the Python path: decode to tuples and stream.
+        writeArrowStream(mutable.Queue(ArrowUtils.deserializeTuples(bytes).toSeq: _*), from, "Data")
       case StateFrame(state) =>
         writeArrowStream(mutable.Queue(state.toTuple()), from, "State")
     }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala
index 39065ca6936..6c832e2d729 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala
@@ -24,7 +24,8 @@ import org.apache.texera.amber.core.state.State
 import org.apache.texera.amber.core.tuple.Tuple
 import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
 import org.apache.texera.amber.engine.architecture.messaginglayer.NetworkOutputGateway
-import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, StateFrame}
+import org.apache.texera.amber.engine.common.ambermessage.{ColumnarFrame, DataFrame, StateFrame}
+import org.apache.texera.amber.util.ArrowUtils
 
 import scala.collection.mutable.ArrayBuffer
 
@@ -57,9 +58,20 @@ class NetworkOutputBuffer(
 
   def flush(): Unit = {
     if (buffer.nonEmpty) {
-      dataOutputPort.sendTo(to, DataFrame(buffer.toArray))
+      val batch = buffer.toArray
+      // Columnar wire (flagged): send the batch as an Arrow IPC ColumnarFrame.
+      val payload =
+        if (NetworkOutputBuffer.columnarWire)
+          ColumnarFrame(ArrowUtils.serializeTuples(batch.head.getSchema, batch), batch.length, batch.head.getSchema)
+        else DataFrame(batch)
+      dataOutputPort.sendTo(to, payload)
       buffer = new ArrayBuffer[Tuple]()
     }
   }
 
 }
+
+object NetworkOutputBuffer {
+  // Global opt-in for the Arrow columnar wire format (default off = row DataFrame).
+  val columnarWire: Boolean = sys.env.getOrElse("COLUMNAR_WIRE", "0") == "1"
+}
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala
index f845836843d..d073c0f0f6d 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala
@@ -165,7 +165,8 @@ class DPThread(
       var channelId: ChannelIdentity = null
       var msgOpt: Option[WorkflowFIFOMessage] = None
       if (
-        dp.inputManager.hasUnfinishedInput || dp.outputManager.hasUnfinishedOutput || dp.pauseManager.isPaused
+        dp.inputManager.hasUnfinishedInput || dp.outputManager.hasUnfinishedOutput ||
+        dp.outputManager.hasUnfinishedColumnarOutput || dp.pauseManager.isPaused
       ) {
         dp.inputGateway.tryPickControlChannel match {
           case Some(channel) =>
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
index e88b0016101..32ae1381ef6 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
@@ -21,7 +21,8 @@ package org.apache.texera.amber.engine.architecture.worker
 
 import com.softwaremill.macwire.wire
 import io.grpc.MethodDescriptor
-import org.apache.texera.amber.core.executor.OperatorExecutor
+import org.apache.texera.amber.core.executor.{ColumnarOperatorExecutor, OperatorExecutor}
+import org.apache.texera.amber.engine.architecture.sendsemantics.partitioners.NetworkOutputBuffer
 import org.apache.texera.amber.core.state.State
 import org.apache.texera.amber.core.tuple._
 import org.apache.texera.amber.core.virtualidentity.{
@@ -59,6 +60,7 @@ import org.apache.texera.amber.engine.common.ambermessage._
 import org.apache.texera.amber.engine.common.statetransition.WorkerStateManager
 import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
 import org.apache.texera.amber.error.ErrorUtils.{mkConsoleMessage, safely}
+import org.apache.texera.amber.util.ArrowUtils
 
 import java.util.concurrent.LinkedBlockingQueue
 
@@ -192,7 +194,9 @@ class DataProcessor(
 
   def continueDataProcessing(): Unit = {
     val dataProcessingStartTime = System.nanoTime()
-    if (outputManager.hasUnfinishedOutput) {
+    if (outputManager.hasUnfinishedColumnarOutput) {
+      outputManager.emitOneColumnarBatch()
+    } else if (outputManager.hasUnfinishedOutput) {
       outputOneTuple()
     } else {
       processInputTuple(inputManager.getNextTuple)
@@ -208,24 +212,62 @@ class DataProcessor(
     val portId = this.inputGateway.getChannel(channelId).getPortId
     dataPayload match {
       case DataFrame(tuples) =>
-        stateManager.conditionalTransitTo(
-          READY,
-          RUNNING,
-          () => {
-            asyncRPCClient.coordinatorInterface.workerStateUpdated(
-              WorkerStateUpdatedRequest(stateManager.getCurrentState),
-              asyncRPCClient.mkContext(COORDINATOR)
-            )
-          }
-        )
-        inputManager.initBatch(channelId, tuples)
-        processInputTuple(inputManager.getNextTuple)
+        processTupleBatch(channelId, portId, tuples)
+      case ColumnarFrame(bytes, _, _) =>
+        executor match {
+          // Native-Arrow path: consume the batch directly, emit a filtered batch.
+          case c: ColumnarOperatorExecutor
+              if NetworkOutputBuffer.columnarWire && outputManager.canEmitColumnar =>
+            c.processColumnarBatch(bytes) match {
+              case Some(result) =>
+                statisticsManager.increaseInputStatistics(portId, bytes.length.toLong)
+                outputManager.setColumnarOutput(Iterator.single(result))
+              case None =>
+                processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
+            }
+          case _ =>
+            processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
+        }
       case StateFrame(state) =>
         processInputState(state, portId.id)
     }
     statisticsManager.increaseDataProcessingTime(System.nanoTime() - dataProcessingStartTime)
   }
 
+  private def processTupleBatch(
+      channelId: ChannelIdentity,
+      portId: PortIdentity,
+      tuples: Array[Tuple]
+  ): Unit = {
+    stateManager.conditionalTransitTo(
+      READY,
+      RUNNING,
+      () => {
+        asyncRPCClient.coordinatorInterface.workerStateUpdated(
+          WorkerStateUpdatedRequest(stateManager.getCurrentState),
+          asyncRPCClient.mkContext(COORDINATOR)
+        )
+      }
+    )
+    inputManager.initBatch(channelId, tuples)
+    // Whole-batch path if the executor supports it, else per-tuple; errors fall back.
+    val vectorized: Option[Iterator[(TupleLike, Option[PortIdentity])]] =
+      try executor.processBatchMultiPort(tuples, portId.id)
+      catch safely { case _ => None }
+    vectorized match {
+      case Some(outputIter) =>
+        var i = 0
+        while (i < tuples.length) {
+          statisticsManager.increaseInputStatistics(portId, tuples(i).inMemSize)
+          i += 1
+        }
+        inputManager.skipToEnd()
+        outputManager.outputIterator.setTupleOutput(outputIter)
+      case None =>
+        processInputTuple(inputManager.getNextTuple)
+    }
+  }
+
   def processECM(
       channelId: ChannelIdentity,
       ecm: EmbeddedControlMessage,
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
index 7794342690b..9ca6bbaa0ff 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
@@ -20,12 +20,14 @@
 package org.apache.texera.amber.engine.architecture.worker.promisehandlers
 
 import com.twitter.util.Future
+import org.apache.texera.amber.core.executor.SourceOperatorExecutor
 import org.apache.texera.amber.core.tuple.FinalizePort
 import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
   AsyncRPCContext,
   EmptyRequest
 }
 import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn
+import org.apache.texera.amber.engine.architecture.sendsemantics.partitioners.NetworkOutputBuffer
 import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer
 import org.apache.texera.amber.error.ErrorUtils.safely
 
@@ -45,9 +47,26 @@ trait EndChannelHandler {
       if (outputState.isDefined) {
         dp.outputManager.emitState(outputState.get)
       }
-      dp.outputManager.outputIterator.setTupleOutput(
-        dp.executor.onFinishMultiPort(portId.id)
-      )
+      // Columnar source path: emit Arrow batches directly (no per-row Tuple),
+      // when enabled and the output is single-receiver-per-link. Else the
+      // row-oriented onFinishMultiPort path.
+      val columnarBatches =
+        if (NetworkOutputBuffer.columnarWire && dp.outputManager.canEmitColumnar)
+          dp.executor match {
+            case s: SourceOperatorExecutor => s.produceColumnarBatch()
+            case _                         => None
+          }
+        else None
+      columnarBatches match {
+        case Some(batchIter) =>
+          // Drained one batch per DP-loop step (backpressure applies between).
+          dp.outputManager.setColumnarOutput(batchIter)
+          dp.outputManager.outputIterator.setTupleOutput(Iterator.empty)
+        case None =>
+          dp.outputManager.outputIterator.setTupleOutput(
+            dp.executor.onFinishMultiPort(portId.id)
+          )
+      }
     } catch safely {
       case e =>
         // forward input tuple to the user and pause DP thread
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala
index 54f577a0beb..7fc7b9e823e 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala
@@ -20,12 +20,18 @@
 package org.apache.texera.amber.engine.common.ambermessage
 
 import org.apache.texera.amber.core.state.State
-import org.apache.texera.amber.core.tuple.Tuple
+import org.apache.texera.amber.core.tuple.{Schema, Tuple}
 
 sealed trait DataPayload extends WorkflowFIFOMessagePayload {}
 
 final case class StateFrame(frame: State) extends DataPayload
 
+// Columnar wire payload: a batch encoded as Arrow IPC stream bytes.
+final case class ColumnarFrame(arrowIpcBytes: Array[Byte], rowCount: Int, schema: Schema)
+    extends DataPayload {
+  val inMemSize: Long = arrowIpcBytes.length.toLong
+}
+
 final case class DataFrame(frame: Array[Tuple]) extends DataPayload {
   val inMemSize: Long = {
     frame.map(_.inMemSize).sum
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/WorkflowMessage.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/WorkflowMessage.scala
index a54d4c20310..e67878476ab 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/WorkflowMessage.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/WorkflowMessage.scala
@@ -26,8 +26,9 @@ case object WorkflowMessage {
     msg match {
       case dataMsg: WorkflowFIFOMessage =>
         dataMsg.payload match {
-          case df: DataFrame => df.inMemSize
-          case _             => 200L
+          case df: DataFrame     => df.inMemSize
+          case cf: ColumnarFrame => cf.inMemSize
+          case _                 => 200L
         }
       case _ => 200L
     }
diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarReproSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarReproSpec.scala
new file mode 100644
index 00000000000..7660c257f7f
--- /dev/null
+++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarReproSpec.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.engine.e2e
+
+import org.apache.pekko.actor.{ActorSystem, Props}
+import org.apache.pekko.testkit.{ImplicitSender, TestKit}
+import org.apache.pekko.util.Timeout
+import org.apache.texera.amber.clustering.SingleNodeListener
+import org.apache.texera.amber.core.tuple.Tuple
+import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
+import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext}
+import org.apache.texera.amber.engine.architecture.coordinator._
+import org.apache.texera.amber.engine.common.AmberRuntime
+import org.apache.texera.amber.engine.e2e.TestUtils.{
+  buildWorkflow,
+  cleanupWorkflowExecutionData,
+  initiateTexeraDBForTestCases,
+  runWorkflowAndReadTerminalResults,
+  setUpWorkflowExecutionData
+}
+import org.apache.texera.amber.operator.TestOperators
+import org.apache.texera.amber.operator.filter.{ComparisonType, FilterPredicate, SpecializedFilterOpDesc}
+import org.apache.texera.workflow.LogicalLink
+import org.scalatest.flatspec.AnyFlatSpecLike
+import org.scalatest.BeforeAndAfterAll
+
+import com.twitter.util.Duration
+import java.util.concurrent.atomic.AtomicInteger
+import scala.concurrent.duration.DurationInt
+
+class ColumnarReproSpec
+    extends TestKit(ActorSystem("ColumnarReproSpec", AmberRuntime.pekkoConfig))
+    with ImplicitSender
+    with AnyFlatSpecLike
+    with BeforeAndAfterAll {
+
+  implicit val timeout: Timeout = Timeout(5.seconds)
+  private val ids = new AtomicInteger(70000)
+  private val tag = if (sys.env.get("COLUMNAR_WIRE").contains("1")) "COLUMNAR" else "ROW"
+
+  override def beforeAll(): Unit = {
+    system.actorOf(Props[SingleNodeListener](), "cluster-info")
+    Class.forName("org.postgresql.Driver")
+    initiateTexeraDBForTestCases()
+  }
+  override def afterAll(): Unit = TestKit.shutdownActorSystem(system)
+
+  private def filt(attr: String, v: String): SpecializedFilterOpDesc = {
+    val op = new SpecializedFilterOpDesc()
+    op.predicates = List(new FilterPredicate(attr, ComparisonType.GREATER_THAN, v)); op
+  }
+
+  private def report(name: String, res: Map[OperatorIdentity, List[Tuple]]): Unit = {
+    val rows = res.values.headOption.getOrElse(Nil)
+    println(s"REPRO[$tag] $name: ${rows.size} rows")
+    rows.take(3).foreach { t =>
+      println(s"REPRO[$tag]   Region=${t.getField[Any]("Region")} UnitsSold=${t.getField[Any]("Units Sold")}")
+    }
+  }
+
+  "columnar source" should "scan -> filter(terminal)" in {
+    val id = ids.incrementAndGet(); setUpWorkflowExecutionData(id)
+    try {
+      val ctx: WorkflowContext = TestUtils.workflowContext(id)
+      val scan = TestOperators.smallCsvScanOpDesc()
+      val f = filt("Units Sold", "5000")
+      val wf = buildWorkflow(
+        List(scan, f),
+        List(LogicalLink(scan.operatorIdentifier, PortIdentity(), f.operatorIdentifier, PortIdentity())),
+        ctx
+      )
+      report("scan->filter", runWorkflowAndReadTerminalResults(system, wf, Duration.fromMinutes(5)))
+    } finally cleanupWorkflowExecutionData(id)
+  }
+
+  "native filter" should "scan -> filter -> filter(terminal)" in {
+    val id = ids.incrementAndGet(); setUpWorkflowExecutionData(id)
+    try {
+      val ctx: WorkflowContext = TestUtils.workflowContext(id)
+      val scan = TestOperators.smallCsvScanOpDesc()
+      val f1 = filt("Units Sold", "5000")
+      val f2 = filt("Units Sold", "0")
+      val wf = buildWorkflow(
+        List(scan, f1, f2),
+        List(
+          LogicalLink(scan.operatorIdentifier, PortIdentity(), f1.operatorIdentifier, PortIdentity()),
+          LogicalLink(f1.operatorIdentifier, PortIdentity(), f2.operatorIdentifier, PortIdentity())
+        ),
+        ctx
+      )
+      report("scan->filter->filter", runWorkflowAndReadTerminalResults(system, wf, Duration.fromMinutes(5)))
+    } finally cleanupWorkflowExecutionData(id)
+  }
+}
diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarWorkflowBenchSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarWorkflowBenchSpec.scala
new file mode 100644
index 00000000000..99943b58406
--- /dev/null
+++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarWorkflowBenchSpec.scala
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.engine.e2e
+
+import com.twitter.util.{Await, Duration, Promise, Return, Throw}
+import org.apache.pekko.actor.{ActorSystem, Props}
+import org.apache.pekko.testkit.{ImplicitSender, TestKit}
+import org.apache.pekko.util.Timeout
+import org.apache.texera.amber.clustering.SingleNodeListener
+import org.apache.texera.amber.core.storage.FileResolver
+import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext}
+import org.apache.texera.amber.engine.architecture.coordinator._
+import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmptyRequest
+import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.COMPLETED
+import org.apache.texera.amber.engine.common.AmberRuntime
+import org.apache.texera.amber.engine.common.client.AmberClient
+import org.apache.texera.amber.engine.e2e.TestUtils.{
+  buildWorkflow,
+  cleanupWorkflowExecutionData,
+  initiateTexeraDBForTestCases,
+  setUpWorkflowExecutionData
+}
+import org.apache.texera.amber.operator.LogicalOp
+import org.apache.texera.amber.operator.aggregate.{
+  AggregateOpDesc,
+  AggregationFunction,
+  AggregationOperation
+}
+import org.apache.texera.amber.operator.filter.{
+  ComparisonType,
+  FilterPredicate,
+  SpecializedFilterOpDesc
+}
+import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc
+import org.apache.texera.workflow.LogicalLink
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.flatspec.AnyFlatSpecLike
+
+import java.util.concurrent.atomic.AtomicInteger
+import scala.concurrent.duration.DurationInt
+
+/**
+  * In-engine A/B for the columnar path (M2): scan -> selective filter -> count.
+  * With COLUMNAR_WIRE=1 the scan and filter build no Tuples; only the filter's
+  * survivors are decoded downstream, so a selective filter builds far fewer
+  * Tuples end-to-end. Run single-worker (CONSTANTS_NUM_WORKER_PER_OPERATOR=1) so
+  * all edges are one-to-one and the columnar path stays active. Prints CWBENCH.
+  */
+class ColumnarWorkflowBenchSpec
+    extends TestKit(ActorSystem("ColumnarWorkflowBenchSpec", AmberRuntime.pekkoConfig))
+    with ImplicitSender
+    with AnyFlatSpecLike
+    with BeforeAndAfterAll {
+
+  implicit val timeout: Timeout = Timeout(5.seconds)
+  private val dataDir = sys.env.getOrElse("TPCH_DATA", "/tmp/tpch/data_sf1")
+  private val runs = sys.env.getOrElse("CW_RUNS", "2").toInt
+  private val ids = new AtomicInteger(80000)
+
+  override def beforeAll(): Unit = {
+    system.actorOf(Props[SingleNodeListener](), "cluster-info")
+    Class.forName("org.postgresql.Driver")
+    initiateTexeraDBForTestCases()
+  }
+  override def afterAll(): Unit = TestKit.shutdownActorSystem(system)
+
+  private def csvScan(path: String): CSVScanSourceOpDesc = {
+    val op = new CSVScanSourceOpDesc()
+    op.fileName = Some(path); op.customDelimiter = Some(","); op.hasHeader = true
+    op.setResolvedFileName(FileResolver.resolve(path)); op
+  }
+  private def filterGT(attr: String, v: String): SpecializedFilterOpDesc = {
+    val op = new SpecializedFilterOpDesc()
+    op.predicates = List(new FilterPredicate(attr, ComparisonType.GREATER_THAN, v)); op
+  }
+  private def countAgg(): AggregateOpDesc = {
+    val op = new AggregateOpDesc(); val a = new AggregationOperation()
+    a.aggFunction = AggregationFunction.COUNT; a.attribute = ""; a.resultAttribute = "cnt"
+    op.aggregations = List(a); op.groupByKeys = List(); op
+  }
+  private def link(f: LogicalOp, t: LogicalOp): LogicalLink =
+    LogicalLink(f.operatorIdentifier, PortIdentity(), t.operatorIdentifier, PortIdentity())
+
+  private def timeRun(): Double = {
+    val id = ids.incrementAndGet()
+    setUpWorkflowExecutionData(id)
+    try {
+      val ctx: WorkflowContext = TestUtils.workflowContext(id)
+      val scan = csvScan(s"$dataDir/lineitem.csv")
+      val filt = filterGT("l_quantity", "45")
+      val agg = countAgg()
+      val wf = buildWorkflow(List(scan, filt, agg), List(link(scan, filt), link(filt, agg)), ctx)
+      val completion = Promise[Unit]()
+      val client = new AmberClient(
+        system, wf.context, wf.physicalPlan, CoordinatorConfig.default,
+        e => completion.updateIfEmpty(Throw(e))
+      )
+      try {
+        client.registerCallback[FatalError](evt => completion.updateIfEmpty(Throw(evt.e)))
+        client.registerCallback[ExecutionStateUpdate](evt =>
+          if (evt.state == COMPLETED) completion.updateIfEmpty(Return(()))
+        )
+        val t0 = System.nanoTime()
+        Await.result(client.coordinatorInterface.startWorkflow(EmptyRequest(), ()), Duration.fromSeconds(60))
+        Await.result(completion, Duration.fromMinutes(15))
+        (System.nanoTime() - t0) / 1e6
+      } finally client.shutdown()
+    } finally cleanupWorkflowExecutionData(id)
+  }
+
+  private val label = if (sys.env.get("COLUMNAR_WIRE").contains("1")) "COLUMNAR" else "ROW"
+
+  "Workflow" should "time scan->filter->count" in {
+    timeRun() // warmup
+    val times = (1 to runs).map { i => val ms = timeRun(); println(f"CWBENCH $label run $i: ${ms / 1000}%.2fs"); ms }
+    println(f"CWBENCH $label MEDIAN=${times.sorted.apply(times.length / 2) / 1000}%.2fs MIN=${times.min / 1000}%.2fs")
+  }
+}
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
new file mode 100644
index 00000000000..166f9330387
--- /dev/null
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.core.executor
+
+import org.apache.arrow.vector.VectorSchemaRoot
+
+/**
+  * An operator that can consume an Arrow columnar batch directly (no per-row
+  * Tuple decode) and produce one. Given the incoming batch as Arrow IPC bytes,
+  * returns the result batch, or None if this batch shape is not supported (the
+  * caller then falls back to the row path). The returned root is owned by the
+  * caller, which serializes and closes it.
+  */
+trait ColumnarOperatorExecutor {
+  def processColumnarBatch(arrowIpcBytes: Array[Byte]): Option[VectorSchemaRoot]
+}
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala
index 9837213abbb..1440c5eedd2 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/OperatorExecutor.scala
@@ -40,6 +40,12 @@ trait OperatorExecutor {
 
   def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike]
 
+  /** Optional whole-batch processing; None means process per-tuple via processTuple. */
+  def processBatchMultiPort(
+      batch: Array[Tuple],
+      port: Int
+  ): Option[Iterator[(TupleLike, Option[PortIdentity])]] = None
+
   def produceStateOnFinish(port: Int): Option[State] = None
 
   def onFinishMultiPort(port: Int): Iterator[(TupleLike, Option[PortIdentity])] = {
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/SourceOperatorExecutor.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/SourceOperatorExecutor.scala
index 633314679a4..24c21b9997c 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/SourceOperatorExecutor.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/SourceOperatorExecutor.scala
@@ -19,12 +19,20 @@
 
 package org.apache.texera.amber.core.executor
 
+import org.apache.arrow.vector.VectorSchemaRoot
 import org.apache.texera.amber.core.tuple.{Tuple, TupleLike}
 import org.apache.texera.amber.core.workflow.PortIdentity
 
 trait SourceOperatorExecutor extends OperatorExecutor {
   override def open(): Unit = {}
 
+  /**
+    * Optional columnar production: yields Arrow batches directly (no per-row
+    * Tuple objects). None means the engine uses the row-oriented produceTuple.
+    * The caller serializes and closes each returned root.
+    */
+  def produceColumnarBatch(): Option[Iterator[VectorSchemaRoot]] = None
+
   override def close(): Unit = {}
   override def processTupleMultiPort(
       tuple: Tuple,
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/ColumnarBatch.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/ColumnarBatch.scala
new file mode 100644
index 00000000000..2161b16c8ef
--- /dev/null
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/ColumnarBatch.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.core.tuple
+
+/**
+  * Column-oriented view of a batch of tuples, for vectorized operator execution.
+  * Wraps the original rows and extracts one column at a time into a contiguous
+  * array, caching each extraction. A numeric column can be pulled into a
+  * primitive `Array[Double]` plus a null mask for tight, JIT/SIMD-friendly loops.
+  */
+class ColumnarBatch(val rows: Array[Tuple]) {
+
+  def size: Int = rows.length
+
+  def schema: Schema = if (rows.isEmpty) null else rows(0).getSchema
+
+  private val cache = scala.collection.mutable.HashMap[String, Array[Any]]()
+
+  /** Column values in row order (boxed), extracted once and cached. */
+  def column(name: String): Array[Any] = {
+    cache.getOrElseUpdate(
+      name, {
+        val a = new Array[Any](rows.length)
+        var i = 0
+        while (i < rows.length) { a(i) = rows(i).getField[Any](name); i += 1 }
+        a
+      }
+    )
+  }
+
+  /**
+    * Numeric column as primitive doubles plus a per-row null mask. NaN is stored
+    * where the value is null; callers must consult the mask. Only valid when the
+    * column type is INTEGER or DOUBLE.
+    */
+  def doubleColumn(name: String): (Array[Double], Array[Boolean]) = {
+    val vals = new Array[Double](rows.length)
+    val isNull = new Array[Boolean](rows.length)
+    var i = 0
+    while (i < rows.length) {
+      val f = rows(i).getField[Any](name)
+      if (f == null) { isNull(i) = true }
+      else vals(i) = f.asInstanceOf[Number].doubleValue()
+      i += 1
+    }
+    (vals, isNull)
+  }
+
+  /** The rows selected by `mask` (mask.length must equal size), as an iterator. */
+  def select(mask: Array[Boolean]): Iterator[Tuple] = {
+    val out = scala.collection.mutable.ArrayBuffer[Tuple]()
+    var i = 0
+    while (i < rows.length) {
+      if (mask(i)) out += rows(i)
+      i += 1
+    }
+    out.iterator
+  }
+}
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
index af14ae9acd0..e6b1df6f937 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
@@ -38,9 +38,13 @@ import org.apache.arrow.vector.{
   VarCharVector,
   VectorSchemaRoot
 }
+import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter}
 
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+import java.nio.channels.Channels
 import java.nio.charset.StandardCharsets
 import java.util
+import scala.collection.mutable.ArrayBuffer
 import scala.jdk.CollectionConverters.CollectionHasAsScala
 import scala.language.implicitConversions
 
@@ -52,6 +56,77 @@ object ArrowUtils extends LazyLogging {
 
   implicit def bool2int(b: Boolean): Int = if (b) 1 else 0
 
+  // Serialize a VectorSchemaRoot to Arrow IPC stream bytes. Does not close root.
+  def serializeRoot(root: VectorSchemaRoot): Array[Byte] = {
+    val out = new ByteArrayOutputStream()
+    val writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))
+    try { writer.start(); writer.writeBatch(); writer.end() }
+    finally writer.close()
+    out.toByteArray
+  }
+
+  // Serialize a batch of tuples to Arrow IPC stream bytes (columnar wire format).
+  def serializeTuples(schema: Schema, tuples: Array[Tuple]): Array[Byte] = {
+    val root = VectorSchemaRoot.create(fromTexeraSchema(schema), allocator)
+    try {
+      var i = 0
+      while (i < tuples.length) { setTexeraTuple(tuples(i), i, root); i += 1 }
+      root.setRowCount(tuples.length)
+      serializeRoot(root)
+    } finally root.close()
+  }
+
+  // Read one Arrow IPC batch, apply `f` to the live root, then close the reader.
+  // `f` must copy anything it needs out (the root is freed on return).
+  def deserializeRootFold[T](bytes: Array[Byte], allocator: BufferAllocator)(
+      f: VectorSchemaRoot => T
+  ): T = {
+    val reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), allocator)
+    try {
+      reader.loadNextBatch()
+      f(reader.getVectorSchemaRoot)
+    } finally reader.close()
+  }
+
+  // Copy the rows of `src` where keep(i) is true into a new independent root.
+  def selectRows(
+      src: VectorSchemaRoot,
+      keep: Array[Boolean],
+      allocator: BufferAllocator
+  ): VectorSchemaRoot = {
+    val dest = VectorSchemaRoot.create(src.getSchema, allocator)
+    val srcVecs = src.getFieldVectors
+    val destVecs = dest.getFieldVectors
+    var destIdx = 0
+    var i = 0
+    val n = src.getRowCount
+    while (i < n) {
+      if (keep(i)) {
+        var c = 0
+        while (c < srcVecs.size) { destVecs.get(c).copyFromSafe(i, destIdx, srcVecs.get(c)); c += 1 }
+        destIdx += 1
+      }
+      i += 1
+    }
+    dest.setRowCount(destIdx)
+    dest
+  }
+
+  // Inverse of serializeTuples: decode Arrow IPC stream bytes back to tuples.
+  def deserializeTuples(bytes: Array[Byte]): Array[Tuple] = {
+    val reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), allocator)
+    try {
+      val out = ArrayBuffer[Tuple]()
+      val root = reader.getVectorSchemaRoot
+      while (reader.loadNextBatch()) {
+        val n = root.getRowCount
+        var i = 0
+        while (i < n) { out += getTexeraTuple(i, root); i += 1 }
+      }
+      out.toArray
+    } finally reader.close()
+  }
+
   /**
     * Reads a row of the given Arrow Vectors into a Texera.Tuple
     * e.g.,
@@ -176,13 +251,17 @@ object ArrowUtils extends LazyLogging {
     * @param vectorSchemaRoot The root of the Vectors that stores the Arrow Fields. It contains
     *                         multiple Vectors.
     */
-  def setTexeraTuple(tuple: Tuple, index: Int, vectorSchemaRoot: VectorSchemaRoot): Unit = {
+  def setTexeraTuple(tuple: Tuple, index: Int, vectorSchemaRoot: VectorSchemaRoot): Unit =
+    setRow(tuple.getFields, index, vectorSchemaRoot)
+
+  // Fill row `index` of the Arrow vectors from a parsed field array (schema order).
+  def setRow(fields: Array[Any], index: Int, vectorSchemaRoot: VectorSchemaRoot): Unit = {
     val arrowSchema = vectorSchemaRoot.getSchema
     val arrowFields = arrowSchema.getFields.asScala.toList
 
     for (i <- arrowFields.indices) {
       val vector: FieldVector = vectorSchemaRoot.getVector(i)
-      val value = tuple.getField[AnyRef](i)
+      val value = fields(i)
       val isNull = value == null
       arrowFields.apply(i).getFieldType.getType match {
         case _: ArrowType.Int =>
diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala
new file mode 100644
index 00000000000..c6b932fb009
--- /dev/null
+++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.util
+
+import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple}
+import org.scalatest.flatspec.AnyFlatSpec
+
+import java.sql.Timestamp
+
+/**
+  * Foundation for the Arrow columnar wire format: a batch of tuples must survive
+  * serializeTuples -> Arrow IPC bytes -> deserializeTuples unchanged.
+  */
+class ArrowIpcRoundTripSpec extends AnyFlatSpec {
+
+  private val schema: Schema = Schema()
+    .add(new Attribute("i", AttributeType.INTEGER))
+    .add(new Attribute("l", AttributeType.LONG))
+    .add(new Attribute("d", AttributeType.DOUBLE))
+    .add(new Attribute("s", AttributeType.STRING))
+    .add(new Attribute("b", AttributeType.BOOLEAN))
+    .add(new Attribute("t", AttributeType.TIMESTAMP))
+
+  private def tuple(i: Int): Tuple =
+    Tuple
+      .builder(schema)
+      .addSequentially(
+        Array(
+          if (i % 10 == 0) null else Int.box(i),
+          Long.box(i.toLong * 1000L),
+          Double.box(i * 1.5),
+          if (i % 7 == 0) null else s"row-$i",
+          Boolean.box(i % 2 == 0),
+          new Timestamp(1_600_000_000_000L + i.toLong * 86_400_000L)
+        )
+      )
+      .build()
+
+  "Arrow IPC" should "round-trip a batch of tuples unchanged across all types" in {
+    val n = 5000
+    val rows = (0 until n).map(tuple).toArray
+    val bytes = ArrowUtils.serializeTuples(schema, rows)
+    val back = ArrowUtils.deserializeTuples(bytes)
+    assert(back.length == n)
+    assert(back.sameElements(rows))
+  }
+
+  "Arrow IPC" should "round-trip an empty batch" in {
+    val bytes = ArrowUtils.serializeTuples(schema, Array.empty[Tuple])
+    assert(ArrowUtils.deserializeTuples(bytes).isEmpty)
+  }
+}
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
index 30f1d955865..d996c98f52c 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
@@ -19,11 +19,206 @@
 
 package org.apache.texera.amber.operator.filter
 
-import org.apache.texera.amber.core.tuple.Tuple
+import org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.vector.{Float8Vector, IntVector, VectorSchemaRoot}
+import org.apache.texera.amber.core.executor.ColumnarOperatorExecutor
+import org.apache.texera.amber.core.tuple.{
+  AttributeType,
+  AttributeTypeUtils,
+  ColumnarBatch,
+  Tuple,
+  TupleLike
+}
+import org.apache.texera.amber.core.workflow.PortIdentity
+import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
 
-class SpecializedFilterOpExec(descString: String) extends FilterOpExec {
+import java.sql.Timestamp
+
+class SpecializedFilterOpExec(descString: String)
+    extends FilterOpExec
+    with ColumnarOperatorExecutor {
   private val desc: SpecializedFilterOpDesc =
     objectMapper.readValue(descString, classOf[SpecializedFilterOpDesc])
+  // Row path: OR over predicates, per tuple.
   setFilterFunc((tuple: Tuple) => desc.predicates.exists(_.evaluate(tuple)))
+
+  private val vectorizedEnabled: Boolean =
+    sys.env.getOrElse("FILTER_VECTORIZED", "1") == "1"
+
+  // Per-predicate row test with per-batch invariants hoisted, resolved from the
+  // batch schema on first use. null = a predicate shape is not vectorizable, so
+  // the whole batch falls back to the row path.
+  @transient private var compiled: Array[Tuple => Boolean] = _
+  @transient private var compileAttempted: Boolean = false
+
+  private def cmpOf(c: ComparisonType): Int => Boolean =
+    c match {
+      case ComparisonType.GREATER_THAN             => (r: Int) => r > 0
+      case ComparisonType.GREATER_THAN_OR_EQUAL_TO => (r: Int) => r >= 0
+      case ComparisonType.LESS_THAN                => (r: Int) => r < 0
+      case ComparisonType.LESS_THAN_OR_EQUAL_TO    => (r: Int) => r <= 0
+      case ComparisonType.EQUAL_TO                 => (r: Int) => r == 0
+      case ComparisonType.NOT_EQUAL_TO             => (r: Int) => r != 0
+      case _                                       => null
+    }
+
+  // Compile one predicate to a Tuple => Boolean matching FilterPredicate.evaluate,
+  // hoisting the schema lookup and constant parse out of the per-row work.
+  // Returns null if the shape isn't supported.
+  private def compileOne(p: FilterPredicate, tpe: AttributeType): Tuple => Boolean = {
+    val attr = p.attribute
+    p.condition match {
+      case ComparisonType.IS_NULL     => (t: Tuple) => t.getField[Any](attr) == null
+      case ComparisonType.IS_NOT_NULL => (t: Tuple) => t.getField[Any](attr) != null
+      case cond =>
+        val cmp = cmpOf(cond)
+        if (cmp == null) return null
+        tpe match {
+          case AttributeType.INTEGER | AttributeType.DOUBLE =>
+            val c = try p.value.toDouble catch { case _: NumberFormatException => return null }
+            (t: Tuple) => {
+              val f = t.getField[Any](attr)
+              if (f == null) false else cmp(java.lang.Double.compare(f.asInstanceOf[Number].doubleValue(), c))
+            }
+          case AttributeType.LONG =>
+            val c = try java.lang.Long.valueOf(p.value.trim) catch { case _: NumberFormatException => return null }
+            (t: Tuple) => {
+              val f = t.getField[Any](attr)
+              if (f == null) false else cmp(java.lang.Long.compare(f.asInstanceOf[Number].longValue(), c))
+            }
+          case AttributeType.TIMESTAMP =>
+            val c = AttributeTypeUtils.parseTimestamp(p.value.trim).getTime
+            (t: Tuple) => {
+              val f = t.getField[Any](attr)
+              if (f == null) false else cmp(java.lang.Long.compare(f.asInstanceOf[Timestamp].getTime, c))
+            }
+          case AttributeType.BOOLEAN =>
+            val c = p.value.trim.toLowerCase
+            (t: Tuple) => {
+              val f = t.getField[Any](attr)
+              if (f == null) false else cmp(f.toString.toLowerCase.compareTo(c))
+            }
+          case AttributeType.STRING | AttributeType.ANY =>
+            // Mirror FilterPredicate.evaluateFilterString: numeric compare when
+            // both field and value parse as double, else lexicographic.
+            val valNum: java.lang.Double =
+              try java.lang.Double.valueOf(p.value) catch { case _: NumberFormatException => null }
+            (t: Tuple) => {
+              val f = t.getField[Any](attr)
+              if (f == null) false
+              else {
+                val s = f.toString
+                if (valNum == null) cmp(s.compareTo(p.value))
+                else {
+                  val fd = try java.lang.Double.valueOf(s.trim) catch { case _: NumberFormatException => null }
+                  if (fd == null) cmp(s.compareTo(p.value))
+                  else cmp(java.lang.Double.compare(fd, valNum))
+                }
+              }
+            }
+          case _ => null
+        }
+    }
+  }
+
+  private def compileAll(sample: Tuple): Unit = {
+    compileAttempted = true
+    val schema = sample.getSchema
+    val fns = desc.predicates.map { p =>
+      val tpe = try schema.getAttribute(p.attribute).getType catch { case _: Throwable => null }
+      if (tpe == null) null else compileOne(p, tpe)
+    }
+    if (fns.nonEmpty && fns.forall(_ != null)) compiled = fns.toArray
+  }
+
+  override def processBatchMultiPort(
+      batch: Array[Tuple],
+      port: Int
+  ): Option[Iterator[(TupleLike, Option[PortIdentity])]] = {
+    if (!vectorizedEnabled) return None
+    if (batch.isEmpty) return Some(Iterator.empty)
+    if (!compileAttempted) compileAll(batch(0))
+    if (compiled == null) return None
+
+    val cb = new ColumnarBatch(batch)
+    val n = batch.length
+    val mask = new Array[Boolean](n)
+
+    if (compiled.length == 1 && desc.predicates.size == 1 && isNumericSimd(batch(0))) {
+      // SIMD fast path: single numeric ordered/eq predicate over a primitive column.
+      val p = desc.predicates.head
+      val cmp = cmpOf(p.condition)
+      val c = p.value.toDouble
+      val (vals, isNull) = cb.doubleColumn(p.attribute)
+      var i = 0
+      while (i < n) {
+        mask(i) = !isNull(i) && cmp(java.lang.Double.compare(vals(i), c))
+        i += 1
+      }
+    } else {
+      // General path: OR the compiled per-predicate tests over the rows.
+      var i = 0
+      while (i < n) {
+        val t = batch(i)
+        var keep = false
+        var k = 0
+        while (!keep && k < compiled.length) { keep = compiled(k)(t); k += 1 }
+        mask(i) = keep
+        i += 1
+      }
+    }
+    Some(cb.select(mask).map(t => (t, None)))
+  }
+
+  private def isNumericSimd(sample: Tuple): Boolean = {
+    val p = desc.predicates.head
+    if (cmpOf(p.condition) == null) return false
+    val tpe = try sample.getSchema.getAttribute(p.attribute).getType catch { case _: Throwable => return false }
+    (tpe == AttributeType.INTEGER || tpe == AttributeType.DOUBLE) &&
+    (try { p.value.toDouble; true } catch { case _: NumberFormatException => false })
+  }
+
+  // ---- Native-Arrow path (M2): consume the Arrow batch directly, no tuple decode.
+  @transient private var columnarAllocator: RootAllocator = _
+
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): Option[VectorSchemaRoot] = {
+    if (!vectorizedEnabled || desc.predicates.size != 1) return None
+    val p = desc.predicates.head
+    val cmp = cmpOf(p.condition)
+    if (cmp == null) return None
+    val c = try p.value.toDouble catch { case _: NumberFormatException => return None }
+    if (columnarAllocator == null) columnarAllocator = new RootAllocator()
+    ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
+      val fields = root.getSchema.getFields
+      var fieldIdx = -1
+      var k = 0
+      while (k < fields.size && fieldIdx < 0) {
+        if (fields.get(k).getName == p.attribute) fieldIdx = k
+        k += 1
+      }
+      if (fieldIdx < 0) None
+      else {
+        val n = root.getRowCount
+        val mask = new Array[Boolean](n)
+        root.getVector(fieldIdx) match {
+          case v: Float8Vector =>
+            var i = 0
+            while (i < n) { mask(i) = !v.isNull(i) && cmp(java.lang.Double.compare(v.get(i), c)); i += 1 }
+            Some(ArrowUtils.selectRows(root, mask, columnarAllocator))
+          case v: IntVector =>
+            var i = 0
+            while (i < n) {
+              mask(i) = !v.isNull(i) && cmp(java.lang.Double.compare(v.get(i).toDouble, c)); i += 1
+            }
+            Some(ArrowUtils.selectRows(root, mask, columnarAllocator))
+          case _ => None // non-numeric column: fall back to the row path
+        }
+      }
+    }
+  }
+
+  override def close(): Unit = {
+    if (columnarAllocator != null) columnarAllocator.close()
+  }
 }
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpExec.scala
index 147238536d8..22a18bf7541 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpExec.scala
@@ -21,9 +21,12 @@ package org.apache.texera.amber.operator.source.scan.csv
 
 import com.univocity.parsers.common.TextParsingException
 import com.univocity.parsers.csv.{CsvFormat, CsvParser, CsvParserSettings}
+import org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.vector.VectorSchemaRoot
 import org.apache.texera.amber.core.executor.SourceOperatorExecutor
 import org.apache.texera.amber.core.storage.DocumentFactory
 import org.apache.texera.amber.core.tuple.{AttributeTypeUtils, Schema, TupleLike}
+import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
 import org.apache.texera.dao.SiteSettings
 
@@ -40,6 +43,8 @@ class CSVScanSourceOpExec private[csv] (descString: String) extends SourceOperat
   var numRowGenerated = 0
   private var maxColumns: Int = CSVScanSourceOpExec.DEFAULT_MAX_COLUMNS
   private val schema: Schema = desc.sourceSchema()
+  private var columnarAllocator: RootAllocator = _
+  private val columnarBatchRows = 10000
 
   override def produceTuple(): Iterator[TupleLike] = {
 
@@ -80,6 +85,37 @@ class CSVScanSourceOpExec private[csv] (descString: String) extends SourceOperat
     tupleIterator
   }
 
+  // Columnar produce: parse CSV straight into Arrow batches (no per-row Tuple).
+  override def produceColumnarBatch(): Option[Iterator[VectorSchemaRoot]] = {
+    columnarAllocator = new RootAllocator()
+    var toSkip = desc.offset.getOrElse(0)
+    while (toSkip > 0) { CSVScanSourceOpExec.parseNextRow(parser, maxColumns); toSkip -= 1 }
+    val limit = desc.limit.getOrElse(Int.MaxValue)
+    Some(new Iterator[VectorSchemaRoot] {
+      private var pending: Array[String] = CSVScanSourceOpExec.parseNextRow(parser, maxColumns)
+      private var emitted = 0
+
+      override def hasNext: Boolean = pending != null && emitted < limit
+
+      override def next(): VectorSchemaRoot = {
+        val root = VectorSchemaRoot.create(ArrowUtils.fromTexeraSchema(schema), columnarAllocator)
+        var filled = 0
+        while (pending != null && filled < columnarBatchRows && emitted < limit) {
+          try {
+            val fields = AttributeTypeUtils.parseFields(pending.asInstanceOf[Array[Any]], schema)
+            ArrowUtils.setRow(fields, filled, root)
+            filled += 1
+          } catch { case _: Throwable => } // skip malformed row (matches row path)
+          emitted += 1
+          numRowGenerated += 1
+          pending = CSVScanSourceOpExec.parseNextRow(parser, maxColumns)
+        }
+        root.setRowCount(filled)
+        root
+      }
+    })
+  }
+
   override def open(): Unit = {
     inputReader = new InputStreamReader(
       DocumentFactory.openReadonlyDocument(new URI(desc.fileName.get)).asInputStream(),
@@ -110,6 +146,9 @@ class CSVScanSourceOpExec private[csv] (descString: String) extends SourceOperat
     if (inputReader != null) {
       inputReader.close()
     }
+    if (columnarAllocator != null) {
+      columnarAllocator.close()
+    }
   }
 }
 
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ArrowColumnarFilterBenchSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ArrowColumnarFilterBenchSpec.scala
new file mode 100644
index 00000000000..d73b1674283
--- /dev/null
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ArrowColumnarFilterBenchSpec.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.operator.filter
+
+import org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.vector.{Float8Vector, VectorSchemaRoot}
+import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, ColumnarBatch, Schema, Tuple}
+import org.apache.texera.amber.util.ArrowUtils
+import org.scalatest.flatspec.AnyFlatSpec
+
+/**
+  * Spike for the columnar direction: how fast does `l_quantity > 25` run over 6M
+  * rows when the data arrives ROW-oriented vs COLUMNAR (Apache Arrow)?
+  *   ROW   : stock FilterPredicate per Tuple.
+  *   T-COL : ColumnarBatch extracts the column from the Tuples, then a tight loop
+  *           (today's vectorized filter -- still pays the extraction).
+  *   ARROW : the data already lives in an Arrow Float8Vector (what a columnar
+  *           wire format would give), filtered by a tight loop, no extraction.
+  * The ARROW number is the end-to-end ceiling a columnar data flow unlocks.
+  * Prints ARROWBENCH lines. All three produce the same survivor count.
+  */
+class ArrowColumnarFilterBenchSpec extends AnyFlatSpec {
+
+  private val n = sys.env.getOrElse("ARROWBENCH_N", "6000000").toInt
+  private val warmups = sys.env.getOrElse("ARROWBENCH_WARMUPS", "3").toInt
+  private val runs = sys.env.getOrElse("ARROWBENCH_RUNS", "5").toInt
+  private val threshold = 25.0
+
+  private val schema: Schema = Schema().add(new Attribute("l_quantity", AttributeType.DOUBLE))
+  private val attr = schema.getAttribute("l_quantity")
+
+  private val rows: Array[Tuple] = {
+    val a = new Array[Tuple](n)
+    var i = 0
+    while (i < n) { a(i) = Tuple.builder(schema).add(attr, Double.box((i % 50) + 1.0)).build(); i += 1 }
+    a
+  }
+
+  // Fill an Arrow columnar batch once (the cost a columnar SOURCE would absorb).
+  private val allocator = new RootAllocator()
+  private val root: VectorSchemaRoot = {
+    val r = VectorSchemaRoot.create(ArrowUtils.fromTexeraSchema(schema), allocator)
+    var i = 0
+    while (i < n) { ArrowUtils.setTexeraTuple(rows(i), i, r); i += 1 }
+    r.setRowCount(n)
+    r
+  }
+  private val qty: Float8Vector = root.getVector("l_quantity").asInstanceOf[Float8Vector]
+
+  private def rowPath(): Int = {
+    val pred = new FilterPredicate("l_quantity", ComparisonType.GREATER_THAN, threshold.toString)
+    var c = 0; var i = 0
+    while (i < n) { if (pred.evaluate(rows(i))) c += 1; i += 1 }
+    c
+  }
+
+  private def tupleColPath(): Int = {
+    val (vals, isNull) = new ColumnarBatch(rows).doubleColumn("l_quantity")
+    var c = 0; var i = 0
+    while (i < n) { if (!isNull(i) && vals(i) > threshold) c += 1; i += 1 }
+    c
+  }
+
+  private def arrowColPath(): Int = {
+    var c = 0; var i = 0
+    while (i < n) { if (!qty.isNull(i) && qty.get(i) > threshold) c += 1; i += 1 }
+    c
+  }
+
+  private def time(name: String, fn: () => Int): (Double, Int) = {
+    var last = 0
+    for (_ <- 0 until warmups) last = fn()
+    var best = Double.MaxValue
+    for (_ <- 0 until runs) {
+      val t = System.nanoTime(); last = fn(); best = math.min(best, (System.nanoTime() - t) / 1e6)
+    }
+    println(f"ARROWBENCH $name: best=${best}%.1fms (n=$n, survivors=$last)")
+    (best, last)
+  }
+
+  "Filter over 6M rows" should "compare row vs Tuple-columnar vs Arrow-columnar" in {
+    val (rowMs, rc) = time("ROW  ", () => rowPath())
+    val (tMs, tc) = time("T-COL", () => tupleColPath())
+    val (aMs, ac) = time("ARROW", () => arrowColPath())
+    assert(rc == tc && tc == ac) // identical results
+    println(f"ARROWBENCH SPEEDUP  row/T-COL=${rowMs / tMs}%.1fx  row/ARROW=${rowMs / aMs}%.1fx  T-COL/ARROW=${tMs / aMs}%.1fx")
+    root.close(); allocator.close()
+  }
+}
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ColumnarScanFilterBenchSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/ColumnarScanFilterBenchSpec.scala
new file mode 100644
index 0000000000000000000000000000000000000000..af33861f4cd1ed85d6946392133f9e318fe20736
GIT binary patch
literal 5614
zcmcIoZExE+628y=6|+WfE4Pl^_I8Uyx+&l|&epiZ4kEY3Vy{RUnYMM+TS>}}H^_hA
zXGkiRle)KGwkQxwBxjzPdFF-cJ!{c3I!j`iYdIrRkXcIVmm<cWsF<6zP?AoIDxV3H
z6gdU`=(Iz4l!}U6^65fRR$y0L<VGdemBEYDNhGmQQf4wYIwXpuwBpB;>1Z&dd6L@m
zvqZad0I*KX5_=L&>q32@IYf!sEMe6mr6k9T%mQJ>N-l(&CHVrX%Z*AFOGCvvmr5^_
z68ffWN_1L-(N1u!7PfAR%A3-#+B>65A0UWLeigo;02^3k+po4eZ|vzzY$z`bRhmr$
zY?e;)CYGfETmZ|;G!c1hPkZZR?QmfH4G>sJacu<l3u_7$bL%LtjD&%1R!_7vrabKR
z*6VdBY?Q)6EqdPFx_1W0j-%lLF#99#^HgdLUwy3-1rJ|uNR+@7i)$cF#hT-1!(_t`
z3~L2{=Zh{Y)HMZ~L2}V&m(>7acyj}sCUQRB?ng9=+H~BHMp2hOj;8M>@2B*!e|gy-
zPe;RuCYLmrj88_>(PWILQ|ga@<CFgxjZeBHK?e{`-js@s0Mdm0DQ68f)fkv7K1#Zj
zaWYS0n3pdqv5>SVR#N3ml~SrK(HvL}kTZ}YON^zIX1$MW3-PwsYQfiESP;BjJKo3}
zsYEEUYpKElas`+lf&fOURtrE2WtaeAl_#qrPRu4Oh0<VE80%GdJJ73BOXn`(=Y?Hr
z{EVdGFzCBlR715;s#u4aMBZ(}%c3y-G%aG*zjyOW#tiTFG%bYr_XqpL9Ot(YvON<l
z=Ud8l?2M7=p{dGLhPQp=^LDx^WtVmy-<u@WU2>AUH02ZjS9pa(VV`ZY$D6k3Sfs*8
zZNfSJa)85rzB$G32*2;${Xel1VN$?M<l!hUD-#(dMb@Rp!=)4`Uw5rm?-{ZRK|g;*
za+S;^()5s^DSz&wSmz-mr1bjy0KmYP9ITWX4WbWS`kY>Utwe4RTzX5de*TQCfk3k^
zXe~A<ddCF{nqG1h<%E$pA~vK&Ciy&BK3EzWxDaH?K~v^t7n$eS3~4i&8bQdhvQjeL
z5X1OvJ^Goht0bM#bNkMNUTLyEgt)ZIaxO_Jb~pl9cy42e0tun`&&b%u<PTedT^ZuD
zLRohJOr>Ip<N(XQXru&CfuD3X8V^U);rSIC<C_PBOBiOBAzi2hm7k(Pe|$PRn+`9J
zhvUIJ>p~}_*NRht#2#3r0z?$~sZF3`1Y^meh0Bc0XRv7-Zn|v&LaCA!C_yU(2MKin
zm2^ZJUYGeQTu3uf!&J)vTM31`0kzFhqHk<%PcH`Vu1@;Xe!ENUp2^CdDdS}i%}-qE
z`O9_(j%=gcIu39iw;`(Y0L`^9@$#}*Yv6hL0{<SYn<dKGvOwUDz!{WktoM3bYH6E~
zt{E;ZS>F&7M59$evW?eC6@NK7K3GWfJ9uH{qiy}#^o)~r=ra)tT}lRwnJ7ee4ch5d
zp^y<@<OXK%q;YsQ8UNa;cY&^E@1`Ajx+=@PJ8N<5#=PS4`E9j366-X3wDn~2{`hQo
zkI~UZ&ftSs?%TX08J#P}fMfP;uHhPSvv13eJS9z~a+ImXqwUf3a@6oAJ3yoPRC?de
z9l<=-fQgT-dLR}(2R-e3J9oq?8Ox;n?*7^Zj3CJ|u2u14cX<>$viU$(%>EK&Oo`ud
zLzB}k7m({VPdL@@NvlIsp!$hTn29pD;x+D04>R<u*^Fj4$cCSqm1HL8l*_ei#X;wA
z+n2HP)*8meW}X>5V<WG-79L<8#OqK?b0Y0XAyx3S`*i#I46%+RGKjl$(Ee}EBcwkv
zK)_&q@bn)~U0SjhE*@To15^(LtV^?|b8$03XVDi@xdIv-ylk%7qN;<}Kh-ZiR%mr<
ztdw`QzzlDULTf_n8Rk40&H@D4x{YjsEgzx@w3AVW*K(2M3=z{xz=$8UTKg7tz1fii
zU$Q$LF9+^LOSXkD;(j^v-SN3CeU08of$g96f`D=Lnma7_8{Of?RP%&m32$r3;hzQQ
znTOmK#pZWR1VH);;dRewET8DeM)#gZe8|oqZe%-6WIEF<Boumt1mKB&GXH%m;1fEw
zWn<?dp>T*>6VqvwGjX;=!c@yGfPLn=-UVJ@GB=WIa--sbAh0FTMb5y&UoVG!bknQp
zyZ#t`>ebKfEp2?D9A$i!U!jlN5(j16i#VjpjtoasIt$0Bn|}9t$sL@Hr5^yUO4E+V
z?5Hre&VZ<0_c{n}3!7;Nl+ey6$OZ2lE+i4IB4B$W(sUCzVI8;610YJ~6tHtbiA<>2
zAEwjsCOk(^wKmXR3-0WC8ER9M><A_%a=^L(OO|F$p146tsC{t)Qa4@uVC_uY_dZU6
z2Uo$PM?-ZOW4MA1V2ht@!?`N5sl7k=cI^GrrF&i5kfK;YDhuaw=0^6wRWP)@CLf!_
z{U;yjl>Ry;FuNi3q3myG3C0~(hHnTMXnFp`6Yfojk)2sFHx`s&cC<e8vnvI4pb$(!
zb1>76r=ATExh^UURaUUXB)#$Cx)_rUwAg94u=fZ*^YXnCTy)<y;sVzTFN`bj3XX96
ztiA0QnVwjzlLMjPay$ke@R>c>A&JS8{6Zw=(l2yrwHvXqoqs-sv!X)P{Q$Phz*j`F
zh9Ddb#<OOpg4N&8ncD<)sh!-1kdxgfbV9#)iP&ya><0#zXbU{)(2qY_#Nb$fr?+sC
zA5)vD_l6P+2wB9k`CkplvxXDQEpQ3>R)ulvT!Xei|4xvPDuK<VI4y8HU@Fz=9p1m%
zTPjSF$sou54HLEPBVC&3hZJ<^=q=ZGRQce<#*(j`2CseXoaflX6cs9~N(w}Z8y-Js
znnWn2__h%9SGPLt6<uRC#BnDrM6<W-cw=ifZcV)A(v-h^1XuLS0lmN#4R>hz7e|;W
z%$uT(#(g!+MP5u9JuLRe8Pl6u53Ux=FiY})^>!(Ebc_xN_VnnbeBJROMiI?Tng{dt
z!#my+=OdPN^yHTR-2F3rF^9E*uG9)2M3g>y!Z<p<d3n9|75HO|bek`P{xaUiHQJ_k
z8}lt%53a1m`oMM>s;|5Ub%8)#@fi*$;BwmLk8#_^%gyZ^pC?qz0kSb|Urs*anh3*f
zD6zqHgE}6mtv-AxZX6m+&d$g8hidK3lepeWWdap*!s4h+js1R$E{4OC_ZMtiuNDW0
mPi`$@cpop%HGKJUeq(zyyla<g_hKEd>Rs${=is~6UF%QA*dpWr

literal 0
HcmV?d00001

diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
new file mode 100644
index 00000000000..8d7be8a2503
--- /dev/null
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.operator.filter
+
+import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple}
+import org.apache.texera.amber.util.ArrowUtils
+import org.apache.texera.amber.util.JSONUtils.objectMapper
+import org.scalatest.flatspec.AnyFlatSpec
+
+import java.sql.Timestamp
+import scala.util.Random
+
+/**
+  * Lever 8.2 correctness: the vectorized batch path (processBatchMultiPort) must
+  * return exactly the same surviving tuples as the row path (the stock
+  * FilterPredicate), across every attribute type, every comparison operator,
+  * nulls, IS NULL / IS NOT NULL, and multi-predicate OR.
+  */
+class VectorizedFilterCorrectnessSpec extends AnyFlatSpec {
+
+  private val orderedOps = Seq(
+    ComparisonType.GREATER_THAN,
+    ComparisonType.GREATER_THAN_OR_EQUAL_TO,
+    ComparisonType.LESS_THAN,
+    ComparisonType.LESS_THAN_OR_EQUAL_TO,
+    ComparisonType.EQUAL_TO,
+    ComparisonType.NOT_EQUAL_TO
+  )
+
+  private def execFor(preds: (String, ComparisonType, String)*): SpecializedFilterOpExec = {
+    val desc = new SpecializedFilterOpDesc()
+    desc.predicates = preds.map { case (a, c, v) => new FilterPredicate(a, c, v) }.toList
+    new SpecializedFilterOpExec(objectMapper.writeValueAsString(desc))
+  }
+
+  private def build(schema: Schema, attr: String, values: Seq[Any]): Array[Tuple] =
+    values.map(v => Tuple.builder(schema).add(schema.getAttribute(attr), v).build()).toArray
+
+  // For each operator, assert vectorized survivors == row survivors AND the fast path ran.
+  private def checkAllOps(schema: Schema, rows: Array[Tuple], attr: String, value: String): Unit =
+    orderedOps.foreach { op =>
+      val exec = execFor((attr, op, value))
+      val row = rows.filter(exec.filterFunc).toList
+      val vec = exec.processBatchMultiPort(rows, 0)
+      assert(vec.isDefined, s"expected vectorized path for $op on $attr")
+      assert(vec.get.map(_._1).toList == row, s"mismatch: op=$op attr=$attr value=$value")
+    }
+
+  private def col(name: String, t: AttributeType) = Schema().add(new Attribute(name, t))
+
+  "Vectorized filter" should "match the row path on DOUBLE (with nulls)" in {
+    val s = col("v", AttributeType.DOUBLE)
+    val rng = new Random(1)
+    val rows = build(s, "v", (0 until 4000).map(i => if (i % 17 == 0) null else Double.box((rng.nextInt(100) - 10).toDouble)))
+    checkAllOps(s, rows, "v", "25")
+  }
+
+  "Vectorized filter" should "match the row path on INTEGER (with nulls)" in {
+    val s = col("v", AttributeType.INTEGER)
+    val rng = new Random(2)
+    val rows = build(s, "v", (0 until 4000).map(i => if (i % 13 == 0) null else Int.box(rng.nextInt(60))))
+    checkAllOps(s, rows, "v", "30")
+  }
+
+  "Vectorized filter" should "match the row path on LONG (with nulls)" in {
+    val s = col("v", AttributeType.LONG)
+    val rng = new Random(3)
+    val rows = build(s, "v", (0 until 4000).map(i => if (i % 11 == 0) null else Long.box(rng.nextInt(1000).toLong)))
+    checkAllOps(s, rows, "v", "500")
+  }
+
+  "Vectorized filter" should "match the row path on STRING, numeric value (coercion)" in {
+    val s = col("v", AttributeType.STRING)
+    val rng = new Random(4)
+    // mix of numeric-looking and non-numeric strings
+    val rows = build(s, "v", (0 until 4000).map { i =>
+      if (i % 19 == 0) null
+      else if (i % 3 == 0) s"item-${rng.nextInt(50)}"
+      else rng.nextInt(50).toString
+    })
+    checkAllOps(s, rows, "v", "25")
+  }
+
+  "Vectorized filter" should "match the row path on STRING, non-numeric value (lexicographic)" in {
+    val s = col("v", AttributeType.STRING)
+    val rng = new Random(5)
+    val rows = build(s, "v", (0 until 3000).map(i => if (i % 23 == 0) null else s"${('A' + rng.nextInt(26)).toChar}${rng.nextInt(10)}"))
+    checkAllOps(s, rows, "v", "M5")
+  }
+
+  "Vectorized filter" should "match the row path on BOOLEAN (with nulls)" in {
+    val s = col("v", AttributeType.BOOLEAN)
+    val rng = new Random(6)
+    val rows = build(s, "v", (0 until 2000).map(i => if (i % 7 == 0) null else Boolean.box(rng.nextBoolean())))
+    Seq(ComparisonType.EQUAL_TO, ComparisonType.NOT_EQUAL_TO).foreach { op =>
+      val exec = execFor(("v", op, "true"))
+      assert(exec.processBatchMultiPort(rows, 0).get.map(_._1).toList == rows.filter(exec.filterFunc).toList)
+    }
+  }
+
+  "Vectorized filter" should "match the row path on TIMESTAMP (with nulls)" in {
+    val s = col("v", AttributeType.TIMESTAMP)
+    val base = 1_600_000_000_000L
+    val rows = build(s, "v", (0 until 2000).map(i => if (i % 9 == 0) null else new Timestamp(base + i.toLong * 86_400_000L)))
+    checkAllOps(s, rows, "v", "2020-10-01 00:00:00")
+  }
+
+  "Vectorized filter" should "match the row path for IS NULL / IS NOT NULL" in {
+    val s = col("v", AttributeType.DOUBLE)
+    val rows = build(s, "v", (0 until 2000).map(i => if (i % 5 == 0) null else Double.box(i.toDouble)))
+    Seq(ComparisonType.IS_NULL, ComparisonType.IS_NOT_NULL).foreach { op =>
+      val exec = execFor(("v", op, ""))
+      assert(exec.processBatchMultiPort(rows, 0).get.map(_._1).toList == rows.filter(exec.filterFunc).toList)
+    }
+  }
+
+  "Vectorized filter" should "match the row path for multi-predicate OR" in {
+    val s = Schema().add(new Attribute("a", AttributeType.DOUBLE)).add(new Attribute("b", AttributeType.INTEGER))
+    val rng = new Random(8)
+    val rows = (0 until 3000).map { i =>
+      val a: Any = if (i % 15 == 0) null else Double.box((rng.nextInt(100)).toDouble)
+      val b: Any = if (i % 21 == 0) null else Int.box(rng.nextInt(100))
+      Tuple.builder(s).add(s.getAttribute("a"), a).add(s.getAttribute("b"), b).build()
+    }.toArray
+    val exec = execFor(("a", ComparisonType.GREATER_THAN, "80"), ("b", ComparisonType.LESS_THAN, "10"))
+    assert(exec.processBatchMultiPort(rows, 0).get.map(_._1).toList == rows.filter(exec.filterFunc).toList)
+  }
+
+  "Vectorized filter" should "fall back (None) when a numeric value cannot be parsed" in {
+    val s = col("v", AttributeType.DOUBLE)
+    val rows = build(s, "v", Seq(Double.box(1.0), Double.box(2.0)))
+    val exec = execFor(("v", ComparisonType.GREATER_THAN, "not-a-number"))
+    assert(exec.processBatchMultiPort(rows, 0).isEmpty)
+  }
+
+  "Native-Arrow filter" should "match the row path via processColumnarBatch (M2)" in {
+    Seq(("v", AttributeType.DOUBLE, "50"), ("w", AttributeType.INTEGER, "40")).foreach {
+      case (name, tpe, value) =>
+        val s = col(name, tpe)
+        val rng = new Random(99)
+        val rows = build(
+          s,
+          name,
+          (0 until 4000).map { i =>
+            if (i % 17 == 0) null
+            else if (tpe == AttributeType.DOUBLE) Double.box((rng.nextInt(100)).toDouble)
+            else Int.box(rng.nextInt(100))
+          }
+        )
+        orderedOps.foreach { op =>
+          val exec = execFor((name, op, value))
+          val bytes = ArrowUtils.serializeTuples(s, rows)
+          val resultRoot = exec.processColumnarBatch(bytes).get
+          val survivors =
+            (0 until resultRoot.getRowCount).map(i => ArrowUtils.getTexeraTuple(i, resultRoot)).toList
+          resultRoot.close()
+          exec.close()
+          val rowSurvivors = rows.filter(exec.filterFunc).toList
+          assert(
+            survivors.map(_.getField[Any](name)) == rowSurvivors.map(_.getField[Any](name)),
+            s"native-arrow mismatch: $name $op $value"
+          )
+        }
+    }
+  }
+}
diff --git a/todo.md b/todo.md
new file mode 100644
index 00000000000..c7c2af92289
--- /dev/null
+++ b/todo.md
@@ -0,0 +1,4 @@
+1. Create a commmitters page on the incubator texera website
+2. have the website pull from each branch for the releases respectively
+3. fix the ci auto sync docs and remove off of the incubator website for texera
+4. summarize meeting yesterday

From 9dc472a09a060b609ee3200832ddcb1f50b1dad1 Mon Sep 17 00:00:00 2001
From: Matthew Ball <mgball@uci.edu>
Date: Sun, 13 Sep 2026 01:00:40 -0700
Subject: [PATCH 2/4] feat(engine): columnar shuffle partitioning, aggregate,
 projection, and config

  Extends the Arrow columnar path (Tier 1 + Tier 2 hardening) so a
  scan -> projection -> filter -> aggregate workflow stays columnar end
to
  end, including across hash-shuffle edges and multiple workers.

  Tier 1 (correctness/safety):
  - OutputManager.emitColumnarBatch now splits each Arrow batch per
receiver
    using the link partitioner (one-to-one/broadcast ship whole;
hash/range/
    round-robin slice via selectRows), so multi-receiver edges stay
columnar
    instead of silently falling back to the row path
  - canEmitColumnar relaxed to allow multi-receiver links
  - one-time log per worker makes any row-path fallback visible
  - OutputManager owns a lazily-created emit allocator, freed on
FinalizeExecutor

  Tier 2 (coverage + config):
  - ColumnarResult (Emit/Consumed/Unsupported) replaces Option so
blocking
    operators can consume a batch with no output without a false
re-process
  - AggregateOpExec consumes Arrow directly: decodes only the group-key
and
    aggregated columns per row
(ArrowUtils.projectionIndices/getProjectedTuple)
    and reuses the existing accumulation, so COUNT/SUM/MIN/MAX/AVERAGE
are
    identical to the row path
  - ProjectionOpExec selects/renames/drops columns on the Arrow batch
    (ArrowUtils.selectColumns), no per-row decode
  - COLUMNAR_WIRE and the vectorized-operator flag move to
application.conf
    (columnar.*) via ApplicationConfig; the env vars still override

  Tests:
  - ColumnarShuffleCorrectnessSpec: 2-worker hash shuffle, ROW ==
COLUMNAR for
    COUNT/SUM/MIN/MAX/AVERAGE and for project(rename) -> filter ->
aggregate
  - ProjectionOpExecSpec: columnar rename/reorder and drop match the row
path
  - VectorizedFilterCorrectnessSpec updated to the new contract (11/11)
---
 .../messaginglayer/OutputManager.scala        |  72 +++++++-
 .../partitioners/Partitioner.scala            |   5 +-
 .../architecture/worker/DataProcessor.scala   |  30 ++-
 .../promisehandlers/EndChannelHandler.scala   |   8 +-
 .../e2e/ColumnarShuffleCorrectnessSpec.scala  | 172 ++++++++++++++++++
 .../src/main/resources/application.conf       |   9 +
 .../common/config/ApplicationConfig.scala     |  13 ++
 .../executor/ColumnarOperatorExecutor.scala   |  22 ++-
 .../apache/texera/amber/util/ArrowUtils.scala |  59 ++++++
 .../operator/aggregate/AggregateOpExec.scala  |  45 ++++-
 .../filter/SpecializedFilterOpExec.scala      |  24 +--
 .../projection/ProjectionOpExec.scala         |  39 +++-
 .../VectorizedFilterCorrectnessSpec.scala     |   6 +-
 .../projection/ProjectionOpExecSpec.scala     |  49 +++++
 14 files changed, 515 insertions(+), 38 deletions(-)
 create mode 100644 amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala

diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
index 6b5cb7f5200..9dad536e06a 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
@@ -39,6 +39,7 @@ import org.apache.texera.amber.engine.architecture.worker.managers.{
 import org.apache.texera.amber.engine.common.AmberLogging
 import org.apache.texera.amber.engine.common.ambermessage.ColumnarFrame
 import org.apache.texera.amber.util.{ArrowUtils, VirtualIdentityUtils}
+import org.apache.arrow.memory.RootAllocator
 import org.apache.arrow.vector.VectorSchemaRoot
 
 import java.net.URI
@@ -199,20 +200,73 @@ class OutputManager(
     saveStateToStorageIfNeeded(state)
   }
 
-  // Columnar emit is safe only when each output link has a single receiver
-  // (e.g. one-to-one); hash/range shuffles would need per-key column splitting.
-  def canEmitColumnar: Boolean =
-    networkOutputBuffers.nonEmpty && networkOutputBuffers.groupBy(_._1._1).forall(_._2.size == 1)
+  // Columnar emit is possible whenever there are output links; multi-receiver
+  // links (hash/range/round-robin shuffles) are split per-receiver in emit.
+  def canEmitColumnar: Boolean = partitioners.nonEmpty
 
-  // Send a whole Arrow batch downstream as a ColumnarFrame (no per-tuple path).
+  // Allocator for the per-receiver sub-batches built during columnar shuffle.
+  // Created lazily on first shuffle emit, freed by closeColumnarResources().
+  private var columnarEmitAllocator: RootAllocator = _
+  private def emitAllocator(): RootAllocator = {
+    if (columnarEmitAllocator == null) columnarEmitAllocator = new RootAllocator()
+    columnarEmitAllocator
+  }
+  def closeColumnarResources(): Unit = {
+    if (columnarEmitAllocator != null) {
+      columnarEmitAllocator.close()
+      columnarEmitAllocator = null
+    }
+  }
+
+  // Emit an Arrow batch downstream as ColumnarFrames, honoring each link's
+  // partitioning. Single-receiver links and broadcast ship the whole batch;
+  // shuffles slice it into one sub-batch per receiver.
   def emitColumnarBatch(root: VectorSchemaRoot): Unit = {
-    val bytes = ArrowUtils.serializeRoot(root)
     val rowCount = root.getRowCount
     val texeraSchema = ArrowUtils.toTexeraSchema(root.getSchema)
-    networkOutputBuffers.keys.foreach {
-      case (_, receiver) =>
-        outputGateway.sendTo(receiver, ColumnarFrame(bytes, rowCount, texeraSchema))
+    lazy val wholeBatchBytes = ArrowUtils.serializeRoot(root)
+
+    partitioners.foreach {
+      case (_, partitioner) =>
+        val receivers = partitioner.allReceivers
+        partitioner match {
+          case _ if receivers.size == 1 =>
+            outputGateway.sendTo(
+              receivers.head,
+              ColumnarFrame(wholeBatchBytes, rowCount, texeraSchema)
+            )
+          case _: BroadcastPartitioner =>
+            receivers.foreach(r =>
+              outputGateway.sendTo(r, ColumnarFrame(wholeBatchBytes, rowCount, texeraSchema))
+            )
+          case _ =>
+            // Shuffle/round-robin: assign each row to a receiver via the
+            // partitioner, then ship one Arrow sub-batch per receiver. The
+            // row decode here is only to compute the partition key; downstream
+            // still receives columnar data.
+            val masks = Array.fill(receivers.size)(new Array[Boolean](rowCount))
+            var i = 0
+            while (i < rowCount) {
+              val t = ArrowUtils.getTexeraTuple(i, root)
+              partitioner.getBucketIndex(t).foreach(b => masks(b)(i) = true)
+              i += 1
+            }
+            var b = 0
+            while (b < receivers.size) {
+              val sub = ArrowUtils.selectRows(root, masks(b), emitAllocator())
+              try {
+                if (sub.getRowCount > 0) {
+                  outputGateway.sendTo(
+                    receivers(b),
+                    ColumnarFrame(ArrowUtils.serializeRoot(sub), sub.getRowCount, texeraSchema)
+                  )
+                }
+              } finally sub.close()
+              b += 1
+            }
+        }
     }
+
     // Materialize to result storage for the GUI result panel, matching the row
     // path's saveTupleToStorageIfNeeded. Only decodes when a port needs storage.
     if (outputPortResultWriterThreads.nonEmpty) {
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala
index 6c832e2d729..3c62c1338eb 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/sendsemantics/partitioners/Partitioner.scala
@@ -72,6 +72,7 @@ class NetworkOutputBuffer(
 }
 
 object NetworkOutputBuffer {
-  // Global opt-in for the Arrow columnar wire format (default off = row DataFrame).
-  val columnarWire: Boolean = sys.env.getOrElse("COLUMNAR_WIRE", "0") == "1"
+  // Opt-in for the Arrow columnar wire format (default off = row DataFrame),
+  // from application.conf (columnar.enable-columnar-wire), COLUMNAR_WIRE overrides.
+  val columnarWire: Boolean = ApplicationConfig.enableColumnarWire
 }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
index 32ae1381ef6..ecf892f23e4 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
@@ -21,7 +21,11 @@ package org.apache.texera.amber.engine.architecture.worker
 
 import com.softwaremill.macwire.wire
 import io.grpc.MethodDescriptor
-import org.apache.texera.amber.core.executor.{ColumnarOperatorExecutor, OperatorExecutor}
+import org.apache.texera.amber.core.executor.{
+  ColumnarOperatorExecutor,
+  ColumnarResult,
+  OperatorExecutor
+}
 import org.apache.texera.amber.engine.architecture.sendsemantics.partitioners.NetworkOutputBuffer
 import org.apache.texera.amber.core.state.State
 import org.apache.texera.amber.core.tuple._
@@ -162,6 +166,7 @@ class DataProcessor(
         sendECMToDataChannels(METHOD_END_CHANNEL, PORT_ALIGNMENT)
         // Send Completed signal to worker actor.
         executor.close()
+        outputManager.closeColumnarResources()
         adaptiveBatchingMonitor.stopAdaptiveBatching()
         stateManager.transitTo(COMPLETED)
         logger.info(
@@ -219,12 +224,20 @@ class DataProcessor(
           case c: ColumnarOperatorExecutor
               if NetworkOutputBuffer.columnarWire && outputManager.canEmitColumnar =>
             c.processColumnarBatch(bytes) match {
-              case Some(result) =>
+              case ColumnarResult.Emit(result) =>
+                logColumnarModeOnce(active = true, "")
                 statisticsManager.increaseInputStatistics(portId, bytes.length.toLong)
                 outputManager.setColumnarOutput(Iterator.single(result))
-              case None =>
+              case ColumnarResult.Consumed =>
+                logColumnarModeOnce(active = true, "")
+                statisticsManager.increaseInputStatistics(portId, bytes.length.toLong)
+              case ColumnarResult.Unsupported =>
+                logColumnarModeOnce(active = false, "operator has no native columnar path for this batch")
                 processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
             }
+          case _: ColumnarOperatorExecutor if NetworkOutputBuffer.columnarWire =>
+            logColumnarModeOnce(active = false, "output has no partitioned receivers")
+            processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
           case _ =>
             processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
         }
@@ -341,6 +354,17 @@ class DataProcessor(
       }
   }
 
+  // Log the columnar-wire decision once per worker so a silent fall back to the
+  // row path is visible in the logs.
+  @transient private var columnarModeLogged = false
+  private[architecture] def logColumnarModeOnce(active: Boolean, reason: String): Unit = {
+    if (!columnarModeLogged) {
+      columnarModeLogged = true
+      if (active) logger.info(s"columnar wire active for $executor")
+      else logger.info(s"columnar wire requested but using row path for $executor: $reason")
+    }
+  }
+
   def handleExecutorException(e: Throwable): Unit = {
     asyncRPCClient.coordinatorInterface.consoleMessageTriggered(
       ConsoleMessageTriggeredRequest(mkConsoleMessage(actorId, e)),
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
index 9ca6bbaa0ff..5791e30282d 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
@@ -48,8 +48,8 @@ trait EndChannelHandler {
         dp.outputManager.emitState(outputState.get)
       }
       // Columnar source path: emit Arrow batches directly (no per-row Tuple),
-      // when enabled and the output is single-receiver-per-link. Else the
-      // row-oriented onFinishMultiPort path.
+      // when enabled and there are output links (shuffles are split in emit).
+      // Else the row-oriented onFinishMultiPort path.
       val columnarBatches =
         if (NetworkOutputBuffer.columnarWire && dp.outputManager.canEmitColumnar)
           dp.executor match {
@@ -59,10 +59,14 @@ trait EndChannelHandler {
         else None
       columnarBatches match {
         case Some(batchIter) =>
+          dp.logColumnarModeOnce(active = true, "")
           // Drained one batch per DP-loop step (backpressure applies between).
           dp.outputManager.setColumnarOutput(batchIter)
           dp.outputManager.outputIterator.setTupleOutput(Iterator.empty)
         case None =>
+          if (NetworkOutputBuffer.columnarWire && dp.executor.isInstanceOf[SourceOperatorExecutor]) {
+            dp.logColumnarModeOnce(active = false, "source has no native columnar batch producer")
+          }
           dp.outputManager.outputIterator.setTupleOutput(
             dp.executor.onFinishMultiPort(portId.id)
           )
diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala
new file mode 100644
index 00000000000..f84efe171e1
--- /dev/null
+++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.engine.e2e
+
+import org.apache.pekko.actor.{ActorSystem, Props}
+import org.apache.pekko.testkit.{ImplicitSender, TestKit}
+import org.apache.pekko.util.Timeout
+import org.apache.texera.amber.clustering.SingleNodeListener
+import org.apache.texera.amber.core.tuple.Tuple
+import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
+import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext}
+import org.apache.texera.amber.engine.common.AmberRuntime
+import org.apache.texera.amber.engine.e2e.TestUtils.{
+  buildWorkflow,
+  cleanupWorkflowExecutionData,
+  initiateTexeraDBForTestCases,
+  runWorkflowAndReadTerminalResults,
+  setUpWorkflowExecutionData
+}
+import org.apache.texera.amber.operator.TestOperators
+import org.apache.texera.amber.operator.aggregate.{
+  AggregateOpDesc,
+  AggregationFunction,
+  AggregationOperation
+}
+import org.apache.texera.amber.operator.filter.{ComparisonType, FilterPredicate, SpecializedFilterOpDesc}
+import org.apache.texera.amber.operator.projection.{AttributeUnit, ProjectionOpDesc}
+import org.apache.texera.workflow.LogicalLink
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.flatspec.AnyFlatSpecLike
+
+import com.twitter.util.Duration
+import java.util.concurrent.atomic.AtomicInteger
+import scala.concurrent.duration.DurationInt
+
+/**
+  * Multi-worker columnar correctness: scan -> filter -> count-by-group. The
+  * aggregate's group-by induces a hash-shuffle edge upstream, so with more than
+  * one worker the columnar path must split each Arrow batch per receiver. Run
+  * this under COLUMNAR_WIRE=0 and =1 (with CONSTANTS_NUM_WORKER_PER_OPERATOR=2)
+  * and diff the SHUFFLE lines: the group counts must match. Proves the columnar
+  * shuffle partitioner produces the same result as the row path.
+  */
+class ColumnarShuffleCorrectnessSpec
+    extends TestKit(ActorSystem("ColumnarShuffleCorrectnessSpec", AmberRuntime.pekkoConfig))
+    with ImplicitSender
+    with AnyFlatSpecLike
+    with BeforeAndAfterAll {
+
+  implicit val timeout: Timeout = Timeout(5.seconds)
+  private val ids = new AtomicInteger(90000)
+  private val tag = if (sys.env.get("COLUMNAR_WIRE").contains("1")) "COLUMNAR" else "ROW"
+  private val workers = sys.env.getOrElse("CONSTANTS_NUM_WORKER_PER_OPERATOR", "1")
+
+  override def beforeAll(): Unit = {
+    system.actorOf(Props[SingleNodeListener](), "cluster-info")
+    Class.forName("org.postgresql.Driver")
+    initiateTexeraDBForTestCases()
+  }
+  override def afterAll(): Unit = TestKit.shutdownActorSystem(system)
+
+  private def filterGT(attr: String, v: String): SpecializedFilterOpDesc = {
+    val op = new SpecializedFilterOpDesc()
+    op.predicates = List(new FilterPredicate(attr, ComparisonType.GREATER_THAN, v)); op
+  }
+  private def agg(fn: AggregationFunction, attr: String, res: String): AggregationOperation = {
+    val a = new AggregationOperation(); a.aggFunction = fn; a.attribute = attr; a.resultAttribute = res; a
+  }
+  private def aggByRegion(): AggregateOpDesc = {
+    val op = new AggregateOpDesc()
+    op.aggregations = List(
+      agg(AggregationFunction.COUNT, "Region", "cnt"),
+      agg(AggregationFunction.SUM, "Units Sold", "sum_units"),
+      agg(AggregationFunction.MIN, "Units Sold", "min_units"),
+      agg(AggregationFunction.MAX, "Units Sold", "max_units"),
+      agg(AggregationFunction.AVERAGE, "Units Sold", "avg_units")
+    )
+    op.groupByKeys = List("Region"); op
+  }
+
+  private def report(res: Map[OperatorIdentity, List[Tuple]]): Unit = {
+    val rows = res.values.headOption.getOrElse(Nil)
+    val lines = rows
+      .map(t =>
+        s"${t.getField[Any]("Region")} cnt=${t.getField[Any]("cnt")} sum=${t.getField[Any]("sum_units")} " +
+          s"min=${t.getField[Any]("min_units")} max=${t.getField[Any]("max_units")} avg=${t.getField[Any]("avg_units")}"
+      )
+      .sorted
+    println(s"SHUFFLE[$tag workers=$workers] groups=${rows.size}")
+    lines.foreach(l => println(s"SHUFFLE[$tag]   $l"))
+  }
+
+  "columnar shuffle" should "scan -> filter -> agg-by-Region match the row path" in {
+    val id = ids.incrementAndGet(); setUpWorkflowExecutionData(id)
+    try {
+      val ctx: WorkflowContext = TestUtils.workflowContext(id)
+      val scan = TestOperators.smallCsvScanOpDesc()
+      val f = filterGT("Units Sold", "5000")
+      val aggOp = aggByRegion()
+      val wf = buildWorkflow(
+        List(scan, f, aggOp),
+        List(
+          LogicalLink(scan.operatorIdentifier, PortIdentity(), f.operatorIdentifier, PortIdentity()),
+          LogicalLink(f.operatorIdentifier, PortIdentity(), aggOp.operatorIdentifier, PortIdentity())
+        ),
+        ctx
+      )
+      report(runWorkflowAndReadTerminalResults(system, wf, Duration.fromMinutes(5)))
+    } finally cleanupWorkflowExecutionData(id)
+  }
+
+  private def projectRegionUnits(): ProjectionOpDesc = {
+    val op = new ProjectionOpDesc()
+    op.attributes = List(new AttributeUnit("Region", "Region"), new AttributeUnit("Units Sold", "units"))
+    op
+  }
+  private def countSumByRegionOn(unitsCol: String): AggregateOpDesc = {
+    val op = new AggregateOpDesc()
+    op.aggregations = List(
+      agg(AggregationFunction.COUNT, "Region", "cnt"),
+      agg(AggregationFunction.SUM, unitsCol, "sum_units")
+    )
+    op.groupByKeys = List("Region"); op
+  }
+  private def reportCntSum(res: Map[OperatorIdentity, List[Tuple]]): Unit = {
+    val rows = res.values.headOption.getOrElse(Nil)
+    val lines =
+      rows.map(t => s"${t.getField[Any]("Region")} cnt=${t.getField[Any]("cnt")} sum=${t.getField[Any]("sum_units")}").sorted
+    println(s"PROJ[$tag workers=$workers] groups=${rows.size}")
+    lines.foreach(l => println(s"PROJ[$tag]   $l"))
+  }
+
+  // Projection renames "Units Sold" -> "units"; the downstream filter and
+  // aggregate then reference the renamed column, all over the columnar wire.
+  "columnar projection" should "scan -> project(rename) -> filter -> agg match the row path" in {
+    val id = ids.incrementAndGet(); setUpWorkflowExecutionData(id)
+    try {
+      val ctx: WorkflowContext = TestUtils.workflowContext(id)
+      val scan = TestOperators.smallCsvScanOpDesc()
+      val proj = projectRegionUnits()
+      val f = filterGT("units", "5000")
+      val aggOp = countSumByRegionOn("units")
+      val wf = buildWorkflow(
+        List(scan, proj, f, aggOp),
+        List(
+          LogicalLink(scan.operatorIdentifier, PortIdentity(), proj.operatorIdentifier, PortIdentity()),
+          LogicalLink(proj.operatorIdentifier, PortIdentity(), f.operatorIdentifier, PortIdentity()),
+          LogicalLink(f.operatorIdentifier, PortIdentity(), aggOp.operatorIdentifier, PortIdentity())
+        ),
+        ctx
+      )
+      reportCntSum(runWorkflowAndReadTerminalResults(system, wf, Duration.fromMinutes(5)))
+    } finally cleanupWorkflowExecutionData(id)
+  }
+}
diff --git a/common/config/src/main/resources/application.conf b/common/config/src/main/resources/application.conf
index c7a7af24180..4584886c2ee 100644
--- a/common/config/src/main/resources/application.conf
+++ b/common/config/src/main/resources/application.conf
@@ -60,6 +60,15 @@ reconfiguration {
     enable-transactional-reconfiguration = ${?RECONFIGURATION_ENABLE_TRANSACTIONAL_RECONFIGURATION}
 }
 
+columnar {
+    # Arrow columnar wire format between operators (opt-in). The COLUMNAR_WIRE
+    # env var ("1"/"0") still overrides this, applied in ApplicationConfig.
+    enable-columnar-wire = false
+
+    # Vectorized path in operators that support it (e.g. the filter).
+    enable-vectorized-operators = true
+}
+
 cache {
     # [false, true]
     enabled = true
diff --git a/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala b/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala
index dea1e169d6a..017b020abf4 100644
--- a/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala
+++ b/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala
@@ -71,6 +71,19 @@ object ApplicationConfig {
   val enableTransactionalReconfiguration: Boolean =
     getConfSource.getBoolean("reconfiguration.enable-transactional-reconfiguration")
 
+  // Columnar execution. The COLUMNAR_WIRE env ("1"/"0") is honored for backward
+  // compatibility and overrides the config value when set.
+  val enableColumnarWire: Boolean =
+    sys.env.get("COLUMNAR_WIRE") match {
+      case Some(v) => v == "1" || v.equalsIgnoreCase("true")
+      case None    => getConfSource.getBoolean("columnar.enable-columnar-wire")
+    }
+  val enableVectorizedOperators: Boolean =
+    sys.env.get("FILTER_VECTORIZED") match {
+      case Some(v) => v == "1" || v.equalsIgnoreCase("true")
+      case None    => getConfSource.getBoolean("columnar.enable-vectorized-operators")
+    }
+
   // Fault tolerance
   val faultToleranceLogFlushIntervalInMs: Long =
     getConfSource.getLong("fault-tolerance.log-flush-interval-ms")
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
index 166f9330387..79a3bfcb832 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
@@ -21,13 +21,25 @@ package org.apache.texera.amber.core.executor
 
 import org.apache.arrow.vector.VectorSchemaRoot
 
+/**
+  * Outcome of consuming one Arrow batch:
+  *   - Emit: consumed, emit this result batch downstream (caller serializes and closes the root).
+  *   - Consumed: consumed, nothing to emit now (a blocking operator accumulating state).
+  *   - Unsupported: not handled, caller falls back to the row path (decode + processTuple).
+  */
+sealed trait ColumnarResult
+object ColumnarResult {
+  final case class Emit(root: VectorSchemaRoot) extends ColumnarResult
+  case object Consumed extends ColumnarResult
+  case object Unsupported extends ColumnarResult
+}
+
 /**
   * An operator that can consume an Arrow columnar batch directly (no per-row
-  * Tuple decode) and produce one. Given the incoming batch as Arrow IPC bytes,
-  * returns the result batch, or None if this batch shape is not supported (the
-  * caller then falls back to the row path). The returned root is owned by the
-  * caller, which serializes and closes it.
+  * Tuple decode). Given the incoming batch as Arrow IPC bytes, returns a
+  * ColumnarResult telling the caller whether it emitted a batch, consumed the
+  * batch with no output, or could not handle it (fall back to the row path).
   */
 trait ColumnarOperatorExecutor {
-  def processColumnarBatch(arrowIpcBytes: Array[Byte]): Option[VectorSchemaRoot]
+  def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult
 }
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
index e6b1df6f937..65bbb445585 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
@@ -112,6 +112,30 @@ object ArrowUtils extends LazyLogging {
     dest
   }
 
+  // Build a new root keeping only the source columns at `srcIndices`, renamed to
+  // `outSchema`'s attribute names (columnar projection). Copies all rows.
+  def selectColumns(
+      src: VectorSchemaRoot,
+      srcIndices: Array[Int],
+      outSchema: Schema,
+      allocator: BufferAllocator
+  ): VectorSchemaRoot = {
+    val dest = VectorSchemaRoot.create(fromTexeraSchema(outSchema), allocator)
+    val n = src.getRowCount
+    val srcVecs = src.getFieldVectors
+    val destVecs = dest.getFieldVectors
+    var c = 0
+    while (c < srcIndices.length) {
+      val sv = srcVecs.get(srcIndices(c))
+      val dv = destVecs.get(c)
+      var i = 0
+      while (i < n) { dv.copyFromSafe(i, i, sv); i += 1 }
+      c += 1
+    }
+    dest.setRowCount(n)
+    dest
+  }
+
   // Inverse of serializeTuples: decode Arrow IPC stream bytes back to tuples.
   def deserializeTuples(bytes: Array[Byte]): Array[Tuple] = {
     val reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), allocator)
@@ -168,6 +192,41 @@ object ArrowUtils extends LazyLogging {
       .build()
   }
 
+  // Source-column indices in `root` for the given attribute names, in order.
+  // Throws if a name is missing. Compute once per batch, reuse across rows.
+  def projectionIndices(root: VectorSchemaRoot, names: Seq[String]): Array[Int] = {
+    val fields = root.getSchema.getFields
+    names.map { name =>
+      var idx = -1
+      var k = 0
+      while (k < fields.size && idx < 0) { if (fields.get(k).getName == name) idx = k; k += 1 }
+      if (idx < 0) throw new IllegalArgumentException(s"column '$name' not found in Arrow batch")
+      idx
+    }.toArray
+  }
+
+  // Decode one row into a Tuple containing only the projected columns (cheaper
+  // than getTexeraTuple for wide inputs). `projectedSchema` and `srcIndices`
+  // (from projectionIndices) align by position.
+  def getProjectedTuple(
+      rowIndex: Int,
+      root: VectorSchemaRoot,
+      projectedSchema: Schema,
+      srcIndices: Array[Int]
+  ): Tuple = {
+    val vectors = root.getFieldVectors
+    val values = new Array[Any](srcIndices.length)
+    var i = 0
+    while (i < srcIndices.length) {
+      val raw = vectors.get(srcIndices(i)).getObject(rowIndex)
+      values(i) =
+        try AttributeTypeUtils.parseField(raw, projectedSchema.getAttributes(i).getType)
+        catch { case _: Exception => null }
+      i += 1
+    }
+    Tuple.builder(projectedSchema).addSequentially(values).build()
+  }
+
   /**
     * Converts an Arrow Schema into Texera Schema.
     * Checks field metadata to recover types that share an Arrow representation
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala
index 3c26709116f..7d3dd65c328 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala
@@ -19,8 +19,14 @@
 
 package org.apache.texera.amber.operator.aggregate
 
-import org.apache.texera.amber.core.executor.OperatorExecutor
-import org.apache.texera.amber.core.tuple.{Tuple, TupleLike}
+import org.apache.arrow.memory.RootAllocator
+import org.apache.texera.amber.core.executor.{
+  ColumnarOperatorExecutor,
+  ColumnarResult,
+  OperatorExecutor
+}
+import org.apache.texera.amber.core.tuple.{Schema, Tuple, TupleLike}
+import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
 
 import scala.collection.mutable
@@ -28,7 +34,7 @@ import scala.collection.mutable
 /**
   * AggregateOpExec performs aggregation operations on input tuples, optionally grouping them by specified keys.
   */
-class AggregateOpExec(descString: String) extends OperatorExecutor {
+class AggregateOpExec(descString: String) extends OperatorExecutor with ColumnarOperatorExecutor {
   private val desc: AggregateOpDesc = objectMapper.readValue(descString, classOf[AggregateOpDesc])
   private var keyedPartialAggregates: mutable.HashMap[List[Object], List[Object]] = _
   private var distributedAggregations: List[DistributedAggregation[Object]] = _
@@ -41,6 +47,7 @@ class AggregateOpExec(descString: String) extends OperatorExecutor {
   override def close(): Unit = {
     keyedPartialAggregates.clear()
     distributedAggregations = null
+    if (columnarAllocator != null) { columnarAllocator.close(); columnarAllocator = null }
   }
 
   override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = {
@@ -78,6 +85,38 @@ class AggregateOpExec(descString: String) extends OperatorExecutor {
 
   }
 
+  // ---- Native-Arrow path: consume the Arrow batch by decoding only the group
+  // keys and aggregated columns per row, then feeding the same processTuple
+  // accumulation. Blocking operator, so it emits nothing per batch (Consumed);
+  // results are produced at onFinish via the row path.
+  @transient private var columnarAllocator: RootAllocator = _
+  @transient private var neededNames: Seq[String] = _
+  @transient private var projSchema: Schema = _
+  @transient private var projIndices: Array[Int] = _
+
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult = {
+    if (neededNames == null) {
+      neededNames = (desc.groupByKeys ++ desc.aggregations.flatMap(a =>
+        Option(a.attribute).map(_.trim).filter(_.nonEmpty)
+      )).distinct
+    }
+    if (columnarAllocator == null) columnarAllocator = new RootAllocator()
+    ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
+      if (projSchema == null) {
+        val full = ArrowUtils.toTexeraSchema(root.getSchema)
+        projSchema = Schema(neededNames.map(full.getAttribute).toList)
+        projIndices = ArrowUtils.projectionIndices(root, neededNames)
+      }
+      val n = root.getRowCount
+      var i = 0
+      while (i < n) {
+        processTuple(ArrowUtils.getProjectedTuple(i, root, projSchema, projIndices), 0)
+        i += 1
+      }
+      ColumnarResult.Consumed
+    }
+  }
+
   override def onFinish(port: Int): Iterator[TupleLike] = {
     // Finalize aggregation for all keys and produce the result
     keyedPartialAggregates.iterator.map {
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
index d996c98f52c..050e6c10822 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
@@ -21,7 +21,8 @@ package org.apache.texera.amber.operator.filter
 
 import org.apache.arrow.memory.RootAllocator
 import org.apache.arrow.vector.{Float8Vector, IntVector, VectorSchemaRoot}
-import org.apache.texera.amber.core.executor.ColumnarOperatorExecutor
+import org.apache.texera.amber.core.executor.{ColumnarOperatorExecutor, ColumnarResult}
+import org.apache.texera.common.config.ApplicationConfig
 import org.apache.texera.amber.core.tuple.{
   AttributeType,
   AttributeTypeUtils,
@@ -43,8 +44,7 @@ class SpecializedFilterOpExec(descString: String)
   // Row path: OR over predicates, per tuple.
   setFilterFunc((tuple: Tuple) => desc.predicates.exists(_.evaluate(tuple)))
 
-  private val vectorizedEnabled: Boolean =
-    sys.env.getOrElse("FILTER_VECTORIZED", "1") == "1"
+  private val vectorizedEnabled: Boolean = ApplicationConfig.enableVectorizedOperators
 
   // Per-predicate row test with per-batch invariants hoisted, resolved from the
   // batch schema on first use. null = a predicate shape is not vectorizable, so
@@ -182,12 +182,14 @@ class SpecializedFilterOpExec(descString: String)
   // ---- Native-Arrow path (M2): consume the Arrow batch directly, no tuple decode.
   @transient private var columnarAllocator: RootAllocator = _
 
-  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): Option[VectorSchemaRoot] = {
-    if (!vectorizedEnabled || desc.predicates.size != 1) return None
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult = {
+    if (!vectorizedEnabled || desc.predicates.size != 1) return ColumnarResult.Unsupported
     val p = desc.predicates.head
     val cmp = cmpOf(p.condition)
-    if (cmp == null) return None
-    val c = try p.value.toDouble catch { case _: NumberFormatException => return None }
+    if (cmp == null) return ColumnarResult.Unsupported
+    val c =
+      try p.value.toDouble
+      catch { case _: NumberFormatException => return ColumnarResult.Unsupported }
     if (columnarAllocator == null) columnarAllocator = new RootAllocator()
     ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
       val fields = root.getSchema.getFields
@@ -197,7 +199,7 @@ class SpecializedFilterOpExec(descString: String)
         if (fields.get(k).getName == p.attribute) fieldIdx = k
         k += 1
       }
-      if (fieldIdx < 0) None
+      if (fieldIdx < 0) ColumnarResult.Unsupported
       else {
         val n = root.getRowCount
         val mask = new Array[Boolean](n)
@@ -205,14 +207,14 @@ class SpecializedFilterOpExec(descString: String)
           case v: Float8Vector =>
             var i = 0
             while (i < n) { mask(i) = !v.isNull(i) && cmp(java.lang.Double.compare(v.get(i), c)); i += 1 }
-            Some(ArrowUtils.selectRows(root, mask, columnarAllocator))
+            ColumnarResult.Emit(ArrowUtils.selectRows(root, mask, columnarAllocator))
           case v: IntVector =>
             var i = 0
             while (i < n) {
               mask(i) = !v.isNull(i) && cmp(java.lang.Double.compare(v.get(i).toDouble, c)); i += 1
             }
-            Some(ArrowUtils.selectRows(root, mask, columnarAllocator))
-          case _ => None // non-numeric column: fall back to the row path
+            ColumnarResult.Emit(ArrowUtils.selectRows(root, mask, columnarAllocator))
+          case _ => ColumnarResult.Unsupported // non-numeric column: fall back to the row path
         }
       }
     }
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala
index 158d15e274e..641612c6633 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala
@@ -20,15 +20,19 @@
 package org.apache.texera.amber.operator.projection
 
 import com.google.common.base.Preconditions
-import org.apache.texera.amber.core.tuple.{Tuple, TupleLike}
+import org.apache.arrow.memory.RootAllocator
+import org.apache.texera.amber.core.executor.{ColumnarOperatorExecutor, ColumnarResult}
+import org.apache.texera.amber.core.tuple.{Attribute, Schema, Tuple, TupleLike}
 import org.apache.texera.amber.operator.map.MapOpExec
+import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
 
 import scala.collection.mutable
 
 class ProjectionOpExec(
     descString: String
-) extends MapOpExec {
+) extends MapOpExec
+    with ColumnarOperatorExecutor {
 
   val desc: ProjectionOpDesc = objectMapper.readValue(descString, classOf[ProjectionOpDesc])
   setMapFunc(project)
@@ -65,4 +69,35 @@ class ProjectionOpExec(
     TupleLike(fields.toSeq: _*)
   }
 
+  // ---- Native-Arrow path: select (and rename) columns directly on the Arrow
+  // batch, no per-row Tuple decode. Streaming 1:1, so it emits the projected batch.
+  @transient private var columnarAllocator: RootAllocator = _
+
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult = {
+    Preconditions.checkArgument(desc.attributes.nonEmpty)
+    if (columnarAllocator == null) columnarAllocator = new RootAllocator()
+    ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
+      val full = ArrowUtils.toTexeraSchema(root.getSchema)
+      // (originalName, alias) in output order, matching the row `project` path.
+      val selected: List[(String, String)] =
+        if (desc.isDrop) {
+          val drop = desc.attributes.map(_.getOriginalAttribute).toSet
+          full.getAttributeNames.filterNot(drop.contains).map(a => (a, a))
+        } else desc.attributes.map(u => (u.getOriginalAttribute, u.getAlias))
+      val aliases = selected.map(_._2)
+      if (aliases.distinct.size != aliases.size) {
+        throw new RuntimeException("have duplicated attribute name/alias")
+      }
+      val outSchema = Schema(selected.map {
+        case (orig, alias) => new Attribute(alias, full.getAttribute(orig).getType)
+      })
+      val srcIdx = ArrowUtils.projectionIndices(root, selected.map(_._1))
+      ColumnarResult.Emit(ArrowUtils.selectColumns(root, srcIdx, outSchema, columnarAllocator))
+    }
+  }
+
+  override def close(): Unit = {
+    if (columnarAllocator != null) { columnarAllocator.close(); columnarAllocator = null }
+  }
+
 }
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
index 8d7be8a2503..2ee11665d5d 100644
--- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
@@ -19,6 +19,7 @@
 
 package org.apache.texera.amber.operator.filter
 
+import org.apache.texera.amber.core.executor.ColumnarResult
 import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple}
 import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
@@ -167,7 +168,10 @@ class VectorizedFilterCorrectnessSpec extends AnyFlatSpec {
         orderedOps.foreach { op =>
           val exec = execFor((name, op, value))
           val bytes = ArrowUtils.serializeTuples(s, rows)
-          val resultRoot = exec.processColumnarBatch(bytes).get
+          val resultRoot = exec.processColumnarBatch(bytes) match {
+            case ColumnarResult.Emit(r) => r
+            case other                  => fail(s"expected Emit, got $other")
+          }
           val survivors =
             (0 until resultRoot.getRowCount).map(i => ArrowUtils.getTexeraTuple(i, resultRoot)).toList
           resultRoot.close()
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala
index a514f0ab3e8..a544828ffe1 100644
--- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala
@@ -19,7 +19,9 @@
 
 package org.apache.texera.amber.operator.projection
 
+import org.apache.texera.amber.core.executor.ColumnarResult
 import org.apache.texera.amber.core.tuple._
+import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
 import org.scalatest.BeforeAndAfter
 import org.scalatest.flatspec.AnyFlatSpec
@@ -129,6 +131,53 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter {
     }
   }
 
+  private def batch(n: Int): Array[Tuple] =
+    (0 until n).map { i =>
+      Tuple
+        .builder(tupleSchema)
+        .add(new Attribute("field1", AttributeType.STRING), s"s$i")
+        .add(new Attribute("field2", AttributeType.INTEGER), Int.box(i))
+        .add(new Attribute("field3", AttributeType.BOOLEAN), Boolean.box(i % 2 == 0))
+        .build()
+    }.toArray
+
+  it should "match the row path via processColumnarBatch (rename + reorder)" in {
+    val d = new ProjectionOpDesc()
+    d.attributes = List(new AttributeUnit("field2", "f2"), new AttributeUnit("field1", "f1"))
+    val exec = new ProjectionOpExec(objectMapper.writeValueAsString(d))
+    exec.open()
+    val rows = batch(200)
+    val out = exec.processColumnarBatch(ArrowUtils.serializeTuples(tupleSchema, rows)) match {
+      case ColumnarResult.Emit(r) => r
+      case other                  => fail(s"expected Emit, got $other")
+    }
+    val cols = out.getSchema.getFields
+    assert(cols.get(0).getName == "f2" && cols.get(1).getName == "f1")
+    val decoded = (0 until out.getRowCount).map(i => ArrowUtils.getTexeraTuple(i, out)).toList
+    out.close(); exec.close()
+    assert(decoded.map(_.getField[Any]("f2")) == rows.map(_.getField[Any]("field2")).toList)
+    assert(decoded.map(_.getField[Any]("f1")) == rows.map(_.getField[Any]("field1")).toList)
+  }
+
+  it should "match the row path via processColumnarBatch (drop mode)" in {
+    val d = new ProjectionOpDesc()
+    d.isDrop = true
+    d.attributes = List(new AttributeUnit("field2", "field2"))
+    val exec = new ProjectionOpExec(objectMapper.writeValueAsString(d))
+    exec.open()
+    val rows = batch(150)
+    val out = exec.processColumnarBatch(ArrowUtils.serializeTuples(tupleSchema, rows)) match {
+      case ColumnarResult.Emit(r) => r
+      case other                  => fail(s"expected Emit, got $other")
+    }
+    val names = (0 until out.getSchema.getFields.size).map(out.getSchema.getFields.get(_).getName)
+    assert(names == Seq("field1", "field3"))
+    val decoded = (0 until out.getRowCount).map(i => ArrowUtils.getTexeraTuple(i, out)).toList
+    out.close(); exec.close()
+    assert(decoded.map(_.getField[Any]("field1")) == rows.map(_.getField[Any]("field1")).toList)
+    assert(decoded.map(_.getField[Any]("field3")) == rows.map(_.getField[Any]("field3")).toList)
+  }
+
   it should "allow empty alias" in {
     opDesc.attributes = List(
       new AttributeUnit("field2", "f2"),

From 83484827d4ebc82c667fad0dfdb7a548e47a07f8 Mon Sep 17 00:00:00 2001
From: Matthew Ball <mgball@uci.edu>
Date: Sun, 13 Sep 2026 01:32:22 -0700
Subject: [PATCH 3/4] feat(engine): columnar join, sink offload, terminal-op
 consume, and Python passthrough

  Extends the Arrow columnar path to the join and the result sink, and
lets
  terminal operators consume columnar too. Full chain scan -> projection
->
  filter -> join -> aggregate -> sink now runs columnar, validated ROW
== COLUMNAR.

  Contract:
  - ColumnarResult gains EmitRows (consume an Arrow batch, emit
row-shaped
    output that the wire still ships columnar) and processColumnarBatch
takes
    the input port, so multi-input operators can behave per-port

  Operators:
  - HashJoinProbeOpExec: selective columnar probe. Decodes only the join
key
    per row and fully decodes a row only on a match (or outer join); the
    build-load port falls back to the row path. Reuses the row-path join
logic
  - DataProcessor: handle EmitRows; drop the canEmitColumnar
precondition so a
    terminal operator (no downstream) still consumes columnar and
materializes.
    emitColumnarBatch already handles the zero-receiver case

  Sink:
  - Columnar sink offloads the batch decode to the storage writer thread
    (ArrowBatchWriteItem) instead of decoding every row on the DP
thread, and
    reuses the batch bytes already serialized for the wire. Still a
row-based
    Iceberg Record write underneath (a vectorized Arrow->Parquet write
would
    need iceberg-arrow, deferred)

  Python:
  - PythonProxyClient streams a ColumnarFrame straight to Arrow Flight
    (writeArrowRoot) instead of Arrow -> tuples -> Arrow

  Tests:
  - ColumnarShuffleCorrectnessSpec: csv join csv, rows + content
checksum
    ROW == COLUMNAR; probe runs columnar (telemetry)
  - DataProcessingSpec: 16/16 in both ROW and COLUMNAR modes (no
regression)
  - OutputPortStorageWriterThreadSpec updated to the StorageWriteItem
shape
---
 .../messaginglayer/OutputManager.scala        | 34 ++++++------
 .../pythonworker/PythonProxyClient.scala      | 26 +++++++++-
 .../architecture/worker/DataProcessor.scala   | 12 ++---
 .../OutputPortStorageWriterThread.scala       | 18 +++++--
 .../promisehandlers/EndChannelHandler.scala   |  2 +-
 .../OutputPortStorageWriterThreadSpec.scala   |  4 +-
 .../e2e/ColumnarShuffleCorrectnessSpec.scala  | 28 ++++++++++
 .../executor/ColumnarOperatorExecutor.scala   | 16 ++++--
 .../operator/aggregate/AggregateOpExec.scala  |  2 +-
 .../filter/SpecializedFilterOpExec.scala      |  2 +-
 .../hashJoin/HashJoinProbeOpExec.scala        | 52 +++++++++++++++++--
 .../projection/ProjectionOpExec.scala         |  2 +-
 .../VectorizedFilterCorrectnessSpec.scala     |  2 +-
 .../projection/ProjectionOpExecSpec.scala     |  4 +-
 14 files changed, 160 insertions(+), 44 deletions(-)

diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
index 9dad536e06a..2e5a4dfa4d8 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala
@@ -33,8 +33,10 @@ import org.apache.texera.amber.engine.architecture.messaginglayer.OutputManager.
 import org.apache.texera.amber.engine.architecture.sendsemantics.partitioners._
 import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings._
 import org.apache.texera.amber.engine.architecture.worker.managers.{
+  ArrowBatchWriteItem,
   OutputPortStorageWriterThread,
-  PortStorageWriterTerminateSignal
+  PortStorageWriterTerminateSignal,
+  RowWriteItem
 }
 import org.apache.texera.amber.engine.common.AmberLogging
 import org.apache.texera.amber.engine.common.ambermessage.ColumnarFrame
@@ -200,10 +202,6 @@ class OutputManager(
     saveStateToStorageIfNeeded(state)
   }
 
-  // Columnar emit is possible whenever there are output links; multi-receiver
-  // links (hash/range/round-robin shuffles) are split per-receiver in emit.
-  def canEmitColumnar: Boolean = partitioners.nonEmpty
-
   // Allocator for the per-receiver sub-batches built during columnar shuffle.
   // Created lazily on first shuffle emit, freed by closeColumnarResources().
   private var columnarEmitAllocator: RootAllocator = _
@@ -267,14 +265,11 @@ class OutputManager(
         }
     }
 
-    // Materialize to result storage for the GUI result panel, matching the row
-    // path's saveTupleToStorageIfNeeded. Only decodes when a port needs storage.
-    if (outputPortResultWriterThreads.nonEmpty) {
-      var i = 0
-      while (i < rowCount) {
-        saveTupleToStorageIfNeeded(ArrowUtils.getTexeraTuple(i, root))
-        i += 1
-      }
+    // Materialize to result storage for the GUI result panel. Hand the whole
+    // Arrow batch to the writer thread(s), which decode off the DP thread. Reuse
+    // the bytes already serialized for the wire when available.
+    if (outputPortResultWriterThreads.nonEmpty && rowCount > 0) {
+      saveArrowBatchToStorageIfNeeded(wholeBatchBytes)
     }
   }
 
@@ -324,17 +319,26 @@ class OutputManager(
     }).foreach({
       case (portId, writerThread) =>
         // write to storage in a separate thread
-        writerThread.queue.put(Left(tuple))
+        writerThread.queue.put(Left(RowWriteItem(tuple)))
     })
   }
 
+  // Columnar sink: hand the whole Arrow batch to each result writer thread,
+  // which decodes it to rows off the DP thread. Avoids the per-row decode on
+  // the hot path and reuses the batch bytes already produced for the wire.
+  def saveArrowBatchToStorageIfNeeded(arrowIpcBytes: Array[Byte]): Unit = {
+    outputPortResultWriterThreads.values.foreach(
+      _.queue.put(Left(ArrowBatchWriteItem(arrowIpcBytes)))
+    )
+  }
+
   private def saveStateToStorageIfNeeded(state: State): Unit = {
     // The same state row is fanned out to every output port's state
     // table. This mirrors the broadcast-to-all-workers behavior on the
     // emit side: state is shared context, not per-key data, so every
     // downstream operator (and every worker reading the materialization)
     // needs the full set.
-    stateWriterThreads.values.foreach(_.queue.put(Left(state.toTuple())))
+    stateWriterThreads.values.foreach(_.queue.put(Left(RowWriteItem(state.toTuple()))))
   }
 
   /**
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
index bc04a274d38..fd31ad79ff8 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
@@ -125,8 +125,12 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
       case DataFrame(frame) =>
         writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
       case ColumnarFrame(bytes, _, _) =>
-        // Columnar wire reaching the Python path: decode to tuples and stream.
-        writeArrowStream(mutable.Queue(ArrowUtils.deserializeTuples(bytes).toSeq: _*), from, "Data")
+        // Columnar wire reaching the Python path: pass the Arrow batch straight
+        // to Flight (no tuple round-trip). deserializeRootFold keeps the root
+        // alive for the put and closes it after.
+        ArrowUtils.deserializeRootFold(bytes, allocator) { root =>
+          writeArrowRoot(root, from, "Data")
+        }
       case StateFrame(state) =>
         writeArrowStream(mutable.Queue(state.toTuple()), from, "State")
     }
@@ -207,6 +211,24 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
     result
   }
 
+  // Send an already-built Arrow batch to the Python worker over Flight, without
+  // decoding to tuples first. Used for the columnar wire passthrough.
+  private def writeArrowRoot(
+      root: VectorSchemaRoot,
+      from: ChannelIdentity,
+      payloadType: String
+  ): Unit = {
+    val descriptor = FlightDescriptor.command(PythonDataHeader(from, payloadType).toByteArray)
+    val flightListener = new SyncPutListener
+    val writer = flightClient.startPut(descriptor, root, flightListener)
+    writer.putNext()
+    writer.completed()
+    val ackMsgBuf: ArrowBuf = flightListener.poll(5, TimeUnit.SECONDS).getApplicationMetadata
+    pythonQueueInMemSize.set(ackMsgBuf.getLong(0))
+    ackMsgBuf.close()
+    flightListener.close()
+  }
+
   private def writeArrowStream(
       tuples: mutable.Queue[Tuple],
       from: ChannelIdentity,
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
index ecf892f23e4..df95d942695 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
@@ -221,13 +221,16 @@ class DataProcessor(
       case ColumnarFrame(bytes, _, _) =>
         executor match {
           // Native-Arrow path: consume the batch directly, emit a filtered batch.
-          case c: ColumnarOperatorExecutor
-              if NetworkOutputBuffer.columnarWire && outputManager.canEmitColumnar =>
-            c.processColumnarBatch(bytes) match {
+          case c: ColumnarOperatorExecutor if NetworkOutputBuffer.columnarWire =>
+            c.processColumnarBatch(bytes, portId.id) match {
               case ColumnarResult.Emit(result) =>
                 logColumnarModeOnce(active = true, "")
                 statisticsManager.increaseInputStatistics(portId, bytes.length.toLong)
                 outputManager.setColumnarOutput(Iterator.single(result))
+              case ColumnarResult.EmitRows(rows) =>
+                logColumnarModeOnce(active = true, "")
+                statisticsManager.increaseInputStatistics(portId, bytes.length.toLong)
+                outputManager.outputIterator.setTupleOutput(rows)
               case ColumnarResult.Consumed =>
                 logColumnarModeOnce(active = true, "")
                 statisticsManager.increaseInputStatistics(portId, bytes.length.toLong)
@@ -235,9 +238,6 @@ class DataProcessor(
                 logColumnarModeOnce(active = false, "operator has no native columnar path for this batch")
                 processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
             }
-          case _: ColumnarOperatorExecutor if NetworkOutputBuffer.columnarWire =>
-            logColumnarModeOnce(active = false, "output has no partitioned receivers")
-            processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
           case _ =>
             processTupleBatch(channelId, portId, ArrowUtils.deserializeTuples(bytes))
         }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala
index f0b814da1de..be2ddad3a5b 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala
@@ -22,6 +22,7 @@ package org.apache.texera.amber.engine.architecture.worker.managers
 import com.google.common.collect.Queues
 import org.apache.texera.amber.core.storage.model.BufferedItemWriter
 import org.apache.texera.amber.core.tuple.Tuple
+import org.apache.texera.amber.util.ArrowUtils
 
 import java.util.concurrent.LinkedBlockingQueue
 import scala.util.control.NonFatal
@@ -29,12 +30,18 @@ import scala.util.control.NonFatal
 sealed trait TerminateSignal
 case object PortStorageWriterTerminateSignal extends TerminateSignal
 
+// A unit of work for the writer thread: a single row, or a whole Arrow batch
+// (columnar sink) decoded on this thread instead of on the hot DP thread.
+sealed trait StorageWriteItem
+final case class RowWriteItem(tuple: Tuple) extends StorageWriteItem
+final case class ArrowBatchWriteItem(arrowIpcBytes: Array[Byte]) extends StorageWriteItem
+
 class OutputPortStorageWriterThread(
     bufferedItemWriter: BufferedItemWriter[Tuple]
 ) extends Thread {
 
-  val queue: LinkedBlockingQueue[Either[Tuple, TerminateSignal]] =
-    Queues.newLinkedBlockingQueue[Either[Tuple, TerminateSignal]]()
+  val queue: LinkedBlockingQueue[Either[StorageWriteItem, TerminateSignal]] =
+    Queues.newLinkedBlockingQueue[Either[StorageWriteItem, TerminateSignal]]()
 
   // Captured failure from put-one or close() so the worker DP thread can
   // re-throw and let the coordinator's pekko supervisor surface a FatalError
@@ -50,8 +57,11 @@ class OutputPortStorageWriterThread(
       var internalStop = false
       while (!internalStop) {
         queue.take() match {
-          case Left(tuple) => bufferedItemWriter.putOne(tuple)
-          case Right(_)    => internalStop = true
+          case Left(RowWriteItem(tuple)) => bufferedItemWriter.putOne(tuple)
+          case Left(ArrowBatchWriteItem(bytes)) =>
+            // Decode the Arrow batch here (off the DP thread) into rows.
+            ArrowUtils.deserializeTuples(bytes).foreach(bufferedItemWriter.putOne)
+          case Right(_) => internalStop = true
         }
       }
     } catch {
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
index 5791e30282d..d76c863e2d8 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala
@@ -51,7 +51,7 @@ trait EndChannelHandler {
       // when enabled and there are output links (shuffles are split in emit).
       // Else the row-oriented onFinishMultiPort path.
       val columnarBatches =
-        if (NetworkOutputBuffer.columnarWire && dp.outputManager.canEmitColumnar)
+        if (NetworkOutputBuffer.columnarWire)
           dp.executor match {
             case s: SourceOperatorExecutor => s.produceColumnarBatch()
             case _                         => None
diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala
index 9cc8216efab..50b922b8094 100644
--- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala
+++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala
@@ -75,7 +75,7 @@ class OutputPortStorageWriterThreadSpec extends AnyFlatSpec {
     val writer = new StubWriter(onPutOne = throwing("test putOne failure"))
     val thread = new OutputPortStorageWriterThread(writer)
     thread.start()
-    thread.queue.put(Left(null.asInstanceOf[Tuple]))
+    thread.queue.put(Left(RowWriteItem(null.asInstanceOf[Tuple])))
     thread.queue.put(Right(PortStorageWriterTerminateSignal))
     thread.join()
     assert(thread.getFailure.exists(_.getMessage.contains("test putOne failure")))
@@ -91,7 +91,7 @@ class OutputPortStorageWriterThreadSpec extends AnyFlatSpec {
     )
     val thread = new OutputPortStorageWriterThread(writer)
     thread.start()
-    thread.queue.put(Left(null.asInstanceOf[Tuple]))
+    thread.queue.put(Left(RowWriteItem(null.asInstanceOf[Tuple])))
     thread.queue.put(Right(PortStorageWriterTerminateSignal))
     thread.join()
     val captured = thread.getFailure.getOrElse(fail("expected putOne failure"))
diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala
index f84efe171e1..5e8aac6c72a 100644
--- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala
+++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ColumnarShuffleCorrectnessSpec.scala
@@ -147,6 +147,34 @@ class ColumnarShuffleCorrectnessSpec
     lines.foreach(l => println(s"PROJ[$tag]   $l"))
   }
 
+  private def reportJoin(res: Map[OperatorIdentity, List[Tuple]]): Unit = {
+    val rows = res.values.headOption.getOrElse(Nil)
+    val checksum = rows.map(_.getFields.mkString("|")).sorted.mkString("\n").hashCode
+    println(s"JOIN[$tag workers=$workers] rows=${rows.size} checksum=$checksum")
+  }
+
+  // Selective columnar probe: the probe decodes only the join key per row and
+  // fully decodes a row only on a match. Row count + content checksum must match
+  // the row path.
+  "columnar join" should "csv join csv on column-1 match the row path" in {
+    val id = ids.incrementAndGet(); setUpWorkflowExecutionData(id)
+    try {
+      val ctx: WorkflowContext = TestUtils.workflowContext(id)
+      val c1 = TestOperators.headerlessSmallCsvScanOpDesc()
+      val c2 = TestOperators.headerlessSmallCsvScanOpDesc()
+      val join = TestOperators.joinOpDesc("column-1", "column-1")
+      val wf = buildWorkflow(
+        List(c1, c2, join),
+        List(
+          LogicalLink(c1.operatorIdentifier, PortIdentity(), join.operatorIdentifier, PortIdentity()),
+          LogicalLink(c2.operatorIdentifier, PortIdentity(), join.operatorIdentifier, PortIdentity(1))
+        ),
+        ctx
+      )
+      reportJoin(runWorkflowAndReadTerminalResults(system, wf, Duration.fromMinutes(5)))
+    } finally cleanupWorkflowExecutionData(id)
+  }
+
   // Projection renames "Units Sold" -> "units"; the downstream filter and
   // aggregate then reference the renamed column, all over the columnar wire.
   "columnar projection" should "scan -> project(rename) -> filter -> agg match the row path" in {
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
index 79a3bfcb832..83110ac29f9 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/executor/ColumnarOperatorExecutor.scala
@@ -20,26 +20,32 @@
 package org.apache.texera.amber.core.executor
 
 import org.apache.arrow.vector.VectorSchemaRoot
+import org.apache.texera.amber.core.tuple.TupleLike
+import org.apache.texera.amber.core.workflow.PortIdentity
 
 /**
   * Outcome of consuming one Arrow batch:
-  *   - Emit: consumed, emit this result batch downstream (caller serializes and closes the root).
+  *   - Emit: consumed, emit this Arrow result batch downstream (caller serializes and closes the root).
+  *   - EmitRows: consumed, emit these row outputs (e.g. a join whose output is row-shaped);
+  *     the caller ships them, still columnar on the wire if enabled.
   *   - Consumed: consumed, nothing to emit now (a blocking operator accumulating state).
   *   - Unsupported: not handled, caller falls back to the row path (decode + processTuple).
   */
 sealed trait ColumnarResult
 object ColumnarResult {
   final case class Emit(root: VectorSchemaRoot) extends ColumnarResult
+  final case class EmitRows(rows: Iterator[(TupleLike, Option[PortIdentity])]) extends ColumnarResult
   case object Consumed extends ColumnarResult
   case object Unsupported extends ColumnarResult
 }
 
 /**
   * An operator that can consume an Arrow columnar batch directly (no per-row
-  * Tuple decode). Given the incoming batch as Arrow IPC bytes, returns a
-  * ColumnarResult telling the caller whether it emitted a batch, consumed the
-  * batch with no output, or could not handle it (fall back to the row path).
+  * Tuple decode). Given the incoming batch as Arrow IPC bytes and the input
+  * port it arrived on, returns a ColumnarResult telling the caller whether it
+  * emitted a batch/rows, consumed the batch with no output, or could not handle
+  * it (fall back to the row path).
   */
 trait ColumnarOperatorExecutor {
-  def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult
+  def processColumnarBatch(arrowIpcBytes: Array[Byte], port: Int): ColumnarResult
 }
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala
index 7d3dd65c328..dd015c4fbc0 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpExec.scala
@@ -94,7 +94,7 @@ class AggregateOpExec(descString: String) extends OperatorExecutor with Columnar
   @transient private var projSchema: Schema = _
   @transient private var projIndices: Array[Int] = _
 
-  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult = {
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte], port: Int): ColumnarResult = {
     if (neededNames == null) {
       neededNames = (desc.groupByKeys ++ desc.aggregations.flatMap(a =>
         Option(a.attribute).map(_.trim).filter(_.nonEmpty)
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
index 050e6c10822..99f5af249f0 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala
@@ -182,7 +182,7 @@ class SpecializedFilterOpExec(descString: String)
   // ---- Native-Arrow path (M2): consume the Arrow batch directly, no tuple decode.
   @transient private var columnarAllocator: RootAllocator = _
 
-  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult = {
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte], port: Int): ColumnarResult = {
     if (!vectorizedEnabled || desc.predicates.size != 1) return ColumnarResult.Unsupported
     val p = desc.predicates.head
     val cmp = cmpOf(p.condition)
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala
index a27ae1d5e24..000b1e44127 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala
@@ -19,9 +19,16 @@
 
 package org.apache.texera.amber.operator.hashJoin
 
-import org.apache.texera.amber.core.executor.OperatorExecutor
-import org.apache.texera.amber.core.tuple.{Tuple, TupleLike}
+import org.apache.arrow.memory.RootAllocator
+import org.apache.texera.amber.core.executor.{
+  ColumnarOperatorExecutor,
+  ColumnarResult,
+  OperatorExecutor
+}
+import org.apache.texera.amber.core.tuple.{Schema, Tuple, TupleLike}
+import org.apache.texera.amber.core.workflow.PortIdentity
 import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc.HASH_JOIN_INTERNAL_KEY_NAME
+import org.apache.texera.amber.util.ArrowUtils
 import org.apache.texera.amber.util.JSONUtils.objectMapper
 
 import scala.collection.mutable
@@ -62,7 +69,8 @@ object JoinUtils {
 
 class HashJoinProbeOpExec[K](
     descString: String
-) extends OperatorExecutor {
+) extends OperatorExecutor
+    with ColumnarOperatorExecutor {
 
   private val desc: HashJoinOpDesc[K] =
     objectMapper.readValue(descString, classOf[HashJoinOpDesc[K]])
@@ -74,6 +82,44 @@ class HashJoinProbeOpExec[K](
 
   override def close(): Unit = {
     buildTableHashMap.clear()
+    if (columnarAllocator != null) { columnarAllocator.close(); columnarAllocator = null }
+  }
+
+  // ---- Native-Arrow probe: on the probe port, decode only the probe key column
+  // to test each row against the build map, and fully decode a row only when it
+  // matches (or on an outer join). For a selective inner join, most probe rows
+  // are never fully decoded. The build-load port (0) needs all columns, so it
+  // falls back to the row path.
+  @transient private var columnarAllocator: RootAllocator = _
+  @transient private var keySchema: Schema = _
+
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte], port: Int): ColumnarResult = {
+    if (port == 0) return ColumnarResult.Unsupported
+    val isOuter = desc.joinType == JoinType.RIGHT_OUTER || desc.joinType == JoinType.FULL_OUTER
+    if (columnarAllocator == null) columnarAllocator = new RootAllocator()
+    ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
+      if (keySchema == null) {
+        val full = ArrowUtils.toTexeraSchema(root.getSchema)
+        keySchema = Schema(List(full.getAttribute(desc.probeAttributeName)))
+      }
+      val keyIdx = ArrowUtils.projectionIndices(root, Seq(desc.probeAttributeName))
+      val out = new ListBuffer[(TupleLike, Option[PortIdentity])]
+      val n = root.getRowCount
+      var i = 0
+      while (i < n) {
+        val key = ArrowUtils
+          .getProjectedTuple(i, root, keySchema, keyIdx)
+          .getField(desc.probeAttributeName)
+          .asInstanceOf[K]
+        val matched = buildTableHashMap.get(key).exists(_._1.nonEmpty)
+        if (matched || isOuter) {
+          // Reuse the exact row-path join logic (also marks the build side joined).
+          processTuple(ArrowUtils.getTexeraTuple(i, root), 1).foreach(t => out += ((t, None)))
+        }
+        i += 1
+      }
+      ColumnarResult.EmitRows(out.iterator)
+    }
   }
 
   override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] =
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala
index 641612c6633..d163dc5a80b 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpExec.scala
@@ -73,7 +73,7 @@ class ProjectionOpExec(
   // batch, no per-row Tuple decode. Streaming 1:1, so it emits the projected batch.
   @transient private var columnarAllocator: RootAllocator = _
 
-  override def processColumnarBatch(arrowIpcBytes: Array[Byte]): ColumnarResult = {
+  override def processColumnarBatch(arrowIpcBytes: Array[Byte], port: Int): ColumnarResult = {
     Preconditions.checkArgument(desc.attributes.nonEmpty)
     if (columnarAllocator == null) columnarAllocator = new RootAllocator()
     ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
index 2ee11665d5d..6f0a58e2b47 100644
--- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/VectorizedFilterCorrectnessSpec.scala
@@ -168,7 +168,7 @@ class VectorizedFilterCorrectnessSpec extends AnyFlatSpec {
         orderedOps.foreach { op =>
           val exec = execFor((name, op, value))
           val bytes = ArrowUtils.serializeTuples(s, rows)
-          val resultRoot = exec.processColumnarBatch(bytes) match {
+          val resultRoot = exec.processColumnarBatch(bytes, 0) match {
             case ColumnarResult.Emit(r) => r
             case other                  => fail(s"expected Emit, got $other")
           }
diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala
index a544828ffe1..a35370fd36c 100644
--- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala
+++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala
@@ -147,7 +147,7 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter {
     val exec = new ProjectionOpExec(objectMapper.writeValueAsString(d))
     exec.open()
     val rows = batch(200)
-    val out = exec.processColumnarBatch(ArrowUtils.serializeTuples(tupleSchema, rows)) match {
+    val out = exec.processColumnarBatch(ArrowUtils.serializeTuples(tupleSchema, rows), 0) match {
       case ColumnarResult.Emit(r) => r
       case other                  => fail(s"expected Emit, got $other")
     }
@@ -166,7 +166,7 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter {
     val exec = new ProjectionOpExec(objectMapper.writeValueAsString(d))
     exec.open()
     val rows = batch(150)
-    val out = exec.processColumnarBatch(ArrowUtils.serializeTuples(tupleSchema, rows)) match {
+    val out = exec.processColumnarBatch(ArrowUtils.serializeTuples(tupleSchema, rows), 0) match {
       case ColumnarResult.Emit(r) => r
       case other                  => fail(s"expected Emit, got $other")
     }

From 87fa02167b0f91da50c93d8c42e2e44d298ce121 Mon Sep 17 00:00:00 2001
From: Matthew Ball <mgball@uci.edu>
Date: Mon, 14 Sep 2026 12:33:41 -0700
Subject: [PATCH 4/4] feat(engine): columnar hardening (wire versioning, faster
 decode, vectorized sink, Python passthrough)

Tier 3 hardening for the columnar execution path.

Wire format:
- ColumnarFrame carries a formatVersion (default CurrentFormatVersion =
1); both
  consume points (DataProcessor, Python bridge) fail fast on an unknown
version
  instead of misreading bytes. Guards mixed-version workers during a
rollout.

Decode:
- Arrow->row decode resolves the Texera schema once per batch instead of
per row
  (an Attribute allocation per column per row). Speeds up the sink and
every
  columnar-consume fallback (aggregate, join, Python); byte-identical
output.
  getTexeraTuple gains a schema-passing overload; deserializeTuples and
the join
  probe reuse a resolved schema.

Vectorized sink (opt-in, columnar.enable-vectorized-sink /
COLUMNAR_SINK=1):
- IcebergTableWriter implements ArrowVectorizedSink: writeArrowBatch
writes an
  Arrow batch to Iceberg via its own DataWriter (valid file + metrics +
DataFile),
  fed by a reused ArrowRecordView that reads each column straight from
the Arrow
  vectors at a moving row index. No Tuple and no GenericRecord
materialized.
  Falls back to the tuple-decode offload when off. Honest ceiling: the
Parquet
  encode stays Iceberg row-driven (no column-chunk Arrow->Parquet writer
exists
  on this JVM stack), and it commits one file per Arrow batch.

Python passthrough:
- ColumnarFrame reaching the Python bridge streams straight to Arrow
Flight; a
  test proves the Arrow batch Python receives is identical to the row
path.

Tests:
- ArrowIpcRoundTripSpec: round-trip + passthrough equivalence (3/3)
- Vectorized sink validated byte-for-byte: terminal columnar filter
written and
  read back matches the row engine (ColumnarReproSpec 2/2 with
COLUMNAR_SINK=1)
- No regression when the sink flag is off (DataProcessingSpec 16/16 both
modes)
---
 .../pythonworker/PythonProxyClient.scala      |   7 +-
 .../architecture/worker/DataProcessor.scala   |   7 +-
 .../OutputPortStorageWriterThread.scala       |  13 +-
 .../common/ambermessage/DataPayload.scala     |  18 ++-
 .../src/main/resources/application.conf       |   5 +
 .../common/config/ApplicationConfig.scala     |   5 +
 .../storage/model/BufferedItemWriter.scala    |  14 ++
 .../result/iceberg/IcebergTableWriter.scala   | 125 +++++++++++++-----
 .../apache/texera/amber/util/ArrowUtils.scala |  16 ++-
 .../amber/util/ArrowIpcRoundTripSpec.scala    |  28 ++++
 .../hashJoin/HashJoinProbeOpExec.scala        |   6 +-
 11 files changed, 199 insertions(+), 45 deletions(-)

diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
index fd31ad79ff8..64cff232863 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyClient.scala
@@ -124,7 +124,12 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
     dataPayload match {
       case DataFrame(frame) =>
         writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
-      case ColumnarFrame(bytes, _, _) =>
+      case ColumnarFrame(bytes, _, _, ver) =>
+        require(
+          ver == ColumnarFrame.CurrentFormatVersion,
+          s"unsupported ColumnarFrame format version $ver " +
+            s"(this worker supports ${ColumnarFrame.CurrentFormatVersion})"
+        )
         // Columnar wire reaching the Python path: pass the Arrow batch straight
         // to Flight (no tuple round-trip). deserializeRootFold keeps the root
         // alive for the put and closes it after.
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
index df95d942695..1b0aa4e5a01 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala
@@ -218,7 +218,12 @@ class DataProcessor(
     dataPayload match {
       case DataFrame(tuples) =>
         processTupleBatch(channelId, portId, tuples)
-      case ColumnarFrame(bytes, _, _) =>
+      case ColumnarFrame(bytes, _, _, ver) =>
+        require(
+          ver == ColumnarFrame.CurrentFormatVersion,
+          s"unsupported ColumnarFrame format version $ver " +
+            s"(this worker supports ${ColumnarFrame.CurrentFormatVersion})"
+        )
         executor match {
           // Native-Arrow path: consume the batch directly, emit a filtered batch.
           case c: ColumnarOperatorExecutor if NetworkOutputBuffer.columnarWire =>
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala
index be2ddad3a5b..dc9099492bb 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala
@@ -20,9 +20,10 @@
 package org.apache.texera.amber.engine.architecture.worker.managers
 
 import com.google.common.collect.Queues
-import org.apache.texera.amber.core.storage.model.BufferedItemWriter
+import org.apache.texera.amber.core.storage.model.{ArrowVectorizedSink, BufferedItemWriter}
 import org.apache.texera.amber.core.tuple.Tuple
 import org.apache.texera.amber.util.ArrowUtils
+import org.apache.texera.common.config.ApplicationConfig
 
 import java.util.concurrent.LinkedBlockingQueue
 import scala.util.control.NonFatal
@@ -59,8 +60,14 @@ class OutputPortStorageWriterThread(
         queue.take() match {
           case Left(RowWriteItem(tuple)) => bufferedItemWriter.putOne(tuple)
           case Left(ArrowBatchWriteItem(bytes)) =>
-            // Decode the Arrow batch here (off the DP thread) into rows.
-            ArrowUtils.deserializeTuples(bytes).foreach(bufferedItemWriter.putOne)
+            bufferedItemWriter match {
+              // Vectorized sink (opt-in): write the Arrow batch straight to
+              // storage, no per-row object materialization.
+              case sink: ArrowVectorizedSink if ApplicationConfig.enableVectorizedSink =>
+                sink.writeArrowBatch(bytes)
+              // Otherwise decode here (off the DP thread) into rows.
+              case _ => ArrowUtils.deserializeTuples(bytes).foreach(bufferedItemWriter.putOne)
+            }
           case Right(_) => internalStop = true
         }
       }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala
index 7fc7b9e823e..b93ad57f447 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/DataPayload.scala
@@ -26,9 +26,21 @@ sealed trait DataPayload extends WorkflowFIFOMessagePayload {}
 
 final case class StateFrame(frame: State) extends DataPayload
 
-// Columnar wire payload: a batch encoded as Arrow IPC stream bytes.
-final case class ColumnarFrame(arrowIpcBytes: Array[Byte], rowCount: Int, schema: Schema)
-    extends DataPayload {
+object ColumnarFrame {
+  // Bump when the on-wire columnar encoding changes incompatibly. A receiver
+  // that sees an unknown version fails fast (see DataProcessor) rather than
+  // misreading bytes, which matters for mixed-version workers during a rollout.
+  val CurrentFormatVersion: Int = 1
+}
+
+// Columnar wire payload: a batch encoded as Arrow IPC stream bytes, stamped
+// with the format version that produced it.
+final case class ColumnarFrame(
+    arrowIpcBytes: Array[Byte],
+    rowCount: Int,
+    schema: Schema,
+    formatVersion: Int = ColumnarFrame.CurrentFormatVersion
+) extends DataPayload {
   val inMemSize: Long = arrowIpcBytes.length.toLong
 }
 
diff --git a/common/config/src/main/resources/application.conf b/common/config/src/main/resources/application.conf
index 4584886c2ee..aaaa50edc9b 100644
--- a/common/config/src/main/resources/application.conf
+++ b/common/config/src/main/resources/application.conf
@@ -67,6 +67,11 @@ columnar {
 
     # Vectorized path in operators that support it (e.g. the filter).
     enable-vectorized-operators = true
+
+    # Write result Arrow batches to Iceberg via a lazy record view (no row-object
+    # materialization) instead of decoding to tuples first. Opt-in; falls back to
+    # the tuple-decode sink when off.
+    enable-vectorized-sink = false
 }
 
 cache {
diff --git a/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala b/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala
index 017b020abf4..38361b6192e 100644
--- a/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala
+++ b/common/config/src/main/scala/org/apache/texera/common/config/ApplicationConfig.scala
@@ -83,6 +83,11 @@ object ApplicationConfig {
       case Some(v) => v == "1" || v.equalsIgnoreCase("true")
       case None    => getConfSource.getBoolean("columnar.enable-vectorized-operators")
     }
+  val enableVectorizedSink: Boolean =
+    sys.env.get("COLUMNAR_SINK") match {
+      case Some(v) => v == "1" || v.equalsIgnoreCase("true")
+      case None    => getConfSource.getBoolean("columnar.enable-vectorized-sink")
+    }
 
   // Fault tolerance
   val faultToleranceLogFlushIntervalInMs: Long =
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/BufferedItemWriter.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/BufferedItemWriter.scala
index cfcd19fb9be..63a8f60e895 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/BufferedItemWriter.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/model/BufferedItemWriter.scala
@@ -57,3 +57,17 @@ trait BufferedItemWriter[T] {
     */
   def removeOne(item: T): Unit
 }
+
+/**
+  * A writer that can ingest a whole Arrow batch (as Arrow IPC bytes) without the
+  * caller decoding it to items first. Used by the vectorized columnar sink.
+  */
+trait ArrowVectorizedSink {
+
+  /**
+    * Write one Arrow batch to the underlying storage, reading values directly
+    * from the Arrow columns (no per-row object materialization).
+    * @param arrowIpcBytes the batch encoded as an Arrow IPC stream.
+    */
+  def writeArrowBatch(arrowIpcBytes: Array[Byte]): Unit
+}
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala
index 81b27d11398..a3c82e6b225 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala
@@ -20,16 +20,24 @@
 package org.apache.texera.amber.core.storage.result.iceberg
 
 import org.apache.texera.common.config.StorageConfig
-import org.apache.texera.amber.core.storage.model.BufferedItemWriter
-import org.apache.texera.amber.util.IcebergUtil
+import org.apache.texera.amber.core.storage.model.{ArrowVectorizedSink, BufferedItemWriter}
+import org.apache.texera.amber.core.tuple.{AttributeTypeUtils, LargeBinary}
+import org.apache.texera.amber.util.{ArrowUtils, IcebergUtil}
+import org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.vector.VectorSchemaRoot
+import org.apache.arrow.vector.ipc.ArrowStreamReader
 import org.apache.iceberg.catalog.Catalog
 import org.apache.iceberg.data.Record
 import org.apache.iceberg.data.parquet.GenericParquetWriter
 import org.apache.iceberg.io.{DataWriter, OutputFile}
 import org.apache.iceberg.parquet.Parquet
+import org.apache.iceberg.types.Types.StructType
 import org.apache.iceberg.{Schema, Table}
 import org.apache.parquet.schema.MessageType
 
+import java.io.ByteArrayInputStream
+import java.nio.ByteBuffer
+import java.time.ZoneId
 import scala.collection.mutable.ArrayBuffer
 
 /**
@@ -55,7 +63,8 @@ private[storage] class IcebergTableWriter[T](
     val tableName: String,
     val tableSchema: Schema,
     val serde: (org.apache.iceberg.Schema, T) => Record
-) extends BufferedItemWriter[T] {
+) extends BufferedItemWriter[T]
+    with ArrowVectorizedSink {
 
   // Buffer to hold items before flushing to the table
   private val buffer = new ArrayBuffer[T]()
@@ -106,37 +115,51 @@ private[storage] class IcebergTableWriter[T](
     */
   private def flushBuffer(): Unit = {
     if (buffer.nonEmpty) {
-      // Create a unique file path using the writer's identifier and the filename index
-      val location = table.location().stripSuffix("/")
-      val filepathString = s"$location/${writerIdentifier}_$filenameIdx"
-      // Increment the filename index by 1
-      filenameIdx += 1
-      val outputFile: OutputFile = table.io().newOutputFile(filepathString)
-      // Create a Parquet data writer to write a new file
-      val dataWriter: DataWriter[Record] = Parquet
-        .writeData(outputFile)
-        .forTable(table)
-        .createWriterFunc((schema: Schema, messageType: MessageType) =>
-          GenericParquetWriter.create(schema, messageType)
-        )
-        .overwrite()
-        .build()
-      // Write each buffered item to the data file
-      try {
-        buffer.foreach { item =>
-          dataWriter.write(serde(tableSchema, item))
-        }
-      } finally {
-        dataWriter.close()
-      }
-      // Commit the new file to the table
-      val dataFile = dataWriter.toDataFile
-      table.newAppend().appendFile(dataFile).commit()
-      // Clear the item buffer
+      writeRecordsToNewFile(buffer.iterator.map(item => serde(tableSchema, item)))
       buffer.clear()
     }
   }
 
+  // Write a batch of Iceberg Records to a new, uniquely named data file and
+  // commit it. Iceberg's DataWriter produces the file metrics and DataFile, so
+  // the callers (row buffer flush and the Arrow batch path) get a valid table.
+  private def writeRecordsToNewFile(records: Iterator[Record]): Unit = {
+    val location = table.location().stripSuffix("/")
+    val filepathString = s"$location/${writerIdentifier}_$filenameIdx"
+    filenameIdx += 1
+    val outputFile: OutputFile = table.io().newOutputFile(filepathString)
+    val dataWriter: DataWriter[Record] = Parquet
+      .writeData(outputFile)
+      .forTable(table)
+      .createWriterFunc((schema: Schema, messageType: MessageType) =>
+        GenericParquetWriter.create(schema, messageType)
+      )
+      .overwrite()
+      .build()
+    try records.foreach(dataWriter.write)
+    finally dataWriter.close()
+    table.newAppend().appendFile(dataWriter.toDataFile).commit()
+  }
+
+  // Vectorized sink: read the Arrow batch and write it via a reused record view
+  // that pulls values straight from the Arrow columns, materializing no rows.
+  @transient private var columnarAllocator: RootAllocator = _
+  override def writeArrowBatch(arrowIpcBytes: Array[Byte]): Unit = {
+    if (columnarAllocator == null) columnarAllocator = new RootAllocator()
+    val reader = new ArrowStreamReader(new ByteArrayInputStream(arrowIpcBytes), columnarAllocator)
+    try {
+      val root = reader.getVectorSchemaRoot
+      val struct = tableSchema.asStruct()
+      while (reader.loadNextBatch()) {
+        val n = root.getRowCount
+        if (n > 0) {
+          val view = new IcebergTableWriter.ArrowRecordView(root, struct)
+          writeRecordsToNewFile((0 until n).iterator.map { i => view.setRow(i); view })
+        }
+      }
+    } finally reader.close()
+  }
+
   /**
     * Close the writer, ensuring any remaining buffered items are flushed.
     */
@@ -144,5 +167,47 @@ private[storage] class IcebergTableWriter[T](
     if (buffer.nonEmpty) {
       flushBuffer()
     }
+    if (columnarAllocator != null) { columnarAllocator.close(); columnarAllocator = null }
+  }
+}
+
+object IcebergTableWriter {
+
+  // A single mutable Iceberg Record backed by an Arrow batch and a row index.
+  // Only the positional reads the Parquet write path uses (get/get-typed/size)
+  // are implemented; mutation and name lookups are unsupported. Each column's
+  // value is decoded from Arrow to the Texera type, then mapped to the Iceberg
+  // Java type exactly as IcebergUtil.toGenericRecord does.
+  private class ArrowRecordView(root: VectorSchemaRoot, structType: StructType) extends Record {
+    private val vectors = root.getFieldVectors
+    private val texeraTypes = ArrowUtils.toTexeraSchema(root.getSchema).getAttributes.map(_.getType)
+    private var rowIndex = 0
+    def setRow(i: Int): Unit = rowIndex = i
+
+    private def icebergValue(pos: Int): AnyRef = {
+      val raw = vectors.get(pos).getObject(rowIndex)
+      val texeraValue =
+        try AttributeTypeUtils.parseField(raw, texeraTypes(pos))
+        catch { case _: Exception => null }
+      texeraValue match {
+        case null                        => null
+        case ts: java.sql.Timestamp      => ts.toInstant.atZone(ZoneId.systemDefault()).toLocalDateTime
+        case bytes: Array[Byte]          => ByteBuffer.wrap(bytes)
+        case largeBinaryPtr: LargeBinary => largeBinaryPtr.getUri
+        case other                       => other.asInstanceOf[AnyRef]
+      }
+    }
+
+    override def size(): Int = vectors.size
+    override def get(pos: Int): AnyRef = icebergValue(pos)
+    override def get[U](pos: Int, javaClass: Class[U]): U = javaClass.cast(icebergValue(pos))
+    override def struct(): StructType = structType
+
+    private def unsupported = throw new UnsupportedOperationException("ArrowRecordView is read-only")
+    override def set[U](pos: Int, value: U): Unit = unsupported
+    override def getField(name: String): AnyRef = unsupported
+    override def setField(name: String, value: AnyRef): Unit = unsupported
+    override def copy(): Record = unsupported
+    override def copy(overwriteValues: java.util.Map[String, AnyRef]): Record = unsupported
   }
 }
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
index 65bbb445585..6e33880586f 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala
@@ -142,10 +142,12 @@ object ArrowUtils extends LazyLogging {
     try {
       val out = ArrayBuffer[Tuple]()
       val root = reader.getVectorSchemaRoot
+      var schema: Schema = null
       while (reader.loadNextBatch()) {
+        if (schema == null) schema = toTexeraSchema(root.getSchema) // resolve once, reuse per row
         val n = root.getRowCount
         var i = 0
-        while (i < n) { out += getTexeraTuple(i, root); i += 1 }
+        while (i < n) { out += getTexeraTuple(i, root, schema); i += 1 }
       }
       out.toArray
     } finally reader.close()
@@ -166,10 +168,16 @@ object ArrowUtils extends LazyLogging {
   def getTexeraTuple(
       rowIndex: Int,
       vectorSchemaRoot: VectorSchemaRoot
-  ): Tuple = {
-    val arrowSchema = vectorSchemaRoot.getSchema
-    val schema = toTexeraSchema(arrowSchema)
+  ): Tuple = getTexeraTuple(rowIndex, vectorSchemaRoot, toTexeraSchema(vectorSchemaRoot.getSchema))
 
+  // Decode one row using a pre-resolved Texera schema. Callers decoding a whole
+  // batch should resolve the schema once and pass it, instead of rebuilding it
+  // (an Attribute allocation per column) on every row.
+  def getTexeraTuple(
+      rowIndex: Int,
+      vectorSchemaRoot: VectorSchemaRoot,
+      schema: Schema
+  ): Tuple = {
     Tuple
       .builder(schema)
       .addSequentially(
diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala
index c6b932fb009..976d4453290 100644
--- a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala
+++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowIpcRoundTripSpec.scala
@@ -66,4 +66,32 @@ class ArrowIpcRoundTripSpec extends AnyFlatSpec {
     val bytes = ArrowUtils.serializeTuples(schema, Array.empty[Tuple])
     assert(ArrowUtils.deserializeTuples(bytes).isEmpty)
   }
+
+  // Python-passthrough equivalence: the Arrow batch the Python bridge sends via
+  // the columnar passthrough (deserialize the ColumnarFrame bytes to a root) must
+  // match the batch the row path builds (fromTexeraSchema + appendTexeraTuple).
+  // Proven at the data level here; the Flight transport itself is unchanged.
+  "Python passthrough" should "produce the same Arrow batch as the row path" in {
+    val n = 3000
+    val rows = (0 until n).map(tuple).toArray
+    val allocator = new org.apache.arrow.memory.RootAllocator()
+    try {
+      // Row path: build the root the way writeArrowStream does.
+      val rowRoot =
+        org.apache.arrow.vector.VectorSchemaRoot.create(ArrowUtils.fromTexeraSchema(schema), allocator)
+      rowRoot.allocateNew()
+      rows.foreach(t => ArrowUtils.appendTexeraTuple(t, rowRoot))
+      rowRoot.setRowCount(n)
+
+      // Passthrough: the columnar bytes decoded back to tuples.
+      val passthrough = ArrowUtils.deserializeTuples(ArrowUtils.serializeTuples(schema, rows))
+
+      // Same schema, same row count, same values.
+      assert(rowRoot.getSchema == ArrowUtils.fromTexeraSchema(schema))
+      assert(rowRoot.getRowCount == passthrough.length)
+      val rowDecoded = (0 until rowRoot.getRowCount).map(i => ArrowUtils.getTexeraTuple(i, rowRoot))
+      assert(rowDecoded.sameElements(passthrough))
+      rowRoot.close()
+    } finally allocator.close()
+  }
 }
diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala
index 000b1e44127..0cb740155e7 100644
--- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/hashJoin/HashJoinProbeOpExec.scala
@@ -98,9 +98,9 @@ class HashJoinProbeOpExec[K](
     val isOuter = desc.joinType == JoinType.RIGHT_OUTER || desc.joinType == JoinType.FULL_OUTER
     if (columnarAllocator == null) columnarAllocator = new RootAllocator()
     ArrowUtils.deserializeRootFold(arrowIpcBytes, columnarAllocator) { root =>
+      val fullSchema = ArrowUtils.toTexeraSchema(root.getSchema) // resolve once, reuse per row
       if (keySchema == null) {
-        val full = ArrowUtils.toTexeraSchema(root.getSchema)
-        keySchema = Schema(List(full.getAttribute(desc.probeAttributeName)))
+        keySchema = Schema(List(fullSchema.getAttribute(desc.probeAttributeName)))
       }
       val keyIdx = ArrowUtils.projectionIndices(root, Seq(desc.probeAttributeName))
       val out = new ListBuffer[(TupleLike, Option[PortIdentity])]
@@ -114,7 +114,7 @@ class HashJoinProbeOpExec[K](
         val matched = buildTableHashMap.get(key).exists(_._1.nonEmpty)
         if (matched || isOuter) {
           // Reuse the exact row-path join logic (also marks the build side joined).
-          processTuple(ArrowUtils.getTexeraTuple(i, root), 1).foreach(t => out += ((t, None)))
+          processTuple(ArrowUtils.getTexeraTuple(i, root, fullSchema), 1).foreach(t => out += ((t, None)))
         }
         i += 1
       }