From 5b8dd572fd77bbae9b00088dc316dac0aaceea86 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Tue, 1 Sep 2026 16:23:41 -0700 Subject: [PATCH 01/33] feat(workflow-compiling-service): export a workflow as a standalone Python script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workflow can be read in the editor but not taken away: there is no form of it that runs anywhere else, so a user who wants to keep a pipeline, hand it to someone without Texera, or step through it in a notebook has nothing to take. This adds the seam for one and the first few operators through it. An operator says how it reads outside the engine by implementing `StandaloneCodeGenerator`, returning a block of pandas that names its inputs and outputs as `in1df` / `out1df`. The translator walks the plan in topological order, gives every port a variable, substitutes those placeholders, and prints the leaves; `inAlldf` stands for the whole list of upstreams, which is what a variadic port like Union's needs, since any fixed count the code stated would be wrong for some workflow. An operator with no generator yet leaves a commented TODO rather than a line that looks like it works. `GET /workflow-to-python` on the compiling service returns the script for a plan it is given. Five operators implement it here — Distinct, Limit, Projection, Filter and Union — chosen to cover the shapes the translator has to handle: a single input, a config-driven one, one that renames columns, one that builds a predicate, and the variadic port. The rest of the operator set follows in later changes. `pyStringLiteral` renders a value as a Python literal with the escaping that keeps a quote or a newline in a column name from ending the literal early. The generators cannot use the runtime's decode expression, which needs an operator instance to decode through. Co-Authored-By: Claude Opus 5 (1M context) --- .../pybuilder/PythonTemplateBuilder.scala | 23 ++ .../PythonTemplateBuilderApiSpec.scala | 20 ++ .../operator/StandaloneCodeGenerator.scala | 59 ++++++ .../operator/distinct/DistinctOpDesc.scala | 9 +- .../filter/SpecializedFilterOpDesc.scala | 40 +++- .../amber/operator/limit/LimitOpDesc.scala | 8 +- .../projection/ProjectionOpDesc.scala | 35 +++- .../amber/operator/union/UnionOpDesc.scala | 11 +- .../distinct/DistinctOpDescSpec.scala | 8 + .../filter/SpecializedFilterOpDescSpec.scala | 11 + .../operator/limit/LimitOpDescSpec.scala | 9 + .../projection/ProjectionOpDescSpec.scala | 13 ++ .../operator/union/UnionOpDescSpec.scala | 13 ++ .../WorkflowToPythonTranslator.scala | 198 ++++++++++++++++++ .../service/WorkflowCompilingService.scala | 9 +- .../resource/WorkflowToPythonResource.scala | 70 +++++++ .../WorkflowToPythonTranslatorSpec.scala | 106 ++++++++++ 17 files changed, 633 insertions(+), 9 deletions(-) create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala create mode 100644 workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala create mode 100644 workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala diff --git a/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala b/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala index 73f4a3846dd..90b4162041b 100644 --- a/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala +++ b/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala @@ -209,6 +209,29 @@ object PythonTemplateBuilder { def wrapWithPythonDecoderExpr(text: String): String = s"self.decode_python_template('$text')" + /** + * Render `text` as a Python double-quoted string literal, quotes included. + * + * For generators that emit standalone Python source rather than an operator + * for the runtime: they cannot use the decode expression (it needs the + * operator's `decode_python_template`, and it is deliberately rejected inside + * quotes), so they need the value as a *literal*. Writing `"$value"` by hand + * instead lets any quote, backslash or newline in the value close the literal + * early and change — or break — the emitted program. + * + * Escapes exactly what can end a double-quoted single-line literal. + */ + def pyStringLiteral(text: String): String = { + val escaped = Option(text) + .getOrElse("") + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\r", "\\r") + .replace("\n", "\\n") + .replace("\t", "\\t") + "\"" + escaped + "\"" + } + sealed trait RenderMode extends Product with Serializable object RenderMode { case object Plain extends RenderMode diff --git a/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala b/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala index acbed3031fc..149a1538574 100644 --- a/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala +++ b/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala @@ -244,4 +244,24 @@ class PythonTemplateBuilderApiSpec extends AnyFunSuite { test("hasUnclosedQuote: three opening single quotes count as unclosed") { assert(PythonLexerUtils.hasUnclosedQuote("'''abc")) } + + // -------- pyStringLiteral -------- + + // Every character that can end a double-quoted single-line literal, since one that + // slips through does not fail here but changes the emitted program. + test("pyStringLiteral: quotes the value and escapes what would close the literal") { + assert(PythonTemplateBuilder.pyStringLiteral("plain") == "\"plain\"") + assert(PythonTemplateBuilder.pyStringLiteral("say \"hi\"") == "\"say \\\"hi\\\"\"") + assert(PythonTemplateBuilder.pyStringLiteral("a\\b") == "\"a\\\\b\"") + assert(PythonTemplateBuilder.pyStringLiteral("one\ntwo") == "\"one\\ntwo\"") + assert(PythonTemplateBuilder.pyStringLiteral("a\tb") == "\"a\\tb\"") + assert(PythonTemplateBuilder.pyStringLiteral("a\rb") == "\"a\\rb\"") + } + + // A column name arrives from JSON and can be absent; an empty literal is a value the + // emitted program can carry, where `null` would reach it as the four letters. + test("pyStringLiteral: renders a null as the empty literal") { + assert(PythonTemplateBuilder.pyStringLiteral(null) == "\"\"") + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala new file mode 100644 index 00000000000..8ca23e55cb5 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator + +import java.net.URLDecoder +import java.nio.charset.StandardCharsets + +trait StandaloneCodeGenerator { + + def generateStandaloneCode(): String + + /** + * The file's own name, for a script that reads it from its own directory + * rather than through Texera's resolved URI. + * + * Taken from the last path segment instead of by parsing the whole string as a + * URI: the resolver percent-encodes the file-relative segments but leaves the + * repository and version names as the user typed them, so a dataset version + * called `v3 - with long text` makes `new URI` throw on the space and no code + * is generated at all. + */ + protected def sourceBasename(rawPath: String): String = { + val segment = rawPath.split("/").lastOption.getOrElse("") + // Percent-decoding only, matching what `URI.getPath` used to return here: form + // decoding would also turn a literal `+` in a file name into a space. + URLDecoder.decode(segment.replace("+", "%2B"), StandardCharsets.UTF_8) + } + + def producesDataFrame(): Boolean = true + + /** + * Definitions this operator's standalone code depends on, emitted once near + * the top of the script rather than inline. + * + * The translator concatenates operator bodies into a single module, so an + * operator needing a helper class has nowhere to put it that another operator + * would not duplicate. Helpers returned here are collected across the whole + * plan and deduplicated by their text, so two sampling operators in one + * workflow yield one copy of the generator they share. + */ + def standaloneHelpers(): Seq[String] = Seq.empty +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala index 9e75e648bb4..17b646bfa10 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala @@ -22,10 +22,10 @@ package org.apache.texera.amber.operator.distinct import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -class DistinctOpDesc extends LogicalOp { +class DistinctOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -54,4 +54,9 @@ class DistinctOpDesc extends LogicalOp { outputPorts = List(OutputPort(blocking = true)) ) + override def generateStandaloneCode(): String = { + // JVM op uses LinkedHashSet to preserve first-occurrence order; + // pandas drop_duplicates does the same by default. + "out1df = in1df.drop_duplicates(ignore_index=True)" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala index 9e86773df7c..cc23a553991 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala @@ -23,10 +23,12 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class SpecializedFilterOpDesc extends FilterOpDesc { +class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator { @JsonProperty(value = "predicates", required = true) @JsonPropertyDescription("multiple predicates in OR") @@ -60,4 +62,40 @@ class SpecializedFilterOpDesc extends FilterOpDesc { supportReconfiguration = true ) } + + override def generateStandaloneCode(): String = { + if (predicates.isEmpty) return "out1df = in1df.copy()" + val conditions = predicates.map { p => + val colLit = pyStringLiteral(p.attribute) + p.condition match { + case ComparisonType.IS_NULL => s"""(in1df[$colLit].isna())""" + case ComparisonType.IS_NOT_NULL => s"""(in1df[$colLit].notna())""" + case other => + val op = other.getName // returns "=", ">=", "<", etc. (see ComparisonType.java) + val pyOp = if (op == "=") "==" else op + // notna mirrors FilterPredicate, which answers false for every condition + // but IS_NULL / IS_NOT_NULL once the field is null. Only `!=` needs it — + // pandas answers True there, where every other operator answers False — + // but guarding all of them keeps the one rule visible in one place. + s"""(in1df[$colLit].notna() & (in1df[$colLit] $pyOp ${coerceValue(p.value)}))""" + } + } + s"out1df = in1df[${conditions.mkString(" | ")}].reset_index(drop=True)" + } + + // Try numeric coercion so generated code compares column values against the right type. + // Strings that don't parse fall through to a quoted string literal. + private def coerceValue(raw: String): String = { + try { + raw.toInt.toString + } catch { + case _: NumberFormatException => + try { + raw.toDouble.toString + } catch { + case _: NumberFormatException => + pyStringLiteral(raw) + } + } + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala index 6e1b7f37af3..bbfb7015036 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala @@ -25,12 +25,12 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StateTransferFunc} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StateTransferFunc} import org.apache.texera.amber.util.JSONUtils.objectMapper import scala.util.{Success, Try} -class LimitOpDesc extends LogicalOp { +class LimitOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("Limit") @@ -80,4 +80,8 @@ class LimitOpDesc extends LogicalOp { } Success(newPhysicalOp, Some(stateTransferFunc)) } + + override def generateStandaloneCode(): String = { + s"out1df = in1df.head($limit).reset_index(drop=True)" + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala index fb9258410cb..c98c61d566f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala @@ -26,17 +26,23 @@ import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PhysicalOp.oneToOnePhysicalOp import org.apache.texera.amber.core.workflow._ +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.map.MapOpDesc import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class ProjectionOpDesc extends MapOpDesc { +class ProjectionOpDesc extends MapOpDesc with StandaloneCodeGenerator { @JsonProperty(required = true, defaultValue = "false") @JsonSchemaTitle("Drop Option") @JsonPropertyDescription("check to drop the selected attributes") var isDrop: Boolean = false + // Named explicitly, without `required`: the form already asks for these and must go + // on accepting an empty list, but a field carrying no annotation is invisible to + // anything reading the operator's config by reflection. + @JsonProperty var attributes: List[AttributeUnit] = List() override def getPhysicalOp( @@ -98,4 +104,31 @@ class ProjectionOpDesc extends MapOpDesc { outputPorts = List(OutputPort()) ) } + + override def generateStandaloneCode(): String = { + val units = Option(attributes).getOrElse(List.empty) + // JVM validates non-empty at runtime via Preconditions; emit passthrough + // as best-effort so the standalone script still runs. + if (units.isEmpty) return "out1df = in1df.copy()" + + if (isDrop) { + // Drop mode ignores aliases (matches ProjectionOpExec). + val cols = units.map(u => pyStringLiteral(u.getOriginalAttribute)).mkString("[", ", ", "]") + s"out1df = in1df.drop(columns=$cols)" + } else { + val originals = + units.map(u => pyStringLiteral(u.getOriginalAttribute)).mkString("[", ", ", "]") + // AttributeUnit.getAlias returns originalAttribute when alias is blank, + // so an explicit rename is only needed when they differ. + val renames = units + .filter(u => u.getAlias != u.getOriginalAttribute) + .map(u => s"""${pyStringLiteral(u.getOriginalAttribute)}: ${pyStringLiteral(u.getAlias)}""") + if (renames.isEmpty) { + s"out1df = in1df[$originals].copy()" + } else { + val renameMap = renames.mkString("{", ", ", "}") + s"out1df = in1df[$originals].rename(columns=$renameMap)" + } + } + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala index 82e292c8f38..460aa7990b1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala @@ -22,10 +22,10 @@ package org.apache.texera.amber.operator.union import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -class UnionOpDesc extends LogicalOp { +class UnionOpDesc extends LogicalOp with StandaloneCodeGenerator { override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -50,4 +50,11 @@ class UnionOpDesc extends LogicalOp { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) + + // UNION ALL: UnionOpExec passes tuples through without dedup. The port is + // variadic, so the code names the whole list of upstreams rather than a fixed + // two — naming two dropped a third and left the second unbound when only one + // was drawn. + override def generateStandaloneCode(): String = + "out1df = pd.concat(inAlldf, ignore_index=True)" } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala index 2aba788acfe..15a9cf5d64e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala @@ -106,4 +106,12 @@ class DistinctOpDescSpec extends AnyFlatSpec with Matchers { val b = new DistinctOpDesc a.operatorIdentifier should not equal b.operatorIdentifier } + + // The JVM operator keeps the first occurrence of a duplicate, which is what + // drop_duplicates does by default, so the emitted line says nothing about order. + "DistinctOpDesc.generateStandaloneCode" should "drop duplicates in place" in { + (new DistinctOpDesc).generateStandaloneCode() shouldBe + "out1df = in1df.drop_duplicates(ignore_index=True)" + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala index 84c7ec93779..16b7e424ea7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala @@ -70,4 +70,15 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { restored shouldBe a[SpecializedFilterOpDesc] restored.asInstanceOf[SpecializedFilterOpDesc].predicates shouldBe empty } + + // A null answers false for every condition but IS_NULL / IS_NOT_NULL, which pandas + // does not do on its own for `!=`, so the emitted condition carries the guard. + "SpecializedFilterOpDesc.generateStandaloneCode" should "emit one condition per predicate" in { + val d = new SpecializedFilterOpDesc + d.predicates = List(new FilterPredicate("age", ComparisonType.GREATER_THAN, "18")) + val code = d.generateStandaloneCode() + code should include("in1df[\"age\"]") + code should include("out1df") + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala index f01e8f63f08..d7314668175 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala @@ -94,4 +94,13 @@ class LimitOpDescSpec extends AnyFlatSpec with Matchers { transfer(oldExec, newExec) newExec.count shouldBe 3 } + + // The index is reset because the operator hands its downstream a fresh table + // rather than a view of the one it read. + "LimitOpDesc.generateStandaloneCode" should "take the first N rows" in { + val d = new LimitOpDesc + d.limit = 3 + d.generateStandaloneCode() shouldBe "out1df = in1df.head(3).reset_index(drop=True)" + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala index 66e6f556afb..ee4468665d1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala @@ -216,4 +216,17 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(out == SinglePartition()) } + // Drop mode names the columns to remove; keep mode names the ones to hold on to, + // in the order the user put them in. + "ProjectionOpDesc.generateStandaloneCode" should "select or drop the named columns" in { + val keep = new ProjectionOpDesc + keep.attributes = List(new AttributeUnit("a", ""), new AttributeUnit("b", "")) + assert(keep.generateStandaloneCode().contains("""in1df[["a", "b"]]""")) + + val drop = new ProjectionOpDesc + drop.attributes = List(new AttributeUnit("a", "")) + drop.isDrop = true + assert(drop.generateStandaloneCode() == """out1df = in1df.drop(columns=["a"])""") + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala index a9c58bbcda4..16c7e027bc5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala @@ -87,6 +87,19 @@ class UnionOpDescSpec extends AnyFlatSpec with Matchers { physical.partitionRequirement shouldBe empty } + // --------------------------------------------------------------------------- + // generateStandaloneCode + // --------------------------------------------------------------------------- + + // UNION ALL: UnionOpExec passes tuples through without dedup, so the + // generated concat must not drop duplicates either. It names the whole list + // of upstreams rather than a fixed two, because the port is variadic and any + // count the code stated would be wrong for some workflow. + "UnionOpDesc.generateStandaloneCode" should "concatenate every input without dedup" in { + (new UnionOpDesc).generateStandaloneCode() shouldBe + "out1df = pd.concat(inAlldf, ignore_index=True)" + } + // --------------------------------------------------------------------------- // Independent instances // --------------------------------------------------------------------------- diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala new file mode 100644 index 00000000000..85ce6f98039 --- /dev/null +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.virtualidentity.OperatorIdentity +import org.apache.texera.common.compiler.model.LogicalPlan +import org.apache.texera.amber.operator.StandaloneCodeGenerator + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import scala.jdk.CollectionConverters._ + +class WorkflowToPythonTranslator extends LazyLogging { + + // Output-port-level key. An operator with N output ports gets N entries + // (e.g. Split has port 0 and port 1, each with its own assigned dfN var). + private type PortKey = (String, Int) // (opId, portIdx) + + def translate(logicalPlan: LogicalPlan): String = { + // Track downstream connections per (opId, fromPortIdx). A port is a leaf + // if it has no outgoing edges — operator-level "no outgoing links" is too + // coarse for multi-output ops (Split's port 0 may have downstream while + // port 1 doesn't, or vice versa). + val outgoingFromPort = mutable.Map[PortKey, Int]().withDefaultValue(0) + logicalPlan.links.foreach { link => + outgoingFromPort((link.fromOpId.id, link.fromPortId.id)) += 1 + } + + val outputVar = mutable.Map[PortKey, String]() + var varCounter = 1 + val script = ArrayBuffer[String]() + + script += "import pandas as pd" + script += "import plotly.express as px" + script += "import plotly.graph_objects as go" + script += "import plotly.io" + script += "" + + // getTopologicalOpIds() uses jgrapht internally — no need for a custom topo sort + val topoOrder = logicalPlan.getTopologicalOpIds.asScala.toList + + // Helper definitions the operator bodies below refer to. Collected across the + // whole plan and deduplicated by text, so a workflow holding two operators + // that share one helper still emits it once. Order follows the topological + // order, which keeps the script stable for a given plan. + val helpers = topoOrder + .map(logicalPlan.getOperator) + .collect { case gen: StandaloneCodeGenerator => gen.standaloneHelpers() } + .flatten + .distinct + if (helpers.nonEmpty) { + helpers.foreach { helper => script += helper; script += "" } + } + + for (opIdentity <- topoOrder) { + val opId = opIdentity.id + val op = logicalPlan.getOperator(opIdentity) + val displayName = op.operatorInfo.userFriendlyName + + // Resolve upstream inputs in the consuming operator's input-port order + // (link.toPortId), NOT the order links happen to appear in the plan's + // link list. This makes in1df/in2df/... deterministic and correct for + // multi-input operators (joins, set ops) where port 0 vs port 1 carries + // semantics (e.g. build vs probe side). Ties on the same toPortId keep + // link order — relevant for variadic single-port operators like Union. + // Each upstream link is resolved via (fromOpId, fromPortId) so that a + // multi-output upstream (Split) hands each downstream the correct DF. + val inVars = logicalPlan + .getUpstreamLinks(opIdentity) + .sortBy(link => (link.toPortId.id, link.toPortId.internal)) + .map(link => outputVar((link.fromOpId.id, link.fromPortId.id))) + + // Allocate one dfN per declared output port. Existing single-output + // operators have outputPorts.size == 1, so they get exactly one var and + // their behavior is identical to the previous flat scheme. + val outVars = op.operatorInfo.outputPorts.map { port => + val v = s"df$varCounter" + varCounter += 1 + outputVar((opId, port.id.id)) = v + v + } + + script += s"# [$displayName]" + + // Jackson deserializes each operator into its concrete subclass via @JsonSubTypes on LogicalOp, + // so the pattern match below will resolve to the correct descriptor (e.g. BarChartOpDesc). + op match { + case gen: StandaloneCodeGenerator => + // generateStandaloneCode() returns a code block using in{N}df / out{N}df + // placeholders; substituteVars() replaces them with the assigned vars. + script += substituteVars(gen.generateStandaloneCode(), inVars, outVars, displayName) + + case _ => + logger.warn( + s"Operator '$displayName' does not implement StandaloneCodeGenerator. Skipping." + ) + script += s"# TODO: '$displayName' is not yet supported by the translator." + outVars.zipWithIndex.foreach { + case (v, i) => script += s"# $v = " + } + } + + script += "" + } + + // Leaf detection runs at the port level: a (opId, port) pair is a leaf + // if no link consumes it. For Split with one downstream port and one + // dangling port, only the dangling port is treated as a leaf to print. + val leafPorts = outputVar.keys.toList + .sortBy { case (_, portIdx) => portIdx } + .filter(key => outgoingFromPort(key) == 0) + val dataFrameLeafPorts = leafPorts.filter { + case (opId, _) => + logicalPlan.getOperator(OperatorIdentity(opId)) match { + case gen: StandaloneCodeGenerator => gen.producesDataFrame() + case _ => false + } + } + + if (dataFrameLeafPorts.nonEmpty) { + script += "# --- Output ---" + // Print in topological order of the producing operator so multi-port + // operators print contiguously and the order matches the script flow. + val topoIndex = topoOrder.map(_.id).zipWithIndex.toMap + dataFrameLeafPorts + .sortBy { case (opId, portIdx) => (topoIndex.getOrElse(opId, Int.MaxValue), portIdx) } + .foreach { + case (opId, portIdx) => + val varName = outputVar((opId, portIdx)) + val displayName = + logicalPlan.getOperator(OperatorIdentity(opId)).operatorInfo.userFriendlyName + val portSuffix = if (outputVar.keys.count(_._1 == opId) > 1) s" port $portIdx" else "" + script += s"""print("\\n[$displayName$portSuffix] $varName:")""" + script += s"print($varName.head())" + script += "" + } + } + + script.mkString("\n") + } + + // Replaces in{N}df / out{N}df placeholders with concrete variable names. + // Substitutes in reverse index order to prevent partial matches (e.g. in1df + // inside in10df). After substitution, scans for any leftover placeholders + // and logs a warning — that signals a mismatch between an operator's + // declared port count and what its generateStandaloneCode actually emits. + private def substituteVars( + code: String, + inVars: List[String], + outVars: List[String], + displayName: String + ): String = { + var result = code + + // A variadic port takes as many upstream links as the user draws, and an + // operator reading one cannot name them: `in1df`/`in2df` state a count, and + // whichever count it states is wrong for every other workflow. This one + // placeholder becomes the whole list, so the operator writes the same line + // whether it is fed one table or five. + result = result.replaceAll("""\binAlldf\b""", inVars.mkString("[", ", ", "]")) + inVars.zipWithIndex.reverse.foreach { + case (v, idx) => result = result.replaceAll(s"\\bin${idx + 1}df\\b", v) + } + outVars.zipWithIndex.reverse.foreach { + case (v, idx) => result = result.replaceAll(s"\\bout${idx + 1}df\\b", v) + } + + val leftoverIn = """\bin\d+df\b""".r.findAllIn(result).toSet + val leftoverOut = """\bout\d+df\b""".r.findAllIn(result).toSet + if (leftoverIn.nonEmpty || leftoverOut.nonEmpty) { + logger.warn( + s"Operator '$displayName' emitted placeholders that don't match its port " + + s"count: leftover inputs=$leftoverIn, leftover outputs=$leftoverOut. " + + s"Generated script will reference unbound variables." + ) + } + + result + } +} diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index a69ef545246..46649647a0a 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -27,7 +27,11 @@ import org.apache.texera.common.config.StorageConfig import org.apache.texera.amber.util.ObjectMapperUtils import org.apache.texera.auth.{AuthFeatures, RoleAnnotationEnforcer} import org.apache.texera.dao.SqlServer -import org.apache.texera.service.resource.{HealthCheckResource, WorkflowCompilationResource} +import org.apache.texera.service.resource.{ + HealthCheckResource, + WorkflowCompilationResource, + WorkflowToPythonResource +} import org.eclipse.jetty.servlet.FilterHolder import java.nio.file.Path @@ -67,6 +71,9 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi // register the compilation endpoint environment.jersey.register(classOf[WorkflowCompilationResource]) + // register the workflow-to-python endpoint + environment.jersey.register(classOf[WorkflowToPythonResource]) + RoleAnnotationEnforcer.enforce( environment.jersey.getResourceConfig, "WorkflowCompilingService" diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala new file mode 100644 index 00000000000..18b75ef338f --- /dev/null +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.service.resource + +import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} +import com.typesafe.scalalogging.LazyLogging +import jakarta.annotation.security.RolesAllowed +import jakarta.ws.rs.core.MediaType +import jakarta.ws.rs.{Consumes, POST, Path, Produces} +import org.apache.texera.common.compiler.model.{LogicalPlan, LogicalPlanPojo} +import org.apache.texera.amber.translator.WorkflowToPythonTranslator + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes( + Array( + new JsonSubTypes.Type(value = classOf[WorkflowToPythonSuccess], name = "success"), + new JsonSubTypes.Type(value = classOf[WorkflowToPythonFailure], name = "failure") + ) +) +sealed trait WorkflowToPythonResponse + +case class WorkflowToPythonSuccess(pythonCode: String) extends WorkflowToPythonResponse + +case class WorkflowToPythonFailure(errorMessage: String) extends WorkflowToPythonResponse + +@Consumes(Array(MediaType.APPLICATION_JSON)) +@Produces(Array(MediaType.APPLICATION_JSON)) +@RolesAllowed(Array("REGULAR", "ADMIN")) +@Path("/workflow-to-python") +class WorkflowToPythonResource extends LazyLogging { + + private val translator = new WorkflowToPythonTranslator() + + @POST + @Path("") + def convertWorkflowToPython( + logicalPlanPojo: LogicalPlanPojo + ): WorkflowToPythonResponse = { + try { + val logicalPlan = LogicalPlan(logicalPlanPojo) + val pythonCode = translator.translate(logicalPlan) + WorkflowToPythonSuccess(pythonCode) + } catch { + case e: Exception => + logger.error("Failed to translate workflow to Python", e) + WorkflowToPythonFailure(e.getMessage) + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala new file mode 100644 index 00000000000..f1a37755c25 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator + +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.distinct.DistinctOpDesc +import org.apache.texera.amber.operator.union.UnionOpDesc +import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlan} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** The placeholder substitution, which is where an operator's generated code + * meets the variables the script actually binds. A variadic port is the case + * the numbered placeholders cannot state, so it is the case worth pinning. + */ +class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { + + private def upstream(id: String): LogicalOp = { + val op = new DistinctOpDesc + op.setOperatorId(id) + op + } + + /** `n` upstreams, all drawn into the union's single port, which is what a + * variadic port looks like in a plan. + */ + private def unionOf(n: Int): String = { + val union = new UnionOpDesc + union.setOperatorId("union") + val ups = (1 to n).map(i => upstream(s"up$i")) + val links = ups.map { up => + LogicalLink( + up.operatorIdentifier, + PortIdentity(0), + union.operatorIdentifier, + PortIdentity(0) + ) + } + new WorkflowToPythonTranslator().translate( + LogicalPlan(ups.toList :+ union, links.toList) + ) + } + + "WorkflowToPythonTranslator" should "hand a variadic port every upstream it was drawn" in { + unionOf(3) should include("pd.concat([df1, df2, df3], ignore_index=True)") + } + + it should "hand a variadic port a one-element list when only one link is drawn" in { + // The case the old fixed `[in1df, in2df]` got wrong in the other direction: + // it named a second frame the script never bound. + unionOf(1) should include("pd.concat([df1], ignore_index=True)") + } + + it should "leave no placeholder behind for a variadic port" in { + unionOf(2) should not include "inAlldf" + } + + it should "still resolve a numbered placeholder against its own upstream" in { + // The variadic form is an addition, not a replacement: a chain of ordinary + // single-input operators has to keep reading `in1df` as its predecessor. + val first = upstream("first") + val second = upstream("second") + val script = new WorkflowToPythonTranslator().translate( + LogicalPlan( + List(first, second), + List( + LogicalLink( + first.operatorIdentifier, + PortIdentity(0), + second.operatorIdentifier, + PortIdentity(0) + ) + ) + ) + ) + script should include("df2 = df1.drop_duplicates(ignore_index=True)") + } + + /** The translator's own contract when it meets an operator it cannot render: + * a comment rather than a silently wrong line. + */ + it should "leave a TODO for an operator with no standalone code generator" in { + val op = new org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 + op.setOperatorId("udf") + val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + script should include("# TODO:") + } +} From a7558ebdd67973d1042f1f0d34c3a73f8237d85b Mon Sep 17 00:00:00 2001 From: kary zheng Date: Tue, 1 Sep 2026 16:36:53 -0700 Subject: [PATCH 02/33] test(workflow-compiling-service): run an operator both ways and compare the files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standalone export claims that a generated script does what the operator does. Nothing checks it. This adds the two runners that make the claim checkable, and the file format they meet in. `OpExecHarness` runs a LogicalOp the way the engine does — compiling it to a physical plan and driving the executor — but outside a workflow, against JSONL files rather than a live upstream. `PyOpExecHarness` does the same for a Python operator, through the worker the engine uses. `StandaloneRunner` takes the other path: it asks the operator for its standalone code, wraps it in a script that binds `in1df` from the same files, and runs it. `TupleIO` is what the two meet in. A JSONL row carries values and no types, so the schema travels beside it in a sidecar; without one, a column written as INTEGER reads back as a number and the two paths disagree over a difference neither operator made. Both runners produce files, not assertions, so what to make of a difference is left to a later change. What is here is enough to run one operator both ways and see that the answers match, which is what the spec does with Distinct. Co-Authored-By: Claude Opus 5 (1M context) --- build.sbt | 1 + .../src/test/resources/python/py_op_driver.py | 531 ++++++++++++++++++ .../resources/python/standalone_worker.py | 122 ++++ .../amber/translator/verify/HarnessSpec.scala | 108 ++++ .../translator/verify/OpExecHarness.scala | 454 +++++++++++++++ .../translator/verify/PyOpExecHarness.scala | 406 +++++++++++++ .../translator/verify/StandaloneRunner.scala | 367 ++++++++++++ 7 files changed, 1989 insertions(+) create mode 100644 workflow-compiling-service/src/test/resources/python/py_op_driver.py create mode 100644 workflow-compiling-service/src/test/resources/python/standalone_worker.py create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala diff --git a/build.sbt b/build.sbt index 035edbb68cc..94a5af5606c 100644 --- a/build.sbt +++ b/build.sbt @@ -239,6 +239,7 @@ lazy val WorkflowCompiler = (project in file("common/workflow-compiler")) .dependsOn(WorkflowOperator) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) .dependsOn(WorkflowCompiler, Auth, Config, Resource) + .dependsOn(WorkflowOperator % "test->test") // reuse PythonWorkerPool in verify tests .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( diff --git a/workflow-compiling-service/src/test/resources/python/py_op_driver.py b/workflow-compiling-service/src/test/resources/python/py_op_driver.py new file mode 100644 index 00000000000..1ff2a9000da --- /dev/null +++ b/workflow-compiling-service/src/test/resources/python/py_op_driver.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Driver that runs a Texera Python-native operator without spinning up the +Pekko/Arrow worker stack. + +Symmetric to ``OpExecHarness`` on the JVM side: take an OpDesc's +``generatePythonCode()`` output (which defines a ``UDFOperatorV2`` / +``UDFTableOperator`` / ``UDFBatchOperator`` / ``UDFSourceOperator`` subclass), +load JSONL+sidecar inputs into ``Tuple`` instances, drive +``open -> process_tuple/on_finish per port -> close``, and write the emitted +tuples back as JSONL+sidecar in the same format ``TupleIO`` reads. + +The harness invokes us as:: + + python3 py_op_driver.py + +with ``PYTHONPATH`` pointing at ``amber/src/main/python`` so ``pytexera`` / +``pyamber`` import cleanly. + +Config schema (all paths absolute):: + + { + "operatorCode": "", + "isSource": false, + "portOrder": [0, 1], # input-port dependency order + "inputs": [{"portIndex": 0, "dataPath": "...", "schemaPath": "..."}], + "outputs": [{"portIndex": 0, "dataPath": "...", "schema": + {"attributes": [{"attributeName": "...", + "attributeType": "..."}]}}] + } + +Output schemas come from the JVM side (``PhysicalOp.propagateSchema``) so +this driver never has to infer them. The driver writes the schema back as a +``.jsonl.schema.json`` sidecar next to each ``dataPath``, matching +``TupleIO.writeTuples``. +""" +from __future__ import annotations + +import base64 +import inspect +import json +import pickle +import sys +import traceback +from pathlib import Path +from typing import Any, Iterable, Iterator, List, Mapping, Sequence + +import pandas as pd + +# pytexera re-exports the operator base classes and the Tuple/Table types. +# The Scala side prepends `amber/src/main/python` to PYTHONPATH so these +# resolve. If they don't, raise a clean error rather than a cryptic +# ImportError deep in user code. +try: + from pytexera import ( # noqa: F401 (used dynamically in user code's globals) + Batch, + BatchLike, + Iterator as PyIterator, # noqa: F401 + Optional as PyOptional, # noqa: F401 + Table, + TableLike, + Tuple, + TupleLike, + UDFBatchOperator, + UDFOperatorV2, + UDFSourceOperator, + UDFTableOperator, + Union as PyUnion, # noqa: F401 + logger as pytexera_logger, # noqa: F401 + overrides, # noqa: F401 + ) + from core.models.schema.schema import Schema as TexeraSchema + from core.models.schema.attribute_type import AttributeType, RAW_TYPE_MAPPING +except ImportError as exc: + sys.stderr.write( + "py_op_driver.py: failed to import pytexera/pyamber. The harness must " + "set PYTHONPATH to `amber/src/main/python` and the venv must have all " + "amber Python deps installed (see amber/requirements.txt).\n" + f"Underlying error: {exc!r}\n" + ) + raise + + +# -------------------------------------------------------------------------- +# Schema sidecar I/O. +# -------------------------------------------------------------------------- +# The JVM writes attributes using AttributeType's Jackson @JsonValue ("string", +# "integer", "long", "double", "boolean", "timestamp", "binary", +# "large_binary"). The Python Schema's RAW_TYPE_MAPPING uses uppercase keys +# ("STRING", "INTEGER", ...). Translate at the boundary; keep the rest of +# the pipeline using Python's AttributeType enum. +_SCALA_TO_PY_TYPE: Mapping[str, str] = { + "string": "STRING", + "integer": "INTEGER", + "long": "LONG", + "double": "DOUBLE", + "boolean": "BOOLEAN", + "timestamp": "TIMESTAMP", + "binary": "BINARY", + "large_binary": "LARGE_BINARY", +} + +_PY_TO_SCALA_TYPE: Mapping[AttributeType, str] = { + AttributeType.STRING: "string", + AttributeType.INT: "integer", + AttributeType.LONG: "long", + AttributeType.DOUBLE: "double", + AttributeType.BOOL: "boolean", + AttributeType.TIMESTAMP: "timestamp", + AttributeType.BINARY: "binary", + AttributeType.LARGE_BINARY: "large_binary", +} + + +def _schema_from_dict(payload: Mapping[str, Any]) -> TexeraSchema: + raw: "dict[str, str]" = {} + for attr in payload["attributes"]: + raw_name = attr["attributeName"] + raw_type = attr["attributeType"].lower() + if raw_type not in _SCALA_TO_PY_TYPE: + raise ValueError( + f"py_op_driver: unknown attributeType {attr['attributeType']!r} " + f"for attribute {raw_name!r}" + ) + raw[raw_name] = _SCALA_TO_PY_TYPE[raw_type] + return TexeraSchema(raw_schema=raw) + + +def _schema_to_dict(schema: TexeraSchema) -> "dict[str, Any]": + return { + "attributes": [ + {"attributeName": name, "attributeType": _PY_TO_SCALA_TYPE[attr_type]} + for name, attr_type in schema.as_key_value_pairs() + ] + } + + +def _read_schema_sidecar(data_path: Path) -> TexeraSchema: + sidecar = data_path.with_name(data_path.name + ".schema.json") + with sidecar.open("r", encoding="utf-8") as fh: + return _schema_from_dict(json.load(fh)) + + +def _write_schema_sidecar(data_path: Path, schema: TexeraSchema) -> None: + sidecar = data_path.with_name(data_path.name + ".schema.json") + with sidecar.open("w", encoding="utf-8") as fh: + json.dump(_schema_to_dict(schema), fh) + + +# -------------------------------------------------------------------------- +# Tuple I/O. JSONL with sidecar — same on-disk shape as TupleIO on the JVM. +# -------------------------------------------------------------------------- +def _coerce_field(raw: Any, attr_type: AttributeType) -> Any: + """Coerce a JSON-decoded field to the type the schema expects.""" + if raw is None: + return None + if attr_type == AttributeType.STRING: + return str(raw) + if attr_type == AttributeType.INT: + return int(raw) + if attr_type == AttributeType.LONG: + return int(raw) + if attr_type == AttributeType.DOUBLE: + return float(raw) + if attr_type == AttributeType.BOOL: + return bool(raw) + if attr_type == AttributeType.BINARY: + return base64.b64decode(raw) + if attr_type == AttributeType.TIMESTAMP: + # TupleIO writes java.sql.Timestamp.toString ("YYYY-MM-DD HH:MM:SS[.f]"); + # the native path's schema maps TIMESTAMP -> datetime.datetime, and + # pandas parses the JDBC form robustly. + return pd.Timestamp(raw).to_pydatetime() + # LARGE_BINARY: defer until an operator actually exercises it. Failing loud + # beats silently passing a string through. + raise NotImplementedError( + f"py_op_driver: reading attribute type {attr_type!r} from JSONL is " + f"not implemented yet" + ) + + +def _read_tuples(data_path: Path, schema: TexeraSchema) -> List[Tuple]: + rows: List[Tuple] = [] + if not data_path.exists(): + return rows + with data_path.open("r", encoding="utf-8") as fh: + for line_num, raw_line in enumerate(fh, 1): + line = raw_line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"py_op_driver: invalid JSON on line {line_num} of {data_path}: {exc}" + ) from exc + field_data: "dict[str, Any]" = {} + for name, attr_type in schema.as_key_value_pairs(): + field_data[name] = _coerce_field(obj.get(name), attr_type) + tup = Tuple(field_data) + tup.finalize(schema) + rows.append(tup) + return rows + + +def _emit_as_dicts( + emitted: Iterable[Any], schema: TexeraSchema +) -> Iterator["dict[str, Any]"]: + """ + Flatten whatever the operator yields into per-row dicts keyed by the + output schema's attribute names. The operator may yield: + * pandas.DataFrame (UDFTableOperator's process_table return) + * pandas.Series (single row) + * dict / OrderedDict (e.g. BarChart yields {'html-content': html}) + * Tuple + * None (skip — matches the engine's behavior) + """ + attr_names = schema.get_attr_names() + for item in emitted: + if item is None: + continue + if isinstance(item, pd.DataFrame): + for _, row in item.iterrows(): + yield {col: row[col] for col in attr_names if col in row.index} + elif isinstance(item, pd.Series): + yield {col: item[col] for col in attr_names if col in item.index} + elif isinstance(item, Tuple): + yield {name: item[name] for name in attr_names} + elif isinstance(item, Mapping): + yield {name: item.get(name) for name in attr_names} + else: + raise TypeError( + f"py_op_driver: cannot serialize emitted value of type " + f"{type(item).__name__}: {item!r}" + ) + + +def _jsonify(value: Any, attr_type: AttributeType) -> Any: + """Convert a Python value into something json.dumps will accept.""" + if value is None: + return None + # A missing cell reaches pandas as NaN or NaT, not None, and every branch + # below assumes a real value: the timestamp one formats NaT's float + # microsecond with a "d" code and raises. Emitting null matches the + # standalone path, whose to_json writes NaN and NaT that way. Scalars only — + # an object column can hold a list or an array, where isna answers + # element-wise and the result is not a truth value. + if pd.api.types.is_scalar(value) and pd.isna(value): + return None + # pandas often hands us numpy scalars; .item() collapses them to native. + if hasattr(value, "item") and not isinstance(value, (str, bytes)): + try: + value = value.item() + except (ValueError, AttributeError): + pass + if attr_type == AttributeType.STRING: + return str(value) + if attr_type in (AttributeType.INT, AttributeType.LONG): + return int(value) + if attr_type == AttributeType.DOUBLE: + return float(value) + if attr_type == AttributeType.BOOL: + return bool(value) + if attr_type == AttributeType.BINARY: + # Trained-model / object columns: pickle then base64 so the value + # survives JSONL round-trip. Mirrors the BINARY read path in + # _coerce_field. For deterministic estimators the pickle is byte-stable + # across processes, so the two verification paths compare equal. + raw = value if isinstance(value, (bytes, bytearray)) else pickle.dumps(value) + return base64.b64encode(raw).decode("ascii") + if attr_type == AttributeType.TIMESTAMP: + # Emit the same JDBC string java.sql.Timestamp.toString produces (>=1 + # fractional digit), so a passed-through timestamp column matches the + # standalone path, which carries it as that exact string. + ts = pd.Timestamp(value) + frac = f"{ts.microsecond:06d}".rstrip("0") or "0" + return ts.strftime("%Y-%m-%d %H:%M:%S") + "." + frac + raise NotImplementedError( + f"py_op_driver: writing attribute type {attr_type!r} to JSONL is " + f"not implemented yet" + ) + + +def _write_tuples( + data_path: Path, rows: Iterable["dict[str, Any]"], schema: TexeraSchema +) -> None: + _write_schema_sidecar(data_path, schema) + with data_path.open("w", encoding="utf-8") as fh: + for row in rows: + serialized: "dict[str, Any]" = {} + for name, attr_type in schema.as_key_value_pairs(): + serialized[name] = _jsonify(row.get(name), attr_type) + fh.write(json.dumps(serialized)) + fh.write("\n") + + +# -------------------------------------------------------------------------- +# Operator discovery + lifecycle. +# -------------------------------------------------------------------------- +_OPERATOR_BASES = ( + UDFOperatorV2, + UDFTableOperator, + UDFBatchOperator, + UDFSourceOperator, +) + + +def _exec_user_code(code: str) -> "dict[str, Any]": + """ + Execute the operator code in a fresh namespace seeded with the pytexera + re-exports, the way the real Texera Python worker does it (see + ``InitializeExecutorHandler``). Returning the namespace lets us pick the + user's operator class out of it. + """ + namespace: "dict[str, Any]" = { + "__name__": "__texera_user_op__", + "__builtins__": __builtins__, + } + # pytexera does `from pyamber import *` itself, so this single import is + # equivalent to what the generated code's `from pytexera import *` brings + # into scope. + exec("from pytexera import *", namespace) + try: + exec(code, namespace) + except Exception: + sys.stderr.write("py_op_driver: error executing operator code:\n") + traceback.print_exc() + raise + return namespace + + +def _discover_operator_class(namespace: Mapping[str, Any]) -> type: + candidates: List[type] = [] + for name, obj in namespace.items(): + if not inspect.isclass(obj): + continue + if obj in _OPERATOR_BASES: + continue # the base classes themselves come in via the import + if any(issubclass(obj, base) for base in _OPERATOR_BASES): + candidates.append(obj) + if not candidates: + raise RuntimeError( + "py_op_driver: operator code did not define a subclass of " + "UDFOperatorV2 / UDFTableOperator / UDFBatchOperator / UDFSourceOperator" + ) + if len(candidates) > 1: + names = ", ".join(c.__name__ for c in candidates) + raise RuntimeError( + f"py_op_driver: operator code defined multiple UDF subclasses " + f"({names}); expected exactly one" + ) + return candidates[0] + + +def _run_operator( + op: Any, + is_source: bool, + port_order: Sequence[int], + inputs_by_port: Mapping[int, Sequence[Tuple]], +) -> List[Any]: + """ + Drive the operator's lifecycle. Returns the flat list of emitted values + (anything not-None yielded by process_tuple / on_finish, in emission + order). UDF operators don't expose multi-output ports today, so we don't + bucket by output port — same convention as ``OpExecHarness`` when port + is unset. + """ + emitted: List[Any] = [] + + op.open() + try: + if is_source: + # Source ops: SourceOperator.on_finish iterates produce() and + # yields Tuples. Single synthetic port 0 — see OpExecHarness. + for item in op.on_finish(0): + if item is not None: + emitted.append(item) + return emitted + + for port in port_order: + for tup in inputs_by_port.get(port, ()): # type: ignore[arg-type] + for item in op.process_tuple(tup, port): + if item is not None: + emitted.append(item) + for item in op.on_finish(port): + if item is not None: + emitted.append(item) + finally: + op.close() + + return emitted + + +# -------------------------------------------------------------------------- +# Entry point. +# -------------------------------------------------------------------------- +def run_config(config: Mapping[str, Any]) -> None: + """Run one operator to completion from a parsed config dict. Writes the + output JSONL+sidecar as a side effect. Raises on any failure. Shared by the + CLI (main) and the persistent server (serve) so both behave identically. + + Each call execs the user code in a FRESH namespace and constructs a FRESH + operator instance, so operators don't share Python-level state across jobs + when run through the server (the isolation the per-process CLI gave for + free).""" + # Both paths seed numpy's global RNG with the same value before running, so + # an estimator built without random_state (sklearn reads the global RNG for + # that) draws the same samples on each and the two models come out + # identical. Without it a stochastic estimator makes the parity check + # inconclusive in both directions: a difference could be the translation or + # could be the draw, and a match could be either. Per call, not per process: + # the worker pool reuses this process, so a job that inherited the previous + # job's RNG position would not line up with a fresh standalone one. Keep in + # step with StandaloneRunner.VerifySeed. + import numpy as _texera_np + + _texera_np.random.seed(20260811) + + operator_code: str = config["operatorCode"] + is_source: bool = bool(config.get("isSource", False)) + port_order: Sequence[int] = list(config.get("portOrder", [])) + + inputs_by_port: "dict[int, List[Tuple]]" = {} + for entry in config.get("inputs", []): + port = int(entry["portIndex"]) + data_path = Path(entry["dataPath"]) + schema = _read_schema_sidecar(data_path) + inputs_by_port[port] = _read_tuples(data_path, schema) + + # Default port order: sorted by index. Matches OpExecHarness's fallback + # when getInputPortDependencyPairs is empty. + if not port_order: + port_order = sorted(inputs_by_port.keys()) + + namespace = _exec_user_code(operator_code) + op_class = _discover_operator_class(namespace) + op_instance = op_class() + + emitted = _run_operator(op_instance, is_source, port_order, inputs_by_port) + + outputs = config.get("outputs", []) + if len(outputs) > 1: + raise NotImplementedError( + "py_op_driver: multi-output Python operators are not supported " + "yet (no UDF base class exposes per-port emission)" + ) + if outputs: + out_entry = outputs[0] + out_path = Path(out_entry["dataPath"]) + out_schema = _schema_from_dict(out_entry["schema"]) + rows = list(_emit_as_dicts(emitted, out_schema)) + _write_tuples(out_path, rows, out_schema) + + +def main(argv: Sequence[str]) -> int: + if len(argv) != 2: + sys.stderr.write(f"usage: {argv[0]} \n") + return 2 + config_path = Path(argv[1]) + with config_path.open("r", encoding="utf-8") as fh: + config = json.load(fh) + run_config(config) + return 0 + + +def serve() -> int: + """Persistent driver: import pyamber once, then run many operators. + + pytexera/pyamber import at module load (~300 ms) is the dominant per-call + cost; paying it once here instead of per operator is the whole point. Reads + one JSON job per line on stdin, writes one JSON result per line on stdout: + + request {"configPath": ""}\n + response {"exit": 0|1, "stdout": "...", "stderr": "..."}\n + + exit=1 with the traceback on stderr mirrors a nonzero CLI exit, so the + Scala side's PyOpDriverException path is unchanged. An operator error never + kills the server; only closing stdin (EOF) ends it. The executed script's + stdout/stderr are captured so they can't corrupt the protocol channel. + """ + import io + from contextlib import redirect_stderr, redirect_stdout + + sys.stdout.write(json.dumps({"ready": True}) + "\n") + sys.stdout.flush() + + for line in sys.stdin: + line = line.strip() + if not line: + continue + out_buf, err_buf = io.StringIO(), io.StringIO() + try: + job = json.loads(line) + with Path(job["configPath"]).open("r", encoding="utf-8") as fh: + config = json.load(fh) + with redirect_stdout(out_buf), redirect_stderr(err_buf): + run_config(config) + resp = {"exit": 0, "stdout": out_buf.getvalue(), "stderr": err_buf.getvalue()} + except BaseException: # noqa: BLE001 — a bad job must not kill the server + resp = { + "exit": 1, + "stdout": out_buf.getvalue(), + "stderr": err_buf.getvalue() + traceback.format_exc(), + } + sys.stdout.write(json.dumps(resp) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "--serve": + sys.exit(serve()) + else: + sys.exit(main(sys.argv)) diff --git a/workflow-compiling-service/src/test/resources/python/standalone_worker.py b/workflow-compiling-service/src/test/resources/python/standalone_worker.py new file mode 100644 index 00000000000..946fec23206 --- /dev/null +++ b/workflow-compiling-service/src/test/resources/python/standalone_worker.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Persistent worker for the Path B (standalone) verify path. + +Motivation: forking a fresh interpreter per operator pays the pandas/plotly +import cost (~260-310 ms) on every spawn, while the operator's actual compute +on the tiny canonical fixtures is ~4 ms. Imports dominate ~96% of the per-spawn +cost. This worker imports those heavy libraries ONCE at startup, then executes +many operators' generated scripts over its lifetime — so the import cost is +paid once, not once per operator. + +It is a drop-in replacement for `python `: it runs the exact same +rendered script `StandaloneRunner` already produces (imports + prologue + body ++ epilogue). The script's own top-of-file `import pandas` becomes a ~0 ms +`sys.modules` cache hit. + +Protocol (line-delimited JSON, both directions): + + startup worker -> parent: {"ready": true} + request parent -> worker: {"scriptPath": "", "workDir": ""}\n + response worker -> parent: {"exit": 0, "stdout": "...", "stderr": "..."}\n + +`exit` is 0 on success or 1 if the script raised; on 1, `stderr` carries the +traceback — mirroring a nonzero subprocess exit so the Scala side's +StandaloneExecutionException path is unchanged. The worker keeps running after +a script error (only a hard interpreter crash ends it); parent closes stdin +(EOF) to shut it down. + +Isolation trade-off (accepted, per design discussion): all jobs share one +interpreter, so module-level state (e.g. pandas display options) can leak +between operators. Each job is exec'd in a FRESH namespace and chdir'd to its +own workDir to contain the common cases; this is weaker than the old +process-per-operator isolation. +""" +from __future__ import annotations + +import io +import json +import os +import sys +import traceback +from contextlib import redirect_stderr, redirect_stdout + +# --- Pay the heavy import cost ONCE, here, at startup. ---------------------- +# These mirror the imports StandaloneRunner injects at the top of every +# rendered script. Pre-importing them populates sys.modules, so each executed +# script's own `import pandas as pd` / `import plotly...` is a cache hit. +# numpy is intentionally NOT imported (see StandaloneRunner.renderScript: the +# production translator only provides pandas + plotly, so an operator needing +# numpy must import it itself — we must not mask that). +import pandas as pd # noqa: F401 +import plotly.express as px # noqa: F401 +import plotly.graph_objects as go # noqa: F401 +import plotly.io # noqa: F401 + + +def _run_one(script_path: str, work_dir: str) -> "dict[str, object]": + """Execute one rendered standalone script and capture its output. + + Runs in a fresh namespace with cwd = work_dir (generated code may use + relative paths, e.g. CSVScan's `pd.read_csv("sample.csv")`; absolute paths + written by the prologue/epilogue are unaffected). The script's stdout / + stderr are redirected into buffers so they never corrupt the protocol + channel on real stdout. + """ + out_buf, err_buf = io.StringIO(), io.StringIO() + # __name__ = "__main__" so scripts with a `if __name__ == "__main__"` guard + # still run their body (the translator does not emit one, but it is free + # insurance and matches `python script.py` semantics). + namespace = {"__name__": "__main__", "__file__": script_path} + try: + with open(script_path, "r", encoding="utf-8") as f: + source = f.read() + os.chdir(work_dir) + code = compile(source, script_path, "exec") + with redirect_stdout(out_buf), redirect_stderr(err_buf): + exec(code, namespace) # noqa: S102 (running generated verify code by design) + return {"exit": 0, "stdout": out_buf.getvalue(), "stderr": err_buf.getvalue()} + except BaseException: # noqa: BLE001 — a script error must NOT kill the worker + # Match a nonzero subprocess exit: traceback goes to stderr, exit = 1. + err = err_buf.getvalue() + traceback.format_exc() + return {"exit": 1, "stdout": out_buf.getvalue(), "stderr": err} + + +def main() -> None: + # Signal readiness only after the heavy imports above have completed, so the + # parent can warm a pool and attribute startup cost deterministically. + sys.stdout.write(json.dumps({"ready": True}) + "\n") + sys.stdout.flush() + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + result = _run_one(req["scriptPath"], req["workDir"]) + except Exception: # malformed request — report, keep serving + result = {"exit": 1, "stdout": "", "stderr": traceback.format_exc()} + sys.stdout.write(json.dumps(result) + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala new file mode 100644 index 00000000000..dbeb4829463 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.distinct.DistinctOpDesc +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} + +/** The two ways of running one operator, and the file format they meet in. + * + * `Distinct` is the operator under test throughout, because what is being + * tested is the harness rather than the operator: it takes one input, needs no + * configuration, and its answer is short enough to state in full. + */ +class HarnessSpec extends AnyFlatSpec with Matchers { + + private val schema = new Schema( + new Attribute("id", AttributeType.INTEGER), + new Attribute("name", AttributeType.STRING) + ) + + private def tuple(id: Int, name: String): Tuple = { + val b = Tuple.builder(schema) + b.add(schema.getAttribute("id"), Int.box(id)) + b.add(schema.getAttribute("name"), name) + b.build() + } + + /** Four rows, the last a repeat of the second. */ + private val rows = Seq(tuple(1, "a"), tuple(2, "b"), tuple(3, "c"), tuple(2, "b")) + + private def withInput(test: (Path, Path) => Unit): Unit = { + val dir = Files.createTempDirectory("harness-spec-") + val input = dir.resolve("input_port_0.jsonl") + TupleIO.writeTuples(input, rows.iterator, schema) + test(dir, input) + } + + "TupleIO" should "read back the rows and the schema it wrote" in { + withInput { (_, input) => + // The schema travels in a sidecar rather than in the JSONL, which carries + // values alone and so cannot say a column is INTEGER rather than a number. + TupleIO.readSchemaSidecar(input) shouldBe schema + val read = TupleIO.readTuples(input, schema).toSeq + read should have length 4 + read.map(_.getField[Integer]("id").intValue) shouldBe Seq(1, 2, 3, 2) + } + } + + "OpExecHarness" should "run an operator and write one file per output port" in { + withInput { (dir, input) => + val out = dir.resolve("actual") + val result = + OpExecHarness.execute(new DistinctOpDesc, Map(PortIdentity(0) -> input), out) + + result.outputs should have size 1 + val produced = result.outputs(PortIdentity(0)) + Files.exists(produced) shouldBe true + + val written = TupleIO.readTuples(produced, result.outputSchemas(PortIdentity(0))).toSeq + written.map(_.getField[Integer]("id").intValue) shouldBe Seq(1, 2, 3) + } + } + + "StandaloneRunner" should "run the generated script and reach the same answer" in { + withInput { (dir, input) => + val work = dir.resolve("standalone") + Files.createDirectories(work) + val result = StandaloneRunner.run( + opDesc = new DistinctOpDesc, + inputs = Map(1 -> input), + outputPortCount = 1, + workDir = work + ) + + // The script is kept where it ran, so a failing operator can be opened as + // generated rather than described second-hand. + Files.exists(work.resolve("script.py")) shouldBe true + + val produced = result.outputs(1) + val lines = Files.readAllLines(produced) + lines should have size 3 + lines.get(0) should include("\"id\":1") + lines.get(2) should include("\"id\":3") + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala new file mode 100644 index 00000000000..2bb04515646 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.node.ObjectNode +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.executor.{ExecFactory, OpExecWithClassName, OperatorExecutor} +import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple, TupleLike} +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} +import org.apache.texera.amber.core.workflow.{PhysicalOp, PhysicalPlan, PortIdentity} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import java.nio.file.{Files, Path} +import java.sql.Timestamp +import java.util.Base64 +import scala.collection.mutable +import scala.jdk.CollectionConverters._ + +/** + * Generic harness that drives an OpDesc's OpExec(s) directly, bypassing the + * Pekko/actor runtime. + * + * Works uniformly for: single-OpExec ops (filter, sort, projection), multi- + * OpExec ops (hash join build+probe), multi-output ops (split), and source + * ops (empty input map). All wiring info — number of OpExecs, internal links, + * input-port dependency order — is derived from `opDesc.getPhysicalPlan(...)`, + * so adding a new operator requires no harness changes. + * + * I/O is JSON Lines with sidecar schemas. Each input/output `*.jsonl` file + * has a companion `*.jsonl.schema.json` describing its [[Schema]]. + * + * Limitations (intentional for MVP): + * - Only `OpExecWithClassName` is supported; Python UDFs (`OpExecWithCode`) + * are out of scope because driving them needs a real Python worker. + * - Single worker only (idx=0, workerCount=1). Multi-worker partitioning + * would require coordinating partitioners across executors. + * - JSONL types: STRING / INTEGER / LONG / DOUBLE / BOOLEAN / BINARY / + * TIMESTAMP (the latter two via explicit base64 / JDBC-string codecs). + */ +object OpExecHarness extends LazyLogging { + + // Test-only workflow / execution IDs. The values don't matter — the harness + // never persists state under them — but the PhysicalOp factory needs *some* + // IDs to embed in PhysicalOpIdentity. + private val TestWorkflowId = WorkflowIdentity(0L) + private val TestExecutionId = ExecutionIdentity(0L) + + /** + * @param outputs external output port → JSONL file path + * @param outputSchemas same keys as outputs, gives each port's [[Schema]] + */ + final case class Result( + outputs: Map[PortIdentity, Path], + outputSchemas: Map[PortIdentity, Schema] + ) + + /** + * Run `opDesc` against the given inputs and write the outputs to `outputDir`. + * + * @param inputs map keyed by the *external* input port identifier (the one + * the user-visible LogicalOp exposes). Each path points to a + * `.jsonl` file with a sibling `.schema.json`. + * @param outputDir destination directory; created if missing. Output files + * are named `output_port_.jsonl` per external output. + */ + def execute( + opDesc: LogicalOp, + inputs: Map[PortIdentity, Path], + outputDir: Path + ): Result = { + Files.createDirectories(outputDir) + + // 1. Compile OpDesc → PhysicalPlan. For most ops this is a single-PhysicalOp + // plan; HashJoin and other multi-stage ops return multiple PhysicalOps + // plus internal PhysicalLinks (e.g. build.out → probe.in0). + val plan = opDesc.getPhysicalPlan(TestWorkflowId, TestExecutionId) + + // 2. Identify external input ports. A PhysicalOp port is "external" iff no + // PhysicalLink in this plan terminates at it. The user's `inputs` map + // must cover exactly these (matched by PortIdentity). + val externalInputs: Set[(PhysicalOpIdentity, PortIdentity)] = + plan.operators.flatMap { phOp => + phOp.inputPorts.keys.collect { + case portId if !plan.links.exists(l => l.toOpId == phOp.id && l.toPortId == portId) => + (phOp.id, portId) + } + } + validateInputCoverage(externalInputs, inputs.keySet) + + // 3. Load each input file as (Schema, Iterator[Tuple]). We read schemas + // eagerly but tuples lazily (saves memory on large fixtures). + val inputSchemas: Map[PortIdentity, Schema] = + inputs.map { case (portId, path) => portId -> TupleIO.readSchemaSidecar(path) } + val inputTuples: Map[PortIdentity, () => Iterator[Tuple]] = + inputs.map { + case (portId, path) => + val schema = inputSchemas(portId) + portId -> (() => TupleIO.readTuples(path, schema)) + } + + // 4. Propagate schemas. We CAN'T use `plan.propagateSchema(inputSchemas)` + // directly because it indexes by PortIdentity globally — for HashJoin + // probe's internal port 0 would collide with build's external port 0. + // Instead, set schemas only on the external (phOpId, portId) pairs and + // let `addLink` propagate the internal links' schemas naturally. + val planWithSchemas = + propagateExternalSchemas(plan, externalInputs, inputSchemas) + + // 5. Identify external output ports symmetrically: no outgoing PhysicalLink. + val externalOutputs: Set[(PhysicalOpIdentity, PortIdentity)] = + planWithSchemas.operators.flatMap { phOp => + phOp.outputPorts.keys.collect { + case portId + if !planWithSchemas.links + .exists(l => l.fromOpId == phOp.id && l.fromPortId == portId) => + (phOp.id, portId) + } + } + + // 6. Instantiate OpExec per PhysicalOp. We only support OpExecWithClassName; + // fail loudly otherwise so test authors know to mock Python UDFs out. + val opExecs: Map[PhysicalOpIdentity, OperatorExecutor] = + planWithSchemas.operators.map { phOp => + phOp.opExecInitInfo match { + case OpExecWithClassName(className, descString) => + phOp.id -> ExecFactory.newExecFromJavaClassName( + className, + descString, + idx = 0, + workerCount = 1 + ) + case other => + throw new UnsupportedOperationException( + s"OpExecHarness only supports OpExecWithClassName, got: $other" + ) + } + }.toMap + + // 7. Drive each PhysicalOp in topological order. Buffer outputs in memory + // keyed by (producer phOpId, output port). Downstream PhysicalOps then + // consume from these buffers via the plan's internal links. + val producedBuffer = + mutable.Map.empty[(PhysicalOpIdentity, PortIdentity), mutable.ArrayBuffer[Tuple]] + + planWithSchemas.topologicalIterator().foreach { phOpId => + val phOp = planWithSchemas.getOperator(phOpId) + val opExec = opExecs(phOpId) + runOneOp( + phOp, + opExec, + externalInputProvider = portId => inputTuples.get(portId).map(_.apply()), + upstreamBuffer = producedBuffer, + plan = planWithSchemas, + produced = producedBuffer + ) + } + + // 8. Materialize external outputs to JSONL with their propagated schemas. + val outputPaths = mutable.Map.empty[PortIdentity, Path] + val outputSchemas = mutable.Map.empty[PortIdentity, Schema] + externalOutputs.foreach { + case (phOpId, portId) => + val schema = planWithSchemas + .getOperator(phOpId) + .outputPorts(portId) + ._3 + .toOption + .getOrElse( + throw new IllegalStateException( + s"Output schema for ($phOpId, $portId) was not propagated" + ) + ) + val tuples = + producedBuffer.getOrElse((phOpId, portId), mutable.ArrayBuffer.empty[Tuple]) + val file = outputDir.resolve(s"output_port_${portId.id}.jsonl") + TupleIO.writeTuples(file, tuples.iterator, schema) + outputPaths(portId) = file + outputSchemas(portId) = schema + } + + Result(outputPaths.toMap, outputSchemas.toMap) + } + + /** + * Drives one PhysicalOp's lifecycle: open → input ports in dependency order + * (processTupleMultiPort + onFinishMultiPort per port) → close. Source ops + * (no input ports) get a single onFinishMultiPort(0) call which gives their + * `produceTuple()`-backed implementation a chance to emit. + * + * Outputs are bucketed by output PortIdentity. `processTupleMultiPort`'s + * `Option[PortIdentity]` return: None means port 0 (the default single- + * output convention used by the trait's fallback). Multi-output ops like + * Split set it explicitly. + */ + private def runOneOp( + phOp: PhysicalOp, + opExec: OperatorExecutor, + externalInputProvider: PortIdentity => Option[Iterator[Tuple]], + upstreamBuffer: mutable.Map[ + (PhysicalOpIdentity, PortIdentity), + mutable.ArrayBuffer[Tuple] + ], + plan: PhysicalPlan, + produced: mutable.Map[ + (PhysicalOpIdentity, PortIdentity), + mutable.ArrayBuffer[Tuple] + ] + ): Unit = { + opExec.open() + try { + def bucket(emitted: Iterator[(TupleLike, Option[PortIdentity])]): Unit = { + emitted.foreach { + case (tupleLike, portOpt) => + // Default: the op's single output port. Most operators have one + // output and use the trait's default port-0 wrapping, but + // multi-stage plans (e.g. HashJoin build) put their internal + // output on PortIdentity(0, internal = true) — a hardcoded + // PortIdentity(0, false) would NoSuchElementException here. + val outPortId = portOpt.getOrElse { + if (phOp.outputPorts.size == 1) phOp.outputPorts.keys.head + else PortIdentity(0) + } + val outSchema = phOp + .outputPorts(outPortId) + ._3 + .toOption + .getOrElse( + throw new IllegalStateException( + s"Op ${phOp.id} emitted to port $outPortId before its output schema was propagated" + ) + ) + val tuple = tupleLike + .asInstanceOf[org.apache.texera.amber.core.tuple.SeqTupleLike] + .enforceSchema(outSchema) + produced + .getOrElseUpdate((phOp.id, outPortId), mutable.ArrayBuffer.empty[Tuple]) += tuple + } + } + + // Process each input port in declared dependency order (e.g. HashJoin + // probe's build-side port must finish before the data-side port starts). + val portOrder = + if (phOp.getInputPortDependencyPairs.nonEmpty) + phOp.getInputPortDependencyPairs + else phOp.inputPorts.keys.toList.sortBy(_.id) + + portOrder.foreach { portId => + val tuples: Iterator[Tuple] = + if (externalInputProvider(portId).isDefined) { + externalInputProvider(portId).get + } else { + // Internal port: stitch upstream PhysicalLinks' buffers together + val upstream = plan.links + .filter(l => l.toOpId == phOp.id && l.toPortId == portId) + .toList + .sortBy(l => (l.fromOpId.toString, l.fromPortId.id)) + upstream.iterator + .flatMap(l => + upstreamBuffer + .getOrElse( + (l.fromOpId, l.fromPortId), + mutable.ArrayBuffer.empty[Tuple] + ) + .iterator + ) + } + + tuples.foreach { t => + bucket(opExec.processTupleMultiPort(t, portId.id)) + } + bucket(opExec.onFinishMultiPort(portId.id)) + } + + // Source operator: no input ports. Trigger production via onFinishMultiPort + // on a synthetic port 0 — SourceOperatorExecutor.onFinish ignores the port + // and emits everything from produceTuple(). + if (phOp.inputPorts.isEmpty) { + bucket(opExec.onFinishMultiPort(0)) + } + } finally { + opExec.close() + } + } + + // Walks the plan in topo order, propagating schemas only at the truly external + // input ports. Internal ports get their schema via `addLink` (PhysicalPlan + // re-applies the source's output schema to the destination port). This avoids + // a collision when multiple PhysicalOps share a PortIdentity (e.g. HashJoin + // probe.in0 internal vs build.in0 external both have PortIdentity(0)). + private def propagateExternalSchemas( + plan: PhysicalPlan, + externalPorts: Set[(PhysicalOpIdentity, PortIdentity)], + schemas: Map[PortIdentity, Schema] + ): PhysicalPlan = { + var acc = PhysicalPlan(operators = Set.empty, links = Set.empty) + plan.topologicalIterator().map(plan.getOperator).foreach { phOp => + val updated = phOp.inputPorts.keys.foldLeft(phOp) { (op, portId) => + if (externalPorts.contains((phOp.id, portId)) && schemas.contains(portId)) { + op.propagateSchema(Some((portId, schemas(portId)))) + } else op + } + // .propagateSchema() with no arg re-fires output derivation if all inputs + // are now resolved (source ops trigger immediately since inputPorts empty). + acc = acc.addOperator(updated.propagateSchema()) + plan.getUpstreamPhysicalLinks(phOp.id).foreach { link => + acc = acc.addLink(link) + } + } + acc + } + + private def validateInputCoverage( + external: Set[(PhysicalOpIdentity, PortIdentity)], + provided: Set[PortIdentity] + ): Unit = { + val expected = external.map(_._2) + val missing = expected -- provided + val extra = provided -- expected + require( + missing.isEmpty, + s"Missing input fixtures for external ports: $missing (expected $expected)" + ) + if (extra.nonEmpty) { + logger.warn(s"Input fixtures provided for non-external ports (ignored): $extra") + } + } +} + +/** + * JSON Lines I/O for Tuples. Each `.jsonl` file is paired with a + * `.jsonl.schema.json` sidecar listing [[Attribute]]s in column order. + * + * Format example: + * + * records.jsonl: + * {"id":1,"name":"alice"} + * {"id":2,"name":"bob"} + * + * records.jsonl.schema.json: + * {"attributes":[{"attributeName":"id","attributeType":"integer"}, + * {"attributeName":"name","attributeType":"string"}]} + * + * pandas symmetry: `pd.read_json(path, lines=True)` and + * `df.to_json(path, orient='records', lines=True)` round-trip cleanly for the + * supported types (STRING / INTEGER / LONG / DOUBLE / BOOLEAN). + */ +object TupleIO { + + private def sidecar(path: Path): Path = + path.resolveSibling(path.getFileName.toString + ".schema.json") + + def readSchemaSidecar(path: Path): Schema = { + val text = new String(Files.readAllBytes(sidecar(path))) + objectMapper.readValue(text, classOf[Schema]) + } + + def readTuples(path: Path, schema: Schema): Iterator[Tuple] = { + // readAllLines closes the underlying handle; safer than Files.lines for + // test-scale fixtures where memory cost is negligible. + val lines = Files.readAllLines(path).asScala + lines.iterator.filter(_.trim.nonEmpty).map { line => + val node = objectMapper.readTree(line) + val builder = Tuple.builder(schema) + schema.getAttributes.foreach { attr => + val fieldNode = node.get(attr.getName) + val v: Any = + if (fieldNode == null || fieldNode.isNull) null + else + attr.getType match { + case AttributeType.STRING => fieldNode.asText() + case AttributeType.INTEGER => Int.box(fieldNode.asInt()) + case AttributeType.LONG => Long.box(fieldNode.asLong()) + case AttributeType.DOUBLE => Double.box(fieldNode.asDouble()) + case AttributeType.BOOLEAN => Boolean.box(fieldNode.asBoolean()) + case AttributeType.BINARY => + Base64.getDecoder.decode(fieldNode.asText()) + // Timestamps round-trip through the JDBC string form + // ("yyyy-mm-dd hh:mm:ss[.f]"), the exact inverse of Timestamp.toString + // below — timezone-free, so no shift across write/read. The Python + // side reads this column with convert_dates=False (see + // StandaloneRunner) and treats it as an opaque string, so both paths + // agree on pass-through. + case AttributeType.TIMESTAMP => + Timestamp.valueOf(fieldNode.asText()) + case other => + throw new UnsupportedOperationException( + s"TupleIO MVP doesn't support $other yet" + ) + } + builder.add(attr, v) + } + builder.build() + } + } + + def writeTuples(path: Path, tuples: Iterator[Tuple], schema: Schema): Unit = { + // Sidecar first so a partial main-file write still has a recoverable schema. + Files.write(sidecar(path), objectMapper.writeValueAsBytes(schema)) + val writer = Files.newBufferedWriter(path) + try { + tuples.foreach { t => + val node: ObjectNode = objectMapper.createObjectNode() + schema.getAttributes.zipWithIndex.foreach { + case (attr, idx) => + val v = t.getField[Any](idx) + if (v == null) node.putNull(attr.getName) + else + attr.getType match { + case AttributeType.STRING => node.put(attr.getName, v.toString) + case AttributeType.INTEGER => node.put(attr.getName, v.asInstanceOf[Int]) + case AttributeType.LONG => node.put(attr.getName, v.asInstanceOf[Long]) + case AttributeType.DOUBLE => node.put(attr.getName, v.asInstanceOf[Double]) + case AttributeType.BOOLEAN => node.put(attr.getName, v.asInstanceOf[Boolean]) + case AttributeType.BINARY => + node.put( + attr.getName, + Base64.getEncoder.encodeToString(v.asInstanceOf[Array[Byte]]) + ) + case AttributeType.TIMESTAMP => + node.put(attr.getName, v.asInstanceOf[Timestamp].toString) + case other => + throw new UnsupportedOperationException( + s"TupleIO MVP doesn't support $other yet" + ) + } + } + writer.write(objectMapper.writeValueAsString(node)) + writer.newLine() + } + } finally writer.close() + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala new file mode 100644 index 00000000000..a6ae7b3b373 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.executor.OpExecWithCode +import org.apache.texera.amber.core.tuple.Schema +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + PhysicalOpIdentity, + WorkflowIdentity +} +import org.apache.texera.amber.core.workflow.{PhysicalPlan, PortIdentity} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.util.python.PythonWorkerPool + +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths, StandardCopyOption} +import scala.collection.mutable.ArrayBuffer +import scala.sys.process._ + +/** + * Counterpart to [[OpExecHarness]] for Python-native operators + * ([[OpExecWithCode]] with language="python"). Drives the operator's + * generatePythonCode() output through a thin subprocess driver rather than + * spinning up the Pekko/Arrow worker stack. + * + * Same Result(outputs, outputSchemas) shape as OpExecHarness so the rest of + * the verify pipeline (Comparator, category runners) is harness-agnostic. + * + * Scope (MVP, mirrors OpExecHarness's MVP): + * - Single-PhysicalOp plans only. PythonOperatorDescriptor only emits + * either a `sourcePhysicalOp` or a `oneToOnePhysicalOp`, so multi-op + * plans don't exist for Python-native ops today. If that changes, add + * topo-order driving here the way OpExecHarness does. + * - Single output port. UDFOperatorV2 / UDFTableOperator / UDFBatchOperator + * / UDFSourceOperator all yield TupleLike without specifying a port — + * same convention OpExecHarness uses when port is unset. + * - JSONL types: STRING / INTEGER / LONG / DOUBLE / BOOLEAN. TIMESTAMP / + * BINARY / LARGE_BINARY require explicit codecs in both [[TupleIO]] and + * the driver — add when the first operator needs them. + */ +object PyOpExecHarness extends LazyLogging { + + private val TestWorkflowId = WorkflowIdentity(0L) + private val TestExecutionId = ExecutionIdentity(0L) + + // Same Result shape as OpExecHarness so callers can swap harnesses + // transparently. + final case class Result( + outputs: Map[PortIdentity, Path], + outputSchemas: Map[PortIdentity, Schema] + ) + + // Driver script lives on the test classpath at /python/py_op_driver.py + // (sibling to compare.py). Extracted to a temp file at runtime so it works + // whether the test resources are loose files or sealed in a jar. + private val DriverResourcePath = "/python/py_op_driver.py" + + def execute( + opDesc: LogicalOp, + inputs: Map[PortIdentity, Path], + outputDir: Path, + pythonExe: String = resolvePython(), + amberPythonHome: Path = resolveAmberPythonHome() + ): Result = { + Files.createDirectories(outputDir) + + val plan = opDesc.getPhysicalPlan(TestWorkflowId, TestExecutionId) + + // PythonOperatorDescriptor builds single-op plans; bail loudly if some + // future Python op produces a multi-stage plan (need to extend the driver + // and the per-PhysicalOp config the way OpExecHarness does). + require( + plan.operators.size == 1, + s"PyOpExecHarness only supports single-PhysicalOp plans for now, got " + + s"${plan.operators.size} PhysicalOps" + ) + val phOp = plan.operators.head + + val (pythonCode, language) = phOp.opExecInitInfo match { + case OpExecWithCode(code, lang) => (code, lang) + case other => + throw new UnsupportedOperationException( + s"PyOpExecHarness only supports OpExecWithCode; got ${other.getClass.getSimpleName}. " + + "For OpExecWithClassName, use OpExecHarness." + ) + } + require( + language == "python", + s"""PyOpExecHarness only supports language="python", got "$language".""" + ) + + // External input ports = same definition as OpExecHarness. For a + // single-op plan that's just every input port the op declares. + val externalInputs: Set[(PhysicalOpIdentity, PortIdentity)] = + phOp.inputPorts.keys.map(portId => (phOp.id, portId)).toSet + validateInputCoverage(externalInputs, inputs.keySet) + + val inputSchemas: Map[PortIdentity, Schema] = + inputs.map { case (portId, path) => portId -> TupleIO.readSchemaSidecar(path) } + + val planWithSchemas = propagateExternalSchemas(plan, externalInputs, inputSchemas) + val phOpWithSchemas = planWithSchemas.operators.head + + // Output port schemas come from PhysicalPlan.propagateSchema — same + // ground truth OpExecHarness writes to its own outputs. + val outputPortSchemas: Map[PortIdentity, Schema] = + phOpWithSchemas.outputPorts.map { + case (portId, (_, _, schemaOrErr)) => + portId -> schemaOrErr.toOption.getOrElse( + throw new IllegalStateException( + s"Output schema for ($portId) was not propagated" + ) + ) + } + + require( + outputPortSchemas.size == 1, + s"PyOpExecHarness only supports single-output-port operators, got " + + s"${outputPortSchemas.size} output ports" + ) + val (outputPortId, outputSchema) = outputPortSchemas.head + val outputPath = outputDir.resolve(s"output_port_${outputPortId.id}.jsonl") + + // Port ordering for multi-input ops: respect declared dependencies + // (matches OpExecHarness — e.g. HashJoin probe processes build-side + // first). Default = sorted by port id when no dependencies declared. + val portOrder: Seq[Int] = + if (phOpWithSchemas.getInputPortDependencyPairs.nonEmpty) + phOpWithSchemas.getInputPortDependencyPairs.map(_.id) + else phOpWithSchemas.inputPorts.keys.toList.map(_.id).sorted + + val config = buildConfig( + pythonCode = pythonCode, + isSource = phOpWithSchemas.isSourceOperator, + portOrder = portOrder, + inputs = inputs, + outputPath = outputPath, + outputSchema = outputSchema + ) + + val configPath = outputDir.resolve("py_op_driver_config.json") + Files.write(configPath, config.getBytes(StandardCharsets.UTF_8)) + + val driverPath = extractDriverScript() + runDriver(driverPath, configPath, outputDir, pythonExe, amberPythonHome) + + Result( + outputs = Map(outputPortId -> outputPath), + outputSchemas = Map(outputPortId -> outputSchema) + ) + } + + // -------------------------------------------------------------------------- + // Config serialization. Matches the driver's expected schema (see + // py_op_driver.py's module docstring). + // -------------------------------------------------------------------------- + private def buildConfig( + pythonCode: String, + isSource: Boolean, + portOrder: Seq[Int], + inputs: Map[PortIdentity, Path], + outputPath: Path, + outputSchema: Schema + ): String = { + val root: ObjectNode = objectMapper.createObjectNode() + root.put("operatorCode", pythonCode) + root.put("isSource", isSource) + + val portOrderArr: ArrayNode = root.putArray("portOrder") + portOrder.foreach(portOrderArr.add) + + val inputsArr: ArrayNode = root.putArray("inputs") + inputs.toSeq.sortBy(_._1.id).foreach { + case (portId, dataPath) => + val entry: ObjectNode = inputsArr.addObject() + entry.put("portIndex", portId.id) + entry.put("dataPath", dataPath.toAbsolutePath.toString) + // schemaPath is implicit (data_path + ".schema.json") — the driver + // resolves it the same way TupleIO does. + } + + val outputsArr: ArrayNode = root.putArray("outputs") + val outEntry: ObjectNode = outputsArr.addObject() + outEntry.put("dataPath", outputPath.toAbsolutePath.toString) + // Embed the schema directly. We can't just write the sidecar ahead of + // time and have the driver read it, because writing a sidecar before + // outputs exist would leave a stale sidecar on partial failures. + outEntry.set[ObjectNode]( + "schema", + objectMapper.valueToTree[ObjectNode](outputSchema) + ) + + objectMapper.writeValueAsString(root) + } + + // -------------------------------------------------------------------------- + // Subprocess invocation. + // -------------------------------------------------------------------------- + private def runDriver( + driverPath: Path, + configPath: Path, + cwd: Path, + pythonExe: String, + amberPythonHome: Path + ): Unit = { + // Prepend amber's Python source to PYTHONPATH so `import pytexera` + // resolves. Existing PYTHONPATH (if any) is preserved as the lower- + // priority suffix. + val existing = sys.env.getOrElse("PYTHONPATH", "") + val newPyPath = + if (existing.isEmpty) amberPythonHome.toAbsolutePath.toString + else s"${amberPythonHome.toAbsolutePath}${File.pathSeparator}$existing" + + val (exit, stdout, stderr) = execDriver(driverPath, configPath, cwd, pythonExe, newPyPath) + if (exit != 0) { + throw new PyOpDriverException( + exitCode = exit, + driverPath = driverPath, + configPath = configPath, + stdout = stdout, + stderr = stderr + ) + } + } + + // Prefer a pooled persistent worker (imports pytexera/pyamber once via + // `py_op_driver.py --serve`, the ~300 ms cost that dominates a per-Python-op + // run — see PythonWorkerPool). A rare hard worker crash falls back to a + // one-shot subprocess so behavior is never worse than the original path. Both + // paths use absolute config paths, so cwd only matters to the subprocess + // form; the worker constructs a fresh operator per job for isolation. + private def execDriver( + driverPath: Path, + configPath: Path, + cwd: Path, + pythonExe: String, + pythonPath: String + ): (Int, String, String) = { + if (PythonWorkerPool.enabled) { + try { + val req = objectMapper.createObjectNode() + req.put("configPath", configPath.toAbsolutePath.toString) + val o = PythonWorkerPool.run( + DriverResourcePath, + Seq("--serve"), + pythonExe, + req, + env = Map("PYTHONPATH" -> pythonPath) + ) + return (o.exit, o.stdout, o.stderr) + } catch { + case e: PythonWorkerPool.WorkerDiedException => + logger.warn( + s"py_op_driver worker unavailable; falling back to one-shot subprocess " + + s"for $configPath: ${e.getMessage}" + ) + } + } + runDriverSubprocess(driverPath, configPath, cwd, pythonExe, pythonPath) + } + + // Original one-process-per-operator path. Retained as the fallback and as the + // behavior selected by TEXERA_TEST_PYTHON_WORKER=0. + private def runDriverSubprocess( + driverPath: Path, + configPath: Path, + cwd: Path, + pythonExe: String, + pythonPath: String + ): (Int, String, String) = { + val outBuf = ArrayBuffer.empty[String] + val errBuf = ArrayBuffer.empty[String] + val procLogger = ProcessLogger(line => outBuf += line, line => errBuf += line) + val exit = Process( + Seq(pythonExe, driverPath.toString, configPath.toString), + Some(cwd.toFile), + "PYTHONPATH" -> pythonPath + ).!(procLogger) + (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) + } + + // -------------------------------------------------------------------------- + // Resolution helpers. + // -------------------------------------------------------------------------- + private def resolvePython(): String = + sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") + + /** + * Locate `amber/src/main/python`. Resolution chain: + * 1. Env var TEXERA_AMBER_PYTHON_HOME (set by CI / dev shell). + * 2. Walk up from cwd looking for `amber/src/main/python`. + * sbt runs tests with cwd = the subproject dir (`workflow-compiling-service/`), + * so the walk-up is two levels at most for the normal layout. + */ + private def resolveAmberPythonHome(): Path = { + sys.env.get("TEXERA_AMBER_PYTHON_HOME").filter(_.nonEmpty).map(Paths.get(_)).getOrElse { + val cwd = Paths.get(".").toAbsolutePath.normalize() + val maxDepth = 5 + var current: Path = cwd + var depth = 0 + while (current != null && depth <= maxDepth) { + val candidate = current.resolve("amber/src/main/python") + if (Files.isDirectory(candidate)) return candidate.toAbsolutePath + current = current.getParent + depth += 1 + } + throw new RuntimeException( + s"PyOpExecHarness: could not locate amber/src/main/python from cwd $cwd. " + + "Set TEXERA_AMBER_PYTHON_HOME to the absolute path." + ) + } + } + + private def extractDriverScript(): Path = { + val stream = getClass.getResourceAsStream(DriverResourcePath) + require( + stream != null, + s"py_op_driver.py not found on classpath at $DriverResourcePath" + ) + try { + val tmp = Files.createTempFile("py_op_driver-", ".py") + Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING) + tmp.toFile.deleteOnExit() + tmp + } finally stream.close() + } + + // -------------------------------------------------------------------------- + // Schema propagation — mirrors OpExecHarness.propagateExternalSchemas. + // Kept inline (rather than shared) so the two harnesses stay independently + // readable; consolidate if a third harness shows up. + // -------------------------------------------------------------------------- + private def propagateExternalSchemas( + plan: PhysicalPlan, + externalPorts: Set[(PhysicalOpIdentity, PortIdentity)], + schemas: Map[PortIdentity, Schema] + ): PhysicalPlan = { + var acc = PhysicalPlan(operators = Set.empty, links = Set.empty) + plan.topologicalIterator().map(plan.getOperator).foreach { phOp => + val updated = phOp.inputPorts.keys.foldLeft(phOp) { (op, portId) => + if (externalPorts.contains((phOp.id, portId)) && schemas.contains(portId)) { + op.propagateSchema(Some((portId, schemas(portId)))) + } else op + } + acc = acc.addOperator(updated.propagateSchema()) + plan.getUpstreamPhysicalLinks(phOp.id).foreach { link => + acc = acc.addLink(link) + } + } + acc + } + + private def validateInputCoverage( + external: Set[(PhysicalOpIdentity, PortIdentity)], + provided: Set[PortIdentity] + ): Unit = { + val expected = external.map(_._2) + val missing = expected -- provided + val extra = provided -- expected + require( + missing.isEmpty, + s"Missing input fixtures for external ports: $missing (expected $expected)" + ) + if (extra.nonEmpty) { + logger.warn(s"Input fixtures provided for non-external ports (ignored): $extra") + } + } +} + +final class PyOpDriverException( + val exitCode: Int, + val driverPath: Path, + val configPath: Path, + val stdout: String, + val stderr: String +) extends RuntimeException( + s"""py_op_driver.py exited with code $exitCode. + |Driver: $driverPath + |Config: $configPath + |--- stdout --- + |$stdout + |--- stderr --- + |$stderr""".stripMargin + ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala new file mode 100644 index 00000000000..359bad51e5a --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.tuple.AttributeType +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.util.python.PythonWorkerPool + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import scala.collection.mutable.ArrayBuffer +import scala.sys.process._ + +/** + * Executes the Python code an OpDesc's [[StandaloneCodeGenerator]] emits and + * captures its DataFrame outputs as JSONL files (compatible with + * [[TupleIO]]'s sidecar-schema format on the comparison side). + * + * Wraps the operator's raw generated code with: + * + * ── prologue ────────────────────────────────────────────── + * in1df = pd.read_json("input_port_0.jsonl", lines=True) + * in2df = pd.read_json("input_port_1.jsonl", lines=True) + * ... + * inAlldf = [in1df, in2df] + * ── operator body (verbatim from generateStandaloneCode) ── + * out1df = in1df[in1df["age"] > 18] + * ── epilogue ───────────────────────────────────────────── + * out1df.to_json("output_port_0.jsonl", orient='records', lines=True) + * ... + * + * Port indexing matches the placeholder convention used by the translator: + * `inNdf`/`outNdf` is 1-based and corresponds to the operator's N-th external + * input/output port in declaration order. The harness key (a 1-based Int) is + * what the placeholder uses; the caller is responsible for ordering inputs + * the same way the operator's `generateStandaloneCode()` expects. + * + * The subprocess inherits the caller's environment so the Python interpreter + * picks up whatever pandas/plotly the test fixture installed. + */ +object StandaloneRunner extends LazyLogging { + + /** The value both paths seed numpy's global RNG with. Any fixed number does; + * what matters is that the two agree, so it is declared once here and + * referenced by name from py_op_driver's comment. + */ + private[verify] val VerifySeed: Int = 20260811 + + /** + * @param outputs paths to the per-port output JSONL files. Empty map iff + * the operator's `producesDataFrame()` returned false + * (visualizations, etc.) — caller handles those separately. + * @param stdout raw subprocess stdout (useful for failure diagnostics) + * @param stderr raw subprocess stderr + */ + final case class Result(outputs: Map[Int, Path], stdout: String, stderr: String) + + /** + * Generate, write, and execute the standalone Python script for `opDesc`. + * + * @param opDesc must mix in [[StandaloneCodeGenerator]]; otherwise we throw + * since there's nothing to test. + * @param inputs map from 1-based port index → JSONL fixture path. The + * script reads each into `inNdf`. + * @param outputPortCount how many `outNdf` variables the operator declares. + * Caller derives this from the OpDesc's output ports. + * @param workDir directory used for the generated `script.py` and output + * JSONL files. Created if missing. + * @param pythonExe path to the Python 3.12 interpreter. Defaults to + * the env var `UDF_PYTHON_PATH`, then `python3.12`, then + * `python3`. The same fallback chain used by the rest of + * the Texera test suite for Python-backed operators. + */ + def run( + opDesc: LogicalOp, + inputs: Map[Int, Path], + outputPortCount: Int, + workDir: Path, + pythonExe: String = resolvePython() + ): Result = { + val gen = opDesc match { + case g: StandaloneCodeGenerator => g + case other => + throw new IllegalArgumentException( + s"OpDesc ${other.getClass.getSimpleName} does not implement " + + s"StandaloneCodeGenerator; nothing to verify" + ) + } + + Files.createDirectories(workDir) + val scriptPath = workDir.resolve("script.py") + val outputPaths: Map[Int, Path] = + if (gen.producesDataFrame()) + (1 to outputPortCount).map(i => i -> workDir.resolve(s"output_port_${i - 1}.jsonl")).toMap + else Map.empty + + val source = + renderScript(gen.generateStandaloneCode(), inputs, outputPaths, gen.standaloneHelpers()) + Files.write(scriptPath, source.getBytes(StandardCharsets.UTF_8)) + + val (exit, stdout, stderr) = execute(scriptPath, workDir, pythonExe) + if (exit != 0) { + throw new StandaloneExecutionException(exit, scriptPath, source, stdout, stderr) + } + Result(outputPaths, stdout, stderr) + } + + private val WorkerResourcePath = "/python/standalone_worker.py" + + // Run the rendered script and return (exitCode, stdout, stderr). Prefers a + // pooled persistent worker (imports pandas/plotly once, ~18x faster per op — + // see PythonWorkerPool); a rare hard worker crash falls back to a one-shot + // subprocess so behavior is never worse than the original path. Both paths + // run with cwd = workDir and read results from files, so they are + // interchangeable — the executed script is byte-identical. + private def execute(scriptPath: Path, workDir: Path, pythonExe: String): (Int, String, String) = { + if (PythonWorkerPool.enabled) { + try { + val req = org.apache.texera.amber.util.JSONUtils.objectMapper.createObjectNode() + req.put("scriptPath", scriptPath.toString) + req.put("workDir", workDir.toString) + val o = PythonWorkerPool.run(WorkerResourcePath, Seq.empty, pythonExe, req) + return (o.exit, o.stdout, o.stderr) + } catch { + case e: PythonWorkerPool.WorkerDiedException => + logger.warn( + s"Standalone worker unavailable; falling back to one-shot subprocess " + + s"for $scriptPath: ${e.getMessage}" + ) + } + } + runSubprocess(scriptPath, workDir, pythonExe) + } + + // Original one-process-per-operator path. Retained as the fallback and as the + // behavior selected by TEXERA_TEST_PYTHON_WORKER=0. + private def runSubprocess( + scriptPath: Path, + workDir: Path, + pythonExe: String + ): (Int, String, String) = { + // Capture stdout/stderr separately. ProcessLogger's append is called from + // the subprocess's I/O thread, so we collect into ArrayBuffer (thread-safe + // append is fine for this serial use) and join at the end. + val outBuf = ArrayBuffer.empty[String] + val errBuf = ArrayBuffer.empty[String] + val logger = ProcessLogger(line => outBuf += line, line => errBuf += line) + // cwd = workDir so generated code using *relative* paths (e.g. CSVScan's + // basename-stripped `pd.read_csv("sample.csv")`) resolves against workDir. + // Absolute paths written by the prologue/epilogue are unaffected. + val exit = Process(Seq(pythonExe, scriptPath.toString), Some(workDir.toFile)).!(logger) + (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) + } + + // Builds the full Python source: imports + prologue + verbatim operator body + // + epilogue. We intentionally do NOT substitute the inNdf/outNdf placeholders + // — the body keeps them so the var-bindings the prologue/epilogue introduce + // (also named inNdf/outNdf) reference the same names. + private def renderScript( + body: String, + inputs: Map[Int, Path], + outputs: Map[Int, Path], + helpers: Seq[String] + ): String = { + val sb = new StringBuilder + + sb.append("# Auto-generated by StandaloneRunner. Do not commit.\n") + sb.append("import json\n") + sb.append("import sys\n") + sb.append("import base64\n") + sb.append("import pickle\n") + // NOTE: numpy is intentionally NOT injected here. The production translator + // (WorkflowToPythonTranslator) only provides pandas + plotly to standalone + // scripts, so any operator whose standalone code needs numpy must import it + // itself. Injecting numpy here would mask that class of bug in verify tests. + sb.append("import pandas as pd\n") + sb.append("import plotly.express as px\n") + sb.append("import plotly.graph_objects as go\n") + sb.append("import plotly.io\n") + // Same seed as py_op_driver's run_config, for the reason given there. Bound + // under a private name and deleted so the note above still holds: a script + // that wants numpy has to import it, and this does not hand it one. + sb.append(s"import numpy as _texera_np; _texera_np.random.seed($VerifySeed); del _texera_np\n") + sb.append("\n") + + // Object columns holding non-primitive values (e.g. a trained sklearn model + // in a BINARY output column) can't go through to_json. Pickle+base64 them so + // the JSONL matches py_op_driver's BINARY write path exactly. Primitives + // (str/int/float/bool/None) pass through unchanged, so ordinary DataFrame + // outputs are unaffected. + sb.append("def _texera_encode_obj_cols(df):\n") + sb.append(" for _c in df.columns:\n") + sb.append(" if df[_c].dtype == object:\n") + sb.append( + " df[_c] = df[_c].map(lambda _v: base64.b64encode(pickle.dumps(_v)).decode('ascii') " + + "if not isinstance(_v, (str, int, float, bool, type(None))) else _v)\n" + ) + sb.append(" return df\n") + sb.append("\n") + + // TIMESTAMP columns are handed to the operator as datetime64 (see the + // prologue below) to match the schema-typed runtime path, but the runtime + // path serializes a TIMESTAMP back out with java.sql.Timestamp.toString — + // "yyyy-mm-dd hh:mm:ss.f", trailing zeros trimmed to at least one digit — + // whereas pandas' to_json would emit epoch millis. Convert datetime columns + // back to that exact form before writing so both paths' JSONL agree. + sb.append("def _texera_ts_str(_v):\n") + sb.append(" if pd.isna(_v):\n") + sb.append(" return None\n") + sb.append(" _s = _v.strftime('%Y-%m-%d %H:%M:%S.%f').rstrip('0')\n") + sb.append(" return _s + '0' if _s.endswith('.') else _s\n") + sb.append("\n") + sb.append("def _texera_encode_ts_cols(df):\n") + sb.append(" for _c in df.columns:\n") + sb.append(" if pd.api.types.is_datetime64_any_dtype(df[_c]):\n") + sb.append(" df[_c] = df[_c].map(_texera_ts_str)\n") + sb.append(" return df\n") + sb.append("\n") + + // Prologue: load each external input into in{N}df. Note: pd.read_json with + // lines=True correctly handles empty files (returns empty DataFrame). + // convert_dates=False: pd.read_json otherwise auto-coerces ISO-ish strings + // and columns named like dates ("date", "*_at", …) to datetime64, which the + // schema-typed runtime path (STRING) does not do — that divergence would + // make a plain date string column serialize as "...T00:00:00" on only one + // side. Operators that genuinely need datetimes convert explicitly, so both + // paths stay in sync. + // precise_float=True: pd.read_json's default (ujson) fast double parser is + // lossy in the last few ULPs, so a DOUBLE column would load slightly + // different values than the schema-typed runtime path (which parses doubles + // exactly). Operators that stringify raw cell values (e.g. Radar hover text) + // then diverge; precise_float=True keeps both paths bit-identical. + // The blanket convert_dates=False also leaves genuine TIMESTAMP columns as + // strings, which the runtime path delivers as datetime64 — a divergence for + // any operator that renders or computes on them. The fixture's schema + // sidecar says which columns those are, so cast exactly those back. + inputs.toSeq.sortBy(_._1).foreach { + case (n, path) => + sb.append( + s"in${n}df = pd.read_json(${py(path.toString)}, lines=True, convert_dates=False, precise_float=True)\n" + ) + timestampColumns(path).foreach { col => + sb.append(s"if ${py(col)} in in${n}df.columns:\n") + sb.append(s" in${n}df[${py(col)}] = pd.to_datetime(in${n}df[${py(col)}])\n") + } + doubleColumns(path).foreach { col => + sb.append(s"if ${py(col)} in in${n}df.columns:\n") + sb.append(s" in${n}df[${py(col)}] = in${n}df[${py(col)}].astype('float64')\n") + } + } + // The variadic placeholder, bound here for the same reason the numbered ones + // are: this script leaves the body's placeholders alone and defines names to + // match them, so an operator reading a variadic port finds its list here the + // way the translator would have written one out. + if (inputs.nonEmpty) { + sb.append( + inputs.keys.toSeq.sorted.map(n => s"in${n}df").mkString("inAlldf = [", ", ", "]\n") + ) + } + sb.append("\n") + + // Body verbatim — placeholders left in place. + // Emitted ahead of the body the way the translator does, so an operator that + // declares a helper is exercised here exactly as it runs in a real script. + helpers.foreach { helper => + sb.append(helper) + if (!helper.endsWith("\n")) sb.append('\n') + sb.append('\n') + } + + sb.append("# ── operator body ──\n") + sb.append(body) + if (!body.endsWith("\n")) sb.append('\n') + sb.append("\n") + + // Epilogue: dump each out{N}df to JSONL. When producesDataFrame() is false + // (visualization ops), `outputs` is empty and this block is a no-op — the + // caller is expected to verify viz outputs by other means. + outputs.toSeq.sortBy(_._1).foreach { + case (n, path) => + sb.append( + s"_texera_encode_obj_cols(_texera_encode_ts_cols(out${n}df))" + + s".to_json(${py(path.toString)}, orient='records', lines=True)\n" + ) + } + + sb.toString + } + + // TIMESTAMP-typed column names from a fixture's `.jsonl.schema.json` sidecar. + // A missing or unreadable sidecar means no casts — the prologue then behaves + // exactly as before. + private def timestampColumns(input: Path): Seq[String] = + columnsOfType(input, AttributeType.TIMESTAMP) + + // DOUBLE-typed column names. pd.read_json narrows a float column whose values + // are all integral to int64, while the runtime path keeps the schema's DOUBLE, + // so a column like 7.0 stringifies as "7" on one side and "7.0" on the other — + // invisible to numeric comparison, visible the moment an operator uses the + // column as a label (a trace name, a legend entry, hover text). + private def doubleColumns(input: Path): Seq[String] = + columnsOfType(input, AttributeType.DOUBLE) + + private def columnsOfType(input: Path, attributeType: AttributeType): Seq[String] = + scala.util + .Try(TupleIO.readSchemaSidecar(input)) + .toOption + .toSeq + .flatMap( + _.getAttributes.filter(_.getType == attributeType).map(_.getName) + ) + + // Python string literal, single-quoted with backslashes escaped. We + // deliberately don't use repr() in Scala (no such thing) — JSON.toString + // would also work but introduces double-quote escaping when the path has + // spaces. + private def py(s: String): String = + "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'" + + // Resolution chain mirrors the rest of the Texera test infra: env var first + // (set by CI / the shared-venv setup), then conventional names. + private def resolvePython(): String = { + val fromEnv = sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty) + fromEnv.getOrElse { + // We don't try to probe `which` here — if neither env var nor a literal + // `python3.12` is on PATH, the subprocess invocation will fail and the + // error path below surfaces it. + "python3.12" + } + } +} + +final class StandaloneExecutionException( + val exitCode: Int, + val scriptPath: Path, + val source: String, + val stdout: String, + val stderr: String +) extends RuntimeException( + // The script path goes first in the message so a failing CI log makes it + // immediately obvious which file to open. stderr ends the message because + // the Python traceback (if any) is the most actionable signal. + s"""Standalone Python script exited with code $exitCode. + |Script: $scriptPath + |--- stdout --- + |$stdout + |--- stderr --- + |$stderr""".stripMargin + ) From cba725fec0cc91e1f790bb18c36ce32c7cb223c4 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Tue, 1 Sep 2026 16:48:56 -0700 Subject: [PATCH 03/33] test(workflow-compiling-service): verify a generated script against the operator it came from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standalone export claims a generated script does what the operator does. This is what checks it, for every operator, on every configuration the operator offers. An operator is run twice. `OpExecHarness` drives it the way the engine does, compiled to a physical plan but outside a workflow, reading JSONL files rather than a live upstream; `PyOpExecHarness` does the same for a Python operator through the worker the engine uses. `StandaloneRunner` takes the other path, wrapping the operator's standalone code in a script that binds the same files. Both write files, and `Comparator` reads them back: order-insensitive by default, since the engine interleaves across workers and only the sort family promises an order. A visualization is compared as a figure rather than as a frame. What to run an operator ON is decided rather than written by hand for each. `ConfigGenerator` reads the operator's own schema — its enums, defaults, declared ranges and column pickers — and produces a base configuration plus one variant per branch the operator offers, so a switch nobody thought to try is still tried. `CanonicalFixture` is the table they run against, one column per shape an operator might ask for. `CuratedHandlers` is the escape hatch for an operator whose input cannot be derived, and `TransformVerificationRunner` decides which of the three tiers each operator takes and reports what it could not run and why. `LogicalOp.orderSensitive` and `@SampleColumn` are the two things the operators had to say for this to read them: whether row order is part of the contract, and which column a field should be pointed at when the first unused one would be a poor choice. Most of the operator set does not implement the generator yet — it arrives a family at a time — and the runner reports each of those rather than passing over it. The tier assertions for a family land with the change that gives that family its generator. Co-Authored-By: Claude Opus 5 (1M context) --- .../texera/amber/operator/LogicalOp.scala | 10 + .../metadata/annotations/SampleColumn.java | 45 + workflow-compiling-service/build.sbt | 27 +- .../verify/tags/IntegrationTest.java | 56 + .../src/test/resources/python/compare.py | 444 ++++ .../resources/verify/canonical_fixture.json | 542 +++++ .../translator/verify/CanonicalFixture.scala | 249 ++ .../verify/CanonicalFixtureSpec.scala | 302 +++ .../amber/translator/verify/Comparator.scala | 189 ++ .../translator/verify/ComparatorSpec.scala | 97 + .../verify/ConfigCoverageSpec.scala | 122 + .../translator/verify/ConfigGenerator.scala | 2015 +++++++++++++++++ .../verify/ConfigGeneratorSpec.scala | 98 + .../translator/verify/CuratedHandlers.scala | 666 ++++++ .../amber/translator/verify/HarnessSpec.scala | 9 +- .../verify/OperatorBehaviorSpec.scala | 151 ++ .../translator/verify/SharedFixture.scala | 178 ++ .../verify/SourceCategoryRunner.scala | 471 ++++ .../verify/StandaloneEscapingCheck.scala | 156 ++ .../verify/TransformVerificationRunner.scala | 962 ++++++++ .../TransformVerificationRunnerSpec.scala | 75 + .../verify/VisualizationHtmlComparator.scala | 85 + .../verify/VisualizationJsonComparator.scala | 123 + 23 files changed, 7070 insertions(+), 2 deletions(-) create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java create mode 100644 workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java create mode 100644 workflow-compiling-service/src/test/resources/python/compare.py create mode 100644 workflow-compiling-service/src/test/resources/verify/canonical_fixture.json create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index efa46144180..d28c806ce56 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -459,6 +459,16 @@ abstract class LogicalOp extends PortDescriptor with Serializable { def operatorInfo: OperatorInfo + /** + * Whether the row ORDER of this operator's output is part of its contract. + * Defaults to false: the engine runs operators across parallel workers, so + * for almost every operator the output row order is an implementation-defined + * interleaving that consumers must not rely on. Only operators whose very + * purpose is to establish an order (the sort family) override this to true. + * Consumers that must not rely on a stable row order read this flag. + */ + def orderSensitive: Boolean = false + private def getOperatorVersion: String = { val path = "amber/src/main/scala/" val operatorPath = path + this.getClass.getPackage.getName.replace(".", "/") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java new file mode 100644 index 00000000000..cb35c9aab91 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.metadata.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Test-only metadata for transform verification: names the column in the shared + * verification fixture the operator runs on that should fill this + * {@code @AutofillAttributeName} field when the operator is auto-configured. + * + *

It lets a field declare a semantic sample that the column's + * {@code AttributeType} alone cannot express — e.g. a valid three-letter ISO + * country code, or a genuine OHLC price column — so the parity test exercises + * the operator on realistic input instead of a degenerate first-column pick + * (which can hide translation bugs and produce vacuous passes). + * + *

This has no effect on production: it is not a Jackson / JSON-schema + * annotation and is read only by the test-side ConfigGenerator. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface SampleColumn { + String value(); +} diff --git a/workflow-compiling-service/build.sbt b/workflow-compiling-service/build.sbt index 2af92efc4d4..94d5dedb389 100644 --- a/workflow-compiling-service/build.sbt +++ b/workflow-compiling-service/build.sbt @@ -41,9 +41,34 @@ ThisBuild / semanticdbVersion := scalafixSemanticdb.revision // Manage dependency conflicts by always using the latest revision ThisBuild / conflictManager := ConflictManager.latestRevision -// Restrict parallel execution of tests to avoid conflicts +// Restrict parallel execution of tests to avoid conflicts. This caps how many +// test *suites* run concurrently; ParallelTestExecution still parallelizes the +// tests *within* a suite (e.g. OperatorBehaviorSpec) via ScalaTest's own pool. Global / concurrentRestrictions += Tags.limit(Tags.Test, 1) +// The fast-unit / integration test split; the selection logic itself is shared +// in project/TestFilters.scala. +Test / testOptions ++= TestFilters.integrationSplit( + envVar = "WCS_TEST_FILTER", + tag = "org.apache.texera.amber.translator.verify.tags.IntegrationTest" +) + +// -P4 bounds ScalaTest's ParallelTestExecution pool, and only this module wants +// it: OperatorBehaviorSpec forks a Python subprocess per operator, and at +// core-count concurrency (e.g. 12) resource contention caused rare flakes. A +// fixed 4 stays deterministic across machines (incl. CI runners) while still +// running ~3x faster than serial, and it matches PythonWorkerPool's own default +// worker cap so the two bounds agree rather than multiply. Unconditional, so a +// local run reproduces the concurrency CI runs at instead of a faster one that +// flakes differently; WCS_TEST_FILTER selects which tests run, which is a +// separate question from how many run at once. The fast-unit job is unaffected +// either way, since OperatorBehaviorSpec is the only spec here that +// parallelizes and that job excludes it. It lives here rather than in the +// shared helper so that helper stays identical for every module. sbt +// concatenates the ScalaTest arguments of every testOptions entry, so this +// lands in the same argument list as the -n above. +Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-P4") + ///////////////////////////////////////////////////////////////////////////// // Compiler Options ///////////////////////////////////////////////////////////////////////////// diff --git a/workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java b/workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java new file mode 100644 index 00000000000..4da3aa9cd2d --- /dev/null +++ b/workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** + * Class-level marker tag for workflow-compiling-service ScalaTest specs that + * exercise both Scala and Python end-to-end (they fork a real Python process + * to run and compare the translator-generated code). Routing to the + * {@code workflow-compiling-service-integration} CI job is by ScalaTest tag + * filtering, controlled by the {@code WCS_TEST_FILTER} env var in + * {@code workflow-compiling-service/build.sbt}: the lighter + * {@code workflow-compiling-service} test run uses {@code skip-integration} + * (which passes {@code -l org.apache.texera.amber.translator.verify.tags.IntegrationTest} + * to ScalaTest), and the integration job uses {@code integration-only} (which + * passes {@code -n} for the same tag). + * + *

Mirrors amber's {@code org.apache.texera.amber.tags.IntegrationTest}. Only + * {@code OperatorBehaviorSpec} carries this tag today — it is the sole spec that + * spawns Python; the other verify specs only exercise pure-JVM classification + * and comparison logic. + * + *

Written in Java rather than Scala because ScalaTest detects tag + * annotations via {@code java.lang.annotation} reflection. A Scala + * {@code class extends StaticAnnotation} does not produce a JVM annotation + * interface that {@code @TagAnnotation} can attach to, so the tag would be + * invisible to ScalaTest at runtime. + */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface IntegrationTest { +} diff --git a/workflow-compiling-service/src/test/resources/python/compare.py b/workflow-compiling-service/src/test/resources/python/compare.py new file mode 100644 index 00000000000..518b5fac901 --- /dev/null +++ b/workflow-compiling-service/src/test/resources/python/compare.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Compare the two paths' outputs for one operator: JSONL DataFrames, or the Plotly +figure a visualization operator renders. + +Usage: compare.py [--unordered] [--ignore-cols c1,c2] + [--model-cols c1,c2 --probe features.jsonl] + + compare.py --plotly + + --unordered Sort both DataFrames lexicographically by all columns before + comparing, so rows match as a set/bag rather than positionally. + This is the norm: the engine runs operators across parallel + workers, so output row order is not part of the contract. + Without this flag the comparator matches rows positionally + (after reset_index(drop=True)) — used only for the sort family, + whose output order IS meaningful. + + --ignore-cols Comma-separated column names to drop from both frames before + comparing. For opaque columns whose value isn't compared. + + --model-cols Comma-separated columns holding a base64(pickle) sklearn model. + Rather than byte-compare them (two independently-trained models + are functionally equal but not bit-identical), the comparator + unpickles both sides, has each model predict on the --probe + feature set, and asserts the predictions match — verifying the + two code paths produce behaviorally-equivalent models. The raw + model columns are then dropped before the frame comparison. + + --probe JSONL feature set the --model-cols models predict on. Each + model uses its own feature_names_in_ to select columns, so the + probe may include extra columns (e.g. the training target). + + --plotly Compare Plotly figures instead of DataFrames. The actual side is + a one-row JSONL with `html-content` or `json-content`; for + `html-content` the first `Plotly.newPlot(...)` payload is + extracted. The expected side is the standalone path's + `fig.write_json(...)`. Only data and layout are compared, with + display-only `uid` fields stripped and floats matched by + tolerance. Takes none of the DataFrame flags. + +Exit 0 - Outputs equal (and model predictions match, if --model-cols) +Exit 1 - Outputs differ; detail on stderr +Exit 2 - Bad invocation + +Persistent mode: `compare.py --serve` imports pandas once and then serves many +comparisons over its lifetime, reading one JSON job per line on stdin and +writing one JSON result per line on stdout. This avoids paying the ~214 ms +pandas import on every comparison (the comparison itself is ~ms). It reuses the +exact same functions the CLI calls, so behavior is identical. + + request {"kind": "dataframe", "actual": "", "expected": "", + "unordered": false, "ignoreCols": [], "modelCols": [], + "probe": null}\n + {"kind": "plotly", "actual": "", "expected": ""}\n + response {"exit": 0|1, "stdout": "", "stderr": ""}\n + +`kind` defaults to "dataframe". Both kinds are served by the same worker so a +run needs one comparison pool rather than one per output shape; the Plotly side +needs nothing pandas does not already pull in. + +A mismatch is exit 1 with the diff on `stderr`, mirroring the CLI's nonzero +exit so the Scala side's ComparatorMismatchException path is unchanged. A +comparison error never kills the server; only closing stdin (EOF) ends it. +""" +import sys + +# pandas is imported where it is used, not here: the --plotly comparison needs +# nothing from it, and a module-level import would make that one-shot invocation +# pay ~500 ms for an interpreter that then compares two JSON documents. `serve()` +# imports it eagerly at startup instead, so a pooled worker still pays it once +# rather than once per DataFrame comparison. + + +def _compare_model_predictions(actual, expected, model_cols, probe_path) -> None: + """For each model column, unpickle both sides and assert their predictions + on the probe set match. Raises AssertionError on any divergence.""" + import base64 + import pickle + + import numpy as np + import pandas as pd + + if probe_path is None: + raise AssertionError("--model-cols requires --probe with a feature set") + probe = pd.read_json(probe_path, lines=True) + # The probe is the operator's own input table, so under the nulls scenario it + # carries the holes that scenario punched. What is under test is whether the + # two models agree, and an estimator that refuses a NaN at predict time would + # end the comparison over the probe rather than over either model. Drop those + # rows: both models are asked the same questions either way. + probe = probe.dropna() + if probe.empty: + raise AssertionError( + "probe has no complete row to predict on; the two models cannot be compared" + ) + + for col in model_cols: + if col not in actual.columns or col not in expected.columns: + continue + if len(actual) != len(expected): + raise AssertionError( + f"model column {col!r}: row count differs " + f"({len(actual)} vs {len(expected)})" + ) + for i in range(len(actual)): + m_actual = pickle.loads(base64.b64decode(actual[col].iloc[i])) + m_expected = pickle.loads(base64.b64decode(expected[col].iloc[i])) + + # A model with feature_names_in_ selects its (numeric) feature + # columns from the probe, naturally dropping the training target the + # probe may still carry. A model WITHOUT it was fitted on a 1-D input + # rather than a named frame — i.e. a text pipeline (e.g. + # CountVectorizer) trained on a single text Series — so feed the + # probe's first column as a Series, not the whole frame (predicting + # on a DataFrame would make CountVectorizer iterate column labels). + names = getattr(m_actual, "feature_names_in_", None) + x_a = probe[list(names)] if names is not None else probe.iloc[:, 0] + names_e = getattr(m_expected, "feature_names_in_", None) + x_e = probe[list(names_e)] if names_e is not None else probe.iloc[:, 0] + + pred_a = np.asarray(m_actual.predict(x_a)) + pred_e = np.asarray(m_expected.predict(x_e)) + + if pred_a.shape != pred_e.shape: + raise AssertionError( + f"model column {col!r} row {i}: prediction shape differs " + f"({pred_a.shape} vs {pred_e.shape})" + ) + numeric = np.issubdtype(pred_a.dtype, np.number) and np.issubdtype( + pred_e.dtype, np.number + ) + ok = ( + np.allclose(pred_a, pred_e, rtol=1e-5, atol=1e-8) + if numeric + else np.array_equal(pred_a, pred_e) + ) + if not ok: + raise AssertionError( + f"model column {col!r} row {i}: predictions differ\n" + f" actual: {pred_a}\n" + f" expected: {pred_e}" + ) + + +def _run_comparison( + actual_path: str, + expected_path: str, + unordered: bool, + ignore_cols: list, + model_cols: list, + probe_path, +) -> "str | None": + """Compare two JSONL DataFrames. Returns None if they match, or a human + diff string if they differ (exit-1 condition). Unexpected errors (e.g. a + bad input file) propagate to the caller. This is the single source of + comparison truth shared by the CLI and the --serve loop.""" + import pandas as pd + + actual = pd.read_json(actual_path, lines=True) + expected = pd.read_json(expected_path, lines=True) + + # Model columns: compare behavior (predictions) rather than bytes, then drop + # the raw columns so the frame comparison covers everything else exactly. + if model_cols: + try: + _compare_model_predictions(actual, expected, model_cols, probe_path) + except AssertionError as exc: + return str(exc) + actual = actual.drop(columns=model_cols, errors="ignore") + expected = expected.drop(columns=model_cols, errors="ignore") + + if ignore_cols: + actual = actual.drop(columns=ignore_cols, errors="ignore") + expected = expected.drop(columns=ignore_cols, errors="ignore") + + if unordered: + # Sort both sides by the same column key so set-equal frames collapse + # to the same row sequence. assert_frame_equal still does the actual + # value diff and respects rtol/check_dtype. Mergesort = stable, so + # rows that are tied on all columns keep their relative order — not + # strictly necessary for set equality (no ties → no duplicates after + # the op's dedup step) but cheap insurance. + cols = list(actual.columns) + if cols: + actual = actual.sort_values( + by=cols, kind="mergesort", na_position="last" + ).reset_index(drop=True) + expected = expected.sort_values( + by=cols, kind="mergesort", na_position="last" + ).reset_index(drop=True) + + try: + pd.testing.assert_frame_equal( + actual, + expected, + check_like=True, + check_dtype=False, + rtol=1e-5, + ) + except AssertionError as exc: + return str(exc) + return None + + +def _load_actual_plot(path) -> dict: + import json + + with open(path, "r", encoding="utf-8") as fh: + line = next((raw for raw in fh if raw.strip()), None) + if line is None: + raise AssertionError(f"{path} is empty") + + row = json.loads(line) + if "json-content" in row and row["json-content"]: + value = row["json-content"] + return json.loads(value) if isinstance(value, str) else value + if "html-content" in row and row["html-content"]: + return _plotly_payload_from_html(row["html-content"]) + raise AssertionError(f"{path} has neither html-content nor json-content") + + +def _plotly_payload_from_html(html: str) -> dict: + """Pull the data/layout arguments out of the first Plotly.newPlot(...) call. + + Scanned with a JSON decoder rather than a regex because the payload is + arbitrary nested JSON that no bracket-matching pattern handles reliably. + """ + import json + + marker = "Plotly.newPlot(" + start = html.find(marker) + if start < 0: + raise AssertionError("html-content does not contain Plotly.newPlot(...)") + + decoder = json.JSONDecoder() + index = start + len(marker) + args: list = [] + while len(args) < 4: + while index < len(html) and html[index] in " \t\r\n,": + index += 1 + value, consumed = decoder.raw_decode(html[index:]) + args.append(value) + index += consumed + + return {"data": args[1], "layout": args[2]} + + +def _load_expected_plot(path) -> dict: + import json + + with open(path, "r", encoding="utf-8") as fh: + value = json.load(fh) + return {"data": value.get("data", []), "layout": value.get("layout", {})} + + +def _strip_unstable(value): + """Remove display-only fields that are unrelated to chart semantics.""" + if isinstance(value, dict): + return { + key: _strip_unstable(child) + for key, child in value.items() + if key not in {"uid"} + } + if isinstance(value, list): + return [_strip_unstable(child) for child in value] + return value + + +def _plots_equal(actual, expected) -> bool: + import math + + if isinstance(actual, (int, float)) and isinstance(expected, (int, float)): + return math.isclose(float(actual), float(expected), rel_tol=1e-9, abs_tol=1e-12) + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all( + _plots_equal(actual[key], expected[key]) for key in actual.keys() + ) + if isinstance(actual, list) and isinstance(expected, list): + return len(actual) == len(expected) and all( + _plots_equal(left, right) for left, right in zip(actual, expected) + ) + return actual == expected + + +def _run_plotly_comparison(actual_path, expected_path) -> "str | None": + """Compare two Plotly figures. Returns None if they match, or a human diff + string if they differ — the same contract as `_run_comparison`, so the CLI + and the --serve loop treat both kinds identically.""" + import json + + actual = _strip_unstable(_load_actual_plot(actual_path)) + expected = _strip_unstable(_load_expected_plot(expected_path)) + if _plots_equal(actual, expected): + return None + return "\n".join( + [ + "Plotly JSON mismatch", + "--- actual ---", + json.dumps(actual, indent=2, sort_keys=True), + "--- expected ---", + json.dumps(expected, indent=2, sort_keys=True), + ] + ) + + +def main() -> None: + args = sys.argv[1:] + unordered = False + ignore_cols: list = [] + model_cols: list = [] + probe_path = None + + if args and args[0] == "--plotly": + if len(args) != 3: + print( + f"usage: {sys.argv[0]} --plotly ", + file=sys.stderr, + ) + sys.exit(2) + msg = _run_plotly_comparison(args[1], args[2]) + if msg is not None: + print(msg, file=sys.stderr) + sys.exit(1) + return + + while args and args[0].startswith("--"): + if args[0] == "--unordered": + unordered = True + args = args[1:] + elif args[0] == "--ignore-cols": + if len(args) < 2: + print("--ignore-cols requires an argument", file=sys.stderr) + sys.exit(2) + ignore_cols = [c for c in args[1].split(",") if c] + args = args[2:] + elif args[0] == "--model-cols": + if len(args) < 2: + print("--model-cols requires an argument", file=sys.stderr) + sys.exit(2) + model_cols = [c for c in args[1].split(",") if c] + args = args[2:] + elif args[0] == "--probe": + if len(args) < 2: + print("--probe requires an argument", file=sys.stderr) + sys.exit(2) + probe_path = args[1] + args = args[2:] + else: + print(f"unknown flag: {args[0]}", file=sys.stderr) + sys.exit(2) + if len(args) != 2: + print( + f"usage: {sys.argv[0]} [--unordered] [--ignore-cols c1,c2] " + f"[--model-cols c1,c2 --probe features.jsonl] " + f" ", + file=sys.stderr, + ) + sys.exit(2) + + msg = _run_comparison( + args[0], args[1], unordered, ignore_cols, model_cols, probe_path + ) + if msg is not None: + print(msg, file=sys.stderr) + sys.exit(1) + + +def serve() -> None: + """Persistent comparison server. See the module docstring for the protocol. + + Each job runs the same function the CLI calls for its kind. A comparison + error is reported as exit 1 with the diff on `stderr`; only closing stdin + ends the loop. + """ + import io + import json + import traceback + from contextlib import redirect_stderr, redirect_stdout + + # Eagerly, before signalling ready: the point of a persistent worker is that + # this cost is paid once per worker instead of once per comparison, and + # `ready` should mean the worker is warm. + import pandas # noqa: F401 + + sys.stdout.write(json.dumps({"ready": True}) + "\n") + sys.stdout.flush() + + for line in sys.stdin: + line = line.strip() + if not line: + continue + out_buf, err_buf = io.StringIO(), io.StringIO() + try: + job = json.loads(line) + with redirect_stdout(out_buf), redirect_stderr(err_buf): + if job.get("kind", "dataframe") == "plotly": + msg = _run_plotly_comparison(job["actual"], job["expected"]) + else: + msg = _run_comparison( + job["actual"], + job["expected"], + job.get("unordered", False), + job.get("ignoreCols", []), + job.get("modelCols", []), + job.get("probe"), + ) + resp = { + "exit": 0 if msg is None else 1, + "stdout": out_buf.getvalue(), + "stderr": err_buf.getvalue() + ("" if msg is None else msg), + } + except BaseException: # noqa: BLE001 — a bad job must not kill the server + resp = { + "exit": 1, + "stdout": out_buf.getvalue(), + "stderr": err_buf.getvalue() + traceback.format_exc(), + } + sys.stdout.write(json.dumps(resp) + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "--serve": + serve() + else: + main() diff --git a/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json b/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json new file mode 100644 index 00000000000..59dc4823c14 --- /dev/null +++ b/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json @@ -0,0 +1,542 @@ +[ + { + "id": 7, + "name": "eve", + "score": 0.9, + "open": 7.0, + "high": 8.5, + "low": 6.0, + "close": 7.5, + "iso_country": "IND", + "trade_date": "2024-01-07", + "pvalue": 0.2597402597402597, + "log2fc": 1.4, + "comp_a": 3.0, + "comp_b": 1.0, + "comp_c": 2.0, + "uvec": -3.0, + "edge_pair": "[0, 7]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-07 00:00:00.0", + "finish_ts": "2024-01-07 08:00:00.0", + "uniq_name": "cat_7", + "simplex_a": 35.0, + "simplex_b": 30.0, + "simplex_c": 35.0, + "short_text": "The meeting is scheduled for three o'clock tomorrow.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 3.9000000000000004, + "petal_width": 1.3, + "species": 1, + "csv_list": "a7", + "mixed_case": "abacus", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 3, + "name": "bob", + "score": 1.2, + "open": 3.0, + "high": 4.5, + "low": 2.0, + "close": 3.5, + "iso_country": "JPN", + "trade_date": "2024-01-03", + "pvalue": 0.11188811188811189, + "log2fc": -1.4, + "comp_a": 4.0, + "comp_b": 4.0, + "comp_c": 1.0, + "uvec": 0.0, + "edge_pair": "[0, 3]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-03 00:00:00.0", + "finish_ts": "2024-01-03 04:00:00.0", + "uniq_name": "cat_3", + "simplex_a": 35.0, + "simplex_b": 25.0, + "simplex_c": 40.0, + "short_text": "The meeting is scheduled for three o'clock tomorrow.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 5.2, + "petal_width": 1.85, + "species": 1, + "csv_list": "a3,b3", + "mixed_case": "ABBEY", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 11, + "name": "1", + "score": 1.7, + "open": 11.0, + "high": 12.5, + "low": 10.0, + "close": 11.5, + "iso_country": "USA", + "trade_date": "2024-01-11", + "pvalue": 0.4075924075924076, + "log2fc": -3.5, + "comp_a": 2.0, + "comp_b": 5.0, + "comp_c": 3.0, + "uvec": 1.0, + "edge_pair": "[0, 11]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-11 00:00:00.0", + "finish_ts": "2024-01-11 04:00:00.0", + "uniq_name": "cat_11", + "simplex_a": 35.0, + "simplex_b": 35.0, + "simplex_c": 30.0, + "short_text": "URGENT: your account needs verification, click the link immediately.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 2.6, + "petal_width": 0.75, + "species": 0, + "csv_list": "a11,b11,c11", + "mixed_case": "101", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 1, + "name": "1", + "score": 0.5, + "open": 1.0, + "high": 2.5, + "low": 0.0, + "close": 1.5, + "iso_country": "USA", + "trade_date": "2024-01-01", + "pvalue": 0.03796203796203796, + "log2fc": -2.8, + "comp_a": 2.0, + "comp_b": 2.0, + "comp_c": 2.0, + "uvec": -2.0, + "edge_pair": "[0, 1]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-01 00:00:00.0", + "finish_ts": "2024-01-01 02:00:00.0", + "uniq_name": "cat_1", + "simplex_a": 25.0, + "simplex_b": 30.0, + "simplex_c": 45.0, + "short_text": "I'm really not sure how I feel about it.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 2.6, + "petal_width": 0.75, + "species": 0, + "csv_list": "a1,b1,c1,d1", + "mixed_case": "abdomen", + "species_pred": 1, + "species_name": "setosa", + "species_name_pred": "versicolor" + }, + { + "id": 14, + "name": "carol", + "score": 4.5, + "open": 14.0, + "high": 15.5, + "low": 13.0, + "close": 14.5, + "iso_country": "DEU", + "trade_date": "2024-01-14", + "pvalue": 0.5184815184815185, + "log2fc": -1.4, + "comp_a": 5.0, + "comp_b": 1.0, + "comp_c": 3.0, + "uvec": -3.0, + "edge_pair": "[0, 14]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-14 00:00:00.0", + "finish_ts": "2024-01-14 07:00:00.0", + "uniq_name": "cat_14", + "simplex_a": 30.0, + "simplex_b": 35.0, + "simplex_c": 35.0, + "short_text": "I absolutely love this, it completely made my day!", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 6.5, + "petal_width": 2.4000000000000004, + "species": 1, + "csv_list": "a14", + "mixed_case": "ABILITY", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 5, + "name": "dave", + "score": 2.0, + "open": 5.0, + "high": 6.5, + "low": 4.0, + "close": 5.5, + "iso_country": "GBR", + "trade_date": "2024-01-05", + "pvalue": 0.18581418581418582, + "log2fc": 0.0, + "comp_a": 1.0, + "comp_b": 6.0, + "comp_c": 3.0, + "uvec": 2.0, + "edge_pair": "[0, 5]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-05 00:00:00.0", + "finish_ts": "2024-01-05 06:00:00.0", + "uniq_name": "cat_5", + "simplex_a": 25.0, + "simplex_b": 35.0, + "simplex_c": 40.0, + "short_text": "This is the worst experience I have ever had.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 1.3, + "petal_width": 0.2, + "species": 0, + "csv_list": "a5,b5", + "mixed_case": "202", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 9, + "name": "grace", + "score": 2.4, + "open": 9.0, + "high": 10.5, + "low": 8.0, + "close": 9.5, + "iso_country": "CAN", + "trade_date": "2024-01-09", + "pvalue": 0.3336663336663337, + "log2fc": 2.8, + "comp_a": 5.0, + "comp_b": 3.0, + "comp_c": 1.0, + "uvec": -1.0, + "edge_pair": "[0, 9]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-09 00:00:00.0", + "finish_ts": "2024-01-09 02:00:00.0", + "uniq_name": "cat_9", + "simplex_a": 25.0, + "simplex_b": 25.0, + "simplex_c": 50.0, + "short_text": "I absolutely love this, it completely made my day!", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 6.5, + "petal_width": 2.4000000000000004, + "species": 1, + "csv_list": "a9,b9,c9", + "mixed_case": "abstract", + "species_pred": 0, + "species_name": "versicolor", + "species_name_pred": "setosa" + }, + { + "id": 2, + "name": "alice", + "score": 3.1, + "open": 2.0, + "high": 3.5, + "low": 1.0, + "close": 2.5, + "iso_country": "CHN", + "trade_date": "2024-01-02", + "pvalue": 0.07492507492507493, + "log2fc": -2.0999999999999996, + "comp_a": 3.0, + "comp_b": 3.0, + "comp_c": 3.0, + "uvec": -1.0, + "edge_pair": "[0, 2]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-02 00:00:00.0", + "finish_ts": "2024-01-02 03:00:00.0", + "uniq_name": "cat_2", + "simplex_a": 30.0, + "simplex_b": 35.0, + "simplex_c": 35.0, + "short_text": "Thank you so much, everything was absolutely perfect.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 3.9000000000000004, + "petal_width": 1.3, + "species": 1, + "csv_list": "a2,b2,c2,d2", + "mixed_case": "ABACUS", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 13, + "name": "bob", + "score": 2.8, + "open": 13.0, + "high": 14.5, + "low": 12.0, + "close": 13.5, + "iso_country": "JPN", + "trade_date": "2024-01-13", + "pvalue": 0.48151848151848153, + "log2fc": -2.0999999999999996, + "comp_a": 4.0, + "comp_b": 7.0, + "comp_c": 2.0, + "uvec": 3.0, + "edge_pair": "[0, 13]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-13 00:00:00.0", + "finish_ts": "2024-01-13 06:00:00.0", + "uniq_name": "cat_13", + "simplex_a": 25.0, + "simplex_b": 30.0, + "simplex_c": 45.0, + "short_text": "Thank you so much, everything was absolutely perfect.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 5.2, + "petal_width": 1.85, + "species": 1, + "csv_list": "a13", + "mixed_case": "303", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 6, + "name": "1", + "score": 5.5, + "open": 6.0, + "high": 7.5, + "low": 5.0, + "close": 6.5, + "iso_country": "FRA", + "trade_date": "2024-01-06", + "pvalue": 0.22277722277722278, + "log2fc": 0.7, + "comp_a": 2.0, + "comp_b": 7.0, + "comp_c": 1.0, + "uvec": 3.0, + "edge_pair": "[0, 6]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-06 00:00:00.0", + "finish_ts": "2024-01-06 07:00:00.0", + "uniq_name": "cat_6", + "simplex_a": 30.0, + "simplex_b": 25.0, + "simplex_c": 45.0, + "short_text": "I'm really not sure how I feel about it.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 2.6, + "petal_width": 0.75, + "species": 0, + "csv_list": "a6,b6", + "mixed_case": "abbey", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 10, + "name": "heidi", + "score": 4.2, + "open": 10.0, + "high": 11.5, + "low": 9.0, + "close": 10.5, + "iso_country": "AUS", + "trade_date": "2024-01-10", + "pvalue": 0.3706293706293706, + "log2fc": 3.5, + "comp_a": 1.0, + "comp_b": 4.0, + "comp_c": 2.0, + "uvec": 0.0, + "edge_pair": "[0, 10]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-10 00:00:00.0", + "finish_ts": "2024-01-10 03:00:00.0", + "uniq_name": "cat_10", + "simplex_a": 30.0, + "simplex_b": 30.0, + "simplex_c": 40.0, + "short_text": "This is the worst experience I have ever had.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 1.3, + "petal_width": 0.2, + "species": 0, + "csv_list": "a10,b10,c10", + "mixed_case": "ABDOMEN", + "species_pred": 1, + "species_name": "setosa", + "species_name_pred": "versicolor" + }, + { + "id": 4, + "name": "carol", + "score": 4.8, + "open": 4.0, + "high": 5.5, + "low": 3.0, + "close": 4.5, + "iso_country": "DEU", + "trade_date": "2024-01-04", + "pvalue": 0.14885114885114886, + "log2fc": -0.7, + "comp_a": 5.0, + "comp_b": 5.0, + "comp_c": 2.0, + "uvec": 1.0, + "edge_pair": "[0, 4]", + "node_src": "n0", + "node_dst": "n1", + "start_ts": "2024-01-04 00:00:00.0", + "finish_ts": "2024-01-04 05:00:00.0", + "uniq_name": "cat_4", + "simplex_a": 20.0, + "simplex_b": 30.0, + "simplex_c": 50.0, + "short_text": "Congratulations! You have won a free prize, reply now to claim it.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 6.5, + "petal_width": 2.4000000000000004, + "species": 1, + "csv_list": "a4,b4,c4,d4", + "mixed_case": "404", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 15, + "name": "dave", + "score": 3.3, + "open": 15.0, + "high": 16.5, + "low": 14.0, + "close": 15.5, + "iso_country": "GBR", + "trade_date": "2024-01-15", + "pvalue": 0.5554445554445554, + "log2fc": -0.7, + "comp_a": 1.0, + "comp_b": 2.0, + "comp_c": 1.0, + "uvec": -2.0, + "edge_pair": "[0, 15]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-15 00:00:00.0", + "finish_ts": "2024-01-15 08:00:00.0", + "uniq_name": "cat_15", + "simplex_a": 35.0, + "simplex_b": 25.0, + "simplex_c": 40.0, + "short_text": "URGENT: your account needs verification, click the link immediately.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 1.3, + "petal_width": 0.2, + "species": 0, + "csv_list": "a15", + "mixed_case": "ability", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 8, + "name": "frank", + "score": 3.7, + "open": 8.0, + "high": 9.5, + "low": 7.0, + "close": 8.5, + "iso_country": "BRA", + "trade_date": "2024-01-08", + "pvalue": 0.2967032967032967, + "log2fc": 2.0999999999999996, + "comp_a": 4.0, + "comp_b": 2.0, + "comp_c": 3.0, + "uvec": -2.0, + "edge_pair": "[0, 8]", + "node_src": "n0", + "node_dst": "n1", + "start_ts": "2024-01-08 00:00:00.0", + "finish_ts": "2024-01-08 01:00:00.0", + "uniq_name": "cat_8", + "simplex_a": 20.0, + "simplex_b": 35.0, + "simplex_c": 45.0, + "short_text": "See you at lunch, save me a seat by the window.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 5.2, + "petal_width": 1.85, + "species": 1, + "csv_list": "a8,b8", + "mixed_case": "ABSTRACT", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 12, + "name": "alice", + "score": 5.1, + "open": 12.0, + "high": 13.5, + "low": 11.0, + "close": 12.5, + "iso_country": "CHN", + "trade_date": "2024-01-12", + "pvalue": 0.44455544455544455, + "log2fc": -2.8, + "comp_a": 3.0, + "comp_b": 6.0, + "comp_c": 1.0, + "uvec": 2.0, + "edge_pair": "[0, 12]", + "node_src": "n0", + "node_dst": "n1", + "start_ts": "2024-01-12 00:00:00.0", + "finish_ts": "2024-01-12 05:00:00.0", + "uniq_name": "cat_12", + "simplex_a": 20.0, + "simplex_b": 25.0, + "simplex_c": 55.0, + "short_text": "Congratulations! You have won a free prize, reply now to claim it.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 3.9000000000000004, + "petal_width": 1.3, + "species": 1, + "csv_list": "a12,b12,c12", + "mixed_case": "505", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + } +] diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala new file mode 100644 index 00000000000..a3e4f6a22bd --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} + +import java.sql.Timestamp +import scala.jdk.CollectionConverters._ + +/** + * The shared input dataset for auto-configured transform verification. + * Properties are deliberate (see CanonicalFixtureSpec): enough rows and + * partial port-0/port-1 overlap to defeat hash-coincidence false passes on + * set ops and joins, and the canonical value "1" present in some-but-not-all + * rows so ConfigGenerator-filled free-form predicates match a proper subset. + */ +object CanonicalFixture extends SharedFixture { + + // Columns are semantically named and type-correct so @SampleColumn-tagged or + // type-constrained fields can be filled with realistic input (a valid OHLC + // block, real ISO country codes, real dates) instead of a degenerate + // first-column pick. Ordering is deliberate: id/name/score lead so the + // first-column fallback AND the type-rule tier ("first column of a matching + // type") are unchanged for un-annotated fields — the domain-specific columns + // that follow are only reached via an explicit @SampleColumn. + val schema: Schema = new Schema( + new Attribute("id", AttributeType.INTEGER), + new Attribute("name", AttributeType.STRING), + new Attribute("score", AttributeType.DOUBLE), + new Attribute("open", AttributeType.DOUBLE), + new Attribute("high", AttributeType.DOUBLE), + new Attribute("low", AttributeType.DOUBLE), + new Attribute("close", AttributeType.DOUBLE), + new Attribute("iso_country", AttributeType.STRING), + new Attribute("trade_date", AttributeType.STRING), + // --- Domain-specific columns (reached only via @SampleColumn) --- + new Attribute("pvalue", AttributeType.DOUBLE), // strictly in (0,1): p-values + new Attribute("log2fc", AttributeType.DOUBLE), // signed, centered on 0: fold-change + new Attribute("comp_a", AttributeType.DOUBLE), // >0 ternary simplex component + new Attribute("comp_b", AttributeType.DOUBLE), // >0 ternary simplex component + new Attribute("comp_c", AttributeType.DOUBLE), // >0 ternary simplex component + new Attribute("uvec", AttributeType.DOUBLE), // any real: a 4th numeric (Quiver u/v) + new Attribute( + "edge_pair", + AttributeType.STRING + ), // "[parent, child]" literals, single-rooted tree + new Attribute("node_src", AttributeType.STRING), // edge source id (Sankey/Network) + new Attribute("node_dst", AttributeType.STRING), // edge target id, overlaps node_src (a DAG) + new Attribute( + "start_ts", + AttributeType.TIMESTAMP + ), // real timestamp; Gantt start / TimeSeries axis + new Attribute( + "finish_ts", + AttributeType.TIMESTAMP + ), // always > start_ts; Gantt finish (bar width) + new Attribute( + "uniq_name", + AttributeType.STRING + ), // distinct per row: Pie/name-keyed ops need no duplicates + new Attribute( + "simplex_a", + AttributeType.DOUBLE + ), // >0 and simplex_a+simplex_b+simplex_c == 100 (ternary-contour) + new Attribute("simplex_b", AttributeType.DOUBLE), // >0 simplex component summing to 100 + new Attribute("simplex_c", AttributeType.DOUBLE), // >0 simplex component summing to 100 + // ── text + iris-numeric columns for Hugging Face model operators ── + new Attribute( + "short_text", + AttributeType.STRING + ), // one sentence: sentiment / spam-detection input + new Attribute( + "long_text", + AttributeType.STRING + ), // a multi-sentence paragraph: summarization input + new Attribute("petal_length", AttributeType.DOUBLE), // iris petal length in cm (~1.3–6.5) + new Attribute("petal_width", AttributeType.DOUBLE), // iris petal width in cm (~0.2–2.4) + new Attribute( + "species", + AttributeType.INTEGER + ), // the 0/1 iris class, exactly `petal_length >= 3.9`: the label the sklearn + // families fit against, separable by the two petal columns above + new Attribute( + "csv_list", + AttributeType.STRING + ), // comma-delimited, 1–4 tokens per row: split/explode ops need real fan-out + new Attribute( + "mixed_case", + AttributeType.STRING + ), // a third lower-case, a third upper, a third letterless: a case flag has to + // change WHICH rows match, and on any other column it changes nothing + new Attribute( + "species_pred", + AttributeType.INTEGER + ), // a predictor's guess at `species`: the same 0/1 domain, wrong on a few rows. + // Scoring compares a PAIR of columns, and no single label column supplies one. + // Last so the first-unused fallback reaches it only after every other column + new Attribute("species_name", AttributeType.STRING), // `species` spelled out + new Attribute( + "species_name_pred", + AttributeType.STRING + ) // `species_pred` spelled out. A scorer takes a string label as readily as a + // numeric one and names the class after it rather than after its position, so + // the pair exists a second time in text + ) + + // ── Data source ── + // The rows are NOT generated at runtime — they live in a single, checked-in, + // human-readable JSON file that IS the source of truth: + // src/test/resources/verify/canonical_fixture.json (15 rows, ids 1..15) + // Open it to see the exact table; edit it to change the data. The + // CanonicalFixtureSpec invariants guard every semantic constraint (valid OHLC + // block, pvalue ∈ (0,1), ternary parts summing to 100, finish_ts > start_ts, + // etc.), so a hand-edit that breaks one fails the build. `schema` above stays + // authoritative for column types: JSON has no TIMESTAMP, so start_ts/finish_ts + // are stored as JDBC strings ("2024-01-01 00:00:00.0") and coerced back here. + private val fixtureResource = "/verify/canonical_fixture.json" + + override val allRows: Vector[Tuple] = { + val stream = Option(getClass.getResourceAsStream(fixtureResource)) + .getOrElse(sys.error(s"canonical fixture not found on classpath: $fixtureResource")) + val root = + try new ObjectMapper().readTree(stream) + finally stream.close() + root + .elements() + .asScala + .map { node => + val b = Tuple.builder(schema) + schema.getAttributes.foreach { attr => + val cell = node.get(attr.getName) + require(cell != null, s"fixture row missing column '${attr.getName}'") + val value: AnyRef = attr.getType match { + case AttributeType.INTEGER => Int.box(cell.asInt()) + case AttributeType.LONG => Long.box(cell.asLong()) + case AttributeType.DOUBLE => Double.box(cell.asDouble()) + case AttributeType.BOOLEAN => Boolean.box(cell.asBoolean()) + case AttributeType.TIMESTAMP => Timestamp.valueOf(cell.asText()) + case _ => cell.asText() // STRING + } + b.add(attr, value) + } + b.build() + } + .toVector + } + + // Each port takes two thirds of the table, from opposite ends, so the ports + // overlap by the remaining third and no row sits outside both. The overlap is + // what stops joins and set ops passing by hash coincidence: ports holding the + // same rows make intersect and union the same answer, and disjoint ports make + // both empty, which a broken operator produces too. Two thirds rather than a + // fixed count so a row added to the JSON widens the windows instead of falling + // off the end, and a port stays well under the whole table — these windows are + // what every per-test Python run is sized by. At 15 rows this is positions 0-9 + // and 5-14. Rows sit out of id order in the file, so the windows are + // positional — not id ranges. + private def windowSize: Int = allRows.size * 2 / 3 + def port0Rows: Seq[Tuple] = allRows.take(windowSize) + def port1Rows: Seq[Tuple] = allRows.takeRight(windowSize) + + override def rowsFor(port: Int): Seq[Tuple] = if (port == 0) port0Rows else port1Rows + + /** `id` keeps every value: it is what joins and set operations match on, and + * emptying it would change which rows pair up rather than what a null does. + */ + override val keepFilled: Set[String] = Set("id") + + /** This table as the sklearn families read it: the two petal columns and the + * `species` label, and nothing else, because `X = table.drop(target, axis=1)` + * hands `fit` every column that is not the target. The two features separate + * the classes exactly, so an estimator fits them without a tie to break. + */ + val sklearnNumeric: SharedFixture = ProjectedFixture( + this, + Seq("petal_length", "petal_width", "species"), + keepFilled = Set("species") + ) + + /** [[sklearnNumeric]] plus a column an estimator cannot fit. The families that + * narrow `X` to the fittable columns drop nothing on the numeric table, so the + * narrowing runs there with nothing to do. Here it has a column to drop, and + * the two paths narrow in different places: the operator once, ahead of the + * port branch; the standalone script once per port. Each has to drop it on its + * own. + * + * The text column carries no signal about the label, so the fit is the one the + * two petal columns give on their own. + */ + /** This table minus `score`. An operator whose output column is named `score` + * by default cannot run here otherwise: it would create a column the input + * already holds, and the schema refuses the duplicate before the operator + * runs. Dropping the one column puts the DEFAULT config under test, which is + * the config a user gets, rather than a hand-written name chosen to dodge the + * clash. + */ + val withoutScore: SharedFixture = ProjectedFixture( + this, + schema.getAttributeNames.filterNot(_ == "score"), + keepFilled = keepFilled + ) + + /** This table as a scorer reads it when the labels are text: the same pair as + * `species` / `species_pred`, spelled out, and nothing else. The scenario that + * takes it names the two columns itself, since the operator's `@SampleColumn`s + * name the numeric pair this projection does not carry. + */ + val scorerTextLabels: SharedFixture = ProjectedFixture( + this, + Seq("species_name", "species_name_pred"), + keepFilled = Set.empty + ) + + val sklearnNumericWithText: SharedFixture = ProjectedFixture( + this, + Seq("petal_length", "petal_width", "short_text", "species"), + keepFilled = Set("species") + ) + + /** This table as the `countVectorizer=true` path reads it: one text column and + * the same label. `short_text` leads because the model probe feeds a text + * pipeline the frame's first column as a Series. Every row carrying a given + * sentence carries the same `species` (an invariant of the table), so the + * vectorized classes separate exactly, as the numeric pair does. + */ + val sklearnText: SharedFixture = ProjectedFixture( + this, + Seq("short_text", "long_text", "species"), + keepFilled = Set("species") + ) +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala new file mode 100644 index 00000000000..aa471e30f1d --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.workflow.PortIdentity +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.file.Files + +class CanonicalFixtureSpec extends AnyFlatSpec with Matchers { + + "CanonicalFixture" should "have at least 10 rows per port with partial id overlap" in { + CanonicalFixture.port0Rows.size should be >= 10 + CanonicalFixture.port1Rows.size should be >= 10 + val ids0 = CanonicalFixture.port0Rows.map(_.getField[Integer]("id")).toSet + val ids1 = CanonicalFixture.port1Rows.map(_.getField[Integer]("id")).toSet + (ids0 intersect ids1) should not be empty + (ids0 diff ids1) should not be empty + (ids1 diff ids0) should not be empty + } + + // The windows are a rule over the table's size rather than fixed indices, so + // that a row added to the JSON is read by a port instead of falling off the + // end. Fixed indices made that silent: the row was simply never fed to + // anything, and every assertion above still held. + it should "leave no row outside both ports" in { + val onAPort = (CanonicalFixture.port0Rows ++ CanonicalFixture.port1Rows) + .map(_.getField[Integer]("id")) + .toSet + onAPort shouldBe CanonicalFixture.allRows.map(_.getField[Integer]("id")).toSet + } + + it should "contain the canonical value \"1\" in some but not all name cells" in { + val names = CanonicalFixture.port0Rows.map(_.getField[String]("name")) + names.count(_ == "1") should be > 0 + names.count(_ == "1") should be < names.size + } + + it should "expose a valid OHLC block (high >= open/close >= low) for candlestick-style ops" in { + CanonicalFixture.port0Rows.foreach { t => + val o = t.getField[java.lang.Double]("open").doubleValue + val h = t.getField[java.lang.Double]("high").doubleValue + val l = t.getField[java.lang.Double]("low").doubleValue + val c = t.getField[java.lang.Double]("close").doubleValue + h should be >= math.max(o, c) + l should be <= math.min(o, c) + } + } + + it should "keep pvalue strictly inside (0, 1) for probability-domain fields" in { + CanonicalFixture.port0Rows.foreach { t => + val p = t.getField[java.lang.Double]("pvalue").doubleValue + p should (be > 0.0 and be < 1.0) + } + } + + it should "keep log2fc signed and centered (both a negative and a positive present)" in { + val vals = CanonicalFixture.port0Rows.map(_.getField[java.lang.Double]("log2fc").doubleValue) + vals.min should be < 0.0 + vals.max should be > 0.0 + } + + it should "keep ternary components strictly positive" in { + CanonicalFixture.port0Rows.foreach { t => + t.getField[java.lang.Double]("comp_a").doubleValue should be > 0.0 + t.getField[java.lang.Double]("comp_b").doubleValue should be > 0.0 + t.getField[java.lang.Double]("comp_c").doubleValue should be > 0.0 + } + } + + it should "expose uniq_name as globally distinct so name-keyed ops have no duplicates" in { + val names = CanonicalFixture.port0Rows.map(_.getField[String]("uniq_name")) + names.distinct.size shouldBe names.size + } + + it should "expose a valid ternary simplex (positive parts summing to 100)" in { + CanonicalFixture.port0Rows.foreach { t => + val a = t.getField[java.lang.Double]("simplex_a").doubleValue + val b = t.getField[java.lang.Double]("simplex_b").doubleValue + val c = t.getField[java.lang.Double]("simplex_c").doubleValue + a should be > 0.0 + b should be > 0.0 + c should be > 0.0 + (a + b + c) shouldBe 100.0 +- 1e-9 + } + } + + it should "expose trade_date as a real ISO-8601 date (parseable, not the old day-N)" in { + CanonicalFixture.port0Rows.foreach { t => + val d = t.getField[String]("trade_date") + noException should be thrownBy java.time.LocalDate.parse(d) + } + } + + it should "expose edge_pair as single-rooted 2-element list literals" in { + // Every cell is "[0, child]" → parses to a 2-list rooted at 0, so TreePlot + // builds one connected tree instead of an error page. + CanonicalFixture.port0Rows.foreach { t => + t.getField[String]("edge_pair") should fullyMatch regex """\[0, \d+\]""" + } + } + + it should "expose overlapping node_src/node_dst so graph ops have drawable edges" in { + val src = CanonicalFixture.port0Rows.map(_.getField[String]("node_src")).toSet + val dst = CanonicalFixture.port0Rows.map(_.getField[String]("node_dst")).toSet + (src intersect dst) should not be empty + } + + it should "expose finish_ts strictly after start_ts (non-degenerate Gantt bar)" in { + CanonicalFixture.port0Rows.foreach { t => + val s = t.getField[java.sql.Timestamp]("start_ts") + val f = t.getField[java.sql.Timestamp]("finish_ts") + f.after(s) shouldBe true + } + } + + it should "round-trip TIMESTAMP columns losslessly through TupleIO (write then read)" in { + val root = Files.createTempDirectory("canonical-fixture-ts-") + val path = CanonicalFixture.writeInputs(root, inputPortCount = 1)(PortIdentity(0)) + val schema = TupleIO.readSchemaSidecar(path) + val rows = TupleIO.readTuples(path, schema).toList + rows should not be empty + val read = rows.head + val orig = CanonicalFixture.port0Rows.head + // The JDBC-string codec is the exact inverse of Timestamp.toString, so the + // value read back equals the value written — no timezone drift. + read.getField[java.sql.Timestamp]("start_ts") shouldBe orig.getField[java.sql.Timestamp]( + "start_ts" + ) + read.getField[java.sql.Timestamp]("finish_ts") shouldBe orig.getField[java.sql.Timestamp]( + "finish_ts" + ) + } + + it should "expose non-empty short_text sentences for text-classification ops" in { + CanonicalFixture.port0Rows.foreach { t => + t.getField[String]("short_text").trim should not be empty + } + } + + it should "expose long_text with several sentences so summarization is non-trivial" in { + CanonicalFixture.port0Rows.foreach { t => + val txt = t.getField[String]("long_text") + // multiple sentence-terminating periods → real content to condense + txt.count(_ == '.') should be >= 2 + } + } + + // The sklearn families fit `species` against the petal columns, so the table + // has to hold up as training data. The estimators that cross-validate pass no + // fold count and so take sklearn's default of five, and a class of fewer than + // five rows leaves a fold holding none of it — no error, just a warning and a + // fold that asks nothing. A label the features cannot separate is the other + // half: it leaves the fit breaking ties, which is where two paths drift apart. + it should "expose species as a petal-separable label with enough members to fold on" in { + val rows = CanonicalFixture.allRows + val byClass = rows.groupBy(_.getField[java.lang.Integer]("species").intValue) + byClass.keySet shouldBe Set(0, 1) + byClass.values.foreach(_.size should be >= 5) + rows.foreach { t => + val large = t.getField[java.lang.Double]("petal_length").doubleValue >= 3.9 + t.getField[java.lang.Integer]("species").intValue shouldBe (if (large) 1 else 0) + } + } + + // `species_pred` exists so the scorer has a real pair to compare. All four + // cells of the confusion matrix have to be occupied: a perfect prediction + // scores every metric at 1.0, and one that never calls a class leaves that + // class's precision undefined — either way the metrics stop telling the two + // code paths apart. + it should "hold a species_pred that is right on most rows and wrong on some" in { + val cells = CanonicalFixture.allRows + .map(t => + ( + t.getField[java.lang.Integer]("species").intValue, + t.getField[java.lang.Integer]("species_pred").intValue + ) + ) + .distinct + cells should contain theSameElementsAs Seq((0, 0), (0, 1), (1, 0), (1, 1)) + } + + // The text pair exists to run the scorer's string-label path on the same + // arrangement the numeric pair gives it. Spelling out a different prediction + // would make the two paths score differently for a reason that has nothing to + // do with the label being text. + it should "spell out the species pair without changing what it says" in { + val name = Map(0 -> "setosa", 1 -> "versicolor") + CanonicalFixture.allRows.foreach { t => + t.getField[String]("species_name") shouldBe name( + t.getField[java.lang.Integer]("species").intValue + ) + t.getField[String]("species_name_pred") shouldBe name( + t.getField[java.lang.Integer]("species_pred").intValue + ) + } + } + + // The countVectorizer=true path fits the same label on short_text alone, and a + // sentence appearing under both labels makes that set unlearnable. + it should "keep every short_text sentence inside one species" in { + CanonicalFixture.allRows + .groupBy(_.getField[String]("short_text")) + .foreach { + case (sentence, rows) => + withClue(s"$sentence: ") { + rows.map(_.getField[java.lang.Integer]("species")).distinct.size shouldBe 1 + } + } + } + + it should "expose iris petal columns in a realistic centimetre range" in { + CanonicalFixture.port0Rows.foreach { t => + val len = t.getField[java.lang.Double]("petal_length").doubleValue + val wid = t.getField[java.lang.Double]("petal_width").doubleValue + len should (be > 0.0 and be < 8.0) + wid should (be > 0.0 and be < 3.0) + } + } + + // A single-token row would make split/explode a no-op, so both windows need + // rows that fan out AND a row that doesn't — the two branches of an unnest. + it should "expose csv_list as a clean delimited list with varying token counts" in { + Seq(CanonicalFixture.port0Rows, CanonicalFixture.port1Rows).foreach { rows => + val tokenCounts = rows.map { t => + val raw = t.getField[String]("csv_list") + raw should not startWith "," + raw should not endWith "," + val tokens = raw.split(",", -1) + tokens.foreach(_.trim should not be empty) + tokens.length + } + tokenCounts.min shouldBe 1 + tokenCounts.max should be > 1 + } + } + + // A case flag is only worth sweeping where flipping it changes WHICH rows match, + // which needs rows of all three kinds in EVERY window a test reads — hence the + // per-port check rather than one over the whole table. + it should "expose mixed_case with lower, upper and letterless rows on every port" in { + Seq(CanonicalFixture.port0Rows, CanonicalFixture.port1Rows).foreach { rows => + val values = rows.map(_.getField[String]("mixed_case")) + values.count(v => v.exists(_.isLower)) should be > 0 + values.count(v => v.exists(_.isUpper) && !v.exists(_.isLower)) should be > 0 + values.count(v => !v.exists(_.isLetter)) should be > 0 + } + } + + it should "write one JSONL fixture per requested input port" in { + val root = Files.createTempDirectory("canonical-fixture-") + val inputs = CanonicalFixture.writeInputs(root, inputPortCount = 2) + inputs.keySet shouldBe Set(PortIdentity(0), PortIdentity(1)) + inputs.values.foreach(p => Files.size(p) should be > 0L) + } + + it should "reject unsupported port counts" in { + val root = Files.createTempDirectory("canonical-fixture-") + an[IllegalArgumentException] should be thrownBy + CanonicalFixture.writeInputs(root, inputPortCount = 3) + } + + it should "empty every column but id exactly once in the gapped table" in { + val rows = CanonicalFixture.emptyOneCellPerColumn(CanonicalFixture.port0Rows) + rows.size shouldBe CanonicalFixture.port0Rows.size + + CanonicalFixture.schema.getAttributes.foreach { attr => + val empties = rows.count(_.getField[AnyRef](attr.getName) == null) + // id carries the joins, so it keeps every value; everything else gets one + // hole, which is what makes the case a null case rather than an empty table. + if (attr.getName == "id") empties shouldBe 0 + else empties shouldBe 1 + } + } + + it should "leave every row in the gapped table with something in it" in { + val names = CanonicalFixture.schema.getAttributes.map(_.getName) + CanonicalFixture.emptyOneCellPerColumn(CanonicalFixture.port0Rows).foreach { t => + // A wholly empty row would test the operator's handling of an empty table + // instead, and would say nothing about a null beside a filled neighbour. + names.count(n => t.getField[AnyRef](n) != null) should be > 0 + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala new file mode 100644 index 00000000000..17926e593b2 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.util.python.PythonWorkerPool + +import java.nio.file.{Files, Path, StandardCopyOption} +import scala.collection.mutable.ArrayBuffer +import scala.sys.process._ + +/** + * Runs the Python comparator (`compare.py`) on two JSONL files emitted by + * [[OpExecHarness]] (actual) and [[StandaloneRunner]] (expected). The + * comparator uses `pandas.testing.assert_frame_equal` with `check_like=True` + * and `check_dtype=False` so row/column-order differences and the + * pandas-int64/float64 coercion that happens when JSONL round-trips through + * `pd.read_json` don't trigger false negatives. Float tolerance: `rtol=1e-5`. + * + * Throws [[ComparatorMismatchException]] on any non-zero exit code (the + * pandas diff is in `stderr` on the exception). Successful comparisons + * return unit. + * + * Python resolution mirrors [[StandaloneRunner.resolvePython]]: + * `UDF_PYTHON_PATH` env var first, else `python3.12` on PATH. + */ +object Comparator extends LazyLogging { + + // Resource path is absolute (leading slash) so getResourceAsStream resolves + // against the classpath root regardless of caller's package. + private val ScriptResourcePath = "/python/compare.py" + + def assertEqual( + actual: Path, + expected: Path, + orderSensitive: Boolean = true, + ignoreColumns: Seq[String] = Seq.empty, + modelColumns: Seq[String] = Seq.empty, + probePath: Option[Path] = None, + pythonExe: String = resolvePython() + ): Unit = { + val (exit, stdout, stderr) = + compare(actual, expected, orderSensitive, ignoreColumns, modelColumns, probePath, pythonExe) + if (exit != 0) { + throw new ComparatorMismatchException( + actual = actual, + expected = expected, + exitCode = exit, + stdout = stdout, + stderr = stderr + ) + } + } + + // Prefer a pooled persistent worker (imports pandas once via `compare.py + // --serve`, so the ~214 ms import isn't repaid per comparison — the diff + // itself is ~ms). A rare hard worker crash falls back to the one-shot CLI so + // behavior is never worse than the original path. Both invoke the same + // `_run_comparison`, so results are identical. + private def compare( + actual: Path, + expected: Path, + orderSensitive: Boolean, + ignoreColumns: Seq[String], + modelColumns: Seq[String], + probePath: Option[Path], + pythonExe: String + ): (Int, String, String) = { + if (PythonWorkerPool.enabled) { + try { + val req = objectMapper.createObjectNode() + req.put("actual", actual.toString) + req.put("expected", expected.toString) + req.put("unordered", !orderSensitive) + val ignoreArr = req.putArray("ignoreCols") + ignoreColumns.foreach(ignoreArr.add) + val modelArr = req.putArray("modelCols") + modelColumns.foreach(modelArr.add) + // --probe only applies with --model-cols (mirrors the CLI's guard). + probePath.filter(_ => modelColumns.nonEmpty) match { + case Some(p) => req.put("probe", p.toString) + case None => req.putNull("probe") + } + val o = PythonWorkerPool.run(ScriptResourcePath, Seq("--serve"), pythonExe, req) + return (o.exit, o.stdout, o.stderr) + } catch { + case e: PythonWorkerPool.WorkerDiedException => + logger.warn( + s"Comparator worker unavailable; falling back to one-shot CLI: ${e.getMessage}" + ) + } + } + runCli(actual, expected, orderSensitive, ignoreColumns, modelColumns, probePath, pythonExe) + } + + // Original one-subprocess-per-comparison CLI path. Retained as the fallback + // and as the behavior selected by TEXERA_TEST_PYTHON_WORKER=0. + private def runCli( + actual: Path, + expected: Path, + orderSensitive: Boolean, + ignoreColumns: Seq[String], + modelColumns: Seq[String], + probePath: Option[Path], + pythonExe: String + ): (Int, String, String) = { + val scriptPath = extractScript() + val outBuf = ArrayBuffer.empty[String] + val errBuf = ArrayBuffer.empty[String] + val procLogger = ProcessLogger(line => outBuf += line, line => errBuf += line) + // --unordered tells compare.py to lex-sort both DataFrames by all columns + // before assert_frame_equal — needed for set-semantics ops whose JVM + // emission order doesn't match the pandas equivalent. Default stays + // positional so deterministic-order ops still catch row-order regressions. + // --ignore-cols drops opaque columns whose value isn't compared. + // --model-cols + --probe compare a model column by behavior: unpickle both + // sides and assert their predictions on the probe feature set match (two + // independently-trained models are functionally equal but not bit-equal). + val baseArgs = Seq(pythonExe, scriptPath.toString) + val flagArgs = if (!orderSensitive) Seq("--unordered") else Seq.empty + val ignoreArgs = + if (ignoreColumns.nonEmpty) Seq("--ignore-cols", ignoreColumns.mkString(",")) else Seq.empty + val modelArgs = + if (modelColumns.nonEmpty) Seq("--model-cols", modelColumns.mkString(",")) else Seq.empty + val probeArgs = + probePath + .filter(_ => modelColumns.nonEmpty) + .map(p => Seq("--probe", p.toString)) + .getOrElse(Seq.empty) + val cmd = + baseArgs ++ flagArgs ++ ignoreArgs ++ modelArgs ++ probeArgs ++ Seq( + actual.toString, + expected.toString + ) + val exit = Process(cmd).!(procLogger) + (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) + } + + // Resources may live inside a jar at runtime; copy to a temp file so Python + // can exec it. deleteOnExit so test runs don't accumulate /tmp clutter. + private def extractScript(): Path = { + val stream = getClass.getResourceAsStream(ScriptResourcePath) + require( + stream != null, + s"compare.py not found on classpath at $ScriptResourcePath" + ) + try { + val tmp = Files.createTempFile("compare-", ".py") + Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING) + tmp.toFile.deleteOnExit() + tmp + } finally stream.close() + } + + private def resolvePython(): String = + sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") +} + +final class ComparatorMismatchException( + val actual: Path, + val expected: Path, + val exitCode: Int, + val stdout: String, + val stderr: String +) extends RuntimeException( + s"""DataFrame mismatch (compare.py exit $exitCode): + | actual: $actual + | expected: $expected + |--- stderr --- + |$stderr""".stripMargin + ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala new file mode 100644 index 00000000000..dc5e5fda029 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} +import org.apache.texera.amber.translator.verify.tags.IntegrationTest +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} + +// Tagged @IntegrationTest: Comparator.assertEqual shells out to compare.py, so +// this spec needs Python and must run in the Python-provisioned integration job. +@IntegrationTest +class ComparatorSpec extends AnyFlatSpec with Matchers { + + private val schema: Schema = Schema() + .add(new Attribute("id", AttributeType.INTEGER)) + .add(new Attribute("name", AttributeType.STRING)) + + private val idAttr = new Attribute("id", AttributeType.INTEGER) + private val nameAttr = new Attribute("name", AttributeType.STRING) + + private def row(id: Int, name: String): Tuple = + Tuple + .builder(schema) + .add(idAttr, Int.box(id)) + .add(nameAttr, name) + .build() + + private def writeJsonl(dir: Path, name: String, rows: Seq[Tuple]): Path = { + val p = dir.resolve(name) + TupleIO.writeTuples(p, rows.iterator, schema) + p + } + + "Comparator.assertEqual" should "pass when JSONL files contain identical rows" in { + val dir = Files.createTempDirectory("comparator-spec-equal-") + val rows = Seq(row(1, "alice"), row(2, "bob")) + val a = writeJsonl(dir, "a.jsonl", rows) + val b = writeJsonl(dir, "b.jsonl", rows) + noException should be thrownBy Comparator.assertEqual(a, b) + } + + it should "throw ComparatorMismatchException when JSONL files differ" in { + val dir = Files.createTempDirectory("comparator-spec-diff-") + val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) + val b = writeJsonl(dir, "b.jsonl", Seq(row(1, "alice"), row(2, "carol"))) + intercept[ComparatorMismatchException] { + Comparator.assertEqual(a, b) + } + } + + it should "treat row-reordered files as unequal under positional comparison" in { + val dir = Files.createTempDirectory("comparator-spec-reorder-strict-") + val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) + val b = writeJsonl(dir, "b.jsonl", Seq(row(2, "bob"), row(1, "alice"))) + intercept[ComparatorMismatchException] { + Comparator.assertEqual(a, b) + } + } + + it should "treat row-reordered files as equal under orderSensitive=false" in { + val dir = Files.createTempDirectory("comparator-spec-reorder-loose-") + val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) + val b = writeJsonl(dir, "b.jsonl", Seq(row(2, "bob"), row(1, "alice"))) + noException should be thrownBy Comparator.assertEqual(a, b, orderSensitive = false) + } + + it should "still report value mismatches under orderSensitive=false" in { + // orderSensitive=false relaxes row ORDER, not row CONTENT — a genuinely + // different cell must still fail. + val dir = Files.createTempDirectory("comparator-spec-reorder-content-diff-") + val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) + val b = writeJsonl(dir, "b.jsonl", Seq(row(2, "bob"), row(1, "carol"))) + intercept[ComparatorMismatchException] { + Comparator.assertEqual(a, b, orderSensitive = false) + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala new file mode 100644 index 00000000000..a2a55e4235a --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.source.SourceOperatorDescriptor +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Reports the harness verification tier per discovered [[StandaloneCodeGenerator]] + * operator: RUNNABLE (auto) / RUNNABLE (curated) / FLAG (reason). The `info` + * table printed by this spec is the coverage artifact shown at the handoff demo. + * + * Hard-asserts the must-run set: the operators that must appear as RUNNABLE for + * the demo to be credible. Flagged operators are always reported with a reason — + * never silently passed, and neither is a single withheld KIND of run + * (see [[reportWithheldRuns]]). + */ +class ConfigCoverageSpec extends AnyFlatSpec with Matchers { + + // The ops the demo must show as runnable; extend as triage flips flags. + private val mustRun = Set( + "IntersectOpDesc", + "DifferenceOpDesc", + "SymmetricDifferenceOpDesc", + "HashJoinOpDesc", + "SpecializedFilterOpDesc", + "SortOpDesc", + "LimitOpDesc" + ) + + "the harness" should "classify every discovered operator into a tier and report coverage" in { + val operators = OperatorBehaviorSpec.discoverStandaloneOperators() + + val rows = operators.map { opClass => + val name = opClass.getSimpleName + val kind = + if (classOf[SourceOperatorDescriptor].isAssignableFrom(opClass)) "source" + else if (classOf[PythonOperatorDescriptor].isAssignableFrom(opClass)) "python-udf" + else "jvm" + val tier = + if (kind == "source") { + if (SourceCategoryRunner.canRun(opClass)) + s"RUNNABLE (${SourceCategoryRunner.tier(opClass)})" + else s"FLAG (${SourceCategoryRunner.flagReason(opClass)})" + } else + TransformVerificationRunner.disposition(opClass) match { + case TransformVerificationRunner.Runnable(t) => s"RUNNABLE ($t)" + case TransformVerificationRunner.Flagged(reason) => s"FLAG ($reason)" + } + (name, kind, tier) + } + + val runnable = rows.count(_._3.startsWith("RUNNABLE")) + info(s"Coverage: $runnable/${rows.size} operators runnable, ${rows.size - runnable} flagged") + Seq("jvm", "python-udf", "source").foreach { k => + val of = rows.filter(_._2 == k) + info(s" $k: ${of.count(_._3.startsWith("RUNNABLE"))}/${of.size} runnable") + } + rows.sortBy { case (n, k, t) => (!t.startsWith("RUNNABLE"), k, n) }.foreach { + case (name, kind, tier) => info(f" $tier%-50s [$kind%-10s] $name") + } + + reportWithheldRuns(operators) + + val failedTargets = rows.collect { + case (name, _, tier) if mustRun.contains(name) && !tier.startsWith("RUNNABLE") => + s"$name → $tier" + } + withClue(s"must-run operators not runnable: $failedTargets") { + failedTargets shouldBe empty + } + } + + /** RUNNABLE is per operator, but a runnable operator can still be missing one + * KIND of run. Report those too, or the table reads as fuller than it is. + * + * Split by whether anyone should be waiting: a pending fix is a line someone + * deletes when an issue closes, by design is an answer. Both name the operator, + * so a family entry expands to the estimators it actually covers. + */ + private def reportWithheldRuns(operators: Seq[Class[_ <: LogicalOp]]): Unit = { + import TransformVerificationRunner._ + + val withheld = for { + opClass <- operators + (kind, reason) <- withheldRunsFor(opClass) + } yield (opClass.getSimpleName, kind, reason) + + val pending = withheld.collect { case (n, k, PendingFix(issue)) => (n, k, issue) } + val byDesign = withheld.collect { case (n, k, ByDesign(why)) => (n, k, why) } + info( + s"Runs withheld: ${pending.size} pending a fix, " + + s"${byDesign.size} not applicable by design" + ) + pending.sorted.foreach { + case (name, kind, issue) => info(f" PENDING $kind%-20s $name%-45s $issue") + } + byDesign.sorted.foreach { + case (name, kind, why) => info(f" BY-DESIGN $kind%-20s $name%-45s $why") + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala new file mode 100644 index 00000000000..853c41c2534 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala @@ -0,0 +1,2015 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.annotation.{ + JsonIgnore, + JsonIgnoreProperties, + JsonProperty, + JsonSubTypes +} +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator +import org.apache.texera.amber.operator.metadata.annotations.{ + AutofillAttributeName, + AutofillAttributeNameList, + AutofillAttributeNameOnPort1, + CommonOpDescAnnotation, + HideAnnotation, + SampleColumn +} +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import java.lang.reflect.{Field, Modifier, ParameterizedType, Type, TypeVariable} +import javax.validation.constraints.{DecimalMin, Min} +import scala.collection.mutable +import scala.jdk.CollectionConverters._ +import scala.util.Try + +/** + * Produces a valid configuration for an operator automatically, from the + * metadata the operator already carries — field defaults, enums, and the + * `@AutofillAttributeName` annotation family (which marks a field as "a column + * name from input port N"). This is the baseline layer of the combined + * config-generation plan: every registered operator gets a runnable config with + * no per-operator handler. + * + * We only need a *valid* config, not a *meaningful* one — both verification + * paths get the identical OpDesc and are compared to each other, so a + * degenerate-but-valid config still tests translation fidelity. Free-form value + * fields are filled with a canonical value (see [[CanonicalString]]) that the + * synthetic dataset is built to contain, so the operator actually does + * something rather than matching nothing. + * + * Strategy: reflect over the operator's config fields (those carrying + * `@JsonProperty` or an autofill annotation), build a JSON object of + * field → value, and let Jackson deserialize it into the OpDesc. Using the + * same `objectMapper` Texera uses everywhere means enums (`@JsonValue`), + * `Option`, and `@JsonCreator` nested objects are handled by existing, + * battle-tested deserialization rather than bespoke reflection. + */ +object ConfigGenerator { + + /** Canonical literal for free-form STRING fields; present in the synthetic + * dataset so filters/comparisons actually match rows. + */ + private val CanonicalString = "1" + + /** Row count used to size the numeric "middle of the range" fallback when a + * caller doesn't supply one (real verification callers pass the fixture's + * actual row count). See [[numericFill]]. + */ + val DefaultRowCount = 10 + + /** + * @param opClass the operator descriptor class to configure. + * @param inputSchemas schema present at each 0-based input port; supplies the + * column names that `@AutofillAttributeName*` fields draw + * from. + * @return Right(configured opDesc), or Left(reason) if a required field can't + * be filled from the available metadata (the operator is then + * reported as uncovered rather than silently passed). + */ + def generate( + opClass: Class[_ <: LogicalOp], + inputSchemas: Map[Int, Schema], + rowCount: Int = DefaultRowCount + ): Either[String, LogicalOp] = { + buildObject(opClass, inputSchemas, rowCount).flatMap { node => + // LogicalOp is polymorphic (@JsonTypeInfo on `operatorType`); Jackson needs + // the registered type id to deserialize the concrete subtype. + typeNameByClass.get(opClass) match { + case Some(typeName) => node.put("operatorType", typeName) + case None => + return Left(s"${opClass.getSimpleName} not registered in LogicalOp @JsonSubTypes") + } + Try(objectMapper.treeToValue(node, opClass)).toEither.left + .map(e => s"deserialization failed: ${e.getMessage}") + } + } + + /** + * Like [[generate]], but also sweeps every enum field: returns the base + * config plus one variant per non-default enum value (one enum flipped at a + * time — linear, NOT the combinatorial product). Lets the runner exercise + * each enum branch (e.g. LineChart's line mode = line / dots / line+dots) + * instead of only the default. The label identifies the flipped value. + */ + def generateVariants( + opClass: Class[_ <: LogicalOp], + inputSchemas: Map[Int, Schema], + rowCount: Int = DefaultRowCount, + pinned: Map[String, JsonNode] = Map.empty, + switches: Map[String, JsonNode] = Map.empty + ): Either[String, Seq[(String, LogicalOp)]] = + typeNameByClass.get(opClass) match { + case None => Left(s"${opClass.getSimpleName} not registered in LogicalOp @JsonSubTypes") + case Some(typeName) => + val used = mutable.Set.empty[(Int, String)] + buildObject(opClass, inputSchemas, used, rowCount, pinned = pinned).flatMap { baseNode => + baseNode.put("operatorType", typeName) + pinned.foreach { case (field, value) => baseNode.set[JsonNode](field, value) } + applyAll( + opClass, + baseNode, + None, + allVariants( + opClass, + baseNode, + inputSchemas, + used, + rowCount, + pinned = pinned.keySet, + switches = switches + ) + ) + } + } + + /** + * Enum-sweep an ALREADY-configured op (e.g. a curated handler's OpDesc): + * serialize it to JSON, then return the base op plus one variant per + * non-default enum value found anywhere in it (including inside lists with + * more than one element). Lets curated fixtures cover every enum branch too, + * not just the single value the handler hard-coded. + */ + def variantsOf(opDesc: LogicalOp): Either[String, Seq[(String, LogicalOp)]] = { + val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] + // base = the original op (preserve the curated config exactly); variants are + // deserialized from the JSON with one enum flipped. + nodeOf(opDesc).flatMap(node => + applyAll(opClass, node, Some(opDesc), Variant.Base +: enumVariants(opClass, node)) + ) + } + + /** + * [[variantsOf]] plus the two multi-knob variants [[generateVariants]] gives an + * auto-configured op: `optionals` and `hostileText` (see [[extraVariants]]). + * + * A separate entry point rather than a widening of [[variantsOf]] so a caller + * states which it wants, and a failure points at one of them. Curated fixtures + * are the reason it exists: a hand-written config is the ONLY config its + * operator ever runs, so without this its optional knobs stay at their defaults + * and nothing ever splices a quote into the code it generates. + * + * `inputSchemas` describes the op's OWN inputs — a curated handler writes its + * own fixture, so this is not necessarily the canonical one. + */ + def fullVariantsOf( + opDesc: LogicalOp, + inputSchemas: Map[Int, Schema], + rowCount: Int = DefaultRowCount, + sweepEnums: Boolean = true + ): Either[String, Seq[(String, LogicalOp)]] = { + val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] + for { + node <- nodeOf(opDesc) + variants <- fullVariantEditsOf(opDesc, inputSchemas, rowCount, sweepEnums) + ops <- applyAll(opClass, node, Some(opDesc), variants) + } yield ops + } + + /** + * What [[fullVariantsOf]] runs, as the edits themselves rather than the finished + * ops — for a caller that has to REBUILD its fixture per variant and so must + * apply them to a FRESH op. A source is that caller: its exported script reads + * the file by bare name out of the directory it runs in, so every variant needs + * its own directory and its own copy, produced by calling the handler again. + * + * `sweepEnums = false` keeps the fills but drops the enum sweep, for a fixture + * whose enums are cross-constrained with the data it holds — flipping one then + * describes a table the fixture is not. + */ + def fullVariantEditsOf( + opDesc: LogicalOp, + inputSchemas: Map[Int, Schema], + rowCount: Int = DefaultRowCount, + sweepEnums: Boolean = true + ): Either[String, Seq[Variant]] = { + val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] + nodeOf(opDesc).map { node => + val used = occupiedColumns(opClass, node, inputSchemas) + allVariants(opClass, node, inputSchemas, used, rowCount, sweepEnums) + } + } + + /** `opDesc` with `variant`'s edits applied. The base variant carries no edits and + * hands the instance straight back, so a curated config is never round-tripped + * through JSON just to be left unchanged. + */ + def applyVariant(opDesc: LogicalOp, variant: Variant): Either[String, LogicalOp] = + if (variant.at.isEmpty) Right(opDesc) + else + nodeOf(opDesc).flatMap { node => + variant.at.foreach { case (pointer, value) => setAtPointer(node, pointer, value) } + deserialize(node, opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]]) + } + + /** One named configuration, as the pointer → value edits that turn a base config + * into it. Applied to one clone of that base. + */ + final case class Variant(label: String, at: Seq[(String, JsonNode)]) + + object Variant { + + /** The base config itself — no edits, so [[applyVariant]] returns it unchanged. */ + val Base: Variant = Variant("default", Seq.empty) + } + + /** Every variant of `baseNode`: the config itself, one per non-default enum value, + * then the two multi-knob fills. + */ + private def allVariants( + opClass: Class[_ <: LogicalOp], + baseNode: ObjectNode, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + sweepEnums: Boolean = true, + pinned: Set[String] = Set.empty, + switches: Map[String, JsonNode] = Map.empty + ): Seq[Variant] = + Variant.Base +: ((if (sweepEnums) + enumVariants( + opClass, + baseNode, + pinned, + FillContext(schemas, mutable.Set.empty ++ used, rowCount) + ) + else Seq.empty) ++ + extraVariants(opClass, baseNode, schemas, used, rowCount, switches)) + + /** The two multi-knob variants, so called because each moves every knob of its kind + * at once. An operator's knobs are worth exercising, but bisecting a rare failure + * by hand costs less than a run per field: all optional knobs are filled together, + * and all free-text knobs take the hostile value together. + * + * A row from the UI's `+` button is one of those optional knobs, not a variant of + * its own: for a list the base leaves empty it IS the "now it is set" case, exactly + * like a scalar going from unset to set. + */ + private def extraVariants( + opClass: Class[_ <: LogicalOp], + baseNode: ObjectNode, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + switches: Map[String, JsonNode] = Map.empty + ): Seq[Variant] = + Seq( + merged( + "optionals", { + // One counter and one `used` set for the whole variant: the three walks below + // land in ONE config, so a column taken by any of them is taken for all, and + // restarting the counter per walk gave two rows the same value. A copy, so the + // base pass's own set is left alone. + val ordinal = new Ordinal + val taken = mutable.Set.empty[(Int, String)] ++ used + optionalColumnFills(opClass, schemas, taken, baseNode) ++ + optionalScalarFills(opClass, baseNode, "", schemas, taken, rowCount, ordinal) ++ + extraRowFills(opClass, baseNode, schemas, taken, rowCount, ordinal) ++ + switches.toSeq.sortBy(_._1).map { + case (field, value) => Variant(s"$field=${value.asText}", Seq((s"/$field", value))) + } + } + ), + merged("hostileText", numbered(hostileTextFills(opClass, baseNode, ""))) + ).flatten + + /** Apply each variant to its own clone of `baseNode` and read the result back as an + * op. `base` is the instance to hand back for the unedited variant, when the caller + * has one whose exact state matters (a curated fixture); `None` deserializes it + * from `baseNode` like any other. + */ + private def applyAll( + opClass: Class[_ <: LogicalOp], + baseNode: ObjectNode, + base: Option[LogicalOp], + variants: Seq[Variant] + ): Either[String, Seq[(String, LogicalOp)]] = { + val results = variants.map { variant => + base.filter(_ => variant.at.isEmpty) match { + case Some(op) => Right((variant.label, op)) + case None => + val clone = baseNode.deepCopy() + variant.at.foreach { case (pointer, value) => setAtPointer(clone, pointer, value) } + deserialize(clone, opClass).map((variant.label, _)) + } + } + results.collectFirst { case Left(err) => err }.toLeft(results.collect { case Right(ok) => ok }) + } + + /** An already-configured op as the JSON this generator edits: its serialized form, + * carrying the polymorphic type id Jackson needs to read the concrete subtype back. + */ + private def nodeOf(opDesc: LogicalOp): Either[String, ObjectNode] = { + val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] + objectMapper.valueToTree[JsonNode](opDesc) match { + case node: ObjectNode => + if (!node.has("operatorType")) + typeNameByClass.get(opClass).foreach(node.put("operatorType", _)) + Right(node) + case _ => Left(s"${opClass.getSimpleName} did not serialize to a JSON object") + } + } + + /** The (port, column) pairs an already-configured op's column pickers already hold, + * as the `used` set the optional-knob fill resolves against — so a knob it fills + * lands on a column the fixture is not using yet, the same rule the base pass keeps + * for sibling pickers. Without it a curated x/y and a filled-in optional colour all + * collapse onto one column. + * + * Walks the fixture's nested rows too, not just its top-level fields: the picker an + * appended row has to differ from usually lives in the rows ALREADY there (a + * projection's column list), and re-picking one of those asks the operator for the + * same output column twice. + */ + private def occupiedColumns( + clazz: Class[_], + node: JsonNode, + schemas: Map[Int, Schema], + path: String = "" + ): mutable.Set[(Int, String)] = { + val used = mutable.Set.empty[(Int, String)] + configFields(clazz).foreach { f => + val childPath = pointerOf(f, path) + rowType(f) match { + case Some(row) => + rowPaths(f, node.at(childPath), childPath) + .foreach(rowPath => used ++= occupiedColumns(row, node, schemas, rowPath)) + case None if hasAutofill(f) => + val port = autofillSpec(f).map(_.port).getOrElse(0) + val columns = + schemas.get(port).map(_.getAttributes.map(_.getName).toSet).getOrElse(Set.empty) + val held = node.at(childPath) + val values = if (held.isArray) held.elements().asScala.toSeq else Seq(held) + values.filter(_.isTextual).map(_.asText).filter(columns).foreach(c => used += ((port, c))) + case None => () + } + } + used + } + + /** One fill per OPTIONAL column knob, which [[decide]] leaves unset. Unset is the + * right base config — it is what most workflows carry — but it also means the + * branch each generator emits for a knob that IS set never runs on either path, + * so the two hand-written branches are never compared. + * + * Resolved against the `used` set the whole variant shares, so the column a knob + * takes differs from what the config already reads. A list knob takes a SINGLE + * column: the "every matching column" fill suits a required axes list, not an + * optional narrowing one — all thirty columns as group-by keys would make every row + * its own group. + * + * A knob the config ALREADY points at a column is left alone. That never happens + * to the auto base — [[decide]] skips every optional picker, so each one still + * holds the value a fresh instance has — but a curated config picks its columns + * deliberately, and overwriting one would discard the fixture's whole point. + * + * Only the operator's OWN fields: a picker inside a nested row is filled by + * [[rowFills]], on the row walk that knows which row it belongs to. + */ + private def optionalColumnFills( + clazz: Class[_], + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + baseNode: JsonNode + ): Seq[Variant] = + configFields(clazz).filter(hasAutofill).filterNot(hiddenBySibling(_, baseNode)).flatMap { f => + columnFill(clazz, f, baseNode, pointerOf(f, ""), schemas, used, baseNode).map { + case (pointer, value) => + // A list knob holds its one column in an array; name the column either way. + val col = if (value.isArray) value.path(0).asText else value.asText + Variant(s"${pointer.stripPrefix("/")}=$col", Seq((pointer, value))) + } + } + + /** One fill per `+`-row list, appending ONE MORE row than the base carries: the first + * row for an optional list (empty, as the UI starts it), a second for a required one. + * + * For an optional list that is the point — its rows are otherwise never populated. + * For a required one it reaches only the code BETWEEN rows (the separator each path + * joins them with, whatever an operator does with several at once); NOT a mis-indexed + * value, since both generators read every value off the loop variable. + * + * The row is built by the same pass as the first, against the `used` set the whole + * variant shares, so its column knobs land on columns nothing else is reading. + */ + private def extraRowFills( + clazz: Class[_], + baseNode: JsonNode, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + ordinal: Ordinal + ): Seq[Variant] = { + // The appended row is built outside [[buildObject]]'s walk, so what that walk + // carries down to a row has to be handed over here as well: without them the new + // row's type-variable fields and schema-ruled fields come back empty, and the + // variant then fails on a value this generator left out rather than on anything + // the operator does. + val ownBindings = typeBindingsOf(clazz) + val ownScope = SchemaScope.of(clazz) + configFields(clazz).flatMap { f => + val childPath = pointerOf(f, "") + val rows = baseNode.at(childPath) + for { + row <- if (isList(f.getType)) elementType(f).toOption.filter(isNestedObject) else None + if rows.isArray + next <- buildObject( + row, + schemas, + used, + rowCount, + elementBindings(f, ownBindings), + ownScope.descend(jsonNameOf(f)) + ).toOption + } yield { + // Fill the new row's own optional knobs too — the `optionals` variant is + // computed against the BASE config, where this row does not exist yet, so + // otherwise the row arrives with every free-value knob at its default, a step + // whose bounds are both empty is dropped by the operator, and an optional column + // picker (which [[decide]] skips) stays null. + rowFills(row, next, "", schemas, used, rowCount, ordinal).foreach { + case (pointer, value) => setAtPointer(next, pointer, value) + } + distinguish( + row, + next, + rows, + elementBindings(f, ownBindings), + ownScope.descend(jsonNameOf(f)), + FillContext(schemas, used, rowCount) + ) + Variant(childPath, Seq((s"$childPath/${rows.size()}", next))) + } + } + } + + /** Move the appended row off a value a row beside it already holds, where holding + * the same one makes the two rows the same row. Every row is built the same way and + * so comes back with the same values, which for a second hyperparameter row means + * `C` twice: not two settings but one written twice, which the emitted Python + * rejects outright as a repeated keyword argument. + * + * Only the field the rest of the row is stated in terms of, which is the one a + * `valueRules` condition reads. A row's other choices are its own business, and two + * lines of a chart drawn in the same style are still two lines. That a knob has to + * differ is something only the schema can say, and this is where it says it. + */ + private def distinguish( + rowClass: Class[_], + next: ObjectNode, + siblings: JsonNode, + bindings: TypeBindings, + scope: SchemaScope, + fill: FillContext + ): Unit = { + val deciding = decidingFields(rowClass, scope) + enumSites(rowClass, next, "", bindings, scope, fill) + .filter(site => deciding.contains(site.pointer.stripPrefix("/"))) + .foreach { site => + val taken = siblings.elements().asScala.map(_.at(site.pointer)).toSet + if (taken.contains(next.at(site.pointer))) + site.values.find(v => !taken.contains(v)).foreach { v => + setAtPointer(next, site.pointer, v) + site.companions(v).foreach { + case (pointer, value) => setAtPointer(next, pointer, value) + } + } + } + } + + /** The fields of `clazz` that some other field's `valueRules` is stated in terms of. + * A hyperparameter row has one, `parameter`, and most objects have none. + */ + private def decidingFields(clazz: Class[_], scope: SchemaScope): Set[String] = + configFields(clazz).iterator + .flatMap(f => scope.child(jsonNameOf(f)).path("valueRules").path("allOf").elements().asScala) + .flatMap(_.path("if").fieldNames().asScala) + .toSet + + /** One variant out of many fills, labelled with the fields it sets. `None` when + * there is nothing to fill, so an operator without such knobs gains no variant. + */ + private def merged(kind: String, fills: Seq[Variant]): Option[Variant] = { + val at = fills.flatMap(_.at) + if (at.isEmpty) None + else { + val names = at.map(_._1.stripPrefix("/")).distinct + val shown = names.mkString(",") + val label = if (shown.length <= 60) shown else s"${names.size} fields" + Some(Variant(s"$kind($label)", at)) + } + } + + /** Extra variants for the OPTIONAL free-value scalar knobs — a number or a + * string the user types in, as opposed to a column picker or a dropdown. + * [[decide]] leaves these unset for the same reason [[optionalColumnFills]]'s + * knobs are unset, and they need the same treatment: the branch each generator + * emits for a knob that IS set (a gauge's delta arrow, a step row's range) + * never runs on either path, so the two hand-written branches are never + * compared. + * + * Every knob found here ends up in ONE variant (see [[merged]]), the row ones + * included: a row is what the UI's `+` button adds, and its fields are read as a + * unit anyway (a step's start AND end make one range). + * + * "Unset" is read off `baseNode` rather than re-derived, so a knob the base + * pass DID fill — one carrying a `defaultValue` or a declared enum — is left + * alone. + */ + private def optionalScalarFills( + clazz: Class[_], + baseNode: JsonNode, + path: String, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + ordinal: Ordinal + ): Seq[Variant] = + configFields(clazz).filterNot(hiddenBySibling(_, baseNode.at(path))).flatMap { f => + val childPath = pointerOf(f, path) + rowType(f) match { + case Some(row) => + // Recurse into containers whatever their own required-ness: an optional + // knob often sits inside a required list of rows. + rowPaths(f, baseNode.at(childPath), childPath).flatMap { rowPath => + val fills = rowFills(row, baseNode, rowPath, schemas, used, rowCount, ordinal) + if (fills.isEmpty) None + else Some(Variant(s"${rowPath.stripPrefix("/")}=filled", fills)) + } + case None => + leafFill(clazz, f, baseNode, childPath, schemas, rowCount) + .map(fill => Variant(s"${fill._1.stripPrefix("/")}=${fill._2.asText}", Seq(fill))) + .toSeq + } + } + + /** Value for the hostile variant of a knob that takes arbitrary text. Legal — + * a user can type it into any text box — but it ends a Python string literal, + * which is what a generator splicing it unescaped gets wrong. + */ + private val HostileString = "a\"b" + + /** Every knob that accepts ARBITRARY TEXT, to carry [[HostileString]] — all of + * them in one variant (see [[merged]]). This is the escaping check, and it is + * generic on purpose: a new operator is covered the day it is verified, with + * nothing to register. + * + * "Arbitrary text" excludes every string whose value is constrained, because + * there the hostile value would be rejected before any escaping mattered: a + * column picker, a declared enum, a CSS color, and a number-in-a-string (which + * declares bounds). Unlike [[optionalScalarFills]] this does not care whether + * the base pass filled the knob — a label carrying a default is spliced just the + * same — so the variant replaces whatever value is there. + */ + private def hostileTextFills(clazz: Class[_], baseNode: JsonNode, path: String): Seq[Variant] = + configFields(clazz).filterNot(hiddenBySibling(_, baseNode.at(path))).flatMap { f => + val childPath = pointerOf(f, path) + rowType(f) match { + case Some(row) => + rowPaths(f, baseNode.at(childPath), childPath).flatMap { rowPath => + val fills = hostileTextFills(row, baseNode, rowPath).flatMap(_.at) + if (fills.isEmpty) None + else Some(Variant(s"${rowPath.stripPrefix("/")}=hostileText", fills)) + } + case None => + hostileLeaf(f, childPath) + .map(fill => Variant(s"${fill._1.stripPrefix("/")}=hostileText", Seq(fill))) + .toSeq + } + } + + private def hostileLeaf(f: Field, childPath: String): Option[(String, JsonNode)] = + if ( + hasAutofill(f) || f.getType != classOf[String] || declaredEnumValues(f).nonEmpty || + !patternAccepts(f, HostileString) || declaredRange(f) != Bounds(None, None) + ) None + else Some((childPath, objectMapper.getNodeFactory.textNode(HostileString))) + + /** Number the knobs of the hostile variant so no two carry the same text: the first + * keeps [[HostileString]], the n-th reads `a"b2`, `a"b3`, … Every one still holds + * the quote, so the escaping this variant exists for is unchanged. + * + * Needed because the knobs land in ONE variant (see [[merged]]). Where they are the + * names of columns the operator CREATES, one shared value asks for several columns + * of the same name and the schema rejects the config outright — the run then fails + * on something this generator invented rather than on a divergence. Numbering also + * says which knob a surviving value came from. + * + * Applied here rather than inside [[hostileTextFills]] because that walk recurses + * into nested rows, and the count has to span the whole variant, not restart per + * row the way [[rowFills]]'s ordinal does. + */ + private def numbered(fills: Seq[Variant]): Seq[Variant] = { + var n = 0 + fills.map(f => + Variant( + f.label, + f.at.map { + case (pointer, _) => + n += 1 + val text = if (n == 1) HostileString else s"$HostileString$n" + (pointer, objectMapper.getNodeFactory.textNode(text)) + } + ) + ) + } + + /** Whether a field's declared `pattern` accepts `value` — the field's own answer to + * "can this be typed here", so the declaration decides rather than this generator. + * A field that declares nothing accepts anything. + * + * The point of asking instead of skipping every field that HAS a pattern: a pattern + * exists to exclude what the consumer would reject, which for many fields is nothing + * at all. Such a field still needs the escaping check — and the escaping bugs this + * variant found were in exactly that kind of knob. + * + * `matches` is a full-string match, which is what the property editor applies too + * (`Validators.pattern` wraps a string pattern in `^(?:…)$`). + */ + private def patternAccepts(f: Field, value: String): Boolean = + schemaKey(f, "pattern").filter(_.isTextual).map(_.asText) match { + case Some(p) => Try(value.matches(p)).getOrElse(false) + case None => true + } + + /** A running position shared by every row filled into one variant, so no two of + * those knobs are handed the same value. One counter rather than one per row: + * rows collide with each other as readily as knobs within a row do, and where the + * knob is an output column NAME — Projection's `alias` — two rows carrying the + * same one is a config the operator refuses outright. + */ + private final class Ordinal { + private var n = 0 + + /** The position a knob would take. Advances only once one actually does, so a + * field that yields no fill leaves no gap in the numbering. + */ + def peek: Int = n + def taken(): Unit = n += 1 + } + + /** Every optional knob under one nested row — a column picker as well as a scalar — + * as pointer → value. + * + * The scalar knobs get DISTINCT values, ascending: a row is often a pair that has to + * differ to mean anything — a step's start and end, where the operator drops the + * step unless `start < end` — and one shared value would collapse it. The first + * knob keeps the value it would have had on its own, so a lone knob is unaffected. + * + * The column pickers are resolved against the same `used` set as the top-level ones, + * so a row's column differs from what the rest of the config already reads. The row + * itself is the sibling context: whether a picker is type-constrained can depend on + * another knob of the SAME row (an aggregation's function decides whether its column + * must be numeric), so the rule is evaluated against the row, not the operator. + */ + private def rowFills( + clazz: Class[_], + baseNode: JsonNode, + path: String, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + ordinal: Ordinal + ): Seq[(String, JsonNode)] = + configFields(clazz).flatMap { f => + val childPath = pointerOf(f, path) + rowType(f) match { + case Some(row) => + rowPaths(f, baseNode.at(childPath), childPath) + .flatMap(rowPath => rowFills(row, baseNode, rowPath, schemas, used, rowCount, ordinal)) + case None if hasAutofill(f) => + columnFill(clazz, f, baseNode, childPath, schemas, used, baseNode.at(path)).toSeq + case None => + val fill = leafFill(clazz, f, baseNode, childPath, schemas, rowCount, ordinal.peek) + if (fill.nonEmpty) ordinal.taken() + fill.toSeq + } + } + + /** The fill for ONE optional column knob, or `None` when it is required (the base + * pass filled it), already points at a column, or no column resolves. + * + * Shared by the top-level pass and the row pass so both obey the same rule: an + * optional picker takes the first unused column that fits its declared type. + * + * `owner` is the object the field belongs to, NOT the class that declares it: a + * knob a family shares is declared on the abstract base, which has no instance to + * read a default off, and every such knob then read as one the config had already + * set and was skipped. + */ + private def columnFill( + owner: Class[_], + f: Field, + baseNode: JsonNode, + childPath: String, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + siblings: JsonNode + ): Option[(String, JsonNode)] = { + val required = Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) + val untouched = defaultsOf(owner).path(jsonNameOf(f)) + if (required || baseNode.at(childPath) != untouched) None + else { + val spec = autofillSpec(f).getOrElse(AutofillSpec(port = 0, holdsList = false)) + // A list knob takes every matching column here exactly as it does in the + // base config: `required` decides WHETHER a field is filled, not how many + // values go in once it is. Filled with one, a list knob runs the same code + // as a scalar — the per-element numbering and the joining between elements, + // which is the whole of what a list does differently, is never reached. + if (spec.holdsList) + listColumnFill(f, schemas, spec.port, used, siblings).toOption.map(childPath -> _) + else + resolveColumn(f, schemas, spec.port, used, siblings).toOption + .map(col => (childPath, objectMapper.getNodeFactory.textNode(col): JsonNode)) + } + } + + /** Every column at `port` a list knob may hold: the ones its `attributeTypeRules` + * admits, or all of them when the rule matches nothing (or there is no rule), + * minus the ones a single-column knob beside it already took. + * The same fill for a required and an optional field, so the two cannot drift. + * + * Subtracting `used` is what [[resolveColumn]] already does for a single-column + * knob, and the two knobs answer to the same rule: a column means something + * different to each field that names it, so handing one column to two of them + * writes a config nobody would. Radar Chart's name column arrived inside its own + * value columns that way, and sklearn's label inside the features it is fitted + * against. Not marked used in turn, since a list knob wants every column its rule + * admits and marking them would leave a later single-column knob nothing to take. + */ + private def listColumnFill( + f: Field, + schemas: Map[Int, Schema], + port: Int, + used: collection.Set[(Int, String)], + siblings: JsonNode + ): Either[String, JsonNode] = + columnNames(schemas, port).map { names => + val filtered = allowedTypes(f, siblings) match { + case Some(types) => + val matching = schemas + .get(port) + .map(_.getAttributes.filter(a => types.contains(a.getType)).map(_.getName)) + .getOrElse(Seq.empty) + if (matching.nonEmpty) matching else names + case None => names + } + val free = filtered.filterNot(name => used.contains((port, name))) + val arr = objectMapper.createArrayNode() + (if (free.nonEmpty) free else filtered).foreach(arr.add) + arr + } + + /** A field's JSON Pointer, under the pointer of the object that holds it. */ + private def pointerOf(f: Field, path: String): String = s"$path/${jsonNameOf(f)}" + + /** The key a field carries in the config JSON. */ + private def jsonNameOf(f: Field): String = + Option(f.getAnnotation(classOf[JsonProperty])) + .map(_.value) + .filter(_.nonEmpty) + .getOrElse(f.getName) + + /** Whether the config has to carry a value for this field. Two sources say so and + * both count: the annotation, and a schema branch the siblings have selected. A + * field required only under a branch carries no annotation, so reading the + * annotation alone leaves it unfilled in exactly the configuration that needs it. + */ + private def isRequired(f: Field, scope: SchemaScope, siblings: JsonNode): Boolean = + Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) || + requiredUnder(scope, siblings).contains(jsonNameOf(f)) + + /** The nested-row type a field holds — its `List[Row]` / `Option[Row]` element + * type, or its own type when the field IS the row. `None` for a scalar field. + */ + private def rowType(f: Field): Option[Class[_]] = { + val t = f.getType + if (isList(t) || isOption(t)) elementType(f).toOption.filter(isNestedObject) + else if (isNestedObject(t)) Some(t) + else None + } + + /** The pointer of each row present at `childPath` — one per array element, or + * the node itself when the field holds a single row. Empty when nothing is + * there to fill (an absent `Option`, a scalar list). + */ + private def rowPaths(f: Field, child: JsonNode, childPath: String): Seq[String] = + if (isList(f.getType) || isOption(f.getType)) + if (child.isArray) (0 until child.size()).map(i => s"$childPath/$i") + else if (child.isObject) Seq(childPath) + else Seq.empty + else if (child.isObject) Seq(childPath) + else Seq.empty + + /** The fill for one optional free-value scalar knob, or `None` if this field + * isn't one (a column picker, a required field, or a knob the base pass filled). + * + * `ordinal` is the knob's position among the ones filled in the same row (0 for a + * top-level knob, which has no siblings to differ from): it offsets the value so + * the knobs of one row do not collide — see [[rowFills]]. + */ + private def leafFill( + owner: Class[_], + f: Field, + baseNode: JsonNode, + childPath: String, + schemas: Map[Int, Schema], + rowCount: Int, + ordinal: Int = 0 + ): Option[(String, JsonNode)] = { + val required = Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) + // "Unset" means the base pass did not fill it: the key still carries the value a + // fresh instance has (see [[defaultsOf]] — every key is present, as the UI sends + // them, so absence alone no longer tells us anything). + val current = baseNode.at(childPath) + val unset = current.isMissingNode || + current == defaultsOf(owner).path(jsonNameOf(f)) + // A knob whose values the field DECLARES is left to its declaration: the enum + // sweep covers a declared value list, and a knob offering an `examples` value + // takes that one. Reading `examples` on its own, rather than only alongside a + // `pattern`, is the point: a field can state a realistic value ("https:// + // example.com" for a URL) without having to invent a constraint to hang it on, + // and inventing one to steer this generator would reject values the platform + // accepts. + // An optional knob is typed by what its Option holds, so `start`/`end` declared + // as Option[Double] are swept like the bare numbers they are. + val scalarType = effectiveScalarType(f) + if ( + hasAutofill(f) || required || !unset || + declaredEnumValues(f).size > 1 || !isFreeScalar(scalarType) + ) None + else if (declaredExample(f).isDefined) declaredExample(f).map(v => (childPath, v)) + else if (scalarType == classOf[String]) + // The canonical string is "1", so the n-th knob reads as "1", "2", … — distinct + // and ascending, so the knobs filled in one row do not collide. + Some((childPath, objectMapper.getNodeFactory.textNode((ordinal + 1).toString))) + else + scalarNode( + scalarType, + None, + schemas, + mutable.Set.empty, + NumHint(declaredRange(f), rowCount) + ).toOption + .map { v => + // Step away from the value rather than scaling it: the n-th knob lands next + // to the first instead of at n times it, so a pair stays inside the span the + // fixture actually holds — doubling walked `end` past the last row. + val stepped = + if (ordinal == 0) v + else objectMapper.getNodeFactory.numberNode(v.asDouble() + ordinal) + (childPath, stepped) + } + } + + /** The first value a field offers under `examples` — a legal sample the operator + * states itself, so nothing here has to invent one. + */ + private def declaredExample(f: Field): Option[JsonNode] = + schemaKey(f, "examples").filter(_.isArray).flatMap(_.elements().asScala.toSeq.headOption) + + /** One key out of a field's own `@JsonSchemaInject` JSON. */ + private def schemaKey(f: Field, key: String): Option[JsonNode] = + Option(f.getAnnotation(classOf[JsonSchemaInject])) + .map(_.json) + .filter(_.nonEmpty) + .flatMap(js => Try(objectMapper.readTree(js)).toOption) + .map(_.path(key)) + .filterNot(_.isMissingNode) + + /** A type whose value the user types in freely — the fills of + * [[optionalScalarFills]]. Boolean is excluded: the enum sweep already covers + * both of its values. + */ + private def isFreeScalar(t: Class[_]): Boolean = + t == classOf[String] || t == classOf[Int] || t == classOf[java.lang.Integer] || + t == classOf[Short] || t == classOf[Long] || t == classOf[java.lang.Long] || + t == classOf[Double] || t == classOf[java.lang.Double] || t == classOf[Float] + + private def deserialize( + node: ObjectNode, + opClass: Class[_ <: LogicalOp] + ): Either[String, LogicalOp] = + Try(objectMapper.treeToValue(node, opClass)).toEither.left + .map(e => s"deserialization failed: ${e.getMessage}") + + /** One variant per non-default enum value reachable in `baseNode`. One enum + * flipped at a time — linear, NOT the combinatorial product. + */ + private def enumVariants( + opClass: Class[_ <: LogicalOp], + baseNode: ObjectNode, + pinned: Set[String] = Set.empty, + fill: FillContext = FillContext() + ): Seq[Variant] = + enumSites(opClass, baseNode, "", Map.empty, SchemaScope.of(opClass), fill) + .filterNot(site => pinned.contains(site.pointer.stripPrefix("/"))) + .flatMap { site => + val baseVal = baseNode.at(site.pointer) + site.values.filterNot(_ == baseVal).map { v => + Variant( + s"${site.pointer.stripPrefix("/")}=${v.asText}", + (site.pointer, v) +: site.companions(v) + ) + } + } + + /** An enum-typed position in the config JSON: its JSON Pointer plus every + * possible JSON value (each enum constant serialized via its `@JsonValue`). + * + * `companions` names the edits a value has to arrive with. A hyperparameter's + * `parameter` needs them: the `value` beside it holds something the PREVIOUS + * parameter accepted, and a flip that left it there would ask the operator to + * put a kernel name through `int()`. + * + * A choice the operator offers and no value runs is NOT dropped here: it is + * generated, it fails, and it is withheld by name in + * [[TransformVerificationRunner.variantsNotRun]], where a reader sees it and the row + * goes away when the operator is fixed. + */ + private final case class EnumSite( + pointer: String, + values: Seq[JsonNode], + companions: JsonNode => Seq[(String, JsonNode)] = _ => Seq.empty + ) + + /** Collect every enum-typed leaf reachable in `node`. Walks the operator's + * fields for type info but the ACTUAL JSON for structure, so it honours real + * list lengths (a curated fixture may hold >1 element) and skipped optionals. + * `path` is the JSON Pointer of the sub-node currently typed by `clazz`. + * + * `bindings` and `scope` are what [[buildObject]] filled the config with, and + * are needed here for the same two reasons: a field declared as a type variable + * reports `Object` from [[Field.getType]] and so hides the enum it really holds, + * and a field whose values are stated in the schema rather than on itself has + * none to sweep as far as reflection can see. + */ + private def enumSites( + clazz: Class[_], + node: JsonNode, + path: String, + bindings: TypeBindings, + scope: SchemaScope, + fill: FillContext + ): Seq[EnumSite] = { + val bound = bindings ++ typeBindingsOf(clazz) + val row = node.at(path) + val sites = configFields(clazz).filterNot(hiddenBySibling(_, row)).flatMap { f => + if (hasAutofill(f)) Seq.empty + else { + val jsonName = jsonNameOf(f) + val childPath = s"$path/$jsonName" + val child = node.at(childPath) + if (child.isMissingNode || child.isNull) Seq.empty + else { + val t = f.getType + val declared = declaredEnumValues(f) + val nested = scope.descend(jsonName) + if (declared.size > 1) Seq(EnumSite(childPath, declared)) + else if (isList(t)) + elementType(f).toOption.toSeq.flatMap { elem => + if (child.isArray) + (0 until child.size()).flatMap(i => + enumSiteFor(elem, node, s"$childPath/$i", elementBindings(f, bound), nested, fill) + ) + else Seq.empty + } + else if (isOption(t)) + elementType(f).toOption.toSeq + .flatMap(elem => enumSiteFor(elem, node, childPath, bound, nested, fill)) + else + ruledEnumSite(f, row, childPath, scope) + .map(Seq(_)) + .getOrElse(enumSiteFor(boundType(f, bound), node, childPath, bound, nested, fill)) + } + } + } + withCompanions(clazz, row, path, bound, scope, fill, sites) + } + + private def enumSiteFor( + t: Class[_], + node: JsonNode, + path: String, + bindings: TypeBindings, + scope: SchemaScope, + fill: FillContext + ): Seq[EnumSite] = + if (t.isEnum) { + val vals = t.getEnumConstants.toSeq.map(c => objectMapper.valueToTree[JsonNode](c)) + if (vals.size > 1) Seq(EnumSite(path, vals)) else Seq.empty + } else if (t == classOf[Boolean] || t == classOf[java.lang.Boolean]) { + // A Boolean is a 2-value "enum": sweep both true and false. + val nf = objectMapper.getNodeFactory + Seq(EnumSite(path, Seq(nf.booleanNode(true), nf.booleanNode(false)))) + } else if (isNestedObject(t)) enumSites(t, node, path, bindings, scope, fill) + else Seq.empty + + /** The site a `valueRules` branch gives a field whose own type names no values: + * the set the branch holding for `row` accepts. `None` where that branch names + * none, a numeric hyperparameter's `value` being one value out of a range rather + * than a choice between named ones. + */ + private def ruledEnumSite( + f: Field, + row: JsonNode, + childPath: String, + scope: SchemaScope + ): Option[EnumSite] = + schemaValueRule(f, scope, row) + .map(_.path("enum")) + .filter(e => e.isArray && e.size() > 1) + .map(e => EnumSite(childPath, e.elements().asScala.toSeq)) + + /** Every site paired with the fields that move with it, in the two ways a schema + * says one field's content depends on another's. + * + * The first is `valueRules`: what such a field may hold is decided by the sibling + * the rule reads. The pairing is derived from the rules themselves rather than + * named here, so an operator stating a rule over some other sibling gets the same + * treatment. The paired field is REFILLED rather than left as it was, since what + * sits there belongs to the PREVIOUS choice, and it is refilled the way any field + * is: by the rule where the rule names a value, and by the field's own type where + * it does not. A branch naming nothing is the operator saying it knows of no value + * worth offering, which is not the same as there being none, and a choice this + * generator cannot fill is a choice that has to fail loudly and be withheld by name + * in [[TransformVerificationRunner.variantsNotRun]] rather than disappear here. + * + * The second is a conditional `required`, which is how an object says that exactly + * one of two fields applies and therefore that neither can be marked required on + * its own. A hyperparameter row is written that way: `value` is required while its + * switch is off and `attribute` once it is on. The base pass filled the one the + * base config needs, so flipping the switch has to fill the other, which until then + * was rightly left empty. + */ + private def withCompanions( + clazz: Class[_], + row: JsonNode, + path: String, + bindings: TypeBindings, + scope: SchemaScope, + fill: FillContext, + sites: Seq[EnumSite] + ): Seq[EnumSite] = { + val ruled = configFields(clazz) + .map(f => (f, scope.child(jsonNameOf(f)).path("valueRules").path("allOf"))) + .filter { case (_, branches) => branches.isArray } + val conditional = conditionallyRequiredFields(scope) + if ((ruled.isEmpty && conditional.isEmpty) || !row.isObject) sites + else + sites.map { site => + val sibling = site.pointer.stripPrefix(s"$path/") + val paired = ruled.filter { + case (_, branches) => + branches.elements().asScala.exists(_.path("if").has(sibling)) + } + if (paired.isEmpty && conditional.isEmpty) site + else + site.copy(companions = v => { + val hypothetical = row.deepCopy[ObjectNode]() + hypothetical.set[JsonNode](sibling, v) + val ruleFills = paired.map { + case (f, _) => companionFill(f, path, hypothetical, bindings, scope, fill) + } + // Only what this value newly asks for and the row does not already + // carry: a config that set the field by hand keeps what it set. + val revealed = requiredUnder(scope, hypothetical) + .diff(requiredUnder(scope, row)) + .flatMap(name => configFields(clazz).find(jsonNameOf(_) == name)) + .filter(f => isBlank(hypothetical.path(jsonNameOf(f)))) + .map(f => companionFill(f, path, hypothetical, bindings, scope, fill)) + ruleFills ++ revealed + }) + } + } + + /** One companion edit, through [[valueFor]], which reads the rule first and falls back + * to the field's own type: the order the base pass filled it in. + * + * A field this generator cannot fill ENDS the run rather than quietly costing the + * choice its variant, the auto tier already ending it when a whole config cannot be + * built. Both are the same thing said about a smaller piece, and a generator that + * came up empty is a gap here rather than a statement about the operator. + */ + private def companionFill( + f: Field, + path: String, + row: JsonNode, + bindings: TypeBindings, + scope: SchemaScope, + fill: FillContext + ): (String, JsonNode) = + valueFor(f, fill.schemas, fill.used, fill.rowCount, row, bindings, scope) match { + case Right(value) => (s"$path/${jsonNameOf(f)}", value) + case Left(reason) => + throw new IllegalStateException(s"cannot fill ${jsonNameOf(f)} beside it: $reason") + } + + /** Set `value` at a JSON Pointer inside `root` — used to clone the base config + * and flip one enum leaf. Handles object fields and array indices. + */ + private def setAtPointer(root: ObjectNode, pointer: String, value: JsonNode): Unit = { + val tokens = pointer.stripPrefix("/").split("/").toList + var cur: JsonNode = root + tokens.dropRight(1).foreach { tk => + cur = if (cur.isArray) cur.get(tk.toInt) else cur.get(tk) + } + (cur, tokens.last) match { + case (o: ObjectNode, name) => o.set[JsonNode](name, value) + // One past the end appends — the `+`-row fill adds a row rather than + // replacing one. + case (a: ArrayNode, idx) if idx.toInt == a.size() => a.add(value); () + case (a: ArrayNode, idx) => a.set(idx.toInt, value); () + case _ => () + } + } + + /** Maps each registered operator class to its `operatorType` discriminator, + * read from [[LogicalOp]]'s `@JsonSubTypes` (the same registry Jackson uses). + */ + private val typeNameByClass: Map[Class[_], String] = { + Option(classOf[LogicalOp].getAnnotation(classOf[JsonSubTypes])) + .map(_.value().toSeq.map(t => (t.value(): Class[_]) -> t.name()).toMap) + .getOrElse(Map.empty) + } + + // ── object assembly ────────────────────────────────────────────────────── + + /** Build a JSON object for `clazz` by filling each of its config fields. + * `rowCount` sizes the numeric fallback for range-less fields (e.g. Limit). + */ + private def buildObject( + clazz: Class[_], + schemas: Map[Int, Schema], + rowCount: Int + ): Either[String, ObjectNode] = + buildObject(clazz, schemas, mutable.Set.empty[(Int, String)], rowCount) + + /** `used` tracks (port, column) already assigned within THIS operator, so that + * sibling autofill fields resolve to DISTINCT columns (e.g. a scatter's x and + * y don't both collapse onto the first numeric column, which would be a + * degenerate diagonal). Shared across the operator, nested objects included. + * An explicit `@SampleColumn` always wins even if the column is already taken; + * only the type-match and first-column tiers avoid reuse. + */ + private def buildObject( + clazz: Class[_], + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + bindings: TypeBindings = Map.empty, + scope: SchemaScope = SchemaScope.empty, + pinned: Map[String, JsonNode] = Map.empty + ): Either[String, ObjectNode] = { + // What `clazz` itself supplies is added to what its caller passed in: an operator + // names the arguments for its own supertypes, a row class receives them from the + // field that holds it. + val bound = bindings ++ typeBindingsOf(clazz) + // An operator carries its own schema, so it is derived here rather than at each + // entry point: a caller that forgot would lose every rule the schema states and + // get a config that merely looks filled. A nested class has no schema of its own + // and uses the scope the field holding it handed down. + val doc = if (classOf[LogicalOp].isAssignableFrom(clazz)) SchemaScope.of(clazz) else scope + val node = defaultsOf(clazz) + // Pins go in BEFORE the fields are decided, not after: `node` is the sibling + // context below, so a knob pinned on decides what its dependents do. Set + // afterwards, a pin cannot reach the field it was pinned to steer. + pinned.foreach { case (name, value) => node.set[JsonNode](name, value) } + configFields(clazz).foreach { f => + // A pinned knob keeps the value it was pinned to. Deciding it again would + // refill it from its default and undo the pin before the fields that read it + // are reached. + if (!pinned.contains(jsonNameOf(f))) { + // `node` doubles as the sibling context: a field whose rule depends on another + // field of the same object reads it here, so declaration order decides what is + // visible — the knob a rule branches on is declared before the column it binds. + decide(f, schemas, used, rowCount, node, bound, doc) match { + case Fill(name, value) => node.set[JsonNode](name, value) + case Skip => () + case Fail(reason) => return Left(s"${clazz.getSimpleName}.${f.getName}: $reason") + } + } + } + Right(node) + } + + /** A fresh instance's own values, as the starting JSON — what the UI submits for a + * form nobody touched, where every key is present carrying the operator's default. + * + * Leaving a skipped knob's key OUT instead produces a shape the UI cannot: a + * config object built through a `@JsonCreator` constructor then receives `null` + * for the missing keys, overwriting the field initializers, and a generator that + * reads them crashes on a value no user can enter (BulletChart's step bounds). + * Empty when the class has no usable no-arg constructor. + */ + private def defaultsOf(clazz: Class[_]): ObjectNode = + Try(clazz.getDeclaredConstructor()) + .flatMap { ctor => + ctor.setAccessible(true) + Try(objectMapper.valueToTree[JsonNode](ctor.newInstance())) + } + .toOption + .collect { case o: ObjectNode => o } + .getOrElse(objectMapper.createObjectNode()) + + private sealed trait Decision + private case class Fill(jsonName: String, value: JsonNode) extends Decision + private case object Skip extends Decision + private case class Fail(reason: String) extends Decision + + /** Decide whether/how to fill one field, applying required-vs-optional policy: + * required (or required autofill) fields that can't be filled fail the whole + * operator; optional fields without a meaningful value are skipped (left at the + * operator's default). + */ + private def decide( + f: Field, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + siblings: JsonNode = noSiblings, + bindings: TypeBindings = Map.empty, + scope: SchemaScope = SchemaScope.empty + ): Decision = { + val jp = Option(f.getAnnotation(classOf[JsonProperty])) + val jsonName = jp.map(_.value).filter(_.nonEmpty).getOrElse(f.getName) + val required = isRequired(f, scope, siblings) + val autofill = hasAutofill(f) + // An optional knob is judged by what it WRAPS: `Option[Double]` is a number the + // user may leave blank, not a thing the base config has to carry. + val held = effectiveScalarType(f, bindings) + val isBoolean = held == classOf[Boolean] || held == classOf[java.lang.Boolean] + + // An OPTIONAL column-name field (`@AutofillAttributeName*` with required=false) + // is left at its operator default rather than force-filled. These are the + // "No Selection" grouping/pattern knobs (e.g. BarChart's categoryColumn / + // pattern); forcing a real column into one produces a degenerate config (one + // trace per row) that the native and generated paths disagree on. + if (hiddenBySibling(f, siblings)) Skip + else if (autofill && !required) Skip + else { + // A field declaring its values in the annotation counts as meaningful just as + // an enum-TYPED one does: the sweep flips it from the base config, so it has + // to BE in the base config (a `defaultValue = ""` alone would skip it). So does + // one whose schema states a rule for it: an untyped hyperparameter `value` is + // an optional plain string, which alone would be skipped, but the operator does + // read it and the rule says what it should hold. + val meaningful = required || autofill || held.isEnum || isBoolean || isList(f.getType) || + isNestedObject(held) || declaredEnumValues(f).size > 1 || + schemaValueRule(f, scope, siblings).isDefined || jp + .map(_.defaultValue) + .exists(_.nonEmpty) + + valueFor(f, schemas, used, rowCount, siblings, bindings, scope) match { + case Right(v) if meaningful => Fill(jsonName, v) + case Right(_) => Skip // optional plain scalar w/o default — leave operator default + case Left(reason) if required || autofill => Fail(reason) + case Left(_) => Skip + } + } + } + + // ── value resolution ───────────────────────────────────────────────────── + + /** Resolve a JSON value node for a field: autofill column refs first, then by + * declared type (list / option / scalar / nested object). + */ + private def valueFor( + f: Field, + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + siblings: JsonNode = noSiblings, + bindings: TypeBindings = Map.empty, + scope: SchemaScope = SchemaScope.empty + ): Either[String, JsonNode] = { + val ruled = schemaValueRule(f, scope, siblings).flatMap(ruleFill) + val nested = scope.descend(jsonNameOf(f)) + autofillSpec(f) match { + case Some(spec) if spec.holdsList => + listColumnFill(f, schemas, spec.port, used, siblings) + case Some(spec) => + resolveColumn(f, schemas, spec.port, used, siblings) + .map(objectMapper.getNodeFactory.textNode) + // A rule stated in the schema wins over the type-driven fill below: it names a + // value this field may hold given the sibling chosen beside it, which the type + // alone — a bare `String` — cannot narrow. + case None if ruled.isDefined => Right(ruled.get) + case None => + val t = boundType(f, bindings) + if (isList(t)) + // An OPTIONAL list starts EMPTY, the way the UI does: its `+` button adds the + // first row, so a config nobody touched has none, and the branch an operator + // takes for "no rows at all" is only reached this way. A REQUIRED list gets + // one row — its operator asserts the list is non-empty, so zero is not a + // config it can run. Either way the extra row comes from [[extraRowFills]]. + // + // Required counts the schema's conditional form too: a list the operator + // needs only on one branch is empty-by-annotation, and reading the + // annotation alone hands that branch the empty list it cannot run. + if (!isRequired(f, scope, siblings)) + Right(objectMapper.createArrayNode()) + else + elementType(f) + .flatMap( + scalarOrNested(_, schemas, used, rowCount, elementBindings(f, bindings), nested) + ) + .map { e => + val arr: ArrayNode = objectMapper.createArrayNode(); arr.add(e); arr + } + else if (isOption(t)) + // An optional scalar is filled like the bare type: the `defaultValue` and any + // declared range sit on the field, not on the element, so a Grid Size that + // declares 10 is still filled with 10 rather than a generic number. + elementType(f).flatMap { elem => + if (isNestedObject(elem)) + scalarOrNested(elem, schemas, used, rowCount, elementBindings(f, bindings), nested) + else + scalarNode(elem, baseValueOf(f), schemas, used, NumHint(declaredRange(f), rowCount)) + } + else if (declaredEnumValues(f).size > 1) Right(declaredEnumDefault(f)) + else + scalarNode( + t, + baseValueOf(f), + schemas, + used, + NumHint(declaredRange(f), rowCount), + Map.empty, + nested + ) + } + } + + /** The base value for a field whose values are declared in its annotation: the + * `default` the annotation names, else its first value. Never the canonical + * string — for such a field that is a value the operator does not accept. + */ + private def declaredEnumDefault(f: Field): JsonNode = { + val declared = declaredEnumValues(f) + Option(f.getAnnotation(classOf[JsonSchemaInject])) + .map(_.json) + .filter(_.nonEmpty) + .flatMap(js => Try(objectMapper.readTree(js).path("default")).toOption) + .filterNot(_.isMissingNode) + .filter(declared.contains) + .getOrElse(declared.head) + } + + /** What the base config should carry for a scalar field, before this generator + * invents anything: the operator's own `defaultValue` if it has one, else the + * value it offers under `examples`. + * + * `examples` matters most on a REQUIRED field, which [[leafFill]] never reaches — + * a required knob with no default would otherwise take the canonical "1", and "1" + * is not a URL, a regex or a delimiter. A field can now say what a realistic value + * looks like without declaring a constraint it does not have. + */ + private def baseValueOf(f: Field): Option[String] = + defaultOf(f).orElse(declaredExample(f).filter(_.isTextual).map(_.asText)) + + /** A node for a list element or Option inner type — no field-level default or + * range annotation (those live on the field, not the element type). + */ + private def scalarOrNested( + clazz: Class[_], + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + rowCount: Int, + bindings: TypeBindings = Map.empty, + scope: SchemaScope = SchemaScope.empty + ): Either[String, JsonNode] = + scalarNode(clazz, None, schemas, used, NumHint(Bounds(None, None), rowCount), bindings, scope) + + /** How to fill a numeric field: `@JsonProperty(defaultValue)` if present, else the + * middle of a declared `[min, max]` (an opacity's 0.0–1.0 → 0.5), else twice a + * lower bound declared on its own, else half the row count (the middle of + * `[0, rowCount]`, e.g. Limit). + * + * Twice, because a field that declares `>= 30` usually also defaults to 30, so + * filling the bound itself would just re-run the base config; `max mid` keeps a + * `>= 0` knob off zero. Doubling can only overshoot a ceiling the field does not + * declare, and a field with a ceiling is supposed to declare it — which is why the + * `[min, max]` case must stay: RadarChart's and Scatterplot's opacity declare one, + * and doubling their floor of 0 would hand them 5. + * + * An upper bound declared ALONE is not handled: no field does that, so there would + * be no way to tell whether the code was right. + */ + private final case class NumHint(bounds: Bounds, rowCount: Int) + + private final case class Bounds(min: Option[Double], max: Option[Double]) + + private def numericFill(default: Option[String], hint: NumHint): Double = + default.flatMap(s => Try(s.trim.toDouble).toOption) match { + case Some(d) => d + case None => + val mid = hint.rowCount / 2.0 + hint.bounds match { + case Bounds(Some(mn), Some(mx)) => (mn + mx) / 2.0 + case Bounds(Some(mn), None) => (mn * 2) max mid + case _ => mid + } + } + + /** A node for a concrete (non-list, non-option) type. Numeric fields follow + * [[numericFill]]; enums/strings honor an optional `@JsonProperty(defaultValue)`. + */ + private def scalarNode( + t: Class[_], + default: Option[String], + schemas: Map[Int, Schema], + used: mutable.Set[(Int, String)], + hint: NumHint, + bindings: TypeBindings = Map.empty, + scope: SchemaScope = SchemaScope.empty + ): Either[String, JsonNode] = { + val nf = objectMapper.getNodeFactory + if (t.isEnum) + Right( + default + .map(nf.textNode) + .getOrElse(objectMapper.valueToTree[JsonNode](t.getEnumConstants.head)) + ) + else if (t == classOf[Boolean] || t == classOf[java.lang.Boolean]) + Right(nf.booleanNode(default.map(_.trim.toBoolean).getOrElse(false))) + else if (t == classOf[Int] || t == classOf[java.lang.Integer] || t == classOf[Short]) + Right(nf.numberNode(numericFill(default, hint).round.toInt)) + else if (t == classOf[Long] || t == classOf[java.lang.Long]) + Right(nf.numberNode(numericFill(default, hint).round)) + else if (t == classOf[Double] || t == classOf[java.lang.Double] || t == classOf[Float]) + Right(nf.numberNode(numericFill(default, hint))) + else if (t == classOf[String]) + Right(nf.textNode(default.getOrElse(CanonicalString))) + else if (isNestedObject(t)) + buildObject(t, schemas, used, hint.rowCount, bindings, scope) + else Left(s"unhandled type ${t.getName}") + } + + /** The values a field declares via its own `@JsonSchemaInject(json = ...)` + * `enum` array — a String field the UI renders as a dropdown (e.g. an ECDF's + * cdfMode = standard / reversed / complementary). To the JVM these are plain + * Strings, so [[enumSiteFor]]'s `isEnum` check can't see them, yet each value + * takes a different branch in the generated code exactly as a real enum does. + * Empty unless the annotation carries an array (TimeSeries declares + * `"enum": "autofill"`, a UI directive rather than a value list). + */ + private def declaredEnumValues(f: Field): Seq[JsonNode] = + Option(f.getAnnotation(classOf[JsonSchemaInject])) + .map(_.json) + .filter(_.nonEmpty) + .toSeq + .flatMap { js => + Try(objectMapper.readTree(js).path("enum")).toOption.toSeq + .filter(_.isArray) + .flatMap(_.elements().asScala.toSeq) + } + + /** The bounds a field declares, from either of the two places an operator states + * them: `@JsonSchemaInject`'s `minimum`/`maximum` (an opacity's 0.0–1.0), which the + * UI reads, and javax validation's `@DecimalMin`/`@Min` (a row height's floor of 30), + * which the compiler's validation pass reads. Either bound may be absent. + */ + private def declaredRange(f: Field): Bounds = { + val schema = Option(f.getAnnotation(classOf[JsonSchemaInject])) + .map(_.json) + .filter(_.nonEmpty) + .flatMap(js => Try(objectMapper.readTree(js)).toOption) + def fromSchema(key: String): Option[Double] = + schema.map(_.path(key)).filter(_.isNumber).map(_.asDouble()) + Bounds( + fromSchema("minimum") + .orElse(Option(f.getAnnotation(classOf[DecimalMin])).flatMap(a => asDouble(a.value))) + .orElse(Option(f.getAnnotation(classOf[Min])).map(_.value.toDouble)), + fromSchema("maximum") + ) + } + + private def asDouble(s: String): Option[Double] = Try(s.trim.toDouble).toOption + + // ── reflection helpers ─────────────────────────────────────────────────── + + /** Config fields declared on `clazz` and its superclasses up to (not + * including) [[LogicalOp]] — i.e. the operator's own knobs, not the + * framework's bookkeeping. A field counts if it carries `@JsonProperty` or an + * autofill annotation. + */ + private def configFields(clazz: Class[_]): Seq[Field] = { + val ignored = ignoredProperties(clazz) + val out = mutable.LinkedHashMap.empty[String, Field] // de-dup by name, keep most-derived + var c: Class[_] = clazz + while (c != null && c != classOf[LogicalOp] && c != classOf[Object]) { + c.getDeclaredFields + .filterNot(f => Modifier.isStatic(f.getModifiers)) + .filter(isConfigField) + .filterNot(f => ignored.contains(jsonNameOf(f))) + .foreach(f => out.getOrElseUpdate(f.getName, { f.setAccessible(true); f })) + c = c.getSuperclass + } + out.values.toSeq + } + + /** The properties an operator declares it does NOT carry, via `@JsonIgnoreProperties`. + * + * An operator that inherits a knob it does not read says so this way — FileScanSource + * over `ScanSourceOpDesc`'s `limit`/`offset` — and the annotation sits on the operator + * while the field sits on the parent, so the field alone cannot be judged. Jackson + * drops these on the way back in, so filling one yields the config it started from: + * the variant built from it would run a second time over the same config and report + * the two paths agreeing about nothing. + */ + private def ignoredProperties(clazz: Class[_]): Set[String] = { + val names = mutable.Set.empty[String] + var c: Class[_] = clazz + while (c != null && c != classOf[Object]) { + Option(c.getAnnotation(classOf[JsonIgnoreProperties])).foreach(names ++= _.value) + c = c.getSuperclass + } + names.toSet + } + + private def isConfigField(f: Field): Boolean = + // `@JsonIgnore` is the field's own way of saying the same thing + // [[ignoredProperties]] handles for the class: not part of the config. + !f.isAnnotationPresent(classOf[JsonIgnore]) && + (f.isAnnotationPresent(classOf[JsonProperty]) || hasAutofill(f)) + + private def hasAutofill(f: Field): Boolean = autofillSpec(f).isDefined + + /** How a field says "fill me with a column name from input port N", and whether + * it holds one name or a list of them. + * + * Two spellings mean the same thing: the `@AutofillAttributeName` family, or + * the `@JsonSchemaInject` that family is defined as, which `SklearnModelOpDesc.text` + * writes out so its `hide*` keys sit in one annotation. They emit identical + * schema keys, so reading only the annotations left such a field out of the + * config entirely — which read as the operator having no such knob. + */ + private def autofillSpec(f: Field): Option[AutofillSpec] = + if (f.isAnnotationPresent(classOf[AutofillAttributeNameList])) + Some(AutofillSpec(port = 0, holdsList = true)) + else if (f.isAnnotationPresent(classOf[AutofillAttributeNameOnPort1])) + Some(AutofillSpec(port = 1, holdsList = false)) + else if (f.isAnnotationPresent(classOf[AutofillAttributeName])) + Some(AutofillSpec(port = 0, holdsList = false)) + else injectedAutofill(f) + + private final case class AutofillSpec(port: Int, holdsList: Boolean) + + /** The `@JsonSchemaInject` spelling: an `autofill` string key naming one of + * the two autofill kinds, plus an optional port. Anything else in the + * annotation (titles, `hide*`) is ignored here. + */ + private def injectedAutofill(f: Field): Option[AutofillSpec] = + for { + inject <- Option(f.getAnnotation(classOf[JsonSchemaInject])) + kind <- inject.strings.find(_.path == CommonOpDescAnnotation.autofill).map(_.value) + holdsList <- + if (kind == CommonOpDescAnnotation.attributeNameList) Some(true) + else if (kind == CommonOpDescAnnotation.attributeName) Some(false) + else None + } yield AutofillSpec( + port = inject.ints + .find(_.path == CommonOpDescAnnotation.autofillAttributeOnPort) + .map(_.value) + .getOrElse(0), + holdsList = holdsList + ) + + private def defaultOf(f: Field): Option[String] = + Option(f.getAnnotation(classOf[JsonProperty])).map(_.defaultValue).filter(_.nonEmpty) + + /** Whether the UI hides this field, given what its siblings currently hold. + * + * A `hide*` triple says "hide me when THAT field holds THIS value", and the UI + * honours it, so a config that fills a hidden field is one no user can submit. + * Filling one was harmless where nothing read it and misleading where something + * did: sklearn's `text` was filled off the numeric projection with the + * vectorizer off, a form the UI never shows. + * + * The sibling's value is read from the node being built, which starts as the + * operator's own defaults, so the target is present whatever the declaration + * order. + */ + private def hiddenBySibling(f: Field, siblings: JsonNode): Boolean = + Option(f.getAnnotation(classOf[JsonSchemaInject])).exists { inject => + val by = inject.strings.find(_.path == HideAnnotation.hideTarget).map(_.value) + val expected = inject.strings.find(_.path == HideAnnotation.hideExpectedValue).map(_.value) + val kind = inject.strings + .find(_.path == HideAnnotation.hideType) + .map(_.value) + .getOrElse(HideAnnotation.Type.equals) + (by, expected) match { + case (Some(target), Some(want)) => + val actual = Option(siblings.get(target)).map(_.asText).getOrElse("") + if (kind == HideAnnotation.Type.regex) Try(actual.matches(want)).getOrElse(false) + else actual == want + case _ => false + } + } + + private def isList(t: Class[_]): Boolean = + classOf[scala.collection.Seq[_]].isAssignableFrom(t) || + classOf[java.util.List[_]].isAssignableFrom(t) + + private def isOption(t: Class[_]): Boolean = classOf[Option[_]].isAssignableFrom(t) + + /** The element class of a `List[X]` / `Option[X]` field, from its generic + * signature. + */ + private def elementType(f: Field): Either[String, Class[_]] = + contentAs(f) match { + case Some(c) => Right(c) + case None => + f.getGenericType match { + case p: ParameterizedType => + p.getActualTypeArguments.headOption match { + case Some(c: Class[_]) => Right(c) + case Some(pt: ParameterizedType) => Right(pt.getRawType.asInstanceOf[Class[_]]) + case _ => Left(s"cannot resolve element type of ${f.getName}") + } + case _ => Left(s"${f.getName} has no generic element type") + } + } + + /** What a field holds as a scalar: an `Option`'s element type, else the field type + * itself. Everything that reasons about a knob's type goes through this, so an + * optional knob is treated exactly like the bare value it wraps. + */ + private def effectiveScalarType(f: Field, bindings: TypeBindings = Map.empty): Class[_] = + if (isOption(f.getType)) elementType(f).getOrElse(f.getType) else boundType(f, bindings) + + /** The concrete classes standing in for the type variables in scope, keyed by the + * variable itself so that two classes declaring a `T` cannot be confused. + * + * Needed because a field declared as a type variable — a trainer's hyperparameter + * row holds `var parameter: T` — reports `Object` from [[Field.getType]], which is + * not a type anything can be filled with. The operator does name the class it means, + * one level up in `SklearnMLOperatorDescriptor[SklearnAdvancedKNNParameters]`, and + * these carry that down to the field. + */ + private type TypeBindings = Map[TypeVariable[_], Class[_]] + + /** What `clazz` supplies for the variables its generic supertypes declare, walking up + * the chain so an argument stated several levels above still arrives. An argument + * that is itself a variable is followed through what the subclass already bound, + * which is why the walk goes downward-first. + */ + private def typeBindingsOf(clazz: Class[_]): TypeBindings = { + val acc = mutable.Map.empty[TypeVariable[_], Class[_]] + var t: Type = clazz.getGenericSuperclass + while (t != null) t match { + case p: ParameterizedType => + val raw = p.getRawType.asInstanceOf[Class[_]] + raw.getTypeParameters.zip(p.getActualTypeArguments).foreach { + case (declared, arg: Class[_]) => acc(declared) = arg + case (declared, arg: TypeVariable[_]) => acc.get(arg).foreach(acc(declared) = _) + case _ => () + } + t = raw.getGenericSuperclass + case c: Class[_] => t = c.getGenericSuperclass + case _ => t = null + } + acc.toMap + } + + /** `f`'s type with a type variable resolved against the bindings in scope. Falls back + * to [[Field.getType]], i.e. to `Object`, so an unresolvable variable still reaches + * [[scalarNode]] and is reported there rather than silently mis-filled. + */ + private def boundType(f: Field, bindings: TypeBindings): Class[_] = + f.getGenericType match { + case tv: TypeVariable[_] => bindings.getOrElse(tv, f.getType) + case _ => f.getType + } + + /** What a `List[Row[T]]` field passes down to its row class: `Row`'s own variables + * bound to the arguments the field names. Those arguments are usually the enclosing + * operator's variables rather than classes, so they are resolved against the bindings + * already in scope before being handed on. + */ + private def elementBindings(f: Field, bindings: TypeBindings): TypeBindings = + f.getGenericType match { + case p: ParameterizedType => + p.getActualTypeArguments.headOption match { + case Some(row: ParameterizedType) => + val raw = row.getRawType.asInstanceOf[Class[_]] + raw.getTypeParameters + .zip(row.getActualTypeArguments) + .flatMap { + case (declared, arg: Class[_]) => Some(declared -> arg) + case (declared, arg: TypeVariable[_]) => bindings.get(arg).map(declared -> _) + case _ => None + } + .toMap + case _ => Map.empty + } + case _ => Map.empty + } + + /** The element class `@JsonDeserialize(contentAs = ...)` names, and the only place + * a Scala `Option[Double]`'s element type survives: the generic signature erases + * it to Object, which is why Jackson needs the annotation too. Checked before the + * signature so an operator that carries it is read the way Jackson reads it. + */ + private def contentAs(f: Field): Option[Class[_]] = + Option(f.getAnnotation(classOf[JsonDeserialize])) + .map(_.contentAs()) + .filterNot(c => c == classOf[java.lang.Void] || c == classOf[Void]) + + /** A type we should recurse into and build as a nested JSON object: not a + * primitive/boxed/String/enum/collection, and it actually declares config + * fields or a creator. + */ + private def isNestedObject(t: Class[_]): Boolean = { + val excluded = t.isPrimitive || t.isEnum || t == classOf[String] || + isList(t) || isOption(t) || t.getName.startsWith("java.lang.") + !excluded && (configFields(t).nonEmpty || t.getDeclaredConstructors.exists( + _.getParameterCount > 0 + )) + } + + private def columnNames(schemas: Map[Int, Schema], port: Int): Either[String, Seq[String]] = + schemas.get(port).map(_.getAttributeNames).filter(_.nonEmpty) match { + case Some(names) => Right(names) + case None => Left(s"no input columns at port $port") + } + + /** First column at `port` not yet claimed by a sibling field of the same + * operator (so two un-annotated / same-type fields don't collapse onto the + * same column); the first column if every column is already taken. Marks the + * pick in `used`. + */ + private def firstUnused( + schemas: Map[Int, Schema], + port: Int, + used: mutable.Set[(Int, String)] + ): Either[String, String] = + columnNames(schemas, port).map { names => + val col = names.find(c => !used.contains((port, c))).getOrElse(names.head) + used += ((port, col)); col + } + + /** Pick which input column fills an `@AutofillAttributeName*` field, in + * priority order: + * 1. `@SampleColumn("x")` — an explicit semantic pick (e.g. a valid ISO + * country code or a real OHLC column) that the column's type can't + * express; always honored, even if already used; + * 2. the first *unused* column whose [[AttributeType]] satisfies the field's + * `attributeTypeRules` (falling back to the first matching column if all + * are taken); + * 3. the first unused column (the original first-column behavior, made + * distinct-aware). + * Tiers 1–2 keep the parity test on realistic, type-correct input; the + * distinct-column preference stops sibling fields (x/y, source/target) from + * collapsing onto one column and producing a degenerate result. + */ + private def resolveColumn( + f: Field, + schemas: Map[Int, Schema], + port: Int, + used: mutable.Set[(Int, String)], + siblings: JsonNode = noSiblings + ): Either[String, String] = { + def take(col: String): String = { used += ((port, col)); col } + Option(f.getAnnotation(classOf[SampleColumn])).map(_.value) match { + case Some(col) => + columnNames(schemas, port).flatMap { names => + if (names.contains(col)) Right(take(col)) + else + Left( + s"""@SampleColumn("$col") not present at port $port (have: ${names.mkString(", ")})""" + ) + } + case None => + allowedTypes(f, siblings) match { + case Some(types) => + schemas + .get(port) + .map(_.getAttributes.filter(a => types.contains(a.getType)).map(_.getName)) match { + case Some(cols) if cols.nonEmpty => + Right(take(cols.find(c => !used.contains((port, c))).getOrElse(cols.head))) + case _ => firstUnused(schemas, port, used) // no type-matching column; fall back + } + case None => firstUnused(schemas, port, used) + } + } + } + + /** [[AttributeType]]s permitted for `f` by its declaring class's + * `@JsonSchemaInject(json = ...)` `attributeTypeRules`, keyed by the field's + * JSON name. `None` when the field is unconstrained. + * + * A rule may be CONDITIONAL — an `allOf` of `if`/`then` branches naming a sibling + * field, which is how an operator says "what this column may hold depends on that + * knob" (an aggregation's `attribute` is numeric for sum/min/max, string for + * concat). `siblings` is the JSON object holding `f`, against which each branch's + * condition is tested; branches that do not apply contribute nothing, and `allOf` + * means the ones that do all bind, so their sets intersect. + */ + private def allowedTypes(f: Field, siblings: JsonNode): Option[Set[AttributeType]] = + Option(f.getDeclaringClass.getAnnotation(classOf[JsonSchemaInject])) + .map(_.json) + .filter(_.nonEmpty) + .flatMap(js => Try(objectMapper.readTree(js)).toOption) + .map(_.path("attributeTypeRules").path(jsonNameOf(f))) + .flatMap { rule => + val branches = + if (rule.path("allOf").isArray) rule.path("allOf").elements().asScala.toSeq + else Seq.empty + val bound = typeSet(rule.path("enum")).toSeq ++ branches + .filter(branch => conditionHolds(branch.path("if"), siblings)) + .flatMap(branch => typeSet(branch.path("then").path("enum"))) + bound.reduceOption(_ intersect _).filter(_.nonEmpty) + } + + /** The [[AttributeType]]s an `enum` array names, or `None` if it names none. */ + private def typeSet(enumNode: JsonNode): Option[Set[AttributeType]] = + if (!enumNode.isArray) None + else { + val set = enumNode.elements().asScala.flatMap(n => typeFromString(n.asText())).toSet + if (set.nonEmpty) Some(set) else None + } + + /** Whether every `sibling: { valEnum: [...] }` clause of a rule's `if` holds for the + * object the field sits in. An empty condition holds vacuously; a clause naming a + * sibling the object has not set does not. + */ + private def conditionHolds(cond: JsonNode, siblings: JsonNode): Boolean = + cond.isObject && cond.fields().asScala.forall { clause => + val permitted = clause.getValue.path("valEnum") + permitted.isArray && + permitted.elements().asScala.exists(_.asText == siblings.path(clause.getKey).asText) + } + + /** The empty object, for a caller with no sibling context: only the unconditional + * part of a rule can bind. + */ + private def noSiblings: JsonNode = objectMapper.getNodeFactory.objectNode() + + /** Where a field's constraints are read from when its own annotation cannot carry + * them: the operator's finished JSON schema, and the node within it describing the + * object currently being built. + * + * An operator implementing `JsonSchemaCustomizer` writes rules into that document + * after the annotations have been read. A hyperparameter row's `value` is stated only + * there, because what it may hold depends on the `parameter` chosen beside it and so + * cannot be annotated on a field every parameter shares. Reflection alone does not + * see those, which is why the document travels alongside the walk. + */ + private final case class SchemaScope(root: JsonNode, node: JsonNode) { + + /** The node describing one field of this object. */ + def child(jsonName: String): JsonNode = node.path("properties").path(jsonName) + + /** The scope a nested object or list element is built under, following the `$ref` + * Jackson emits in place of a class it has already defined. + */ + def descend(jsonName: String): SchemaScope = { + val field = child(jsonName) + val target = if (field.path("items").isObject) field.path("items") else field + val ref = target.path("$ref").asText("") + SchemaScope( + root, + if (ref.isEmpty) target + else root.path("definitions").path(ref.stripPrefix("#/definitions/")) + ) + } + } + + private object SchemaScope { + val empty: SchemaScope = { + val nothing = objectMapper.getNodeFactory.objectNode() + SchemaScope(nothing, nothing) + } + + /** An operator's finished schema, or [[empty]] where one cannot be produced — such a + * class is then read from its annotations alone, as every operator was before. + */ + def of(clazz: Class[_]): SchemaScope = + Try( + OperatorMetadataGenerator + .generateOperatorJsonSchema(clazz.asInstanceOf[Class[_ <: LogicalOp]]) + ).toOption.map(s => SchemaScope(s, s)).getOrElse(empty) + } + + /** What filling a field needs beyond the field itself: the input schemas a column + * picker resolves against, the columns already spoken for, and the row count a + * range-less number is sized from. Carried into the enum walk so that a value which + * makes a field apply can fill it the way the base pass would have. + * + * Empty for a caller sweeping an already-configured operator: such a config states + * both sides of a conditional itself, so nothing there is left to fill. + */ + private final case class FillContext( + schemas: Map[Int, Schema] = Map.empty, + used: mutable.Set[(Int, String)] = mutable.Set.empty, + rowCount: Int = DefaultRowCount + ) + + /** Whether a config holds nothing at this key: absent, null, the empty string a + * field initialised to `""` starts as, or the empty list a `List()` field starts + * as. All four mean the same thing to an operator reading it, so a conditional + * `required` is unmet by any of them. + */ + private def isBlank(node: JsonNode): Boolean = + node.isMissingNode || node.isNull || (node.isTextual && node.asText.isEmpty) || + (node.isArray && node.isEmpty) + + /** The fields an object's schema requires only under some condition, named by every + * `required` its `allOf` states in either branch. Empty for an object whose + * requirements are all unconditional, which is every one but a hyperparameter row. + */ + private def conditionallyRequiredFields(scope: SchemaScope): Set[String] = + scope.node + .path("allOf") + .elements() + .asScala + .flatMap(branch => Seq(branch.path("then"), branch.path("else"))) + .flatMap(_.path("required").elements().asScala) + .map(_.asText) + .toSet + + /** The fields an object's conditional `allOf` requires of `row` as it stands. The + * condition is JSON Schema's own `properties`/`const`, not the `valEnum` form a + * Texera rule uses, because this one is read by the validator rather than by the + * form. + */ + private def requiredUnder(scope: SchemaScope, row: JsonNode): Set[String] = + scope.node + .path("allOf") + .elements() + .asScala + .flatMap { branch => + val holds = branch.path("if").path("properties").fields().asScala.forall { clause => + clause.getValue.path("const") == row.path(clause.getKey) + } + val outcome = if (holds) branch.path("then") else branch.path("else") + outcome.path("required").elements().asScala.map(_.asText) + } + .toSet + + /** What a field's `valueRules` call for, given the object it sits in: the one branch + * whose condition holds. `None` for a field declaring no such rule, which is every + * field but a trainer's hyperparameter `value`. + * + * One branch at most: each names a single parameter, so unlike `attributeTypeRules` + * there is nothing to intersect. + */ + private def schemaValueRule( + f: Field, + scope: SchemaScope, + siblings: JsonNode + ): Option[JsonNode] = { + val branches = scope.child(jsonNameOf(f)).path("valueRules").path("allOf") + if (!branches.isArray) None + else + branches + .elements() + .asScala + .find(branch => conditionHolds(branch.path("if"), siblings)) + .map(_.path("then")) + } + + /** The value a `valueRules` branch calls for: the example it offers, else the head of + * its accepted set, which the branch states default-first. Both arrive as text and + * the field they fill is a `String` — the branch's `type` says how the OPERATOR will + * convert that text, not how the config carries it. + */ + private def ruleFill(rule: JsonNode): Option[JsonNode] = + rule + .path("examples") + .elements() + .asScala + .toSeq + .headOption + .orElse(rule.path("enum").elements().asScala.toSeq.headOption) + + private def typeFromString(s: String): Option[AttributeType] = + AttributeType.values().find(_.name.equalsIgnoreCase(s)) +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala new file mode 100644 index 00000000000..d538525266b --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc +import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc +import org.apache.texera.amber.operator.intersect.IntersectOpDesc +import org.apache.texera.amber.operator.visualization.histogram2d.Histogram2DOpDesc +import org.apache.texera.amber.operator.visualization.radarChart.RadarChartOpDesc +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Drives [[ConfigGenerator]] across the range of operator config shapes: + * no-config, flat autofill + enum, and a nested list of objects with a + * free-form value. These are the cases the reflective generator must handle to + * cover the JVM-exec operators automatically. + */ +class ConfigGeneratorSpec extends AnyFlatSpec with Matchers { + + private val schema = new Schema( + new Attribute("id", AttributeType.INTEGER), + new Attribute("name", AttributeType.STRING), + new Attribute("score", AttributeType.DOUBLE) + ) + private val twoPorts = Map(0 -> schema, 1 -> schema) + + "ConfigGenerator" should "configure an operator that has no config fields" in { + val result = ConfigGenerator.generate(classOf[IntersectOpDesc], twoPorts) + withClue(result) { result.isRight shouldBe true } + result.toOption.get shouldBe a[IntersectOpDesc] + } + + it should "fill autofill column refs from the correct port and default the enum" in { + val result = ConfigGenerator.generate(classOf[HashJoinOpDesc[Any]], twoPorts) + withClue(result) { result.isRight shouldBe true } + val op = result.toOption.get.asInstanceOf[HashJoinOpDesc[Any]] + schema.getAttributeNames should contain(op.buildAttributeName) + schema.getAttributeNames should contain(op.probeAttributeName) + op.joinType should not be null + } + + it should "build a non-empty nested predicate list with a valid column and enum" in { + val result = ConfigGenerator.generate(classOf[SpecializedFilterOpDesc], Map(0 -> schema)) + result.isRight shouldBe true + val op = result.toOption.get.asInstanceOf[SpecializedFilterOpDesc] + op.predicates should not be empty + val p = op.predicates.head + schema.getAttributeNames should contain(p.attribute) + p.condition should not be null + } + + // ── semantic column resolution (the @SampleColumn / attributeTypeRules tiers) ── + + it should "assign distinct columns to sibling autofill fields (no x = y collapse)" in { + // Histogram2D's xColumn and yColumn are both plain @AutofillAttributeName + // with no type-rule; before distinct-aware binding both resolved to the + // first column ("id"). They must now differ so the plot isn't a degenerate + // diagonal. + val result = + ConfigGenerator.generate(classOf[Histogram2DOpDesc], CanonicalFixture.schemasByPort) + withClue(result) { result.isRight shouldBe true } + val op = result.toOption.get.asInstanceOf[Histogram2DOpDesc] + op.xColumn.toString should not be empty + op.xColumn.toString should not be op.yColumn.toString + } + + it should "keep a list knob off the column a single-column sibling took" in { + // Radar Chart picks its name column first and its value columns took the whole + // table, so the name arrived inside them and the generated `required_cols`, + // which is the name followed by the values, named it twice. More than one value + // column left, or the list runs the same code a single column would. + val result = + ConfigGenerator.generate(classOf[RadarChartOpDesc], CanonicalFixture.schemasByPort) + withClue(result) { result.isRight shouldBe true } + val op = result.toOption.get.asInstanceOf[RadarChartOpDesc] + op.valueColumns.map(_.toString) should not contain op.nameColumn.toString + op.valueColumns.size should be > 1 + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala new file mode 100644 index 00000000000..52ecf4140c8 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala @@ -0,0 +1,666 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.distinct.DistinctOpDesc +import org.apache.texera.amber.operator.aggregate.{ + AggregateOpDesc, + AggregationFunction, + AggregationOperation +} +import org.apache.texera.amber.operator.filter.{ + ComparisonType, + FilterPredicate, + SpecializedFilterOpDesc +} +import org.apache.texera.amber.operator.hashJoin.{HashJoinOpDesc, JoinType} +import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc +import org.apache.texera.amber.operator.projection.{AttributeUnit, ProjectionOpDesc} +import org.apache.texera.amber.operator.regex.RegexOpDesc +import org.apache.texera.amber.operator.typecasting.{TypeCastingOpDesc, TypeCastingUnit} +import org.apache.texera.amber.operator.visualization.ImageViz.ImageVisualizerOpDesc + +import org.apache.texera.amber.operator.visualization.dumbbellPlot.{ + DumbbellDotConfig, + DumbbellPlotOpDesc +} +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnLinearRegressionOpDesc +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor +import org.apache.texera.amber.operator.ifStatement.IfOpDesc +import java.nio.file.{Files, Path} +import java.util + +/** + * A curated handler ships a configured OpDesc and the input fixtures it + * needs, written once into `testRoot`. Register it in [[CuratedHandlers.all]] + * to override the auto-config tier for that operator. + */ +trait TransformHandler { + def opDescClass: Class[_ <: LogicalOp] + def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) + + /** Extra independent scenarios beyond [[fixture]], each a self-contained + * (label, configured op, its own inputs). The runner runs each as a PINNED + * config (no enum sweep), in its own work subdir. Default: none. + * + * Used where one operator needs structurally different inputs per config + * branch that a single swept fixture can't cover — e.g. the sklearn + * `countVectorizer=true` text path, whose feature column must be text and so + * is incompatible with the numeric default fixture (`X = table.drop(target)` + * would feed a string column to a numeric estimator). Each scenario must + * write its input files somewhere unique (e.g. a `testRoot` subdir) so it + * does not clobber the primary fixture's files. + */ + def extraScenarios(testRoot: Path): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + Seq.empty + + /** Opts this fixture into the `nulls` case, naming the columns it must never + * empty because their VALUE is what the fixture was built to arrange rather + * than data under test: a join key that has to pair, a grouping key that has + * to group. Emptying one of those changes what the test asks instead of asking + * what the operator does with a null. + * + * The default is `Some(Set.empty)`: most curated tables arrange nothing that a + * hole would disturb, so taking part is the normal case and a fixture that + * cannot afford a hole says so. `None` sits the case out entirely, for a table + * whose every column is load-bearing. + */ + def nullsKeepFilled: Option[Set[String]] = Some(Set.empty) +} + +/** + * The curated override tier of the config/fixture resolution chain: an + * operator listed here is verified with its hand-written fixture instead of + * the auto-generated one. This is also the seam where Xuan's curated + * operator-field-values JSON plugs in later, as a second curated source. + */ +object CuratedHandlers { + + /** Concrete `LogicalOp` classes discovered from the `@JsonSubTypes` registry + * on [[LogicalOp]] — the same source [[ConfigGenerator]] enumerates. The + * sklearn handler families below are auto-derived from this list, so a newly + * registered sklearn estimator is picked up with zero per-operator + * boilerplate here. + */ + private val registeredOps: Seq[Class[_ <: LogicalOp]] = + Option(classOf[LogicalOp].getAnnotation(classOf[com.fasterxml.jackson.annotation.JsonSubTypes])) + .map(_.value().toSeq.map(_.value().asInstanceOf[Class[_ <: LogicalOp]])) + .getOrElse(Seq.empty) + + private def isConcrete(cls: Class[_]): Boolean = + !java.lang.reflect.Modifier.isAbstract(cls.getModifiers) + + /** The concrete leaf ops under one sklearn base, excluding the base itself. + * + * No hard-coded baseline: a new sklearn operator is picked up automatically + * the moment it is registered in LogicalOp's @JsonSubTypes — zero per-op code + * here. The test suite (ConfigCoverageSpec / TransformVerificationRunnerSpec) + * is the safety net: a mis-discovered or misbehaving op fails its own parity + * check rather than being frozen by an assertion. + */ + private def sklearnFamily(base: Class[_]): Seq[Class[_ <: LogicalOp]] = + registeredOps.filter(c => base.isAssignableFrom(c) && c != base && isConcrete(c)) + + private def trainingOps = sklearnFamily(classOf[SklearnTrainingOpDesc]) + private def classifierOps = sklearnFamily(classOf[SklearnClassifierOpDesc]) + private def advancedOps = sklearnFamily(classOf[SklearnMLOperatorDescriptor[_]]) + + /** Every sklearn op, whichever tier serves it. `X = table.drop(target)` feeds + * each remaining column to `fit`, so these take canonical's petal-and-label + * projection rather than the whole table, whose string columns end the fit. + * + * Linear Regression is named on its own because it descends from + * `PythonOperatorDescriptor` directly rather than from one of the three + * bases, so no family picks it up. + */ + val sklearnNumericClasses: Set[Class[_ <: LogicalOp]] = + (trainingOps ++ classifierOps ++ advancedOps).toSet + classOf[SklearnLinearRegressionOpDesc] + + val all: Seq[TransformHandler] = Seq( + AggregateTransformHandler, + SpecializedFilterTransformHandler, + DistinctTransformHandler, + ProjectionTransformHandler, + HashJoinTransformHandler, + TypeCastingTransformHandler, + KeywordSearchTransformHandler, + DumbbellPlotVisualizationHandler, + ImageVisualizerVisualizationHandler, + IfTransformHandler, + RegexTransformHandler + ) + + val byClass: Map[Class[_ <: LogicalOp], TransformHandler] = + all.map(h => h.opDescClass -> h).toMap + + /** Generic fixture writer: builds a JSONL file with the given typed columns + * and rows, boxing each value per its declared [[AttributeType]]. Lets a + * curated handler declare bespoke per-operator input data in one call + * instead of hand-rolling a Schema + Tuple.builder loop. + */ + def writeFixture( + path: Path, + columns: Seq[(String, AttributeType)], + rows: Seq[Seq[Any]] + ): Path = { + val schema = new Schema(columns.map { case (n, t) => new Attribute(n, t) }: _*) + val tuples = rows.map { row => + val builder = Tuple.builder(schema) + columns.zip(row).foreach { + case ((name, attrType), value) => + val boxed: AnyRef = (attrType, value) match { + case (_, null) => null + case (AttributeType.INTEGER, x: Int) => Int.box(x) + case (AttributeType.INTEGER, x: Long) => Int.box(x.toInt) + case (AttributeType.INTEGER, x: Double) => Int.box(x.toInt) + case (AttributeType.LONG, x: Long) => Long.box(x) + case (AttributeType.LONG, x: Int) => Long.box(x.toLong) + case (AttributeType.DOUBLE, x: Double) => Double.box(x) + case (AttributeType.DOUBLE, x: Int) => Double.box(x.toDouble) + case (AttributeType.DOUBLE, x: Long) => Double.box(x.toDouble) + case (AttributeType.BOOLEAN, x: Boolean) => Boolean.box(x) + case (AttributeType.STRING, x) => x.toString + case (_, x) => x.toString + } + builder.add(schema.getAttribute(name), boxed) + } + builder.build() + } + TupleIO.writeTuples(path, tuples.iterator, schema) + path + } + +} + +/** + * Handler for `SpecializedFilterOpDesc`. Curated CONFIG over the shared + * canonical fixture: the auto tier fills a free-form predicate `value` with + * the canonical "1", which pins the shape of the comparison but not its + * corners. `id > 8 OR name == "eve"` exercises numeric comparison, string + * equality (the JSON predicate `value` is always a string) and OR-combination + * in one run, and keeps 5 of port 0's 10 rows — a proper subset either way. + * + * Both JVM `SpecializedFilterOpExec` and pandas boolean indexing preserve + * input row order, so positional comparator equality holds. + */ +object SpecializedFilterTransformHandler extends TransformHandler { + + override val opDescClass: Class[_ <: LogicalOp] = classOf[SpecializedFilterOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val desc = new SpecializedFilterOpDesc() + desc.predicates = List( + new FilterPredicate("id", ComparisonType.GREATER_THAN, "8"), + new FilterPredicate("name", ComparisonType.EQUAL_TO, "eve") + ) + + (desc, CanonicalFixture.writeInputs(testRoot, 1)) + } +} + +/** Handler for `DistinctOpDesc`. The canonical auto-fixture is all-distinct + * (uniq_name is globally unique by invariant), so it never exercises dedup. + * This 5-row table repeats two rows so both paths must actually drop + * duplicates; survivors keep first-occurrence order (JVM LinkedHashSet == + * pandas drop_duplicates keep="first"), so the positional comparator holds. + */ +/** + * Curated handler for [[ProjectionOpDesc]]. Its `attributes` list is not declared + * `required`, so the auto tier starts it empty the way the UI does — and + * `getPhysicalOp` refuses an empty list. Pinning one row is all this needs; the + * runner derives the rest of the variants from it. + */ +object ProjectionTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[ProjectionOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val columns = Seq( + ("id", AttributeType.INTEGER), + ("name", AttributeType.STRING), + ("score", AttributeType.DOUBLE) + ) + val rows = Seq( + Seq[Any](1, "a", 1.5), + Seq[Any](2, "b", 2.5), + Seq[Any](3, "c", 3.5) + ) + val inputPath = + CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) + val op = new ProjectionOpDesc() + // A blank alias is the untouched state of the row the `+` button adds, and it is + // the branch where the operator keeps the original name. + op.attributes = List(new AttributeUnit("id", "")) + (op, Map(PortIdentity(0) -> inputPath)) + } +} + +object DistinctTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[DistinctOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val columns = Seq( + ("id", AttributeType.INTEGER), + ("name", AttributeType.STRING) + ) + val rows = Seq( + Seq[Any](1, "a"), + Seq[Any](2, "b"), + Seq[Any](1, "a"), // duplicate of row 0 + Seq[Any](3, "c"), + Seq[Any](2, "b") // duplicate of row 1 + ) + val inputPath = + CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) + (new DistinctOpDesc(), Map(PortIdentity(0) -> inputPath)) + } +} + +/** + * Curated handler for [[RegexOpDesc]]. The auto tier only ever feeds it the + * trivial pattern `"1"` against the first column, which never exercises real + * regex semantics. This handler pins genuine patterns so the JVM↔Python engine + * parity is actually tested: + * + * - Primary fixture: `[a-z]+` over a mixed-case `text` column. The runner + * enum-sweeps the Boolean `caseInsensitive`, so BOTH branches run against + * the same data. The two branches select DIFFERENT row sets (case-sensitive + * keeps only rows with a lowercase letter; case-insensitive also keeps the + * all-caps rows), proving the flag actually flows through to both paths. + * - `extraScenarios`: `\d+` (a backslash class — verifies the escape survives + * `toPyDoubleQuotedLiteral` into Python's engine) and `\.` (an escaped + * metachar — an escaping bug would turn it into "match any char" and change + * the result, so this pins literal-vs-metachar handling). + * + * All fixture data is ASCII, where Java `\d` / `[a-z]` / CASE_INSENSITIVE and + * Python's `re` agree exactly; each pattern yields a proper subset (never + * all/none) so the comparison is meaningful. + */ +object RegexTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[RegexOpDesc] + + private def regexOp(attribute: String, regex: String, caseInsensitive: Boolean): RegexOpDesc = { + val op = new RegexOpDesc() + op.attribute = attribute + op.regex = regex + op.caseInsensitive = caseInsensitive + op + } + + // Rows chosen so `[a-z]+` differs by case flag: "ABC"/"XY9" have no lowercase + // (dropped when case-sensitive) but are all-letter (kept when insensitive). + private val textColumn = Seq(("text", AttributeType.STRING)) + private val caseRows: Seq[Seq[Any]] = + Seq(Seq[Any]("abc"), Seq[Any]("ABC"), Seq[Any]("123"), Seq[Any]("a1B"), Seq[Any]("XY9")) + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val inputPath = + CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), textColumn, caseRows) + (regexOp("text", "[a-z]+", caseInsensitive = false), Map(PortIdentity(0) -> inputPath)) + } + + override def extraScenarios( + testRoot: Path + ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = { + // `\d+`: rows where a digit is present form a proper subset. + val digitDir = testRoot.resolve("digits") + Files.createDirectories(digitDir) + val digitRows: Seq[Seq[Any]] = + Seq(Seq[Any]("abc"), Seq[Any]("a1B"), Seq[Any]("XY9"), Seq[Any]("123"), Seq[Any]("ab")) + val digitInput = + CuratedHandlers.writeFixture(digitDir.resolve("input_port_0.jsonl"), textColumn, digitRows) + + // `\.`: only rows with a literal dot match. If the backslash were lost, the + // pattern would become bare `.` (match any char) and select every row. + val dotDir = testRoot.resolve("dot") + Files.createDirectories(dotDir) + val dotRows: Seq[Seq[Any]] = + Seq(Seq[Any]("a.b"), Seq[Any]("abc"), Seq[Any]("x.y.z"), Seq[Any]("no")) + val dotInput = + CuratedHandlers.writeFixture(dotDir.resolve("input_port_0.jsonl"), textColumn, dotRows) + + Seq( + ( + "regex=\\d+", + regexOp("text", "\\d+", caseInsensitive = false), + Map(PortIdentity(0) -> digitInput) + ), + ( + "regex=\\.", + regexOp("text", "\\.", caseInsensitive = false), + Map(PortIdentity(0) -> dotInput) + ) + ) + } +} + +/** HashJoin INNER on `id`. Build (port 0) and probe (port 1) intentionally + * arrive in different id orders so any probe-major / left-major mismatch + * between the JVM emit and `pd.merge` shows up. HashJoin inherits the + * unordered `LogicalOp.orderSensitive` default, so rows compare as a set. + */ +object HashJoinTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[HashJoinOpDesc[_]] + + /** `id` is what the two sides pair on: empty it and the rows stop matching, so + * the run would be asking about an inner join that finds nothing rather than + * about a null. The payload columns carry no arrangement and take the holes. + */ + override def nullsKeepFilled: Option[Set[String]] = Some(Set("id")) + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val buildSchema = new Schema( + new Attribute("id", AttributeType.INTEGER), + new Attribute("name", AttributeType.STRING) + ) + val probeSchema = new Schema( + new Attribute("id", AttributeType.INTEGER), + new Attribute("score", AttributeType.INTEGER) + ) + + def buildTup(id: Int, name: String): Tuple = { + val b = Tuple.builder(buildSchema) + b.add(buildSchema.getAttribute("id"), Int.box(id)) + b.add(buildSchema.getAttribute("name"), name) + b.build() + } + def probeTup(id: Int, score: Int): Tuple = { + val b = Tuple.builder(probeSchema) + b.add(probeSchema.getAttribute("id"), Int.box(id)) + b.add(probeSchema.getAttribute("score"), Int.box(score)) + b.build() + } + + val buildRows = Seq( + buildTup(3, "carol"), + buildTup(1, "alice"), + buildTup(5, "eve"), + buildTup(2, "bob"), + buildTup(4, "dave") + ) + val probeRows = Seq( + probeTup(1, 95), + probeTup(2, 80), + probeTup(3, 88), + probeTup(4, 72), + probeTup(5, 91) + ) + val buildPath = testRoot.resolve("input_port_0.jsonl") + val probePath = testRoot.resolve("input_port_1.jsonl") + TupleIO.writeTuples(buildPath, buildRows.iterator, buildSchema) + TupleIO.writeTuples(probePath, probeRows.iterator, probeSchema) + + val desc = new HashJoinOpDesc[Integer]() + desc.buildAttributeName = "id" + desc.probeAttributeName = "id" + desc.joinType = JoinType.INNER + + (desc, Map(PortIdentity(0) -> buildPath, PortIdentity(1) -> probePath)) + } +} + +/** + * Handler for `TypeCastingOpDesc`. The auto tier points `attribute` at the + * canonical fixture's first column (`id`, INTEGER) and then sweeps `resultType` + * across ALL `AttributeType` values — but `TypeCastingUnit`'s attributeTypeRules + * only permit certain source types per target (e.g. `timestamp` accepts only + * string/long), and the native `TypeCastingOpExec` throws on an illegal cast + * (INTEGER → Timestamp). So the auto variant `resultType=timestamp` crashes + * Path A before any comparison. + * + * This fixture gives each cast a type-compatible source column and a value that + * round-trips identically on both paths (JVM `AttributeTypeUtils` vs the + * generated pandas), covering the value-comparable branches of + * `generateStandaloneCode`'s `resultType` match: STRING, INTEGER, LONG, DOUBLE, + * BOOLEAN. The op has an `enumSweep` row in + * [[TransformVerificationRunner.variantsNotRun]], suppressing the blind + * one-enum-at-a-time sweep that would re-pair each fixed column with every target + * type; the units below already exercise each branch. Map op: both paths keep + * input row order, so strict positional equality holds. + * + * TIMESTAMP is intentionally omitted: the two runtimes serialize a Timestamp + * differently to JSONL (native emits an ISO string `"2024-01-01 09:00:00.0"`, + * pandas emits epoch millis `1704099600000`), so the dataframe comparator flags + * a representation mismatch even though the instant is identical — a harness-wide + * timestamp-serialization gap, not a TypeCasting translation defect. + */ +object TypeCastingTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[TypeCastingOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + // One dedicated source column per target so the casts don't chain. + val columns = Seq( + ("str_to_int", AttributeType.STRING), // numeric string → INTEGER + ("int_to_dbl", AttributeType.INTEGER), // integer → DOUBLE + ("int_to_str", AttributeType.INTEGER), // integer → STRING + ("int_to_lng", AttributeType.INTEGER), // integer → LONG + ("int_to_bool", AttributeType.INTEGER) // 1/0 → BOOLEAN + ) + val rows = Seq( + Seq[Any]("10", 1, 6, 11, 1), + Seq[Any]("20", 2, 7, 12, 0), + Seq[Any]("30", 3, 8, 13, 1), + Seq[Any]("40", 4, 9, 14, 0), + Seq[Any]("50", 5, 10, 15, 1) + ) + val inputPath = + CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) + + def unit(attr: String, t: AttributeType): TypeCastingUnit = { + val u = new TypeCastingUnit() + u.attribute = attr + u.resultType = t + u + } + val desc = new TypeCastingOpDesc() + desc.typeCastingUnits = List( + unit("str_to_int", AttributeType.INTEGER), + unit("int_to_dbl", AttributeType.DOUBLE), + unit("int_to_str", AttributeType.STRING), + unit("int_to_lng", AttributeType.LONG), + unit("int_to_bool", AttributeType.BOOLEAN) + ) + + (desc, Map(PortIdentity(0) -> inputPath)) + } +} + +/** + * Handler for `KeywordSearchOpDesc`. The auto tier points `attribute` at the + * canonical fixture's first column (`id`) and fills `keyword` with the canonical + * "1", so the search runs against numeric ids and never touches a real text + * column. This fixture searches a genuine free-text column with a two-term + * query, exercising the standalone regex's meaningful branches — multi-term OR, + * whole-word boundaries — that both the JVM Lucene path and the pandas path + * agree on. Query "love day" keeps rows 1 and 2 (contain the whole words + * love/day); row 3 has neither; row 4's "lovely"/"today" are different tokens, + * so the shared word-boundary rule drops it. 4 rows → 2 kept. + * + * The rows are intentionally punctuation-free. The `isCaseSensitive` enum is + * swept (true and false), and the case-sensitive path uses `CaseSensitiveAnalyzer` + * (a `WhitespaceTokenizer` that leaves punctuation attached, e.g. "perfect."), + * which diverges from the standalone regex's `\b`-boundary matching on any + * punctuated word — and the standalone does NOT honor case at all. Clean + * whitespace-delimited words keep both tokenizers (and both case modes) in + * agreement; this is why the canonical fixture's punctuated `short_text` column + * cannot be reused here. Lucene phrase/boolean/wildcard syntax is likewise + * avoided — the regex approximation cannot reproduce it. + */ +object KeywordSearchTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[KeywordSearchOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val columns = Seq(("txt", AttributeType.STRING)) + val rows = Seq( + Seq[Any]("i love this product"), + Seq[Any]("what a great day"), + Seq[Any]("terrible experience"), + Seq[Any]("lovely weather today") + ) + val inputPath = + CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) + + val desc = new KeywordSearchOpDesc() + desc.attribute = "txt" + desc.keyword = "love day" + desc.isCaseSensitive = false + + (desc, Map(PortIdentity(0) -> inputPath)) + } +} + +/** DumbbellPlot: curated CONFIG over the shared canonical fixture. A dumbbell is + * one line per entity between the entity's value in two categories, so the two + * category values have to be values the entity actually has — which the auto tier + * cannot know: it fills both with the canonical string, leaving start == end and + * every line a point. + * + * `node_src` = n3 / n1 is the pair that works on this fixture: `bob` holds both + * (score 1.2 → 2.8) and so does `1` (1.7 → 0.5), giving two real dumbbells, while + * eve, dave and grace hold one each and stay single points — both branches drawn + * at once. `comparedColumnName` is a STRING column on purpose: plotly's trace + * `name` rejects a numpy number, so a numeric column raises there instead of + * plotting (reported upstream, not worked around here). + */ +object DumbbellPlotVisualizationHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[DumbbellPlotOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val dots = new util.ArrayList[DumbbellDotConfig]() + val dot = new DumbbellDotConfig() + dot.dotValue = "open" + dots.add(dot) + + val desc = new DumbbellPlotOpDesc() + desc.categoryColumnName = "node_src" + desc.dumbbellStartValue = "n3" + desc.dumbbellEndValue = "n1" + desc.measurementColumnName = "score" + desc.comparedColumnName = "name" + desc.dots = dots + + (desc, CanonicalFixture.writeInputs(testRoot, 1)) + } +} + +/** ImageVisualizer fixture. Uses deterministic binary payloads; the operator + * base64-encodes them into img tags. + */ +object ImageVisualizerVisualizationHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[ImageVisualizerOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val schema = new Schema(new Attribute("image_bytes", AttributeType.BINARY)) + + def tup(bytes: Array[Byte]): Tuple = { + val builder = Tuple.builder(schema) + builder.add(schema.getAttribute("image_bytes"), bytes) + builder.build() + } + + val rows = Seq( + tup(Array[Byte](1, 2, 3, 4)), + tup(Array[Byte](10, 20, 30, 40)) + ) + val inputPath = testRoot.resolve("input_port_0.jsonl") + TupleIO.writeTuples(inputPath, rows.iterator, schema) + + val desc = new ImageVisualizerOpDesc() + desc.binaryContent = "image_bytes" + + (desc, Map(PortIdentity(0) -> inputPath)) + } +} + +/** If operator: routes the data port (port 1) to the True (port 1) or False + * (port 0) output. We feed an EMPTY Condition port (port 0) so IfOpExec + * forwards no condition rows; with no State message it keeps its default + * active output (True), matching the standalone's default-True branch — so + * the True output gets all data rows and the False output is empty on both + * paths. + */ +/** Aggregate fixture exercising every aggregation function in one op, including + * COUNT(*) (empty attribute). Auto-config can't build this: upstream #5896 made + * `AggregationOperation.attribute` optional (required only for non-count via a + * conditional JSON-schema rule), so ConfigGenerator skips the optional autofill + * field and leaves it null — invalid for a non-count function, which NPEs in + * AggregateOpExec. This pins valid (function, column) pairs. Enum-sweep-exempt + * (see [[TransformVerificationRunner.variantsNotRun]]): the sweep flips each + * element's function in isolation and would re-pair, e.g., concat with a numeric + * column; the fixture already covers each function with a type-compatible column. + * Aggregate inherits the unordered `orderSensitive` default, so + * the comparator lex-sorts rows before comparing. + */ +object AggregateTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[AggregateOpDesc] + + private def agg( + fn: AggregationFunction, + attr: String, + result: String + ): AggregationOperation = { + val a = new AggregationOperation() + a.aggFunction = fn + a.attribute = attr + a.resultAttribute = result + a + } + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val desc = new AggregateOpDesc() + desc.groupByKeys = List("name") + desc.aggregations = List( + agg(AggregationFunction.SUM, "score", "sum_score"), + agg(AggregationFunction.COUNT, "", "count_all"), // empty attribute => COUNT(*) + agg(AggregationFunction.COUNT, "score", "count_score"), + agg(AggregationFunction.AVERAGE, "score", "avg_score"), + agg(AggregationFunction.MIN, "score", "min_score"), + agg(AggregationFunction.MAX, "score", "max_score"), + agg(AggregationFunction.CONCAT, "iso_country", "cat_country") + ) + (desc, CanonicalFixture.writeInputs(testRoot, 1)) + } +} + +object IfTransformHandler extends TransformHandler { + override val opDescClass: Class[_ <: LogicalOp] = classOf[IfOpDesc] + + override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { + val cols = Seq("id" -> AttributeType.INTEGER, "name" -> AttributeType.STRING) + val condition = + CuratedHandlers.writeFixture( + testRoot.resolve("input_port_0.jsonl"), + cols, + Seq.empty[Seq[Any]] + ) + val data = CuratedHandlers.writeFixture( + testRoot.resolve("input_port_1.jsonl"), + cols, + Seq(Seq(1, "a"), Seq(2, "b"), Seq(3, "c")) + ) + val desc = new IfOpDesc() + desc.conditionName = "cond" + (desc, Map(PortIdentity(0) -> condition, PortIdentity(1) -> data)) + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala index dbeb4829463..685b6f056db 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala @@ -22,6 +22,7 @@ package org.apache.texera.amber.translator.verify import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.distinct.DistinctOpDesc +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -35,6 +36,12 @@ import java.nio.file.{Files, Path} */ class HarnessSpec extends AnyFlatSpec with Matchers { + /** Only the standalone run needs an interpreter, so only it is held back from + * the job that provisions none. The other two are JVM-side and run there. + */ + private val NeedsPython = + Tag("org.apache.texera.amber.translator.verify.tags.IntegrationTest") + private val schema = new Schema( new Attribute("id", AttributeType.INTEGER), new Attribute("name", AttributeType.STRING) @@ -83,7 +90,7 @@ class HarnessSpec extends AnyFlatSpec with Matchers { } } - "StandaloneRunner" should "run the generated script and reach the same answer" in { + "StandaloneRunner" should "run the generated script and reach the same answer" taggedAs NeedsPython in { withInput { (dir, input) => val work = dir.resolve("standalone") Files.createDirectories(work) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala new file mode 100644 index 00000000000..f6d8b6759be --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.annotation.JsonSubTypes +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.source.SourceOperatorDescriptor +import org.apache.texera.amber.translator.verify.tags.IntegrationTest +import org.scalatest.ParallelTestExecution +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Auto-discovered behavioral-parity tests: for every operator registered + * with [[LogicalOp]]'s `@JsonSubTypes` that implements + * [[StandaloneCodeGenerator]], emit a test that runs both Path A (Texera + * exec) and Path B (translator-generated Python via [[StandaloneRunner]]) + * and asserts their outputs are equivalent. + * + * Dispatch is auto-first: [[TransformVerificationRunner]] classifies each + * non-source transform as `Runnable("auto")` (auto-configured fixture), + * `Runnable("curated")` (hand-written fixture from [[CuratedHandlers]]), + * or `Flagged(reason)` (shown as ignored with the reason in the test name). + * Sources route to [[SourceCategoryRunner]] unchanged. + * + * No edits to this spec are needed when a new operator is added — reflection + * discovers it automatically via `@JsonSubTypes`. The tier label appears in + * the test name so the report shows which path exercised each operator. + * + * Requires Python 3 with pandas on the [[Comparator]] / [[StandaloneRunner]] + * resolution chain (`UDF_PYTHON_PATH` env var, then `python3.12`). + */ +// Tagged @IntegrationTest: this is the only verify spec that forks a real +// Python process end-to-end, so CI routes it to the Python-provisioned +// integration job (see workflow-compiling-service/build.sbt WCS_TEST_FILTER). +@IntegrationTest +class OperatorBehaviorSpec extends AnyFlatSpec with Matchers with ParallelTestExecution { + + // Build the test list at class construction. Each branch below registers + // one test (`in` for runnable, `ignore` for skipped) so the test report + // shows every translator-eligible operator and why it did or didn't run. + OperatorBehaviorSpec.discoverStandaloneOperators().foreach { opClass => + val name = opClass.getSimpleName + + if (!OperatorBehaviorSpec.isSelected(name)) { + // Narrowed out by VERIFY_ONLY / VERIFY_SKIP, which only a local run sets. + // Still registered, as an `ignore`, so the report lists every operator + // rather than reading as though the narrowed-out ones do not exist. + name should "NARROWED OUT — outside this run's VERIFY_ONLY / VERIFY_SKIP" ignore {} + } else if (classOf[SourceOperatorDescriptor].isAssignableFrom(opClass)) { + // Sources keep their handler-per-source design: each needs a real file + // in its specific format, which a generic fixture can't supply. + if (SourceCategoryRunner.canRun(opClass)) { + name should "produce equivalent output in Texera and standalone Python (source)" in { + SourceCategoryRunner.run(opClass) + } + } else { + name should s"FLAGGED — ${SourceCategoryRunner.flagReason(opClass)}" ignore {} + } + } else { + TransformVerificationRunner.disposition(opClass) match { + case TransformVerificationRunner.Runnable(tier) => + name should s"produce equivalent output in Texera and standalone Python ($tier)" in { + TransformVerificationRunner.run(opClass) + } + case TransformVerificationRunner.Flagged(reason) => + name should s"FLAGGED — $reason" ignore { + // Reason is in the test name so the report carries it; the + // coverage table in ConfigCoverageSpec aggregates these. + } + } + } + } + + // Not one test per operator like the rest of this spec: it is one assertion + // over all of them, and it deliberately ignores the selection knobs above so a + // VERIFY_ONLY run still cannot hide a broken splice site. + "Generated standalone code" should "stay parseable when the column names are hostile" in { + StandaloneEscapingCheck.run() shouldBe empty + } +} + +object OperatorBehaviorSpec { + + // Narrowing knobs for a local run, both unset by default, so the default run + // is every operator: VERIFY_ONLY names the only ones to run, VERIFY_SKIP the + // ones to leave out. Case-sensitive substrings against the operator's simple + // name, comma-separated. Neither is set in CI, which therefore runs the lot. + // + // There is deliberately no third list withholding operators by default. What + // stays withheld is narrower than an operator and lives where it can say why: + // a single variant in [[TransformVerificationRunner.variantsNotRun]], or an + // operator that cannot be run at all in its `knownIssues`, each against an + // issue or a reason. A name here would withdraw an operator's every variant + // and record nothing about what is wrong with it. + private def patterns(envVar: String): Seq[String] = + sys.env.getOrElse(envVar, "").split(",").iterator.map(_.trim).filter(_.nonEmpty).toSeq + + private lazy val onlyPatterns: Seq[String] = patterns("VERIFY_ONLY") + private lazy val skipPatterns: Seq[String] = patterns("VERIFY_SKIP") + + /** True if `name` should run: in VERIFY_ONLY when that is set, and not in + * VERIFY_SKIP. True for everything when neither is set. + */ + def isSelected(name: String): Boolean = { + val included = onlyPatterns.isEmpty || onlyPatterns.exists(name.contains) + val excluded = skipPatterns.exists(name.contains) + included && !excluded + } + + /** + * Enumerates every concrete subclass of [[LogicalOp]] declared in its + * `@JsonSubTypes` annotation, filters to those implementing + * [[StandaloneCodeGenerator]], and returns them sorted by simple name + * (stable test report order). + * + * Uses the same registry Jackson uses to deserialize operators — no + * separate discovery mechanism needed. Adding an operator to + * `LogicalOp.@JsonSubTypes` makes it visible here automatically. + */ + def discoverStandaloneOperators(): Seq[Class[_ <: LogicalOp]] = { + val annotation = classOf[LogicalOp].getAnnotation(classOf[JsonSubTypes]) + if (annotation == null) Seq.empty + else + annotation + .value() + .toSeq + .map(_.value()) + .filter(classOf[StandaloneCodeGenerator].isAssignableFrom) + .map(_.asInstanceOf[Class[_ <: LogicalOp]]) + .distinct + .sortBy(_.getSimpleName) + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala new file mode 100644 index 00000000000..f42c1ca6897 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Schema, Tuple} +import org.apache.texera.amber.core.workflow.PortIdentity + +import java.nio.file.Path + +/** + * A checked-in table a whole FAMILY of operators runs on, as opposed to one + * written for a single operator. + * + * Which table an operator runs on is its own axis, separate from who writes its + * config. [[CanonicalFixture]] is the wide mixed-type table every operator takes; + * the sklearn families take [[ProjectedFixture]] views of it, since + * `X = table.drop(target)` feeds every remaining column to `fit`, which a string + * or timestamp column ends. + */ +trait SharedFixture { + + def schema: Schema + + /** The rows port `port` gets. Ports may take different windows of the table + * (canonical overlaps them partially, to defeat hash-coincidence passes on + * joins) or the same rows twice. + */ + def rowsFor(port: Int): Seq[Tuple] + + /** Every row of the table, ports aside — what a [[ProjectedFixture]] of it + * narrows. A table whose ports read windows says so by overriding; by default + * a port already sees the whole of it. + */ + def allRows: Seq[Tuple] = rowsFor(0) + + /** Columns [[write]] never empties, because their VALUE is what the table was + * built to arrange rather than data under test: canonical's `id` is what joins + * and set operations pair rows on, and a sklearn table's label is what its + * estimator fits against. Emptying one of those changes what the test asks + * instead of asking what an operator does with a null. + */ + def keepFilled: Set[String] + + /** Write one JSONL file per 0-based input port under `dir`. At most 2 ports. */ + final def write( + dir: Path, + inputPortCount: Int, + withGaps: Boolean + ): Map[PortIdentity, Path] = { + require( + inputPortCount >= 1 && inputPortCount <= 2, + s"unsupported input port count: $inputPortCount" + ) + (0 until inputPortCount).map { port => + val rows = rowsFor(port) + val path = dir.resolve(s"input_port_$port.jsonl") + TupleIO.writeTuples( + path, + (if (withGaps) emptyOneCellPerColumn(rows) else rows).iterator, + schema + ) + PortIdentity(port) -> path + }.toMap + } + + /** Schemas ConfigGenerator resolves @AutofillAttributeName fields against. + * Every port sees the same columns: a fixture's ports differ in which ROWS + * they get, not in shape. + */ + final def schemasByPort: Map[Int, Schema] = Map(0 -> schema, 1 -> schema) + + /** Write one JSONL fixture per 0-based input port, every cell filled. */ + final def writeInputs(dir: Path, inputPortCount: Int): Map[PortIdentity, Path] = + write(dir, inputPortCount, withGaps = false) + + /** How many rows port 0 gets — what a row-count-sensitive knob (`limit`, + * `offset`) is sized against so its value keeps some rows and drops some. + */ + final def port0RowCount: Int = rowsFor(0).size + + /** This table's rows with [[SharedFixture.emptyOneCellPerColumn]] applied. */ + private[verify] def emptyOneCellPerColumn(rows: Seq[Tuple]): Seq[Tuple] = + SharedFixture.emptyOneCellPerColumn(rows, schema, keepFilled) +} + +/** + * A column subset of another table: the same rows in the same order, keeping + * only the named columns, in the order named. + * + * The sklearn families need one. Their generated code is + * `X = table.drop(target, axis=1)`, so every column that is not the target + * reaches `fit`, and a string or a timestamp ends it. A projection hands them a + * table an estimator can fit without a second dataset to keep in step: the rows + * are still [[CanonicalFixture]]'s, only narrower. + */ +final case class ProjectedFixture( + source: SharedFixture, + columns: Seq[String], + keepFilled: Set[String] +) extends SharedFixture { + + val schema: Schema = new Schema(columns.map(c => source.schema.getAttribute(c)): _*) + + private val rows: Vector[Tuple] = source.allRows.map { t => + val b = Tuple.builder(schema) + schema.getAttributes.foreach(a => b.add(a, t.getField[AnyRef](a.getName))) + b.build() + }.toVector + + /** Every port gets the whole table. An estimator pair trains on port 0 and + * tests on port 1, and the point of the pair is the two ports rather than two + * datasets: what the comparison sees is the fitted model, which port 1 has no + * hand in, so giving the ports different rows buys nothing. + * + * The whole table rather than the source's ten-row window, because the + * estimators that cross-validate pass no fold count and so take sklearn's + * default of five: the window would leave the smaller class at four, and one + * fold holding none of a class is a fold that asks nothing (sklearn warns and + * splits anyway rather than refusing). + */ + override def rowsFor(port: Int): Seq[Tuple] = rows +} + +object SharedFixture { + + /** One empty cell per column, spread across rows so no row is wholly empty — an + * operator that reads two columns should still meet a row where one is filled + * and the other is not. Placement is by column position, so it is the same on + * every run. + * + * Free-standing rather than a member, because a curated handler's table has no + * [[SharedFixture]] behind it: the runner reads back the rows the handler wrote + * and punches the holes here. + */ + def emptyOneCellPerColumn( + rows: Seq[Tuple], + schema: Schema, + keepFilled: Set[String] + ): Seq[Tuple] = { + if (rows.isEmpty) return rows + val holes: Map[Int, Set[String]] = schema.getAttributes.zipWithIndex + .filterNot { case (attr, _) => keepFilled.contains(attr.getName) } + .map { case (attr, i) => (i % rows.size) -> attr.getName } + .groupBy(_._1) + .map { case (row, pairs) => row -> pairs.map(_._2).toSet } + rows.zipWithIndex.map { + case (t, rowIdx) => + val emptied = holes.getOrElse(rowIdx, Set.empty) + if (emptied.isEmpty) t + else { + val b = Tuple.builder(schema) + schema.getAttributes.foreach { attr => + val v: AnyRef = + if (emptied.contains(attr.getName)) null else t.getField[AnyRef](attr.getName) + b.add(attr, v) + } + b.build() + } + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala new file mode 100644 index 00000000000..01c13bf777a --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala @@ -0,0 +1,471 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Schema, Tuple} +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.source.fetcher.URLFetcherOpDesc +import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc +import org.apache.texera.amber.operator.source.scan.file.{FileScanOpDesc, FileScanSourceOpDesc} +import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.arrow.memory.RootAllocator +import org.apache.arrow.vector.ipc.ArrowFileWriter +import org.apache.arrow.vector.VectorSchemaRoot +import org.apache.texera.amber.util.ArrowUtils + +import java.nio.channels.FileChannel +import java.nio.charset.{Charset, StandardCharsets} +import java.nio.file.{Files, Path, StandardOpenOption} +import scala.collection.mutable +import scala.util.{Try, Using} + +/** + * Per-category test runner for source operators (operators with no input + * ports — they read from an external resource and emit tuples). + * + * Dispatch is auto-first: + * - Auto tier: a scan source declares the file format it reads via + * [[ScanSourceOpDesc.fileTypeName]]. If that tag is in [[encoderByFileType]], + * the operator is fixtured with zero per-operator code — the shared + * [[CanonicalSourceFixture]] is encoded into that format and `fileName` + * points at it. A newly added file-scan source in a known format + * (CSV/JSONL/Arrow/…) is verified the moment it is registered in + * [[LogicalOp]]'s `@JsonSubTypes`, no edit here. + * - Curated tier: sources that can't take the shared table (text-family + * single-`line` output, or inline-config data) keep a hand-written + * [[SourceHandler]] in [[curatedHandlersByClass]]. + * - Otherwise the test is flagged (a [[knownIssues]] reason, an unsupported + * declared format, or no match) — never silently skipped. + * + * The runner itself is operator-agnostic: it builds an OpDesc, drives + * [[OpExecHarness]] (Path A) and [[StandaloneRunner]] (Path B), compares via + * [[Comparator]]. Sources have no input ports so `inputs = Map.empty` for both. + */ +object SourceCategoryRunner { + + /** + * The curated tier: sources that keep a hand-written handler because they + * can't go through the shared-fixture + encoder (auto) path — their output + * isn't the shared 3-column table (text-family, single `line` column) or + * their data is inline config rather than a file. Mirrors the transform + * side's [[CuratedHandlers]] (hand-written vs auto-generated fixture). + */ + private val curatedHandlersByClass: Map[Class[_ <: LogicalOp], SourceHandler] = + Seq[SourceHandler](TextInputHandler, FileScanSourceHandler) + .map(h => h.opDescClass -> h) + .toMap + + /** + * The auto tier. A scan source declares the file format it reads via + * [[ScanSourceOpDesc.fileTypeName]] ("CSV", "JSONL", "Arrow", …). Map that + * tag to the [[CanonicalSourceFixture]] encoder that writes a file in that + * format. Any source whose `fileTypeName` is a key here runs with zero + * per-operator code, so a newly added file-scan source in a known format is + * verified the moment it is registered in `@JsonSubTypes` — no handler, no + * edit here. (ParallelCSV also declares "CSV" and would be covered for free, + * but it is currently commented out of `@JsonSubTypes`, so the suite doesn't + * enumerate it.) + */ + private val encoderByFileType: Map[String, (Path, Charset) => Path] = Map( + "CSV" -> CanonicalSourceFixture.writeCsv, + "CSVOld" -> CanonicalSourceFixture.writeCsv, + "JSONL" -> CanonicalSourceFixture.writeJsonl, + // Arrow is binary and its descriptor declares fileEncoding ignored, so the + // charset a variant asks for has nothing to apply to. + "Arrow" -> ((dir, _) => CanonicalSourceFixture.writeArrow(dir)) + ) + + /** + * Sources this runner cannot verify, with the honest reason. Mirrors + * `TransformVerificationRunner.knownIssues`: the reason surfaces in the + * ignored test's name and the coverage table. + */ + private val knownIssues: Map[Class[_ <: LogicalOp], String] = Map( + classOf[FileScanOpDesc] -> + ("input-driven source: filenames arrive on an input port at runtime, but this runner " + + "feeds sources no inputs — Path B's generated code references an undefined in1df"), + classOf[URLFetcherOpDesc] -> + ("live-network source: the operator fetches a real URL over the network, so its " + + "output is non-deterministic and depends on external connectivity — it cannot be " + + "verified against a fixed fixture in isolation") + ) + + /** The format tag a source declares, or `None` if it isn't an instantiable + * ScanSourceOpDesc (non-scan sources, or ones that fail to construct). + */ + private def declaredFileType(opDescClass: Class[_ <: LogicalOp]): Option[String] = + Try(opDescClass.getDeclaredConstructor().newInstance()).toOption.collect { + case scan: ScanSourceOpDesc => scan.fileTypeName + }.flatten + + def canRun(opDescClass: Class[_ <: LogicalOp]): Boolean = + curatedHandlersByClass.contains(opDescClass) || + declaredFileType(opDescClass).exists(encoderByFileType.contains) + + /** + * Tier label for a runnable source, mirroring the transform side's + * auto/curated distinction: `"curated source"` when a hand-written + * [[SourceHandler]] serves it, else `"auto source"` (a declared-format scan + * source fixtured by an [[encoderByFileType]] encoder with zero per-op code). + */ + def tier(opDescClass: Class[_ <: LogicalOp]): String = + if (curatedHandlersByClass.contains(opDescClass)) "curated source" else "auto source" + + /** Why a non-runnable source is flagged: a specific known issue, an + * unsupported declared format, or no handler/format match at all. + */ + def flagReason(opDescClass: Class[_ <: LogicalOp]): String = + knownIssues.getOrElse( + opDescClass, + declaredFileType(opDescClass) match { + case Some(fileType) => + s"unsupported source format '$fileType' — no encoder registered in SourceCategoryRunner" + case None => "no source handler registered yet" + } + ) + + private def newScanSource(opDescClass: Class[_ <: LogicalOp]): ScanSourceOpDesc = + opDescClass.getDeclaredConstructor().newInstance() match { + case s: ScanSourceOpDesc => s + case other => + throw new IllegalArgumentException( + s"${opDescClass.getSimpleName} has no curated handler and is not a " + + s"ScanSourceOpDesc (${other.getClass.getName})" + ) + } + + /** + * Every configuration of one source worth running, as (label, op, its own + * directory). + * + * Each variant gets a directory of its own holding its OWN copy of the fixture, + * because the generated script reads the file by bare name (`pd.read_csv( + * "sample.csv")`) out of the directory it runs in. Two variants wanting two + * different `sample.csv` files cannot share one. + */ + private def variantsFor( + opDescClass: Class[_ <: LogicalOp], + testRoot: Path + ): Seq[(String, LogicalOp, Path)] = { + // Punctuation collapses to '_', so two labels differing only in punctuation + // would name the same directory and share one fixture and one output dir. Fail + // loudly instead of letting a variant quietly run someone else's file. + val taken = mutable.Set.empty[String] + def dirFor(label: String): Path = { + val name = label.replaceAll("[^A-Za-z0-9]+", "_") + require(taken.add(name), s"two variants of $opDescClass both map to the directory '$name'") + Files.createDirectories(testRoot.resolve(name)) + } + + curatedHandlersByClass.get(opDescClass) match { + case Some(handler) => + val baseDir = dirFor("default") + val base = handler.makeOpDesc(baseDir) + // Every variant calls the handler AGAIN rather than reusing `base`: the handler + // writes its fixture into the directory it is given, and a second op carrying + // the first one's `fileName` would read a file outside the directory it runs in. + // No enum sweep: both curated sources are the text family, whose `attributeType` + // says how to PARSE the fixture (`alice` is not an integer) and whose + // `fileEncoding` describes its BYTES — flipping either without rewriting the + // fixture compares nothing but how the two paths fail. The auto branch below + // rewrites its fixture per variant and does sweep them. + ConfigGenerator + .fullVariantEditsOf(base, Map.empty, handler.rowCount, sweepEnums = false) + .fold( + reason => + throw new IllegalStateException( + s"cannot vary ${opDescClass.getSimpleName}: $reason" + ), + identity + ) + .map { variant => + if (variant.at.isEmpty) ("default", base, baseDir) + else { + val dir = dirFor(variant.label) + val op = ConfigGenerator + .applyVariant(handler.makeOpDesc(dir), variant) + .fold( + reason => + throw new IllegalStateException( + s"cannot build ${opDescClass.getSimpleName} variant '${variant.label}': $reason" + ), + identity + ) + (variant.label, op, dir) + } + } + case None => + val fileType = declaredFileType(opDescClass).getOrElse("") + val encoder = encoderByFileType.getOrElse( + fileType, + throw new IllegalArgumentException( + s"No encoder for ${opDescClass.getSimpleName} (fileTypeName='$fileType')" + ) + ) + val base = { + val dir = dirFor("default") + val op = newScanSource(opDescClass) + op.fileName = Some(encoder(dir, op.fileEncoding.getCharset).toUri.toString) + ("default", op: LogicalOp, dir) + } + base +: generatedVariants(opDescClass, encoder, dirFor) + } + } + + /** + * The variants the shared [[ConfigGenerator]] derives from the operator's own + * fields — the base config with every knob filled, plus one per enum branch + * (`hasHeader`, JSONL's `flatten`). Nothing to register per operator: a knob + * added to a source is swept the day it is added. + * + * `fileEncoding` is swept like any other enum, and the fixture FOLLOWS it: each + * variant's file is written in the charset that variant declares. Encoding is a + * statement about the bytes, so a UTF_16 config over a file left in UTF-8 would + * only compare how each path fails. + * + * Variants that serialize identically are dropped — an operator that ignores + * `fileEncoding` (Arrow declares `@JsonIgnoreProperties`) would otherwise run the + * same config three times. + */ + private def generatedVariants( + opDescClass: Class[_ <: LogicalOp], + encoder: (Path, Charset) => Path, + dirFor: String => Path + ): Seq[(String, LogicalOp, Path)] = { + val seen = mutable.Set.empty[String] + ConfigGenerator + .generateVariants(opDescClass, Map.empty, CanonicalSourceFixture.rows.size) + .fold( + reason => + throw new IllegalStateException( + s"cannot auto-configure ${opDescClass.getSimpleName}: $reason" + ), + identity + ) + .flatMap { + case (label, op) => + val scan = op.asInstanceOf[ScanSourceOpDesc] + val shape = objectMapper.valueToTree[ObjectNode](scan) + shape.remove("fileName") // every variant reads its own copy of the file + if (!seen.add(shape.toString)) None + else { + // "default" is already the bare newInstance config above; this one is + // the generator's, which additionally fills limit and offset. + val name = if (label == "default") "auto-base" else label + val dir = dirFor(name) + scan.fileName = Some(encoder(dir, scan.fileEncoding.getCharset).toUri.toString) + Some((name, scan: LogicalOp, dir)) + } + } + } + + /** Runs the parity test for the operator, once per variant. Throws on mismatch. */ + def run(opDescClass: Class[_ <: LogicalOp]): Unit = { + val testRoot = Files.createTempDirectory(s"op-behavior-${opDescClass.getSimpleName}-") + variantsFor(opDescClass, testRoot).foreach { + case (label, opDesc, workDir) => + try runVariant(opDesc, workDir) + catch { + case e: Throwable => + throw new AssertionError(s"[variant: $label] ${e.getMessage}", e) + } + } + } + + /** Drive one configured source through both paths inside `workDir`, which holds + * that variant's fixture, and assert the two tables match. + */ + private def runVariant(opDesc: LogicalOp, workDir: Path): Unit = { + val actualDir = workDir.resolve("actual") + Files.createDirectories(actualDir) + + val pathA = OpExecHarness.execute(opDesc, inputs = Map.empty, outputDir = actualDir) + val pathB = StandaloneRunner.run( + opDesc = opDesc, + inputs = Map.empty, + outputPortCount = 1, + workDir = workDir + ) + + val actual = pathA.outputs(PortIdentity(0)) + val expected = pathB.outputs(1) + Comparator.assertEqual(actual, expected) + } +} + +/** + * A hand-written recipe for one source that can't use the auto tier + * (fileTypeName + [[CanonicalSourceFixture]] encoder): which OpDesc class it + * handles and how to fixture a working instance. Used for the text-family + * sources ([[TextInputHandler]], [[FileScanSourceHandler]]). + */ +trait SourceHandler { + + /** The concrete OpDesc class this handler tests. */ + def opDescClass: Class[_ <: LogicalOp] + + /** + * Generate the fixture file inside `testRoot` and return a configured + * OpDesc instance whose `fileName` (or analogous URI field) points at it. + */ + def makeOpDesc(testRoot: Path): LogicalOp + + /** How many rows the fixture holds. Only the handler knows — it writes its own, + * rather than the shared [[CanonicalSourceFixture]]. A row-window knob the + * variants fill (`limit`, `offset`) is sized against this, so that the value they + * take keeps some rows and drops some instead of landing past the end. + */ + def rowCount: Int +} + +/** + * The rows every structured-file source reads: [[CanonicalFixture]]'s, whole. + * + * A source has no input port, so the fixture is delivered not as an input JSONL + * but as a file the operator opens itself. Each `writeXxx` encodes these rows + * into one on-disk format (CSV / JSONL / Arrow); a source handler picks the + * encoder its operator understands and points `fileName` at the result. So CSV, + * CSVOld, JSONL and Arrow all verify that the operator reconstructs one shared + * table, instead of each asserting against its own ad-hoc sample. + * + * It reads the canonical table rather than a narrow one of its own. A source + * fixture picked for the types that survive a round trip would be choosing not + * to ask the question this suite exists to ask: these files carry no types, both + * readers infer, and where they infer differently is exactly what should show. A + * date column does part them, and [[StandaloneRunner.sourceCasts]] is where that + * is settled — on Path B's reading, not by leaving the column out. + */ +object CanonicalSourceFixture { + + val schema: Schema = CanonicalFixture.schema + + val rows: Vector[Tuple] = CanonicalFixture.allRows + + /** Write the rows as a header-first, comma-delimited CSV encoded in `charset`. + * + * The charset is a parameter because it describes the BYTES, not the config: a + * variant declaring `fileEncoding = UTF_16` over a file left in UTF-8 would + * compare nothing but how each path fails. + */ + def writeCsv(dir: Path, charset: Charset): Path = { + val path = dir.resolve("sample.csv") + val header = schema.getAttributes.map(a => csvField(a.getName)).mkString(",") + val body = rows.map { t => + schema.getAttributes + .map(a => csvField(Option(t.getField[AnyRef](a.getName)).map(_.toString).orNull)) + .mkString(",") + } + Files.write(path, ((header +: body).mkString("\n") + "\n").getBytes(charset)) + path + } + + /** One CSV field, quoted per RFC 4180. + * + * The table carries commas inside values — a bracketed edge pair, a + * comma-delimited list, an ordinary English sentence — and writing those raw + * shifts every column after them. What the two paths then disagree about is a + * broken file rather than anything either of them does. + */ + private def csvField(value: String): String = + if (value == null) "" + else if (value.exists(c => c == ',' || c == '"' || c == '\n' || c == '\r')) + "\"" + value.replace("\"", "\"\"") + "\"" + else value + + /** Write the rows as JSON Lines (one object per line, keys in schema order). + * Reuses [[TupleIO.writeTuples]] — the same writer the transform fixtures + * use; it also drops a `.schema.json` sidecar the source ignores. + * + * That writer is shared and always writes UTF-8, so a variant asking for another + * charset gets the bytes transcoded afterwards rather than a second writer. + */ + def writeJsonl(dir: Path, charset: Charset): Path = { + val path = dir.resolve("sample.jsonl") + TupleIO.writeTuples(path, rows.iterator, schema) + if (charset != StandardCharsets.UTF_8) { + val text = new String(Files.readAllBytes(path), StandardCharsets.UTF_8) + Files.write(path, text.getBytes(charset)) + } + path + } + + /** Write the rows as an uncompressed Arrow IPC ("file" format) stream — the + * format both `ArrowFileReader` (Path A) and `pd.read_feather` (Path B) + * read. + */ + def writeArrow(dir: Path): Path = { + val path = dir.resolve("sample.arrow") + // Texera's own Schema-to-Arrow mapping and tuple writer, so the file carries + // exactly the types `ArrowUtils.toTexeraSchema` reads back on the other side. + // Hand-listing the fields is what let the table outgrow them unnoticed: the + // columns past the list were simply not written, and both paths went on + // agreeing about the few that were. + val arrowSchema = ArrowUtils.fromTexeraSchema(schema) + Using.Manager { use => + val allocator = use(new RootAllocator()) + val root = use(VectorSchemaRoot.create(arrowSchema, allocator)) + root.allocateNew() + rows.zipWithIndex.foreach { case (t, i) => ArrowUtils.setTexeraTuple(t, i, root) } + root.setRowCount(rows.size) + val channel = use( + FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE) + ) + val writer = use(new ArrowFileWriter(root, null, channel)) + writer.start() + writer.writeBatch() + writer.end() + }.get + path + } +} + +/** Handler for `TextInputSourceOpDesc`. The text lives in the config — no fixture file. */ +object TextInputHandler extends SourceHandler { + + override val opDescClass: Class[_ <: LogicalOp] = classOf[TextInputSourceOpDesc] + + override val rowCount: Int = 3 + + override def makeOpDesc(testRoot: Path): LogicalOp = { + val desc = new TextInputSourceOpDesc() + desc.textInput = "alice\nbob\ncarol" + desc // defaults: attributeType STRING (one row per line), attributeName "line" + } +} + +/** Handler for `FileScanSourceOpDesc`. Plain text file read in default line mode. */ +object FileScanSourceHandler extends SourceHandler { + + override val opDescClass: Class[_ <: LogicalOp] = classOf[FileScanSourceOpDesc] + + override val rowCount: Int = 3 + + override def makeOpDesc(testRoot: Path): LogicalOp = { + val txtPath = testRoot.resolve("sample.txt") + Files.write(txtPath, "alice\nbob\ncarol\n".getBytes(StandardCharsets.UTF_8)) + + val desc = new FileScanSourceOpDesc() + desc.fileName = Some(txtPath.toUri.toString) + desc // defaults: attributeType STRING (one row per line), attributeName "line" + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala new file mode 100644 index 00000000000..5280b4cf1e7 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import org.apache.texera.amber.operator.source.SourceOperatorDescriptor +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ +import scala.sys.process.{Process, ProcessLogger} +import scala.util.Try + +/** + * Configures every operator against hostile column names and parses what + * `generateStandaloneCode` produces. A value spliced in without + * `pyStringLiteral` is silent until someone names a column `a"b`, and then the + * exported script will not compile. + * + * The check has to be behavioural. A generator that builds its quoted literal + * inside a helper shows no quotes in its template, which is how RadarPlot and + * Aggregate survived a source-level sweep that reported zero remaining sites. + */ +object StandaloneEscapingCheck { + + private val schemas = CanonicalFixture.schemasByPort + private val columns = CanonicalFixture.schema.getAttributes.map(_.getName).toSet + + /** Every problem found, empty when the suite is clean. An operator whose config + * cannot be built lands here too: dropping it would check less while still + * passing. + */ + def run(): Seq[String] = { + // Sources have no input schema and so no column knobs. Their free-text knobs + // already get a hostile variant in the normal verify run. + val operators = OperatorBehaviorSpec + .discoverStandaloneOperators() + .filterNot(classOf[SourceOperatorDescriptor].isAssignableFrom) + val dir = Files.createTempDirectory("standalone-escaping-") + dir.toFile.deleteOnExit() + + val (unconfigurable, code) = operators + .map(op => op.getSimpleName -> codeFor(op, dir)) + .partitionMap { + case (name, Left(why)) => Left(s"$name: $why") + case (name, Right(variants)) => + Right(variants.map { case (label, src) => s"$name/$label" -> src }) + } + unconfigurable ++ parse(code.flatten, dir) + } + + /** Holds the four characters that end a Python literal, and carries the column + * it replaces so that two knobs never collide on one name. + */ + private def hostile(column: String): String = "a\"b'c\\d\ne_" + column + + private def hostilize(node: JsonNode): Unit = + node match { + case obj: ObjectNode => + obj.fields().asScala.toSeq.foreach { e => + if (isColumn(e.getValue)) obj.put(e.getKey, hostile(e.getValue.asText)) + else hostilize(e.getValue) + } + case arr: ArrayNode => + (0 until arr.size).foreach { i => + if (isColumn(arr.get(i))) + arr.set(i, objectMapper.getNodeFactory.textNode(hostile(arr.get(i).asText))) + else hostilize(arr.get(i)) + } + case _ => () + } + + private def isColumn(n: JsonNode): Boolean = n.isTextual && columns.contains(n.asText) + + /** Split the way the runner splits: a curated operator's config is hand-written + * because the generator cannot derive one. Auto operators contribute every + * variant, since a knob only `optionals` fills is a site the base never reaches. + */ + private def codeFor( + opClass: Class[_ <: LogicalOp], + dir: Path + ): Either[String, Seq[(String, String)]] = { + val configs = CuratedHandlers.byClass.get(opClass) match { + // The handler's config, not its enum sweep: sweeping moves an enum value, + // never a column name, and would need schemas only the runner holds. + case Some(h) => + Try(Seq("curated" -> h.fixture(dir)._1)).toEither.left.map(e => s"curated: ${e.getMessage}") + case None => ConfigGenerator.generateVariants(opClass, schemas) + } + configs.flatMap { variants => + val results = variants.map { + case (label, op) => + val node = objectMapper.valueToTree[ObjectNode](op) + hostilize(node) + Try( + objectMapper + .treeToValue(node, opClass) + .asInstanceOf[StandaloneCodeGenerator] + .generateStandaloneCode() + ).toEither.left.map(e => s"$label: $e").map(label -> _) + } + results.collectFirst { case Left(why) => why }.toLeft(results.collect { case Right(r) => r }) + } + } + + /** One Python process for all of them; the cost is startup, not parsing. The + * snippets stay in `dir` so a reported operator can be opened as generated. + */ + private def parse(snippets: Seq[(String, String)], dir: Path): Seq[String] = { + val payload = objectMapper.createObjectNode() + snippets.foreach { case (name, src) => payload.put(name, src) } + val input = write(dir, "snippets.json", payload.toString) + val script = write( + dir, + "parse_all.py", + """import ast, json, sys + |for name, code in json.load(open(sys.argv[1])).items(): + | try: + | ast.parse(code) + | except SyntaxError as e: + | print(f"{name}: line {e.lineno}: {e.msg}") + |""".stripMargin + ) + val out = Seq.newBuilder[String] + val err = Seq.newBuilder[String] + val python = sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") + val exit = Process(Seq(python, script.toString, input.toString)) + .!(ProcessLogger(out += _, err += _)) + // Without this a parser that never ran reads as "nothing failed". + require(exit == 0, s"parse_all.py exited $exit: ${err.result().mkString("\n")}") + out.result() + } + + private def write(dir: Path, name: String, content: String): Path = + Files.write(dir.resolve(name), content.getBytes(UTF_8)) +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala new file mode 100644 index 00000000000..590e0c361ce --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -0,0 +1,962 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{BooleanNode, IntNode, TextNode} +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.{ + LogicalOp, + PythonOperatorDescriptor, + StandaloneCodeGenerator +} +import org.apache.texera.amber.operator.aggregate.AggregateOpDesc +import org.apache.texera.amber.operator.dummy.DummyOpDesc +import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc +import org.apache.texera.amber.operator.sleep.SleepOpDesc +import org.apache.texera.amber.operator.split.SplitOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnGaussianNaiveBayesOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnLinearRegressionOpDesc +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor +import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc +import org.apache.texera.amber.operator.huggingFace.HuggingFaceSpamSMSDetectionOpDesc +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingGaussianNaiveBayesOpDesc +import org.apache.texera.amber.operator.regex.RegexOpDesc +import org.apache.texera.amber.operator.sklearn.testing.SklearnTestingOpDesc +import org.apache.texera.amber.operator.typecasting.TypeCastingOpDesc +import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc +import org.apache.texera.amber.operator.visualization.DotPlot.DotPlotOpDesc +import org.apache.texera.amber.operator.visualization.barChart.BarChartOpDesc +import org.apache.texera.amber.operator.visualization.boxViolinPlot.BoxViolinPlotOpDesc +import org.apache.texera.amber.operator.visualization.ImageViz.ImageVisualizerOpDesc +import org.apache.texera.amber.operator.visualization.IcicleChart.IcicleChartOpDesc +import org.apache.texera.amber.operator.visualization.bubbleChart.BubbleChartOpDesc +import org.apache.texera.amber.operator.visualization.bulletChart.BulletChartOpDesc +import org.apache.texera.amber.operator.visualization.candlestickChart.CandlestickChartOpDesc +import org.apache.texera.amber.operator.visualization.carpetPlot.CarpetPlotOpDesc +import org.apache.texera.amber.operator.visualization.choroplethMap.ChoroplethMapOpDesc +import org.apache.texera.amber.operator.visualization.continuousErrorBands.ContinuousErrorBandsOpDesc +import org.apache.texera.amber.operator.visualization.contourPlot.ContourPlotOpDesc +import org.apache.texera.amber.operator.visualization.dendrogram.DendrogramOpDesc +import org.apache.texera.amber.operator.visualization.dumbbellPlot.DumbbellPlotOpDesc +import org.apache.texera.amber.operator.visualization.ecdfPlot.ECDFPlotOpDesc +import org.apache.texera.amber.operator.visualization.figureFactoryTable.FigureFactoryTableOpDesc +import org.apache.texera.amber.operator.visualization.filledAreaPlot.FilledAreaPlotOpDesc +import org.apache.texera.amber.operator.visualization.funnelPlot.FunnelPlotOpDesc +import org.apache.texera.amber.operator.visualization.ganttChart.GanttChartOpDesc +import org.apache.texera.amber.operator.visualization.gaugeChart.GaugeChartOpDesc +import org.apache.texera.amber.operator.visualization.ScatterMatrixChart.ScatterMatrixChartOpDesc + +import org.apache.texera.amber.operator.visualization.heatMap.HeatMapOpDesc +import org.apache.texera.amber.operator.visualization.hierarchychart.HierarchyChartOpDesc +import org.apache.texera.amber.operator.visualization.histogram2d.Histogram2DOpDesc +import org.apache.texera.amber.operator.visualization.histogram.HistogramChartOpDesc +import org.apache.texera.amber.operator.visualization.lineChart.LineChartOpDesc +import org.apache.texera.amber.operator.visualization.nestedTable.NestedTableOpDesc +import org.apache.texera.amber.operator.visualization.networkGraph.NetworkGraphOpDesc +import org.apache.texera.amber.operator.visualization.parallelCoordinatesPlot.ParallelCoordinatesPlotOpDesc +import org.apache.texera.amber.operator.visualization.pieChart.PieChartOpDesc +import org.apache.texera.amber.operator.visualization.polarChart.PolarChartOpDesc +import org.apache.texera.amber.operator.visualization.quiverPlot.QuiverPlotOpDesc +import org.apache.texera.amber.operator.visualization.radarChart.RadarChartOpDesc +import org.apache.texera.amber.operator.visualization.radarPlot.RadarPlotOpDesc +import org.apache.texera.amber.operator.visualization.rangeSlider.RangeSliderOpDesc +import org.apache.texera.amber.operator.visualization.sankeyDiagram.SankeyDiagramOpDesc +import org.apache.texera.amber.operator.visualization.scatter3DChart.Scatter3dChartOpDesc +import org.apache.texera.amber.operator.visualization.scatterplot.ScatterplotOpDesc +import org.apache.texera.amber.operator.visualization.stripChart.StripChartOpDesc +import org.apache.texera.amber.operator.visualization.tablesChart.TablesPlotOpDesc +import org.apache.texera.amber.operator.visualization.ternaryContour.TernaryContourOpDesc +import org.apache.texera.amber.operator.visualization.ternaryPlot.TernaryPlotOpDesc +import org.apache.texera.amber.operator.visualization.timeSeriesplot.TimeSeriesOpDesc +import org.apache.texera.amber.operator.visualization.treeplot.TreePlotOpDesc +import org.apache.texera.amber.operator.visualization.volcanoPlot.VolcanoPlotOpDesc +import org.apache.texera.amber.operator.visualization.waterfallChart.WaterfallChartOpDesc +import org.apache.texera.amber.operator.visualization.windRoseChart.WindRoseChartOpDesc +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try} + +/** + * Unified verification runner for non-source operators implementing + * [[StandaloneCodeGenerator]]. Resolves, per operator: + * - Path A engine: [[PyOpExecHarness]] for PythonOperatorDescriptor, + * [[OpExecHarness]] otherwise; Path B is always [[StandaloneRunner]]. + * - Config + fixture: curated handler ([[CuratedHandlers]]) if registered, + * else [[ConfigGenerator]] against the [[CanonicalFixture]] schemas. + * - Comparison: order-insensitive by default (parallel output order isn't a + * contract); strict positional only when the operator declares + * `orderSensitive = true` (the sort family). All output ports are compared. + * Operators that can't be run are Flagged with a reason — never silently + * skipped. + */ +object TransformVerificationRunner { + + /** + * Per-operator knob handling: a value this operator's variants must carry, + * and where it applies. Two needs, one table, because both answer the same + * question — what does the generator have to be told about this operator's + * knobs that its metadata does not say. + * + * `Pinned` holds a knob at one value and keeps it out of the sweep, for a + * knob whose other value selects non-determinism rather than a different + * behavior to check. Split's "Auto-Generate Seed" is the case: with it on the + * executor seeds from the clock, so that run agrees with nothing — its own + * previous run included — and there is no output for a script to reproduce. + * Everything else about the operator is deterministic, so pinning covers the + * partition rather than abandoning the operator over one switch. The value + * reaches the test name via [[pinnedTierNote]], so the run does not read as + * full coverage. + * + * `WithOptionals` sets a knob inside the `optionals` variant, for a branch + * that needs a switch AND the field it governs. Ternary Plot colours its + * points only when `colorEnabled` is on and `colorDataField` is set, and the + * two belong to different mechanisms: the sweep turns the switch on with the + * column empty, the optional fill supplies the column with the switch off, so + * neither variant generated the coloured branch. Naming the switch here puts + * it in the variant that fills the column. + * + * Named per operator rather than applied wholesale, because switches are not + * generally independent: turning every Boolean on in that variant paired + * Sklearn's `countVectorizer` with `tfidfTransformer` (mutually exclusive + * text pipelines), asked File Scan to extract an archive from a plain file, + * and re-enabled the very auto-seed switch the first scope holds off. + * + * Distinct from an `enumSweep` row in [[variantsNotRun]], which is about an + * operator's enums as a whole rather than one named knob. + */ + sealed trait KnobScope + object KnobScope { + case object Pinned extends KnobScope + case object WithOptionals extends KnobScope + } + + final case class Knob(field: String, value: JsonNode, scope: KnobScope) + + val knobOverrides: Map[Class[_], Seq[Knob]] = Map( + classOf[SplitOpDesc] -> Seq(Knob("random", BooleanNode.FALSE, KnobScope.Pinned)), + // `SleepOpExec` sleeps this many seconds per tuple, and the generator fills a + // required Int with half the row count, so the fixture would spend tens of + // seconds asleep for nothing: the delay never reaches the output being + // compared, and the standalone translation is a passthrough by design. + classOf[SleepOpDesc] -> Seq(Knob("sleepTime", IntNode.valueOf(0), KnobScope.Pinned)), + classOf[TernaryPlotOpDesc] -> Seq( + Knob("colorEnabled", BooleanNode.TRUE, KnobScope.WithOptionals) + ), + // Both text switches are pinned off on the numeric table: the pipeline they + // build reads a column that table does not have, and `tfidfTransformer` has + // no meaning at all outside a CountVectorizer pipeline (the schema hides it + // when the vectorizer is off). Their branches are generated against the text + // table by the [[AltScenario]]s instead. + classOf[SklearnClassifierOpDesc] -> Seq( + Knob("countVectorizer", BooleanNode.FALSE, KnobScope.Pinned), + Knob("tfidfTransformer", BooleanNode.FALSE, KnobScope.Pinned) + ), + classOf[SklearnTrainingOpDesc] -> Seq( + Knob("countVectorizer", BooleanNode.FALSE, KnobScope.Pinned), + Knob("tfidfTransformer", BooleanNode.FALSE, KnobScope.Pinned) + ) + ) + + /** This operator's overrides for one scope, as the generator takes them. + * Exact class first, then base class, so a family can be named once instead + * of per estimator. + */ + private def knobsFor(opClass: Class[_ <: LogicalOp], scope: KnobScope): Map[String, JsonNode] = + knobOverrides + .get(opClass) + .orElse(knobOverrides.collectFirst { + case (base, knobs) if base.isAssignableFrom(opClass) => knobs + }) + .getOrElse(Seq.empty) + .filter(_.scope == scope) + .map(k => k.field -> k.value) + .toMap + + /** A second generation pass for a branch the base config cannot reach. Usually + * that is a branch needing a DIFFERENT table: a swept variant cannot switch + * tables, since [[ConfigGenerator]] resolves every column picker against ONE + * schema, so the branch is generated separately against the table it needs + * with its switch pinned on. + * + * It also reaches a branch whose own knobs the base config leaves empty. The + * sweep offers the values a config already holds, so a list filled only on the + * far side of a switch has nothing to offer until the switch is pinned — and + * then the second pass may well take the same table as the first. + * + * The auto-tier twin of [[TransformHandler.extraScenarios]]: it names only the + * table and the pins, and the generator writes the config. + */ + final case class AltScenario( + label: String, + fixture: SharedFixture, + pinned: Map[String, JsonNode] + ) + + /** Every kind of run a [[variantsNotRun]] row can name: the derived variants, + * plus the [[AltScenario]] labels (an alt scenario IS one kind of run). Named + * so a row cannot misspell one and silently stop suppressing anything. + */ + object RunKind { + val Nulls = "nulls" + val EnumSweep = "enumSweep" + val HostileText = "hostileText" + val CountVectorizerText = "countVectorizer_text" + val TfidfText = "tfidf_text" + val NonFeatureColumn = "nonFeatureColumn" + val TextLabels = "textLabels" + val RegressionBranch = "regressionBranch" + + /** One swept hyperparameter of an advanced trainer, which the sweep labels by the + * pointer it flips. Built here rather than spelled out at each row, since a row + * that misspelled the pointer would suppress nothing and say nothing. + */ + def hyperParameter(name: String): String = s"paraList/0/parameter=$name" + } + + /** The [[RunKind]] a generated variant's label names. A `merged` variant labels + * itself `kind(fields…)`, and the fields it happened to move are not part of + * what is being withheld. + */ + private def kindOf(label: String): String = label.takeWhile(_ != '(') + + /** Keyed by BASE class, not by concrete operator: a newly registered sklearn + * estimator is covered with no entry of its own, matching how the families + * themselves are discovered. + */ + val altFixtureScenarios: Map[Class[_], Seq[AltScenario]] = { + // One scenario per text pipeline rather than a sweep inside one: which + // pipeline is built is the branch under test, and the two are alternatives, + // not a knob crossed with everything else. + val countVectorizerText = AltScenario( + label = RunKind.CountVectorizerText, + fixture = CanonicalFixture.sklearnText, + pinned = Map( + "countVectorizer" -> BooleanNode.TRUE, + "tfidfTransformer" -> BooleanNode.FALSE + ) + ) + val tfidfText = countVectorizerText.copy( + label = RunKind.TfidfText, + pinned = Map( + "countVectorizer" -> BooleanNode.TRUE, + "tfidfTransformer" -> BooleanNode.TRUE + ) + ) + // The vectorizer stays off: what this scenario covers is the branch that + // narrows `X` for an estimator, and Count Vectorizer replaces it rather than + // feeding it, naming the text columns the narrowing would otherwise drop. + val nonFeatureColumn = AltScenario( + label = RunKind.NonFeatureColumn, + fixture = CanonicalFixture.sklearnNumericWithText, + pinned = Map( + "countVectorizer" -> BooleanNode.FALSE, + "tfidfTransformer" -> BooleanNode.FALSE + ) + ) + // The advanced trainers are the ones left out: they name the feature columns + // themselves rather than taking every column but the target, so a column an + // estimator cannot fit is not reachable for them. + Map( + classOf[SklearnClassifierOpDesc] -> Seq(countVectorizerText, tfidfText, nonFeatureColumn), + classOf[SklearnTrainingOpDesc] -> Seq(countVectorizerText, tfidfText, nonFeatureColumn), + // No text scenarios, and nothing pinned: this operator declares neither + // switch, and a pin is set on the config whether or not the field exists, + // so pinning one here would hand it a property it cannot read back. + classOf[SklearnLinearRegressionOpDesc] -> Seq(nonFeatureColumn.copy(pinned = Map.empty)), + // A scorer reads a text label as readily as a numeric one, and names the + // class after the label rather than after its position. Regression is + // pinned off rather than swept: a regression metric puts both columns + // through `float()`, so on this table the sweep would generate the one + // configuration the operator is right to refuse. The two columns are + // pinned because the operator's `@SampleColumn`s name the numeric pair, + // and an annotation naming a column the table does not hold ends the run + // rather than falling back — which is what catches a misspelling. + classOf[MachineLearningScorerOpDesc] -> Seq( + AltScenario( + label = RunKind.TextLabels, + fixture = CanonicalFixture.scorerTextLabels, + pinned = Map( + "isRegression" -> BooleanNode.FALSE, + "actualValueColumn" -> TextNode.valueOf("species_name"), + "predictValueColumn" -> TextNode.valueOf("species_name_pred") + ) + ), + // The regression metrics are unreachable from the base config: the sweep + // reads the sites the config already holds, and on the classification + // branch the regression list is empty, so it offers none. Pinned on, the + // list is filled before the sweep looks, and the other three metrics + // become variants like any other enum. Same table as the default runs — + // this scenario is here for the branch, not for a different set of rows. + AltScenario( + label = RunKind.RegressionBranch, + fixture = CanonicalFixture, + pinned = Map("isRegression" -> BooleanNode.TRUE) + ) + ) + ) + } + + /** The alternate-table scenarios this operator takes, resolved by family and + * minus any [[variantsNotRun]] names. + */ + private def altScenariosFor(opClass: Class[_ <: LogicalOp]): Seq[AltScenario] = + altFixtureScenarios + .collectFirst { case (base, scenarios) if base.isAssignableFrom(opClass) => scenarios } + .getOrElse(Seq.empty) + .filterNot(alt => notRun(opClass, alt.label)) + + /** How a pinned operator's tier reads in the report, e.g. `auto, random=false`. */ + private def pinnedTierNote(opClass: Class[_ <: LogicalOp]): String = { + val pinned = knobsFor(opClass, KnobScope.Pinned) + if (pinned.isEmpty) "" + else pinned.map { case (field, value) => s"$field=${value.asText}" }.mkString(", ", ", ", "") + } + + /** Why one kind of run is left out for an operator. The distinction is whether + * anyone should be waiting for it: [[PendingFix]] is a debt someone closes, + * [[ByDesign]] is an answer that will not change. + */ + sealed trait NotRunReason + final case class PendingFix(issue: String) extends NotRunReason + final case class ByDesign(why: String) extends NotRunReason + + final case class NotRun(op: Class[_], kind: String, reason: NotRunReason) + + /** The runs an operator does not get, and why. + * + * One table rather than one per kind. Every row makes the same statement, so + * the coverage report can print them together, and the next exemption has an + * obvious home instead of arriving as another set somewhere else. + * + * `op` matches its subclasses, so one row covers a family. `kind` is a + * [[RunKind]]. + * + * A curated handler's own [[TransformHandler.unfillableVariants]] stays where + * it is: those describe the table that handler wrote rather than the operator, + * and change when the fixture is rewritten. + */ + val variantsNotRun: Seq[NotRun] = { + // The platform raises on an empty cell, so the two paths cannot be compared + // on one until it stops. + val emptyCellRaises: Seq[(Class[_], String)] = Seq( + // Regex alone: apache/texera#7566 answers the empty cell in Substring Search + // and Unnest String, and this one was not part of it. + classOf[RegexOpDesc] -> "apache/texera#7548" + ) + + // The operator refuses the text pipeline in `getOutputSchemas`, so there is no + // configuration to compare: neither path is generated. An invalid configuration + // rather than a translation gap. + // + // Not sklearn raising, which is what the estimator's own limitation would look + // like. This fixture's word counts repeat enough that `ColumnTransformer` stays + // above its 0.3 sparse threshold and hands over a dense array, which GaussianNB + // fits without complaint. Only a wider vocabulary would reach the limitation + // the operator is guarding against. + val dense = ByDesign("the operator refuses Count Vectorizer at compile time") + val denseOnly = for { + op <- Seq( + classOf[SklearnGaussianNaiveBayesOpDesc], + classOf[SklearnTrainingGaussianNaiveBayesOpDesc] + ) + label <- Seq(RunKind.CountVectorizerText, RunKind.TfidfText) + } yield NotRun(op, label, dense) + + emptyCellRaises.map { + case (op, issue) => NotRun(op, RunKind.Nulls, PendingFix(issue)) + } ++ Seq( + // An enum whose legal values depend on a sibling field: flipping it alone + // builds a config the curated fixture already covers properly. + NotRun( + classOf[TypeCastingOpDesc], + RunKind.EnumSweep, + ByDesign( + "resultType is legal only for certain source column types, and the native " + + "executor throws on an illegal cast; the fixture pairs each type with a " + + "compatible column already" + ) + ), + NotRun( + classOf[AggregateOpDesc], + RunKind.EnumSweep, + ByDesign( + "aggFunction is cross-constrained with its attribute's type and with " + + "COUNT(*)'s empty attribute; the fixture pairs each function with a " + + "compatible column already" + ) + ), + // Stated about the operator's own fixture rather than about the operator: a + // predicate over a string column takes the hostile value fine, so this row goes + // the day that fixture filters on one. + NotRun( + classOf[SpecializedFilterOpDesc], + RunKind.HostileText, + ByDesign( + "`id > 8` compares against an INTEGER column, and the platform parses the " + + "predicate value as that column's type, so the number parser refuses the " + + "hostile string before any escaping could matter" + ) + ), + NotRun( + classOf[SklearnMLOperatorDescriptor[_]], + RunKind.HostileText, + ByDesign( + "what a hyperparameter's value may hold is decided by the parameter beside " + + "it, and every one of those is a number or a word from a fixed set, so a " + + "spliced a\"b fails at the conversion rather than at any escaping" + ) + ) + ) ++ denseOnly + } + + /** Every kind of run withheld from this operator, with why. This is the whole of + * what the coverage report needs, so it never walks the table itself. One entry + * per kind: a family row and an operator row for the same kind are the same + * statement twice, and the first one wins. + */ + def withheldRunsFor(opClass: Class[_ <: LogicalOp]): Seq[(String, NotRunReason)] = + variantsNotRun + .collect { case NotRun(op, kind, reason) if op.isAssignableFrom(opClass) => kind -> reason } + .distinctBy(_._1) + + private def notRun(opClass: Class[_ <: LogicalOp], kind: String): Boolean = + withheldRunsFor(opClass).exists(_._1 == kind) + + /** Visualization operators with deterministic Plotly JSON validation. */ + val visualizationJsonOps: Set[Class[_]] = Set( + classOf[RangeSliderOpDesc], + classOf[HeatMapOpDesc], + classOf[HierarchyChartOpDesc], + classOf[HistogramChartOpDesc], + classOf[Histogram2DOpDesc], + classOf[LineChartOpDesc], + classOf[ParallelCoordinatesPlotOpDesc], + classOf[PieChartOpDesc], + classOf[PolarChartOpDesc], + classOf[QuiverPlotOpDesc], + classOf[RadarChartOpDesc], + classOf[RadarPlotOpDesc], + classOf[SankeyDiagramOpDesc], + classOf[Scatter3dChartOpDesc], + classOf[ScatterplotOpDesc], + classOf[StripChartOpDesc], + classOf[TablesPlotOpDesc], + classOf[TernaryContourOpDesc], + classOf[TernaryPlotOpDesc], + classOf[TimeSeriesOpDesc], + classOf[TreePlotOpDesc], + classOf[VolcanoPlotOpDesc], + classOf[WaterfallChartOpDesc], + classOf[WindRoseChartOpDesc], + classOf[BarChartOpDesc], + classOf[BulletChartOpDesc], + classOf[CandlestickChartOpDesc], + classOf[CarpetPlotOpDesc], + classOf[ChoroplethMapOpDesc], + classOf[ContinuousErrorBandsOpDesc], + classOf[ContourPlotOpDesc], + classOf[DendrogramOpDesc], + classOf[DumbbellPlotOpDesc], + classOf[ECDFPlotOpDesc], + classOf[FigureFactoryTableOpDesc], + classOf[FilledAreaPlotOpDesc], + classOf[FunnelPlotOpDesc], + classOf[GanttChartOpDesc], + classOf[GaugeChartOpDesc], + classOf[DotPlotOpDesc], + classOf[IcicleChartOpDesc], + classOf[BubbleChartOpDesc], + classOf[ScatterMatrixChartOpDesc], + classOf[BoxViolinPlotOpDesc] + ) + + /** Visualization operators with deterministic HTML validation. */ + val visualizationHtmlOps: Set[Class[_]] = Set( + classOf[ImageVisualizerOpDesc], + classOf[NestedTableOpDesc] + ) + + /** Triaged, explicitly-not-run operators: class → honest reason, shown in + * the test report and coverage table. + */ + val knownIssues: Map[Class[_], String] = Map( + classOf[DummyOpDesc] -> + ("harness gap: placeholder operator with no physical execution — " + + "LogicalOp.getPhysicalOp throws NotImplementedError"), + classOf[SklearnPredictionOpDesc] -> + ("trained-model input: the operator consumes a fitted sklearn model on " + + "its model port; a JSONL fixture written from the JVM cannot carry a " + + "live model object, so the operator cannot be run in isolation here"), + classOf[SklearnTestingOpDesc] -> + ("trained-model input: scores a fitted sklearn model read from its model " + + "port; a JVM-written JSONL fixture cannot carry a live model object, so " + + "the operator cannot be run in isolation here"), + classOf[WordCloudOpDesc] -> + ("non-deterministic image: emits a base64 PNG from the wordcloud library " + + "whose word placement is randomized (no seed), so the two paths' images " + + "never match byte-for-byte"), + classOf[NetworkGraphOpDesc] -> + ("non-deterministic layout: the native path calls nx.spring_layout with no " + + "seed, so node coordinates are random per run and differ from the seeded " + + "standalone path, and the two paths' Plotly figures never match numerically") + ) + + sealed trait Disposition + final case class Runnable(tier: String) extends Disposition // "auto" | "curated" + final case class Flagged(reason: String) extends Disposition + + /** When `VERIFY_FORCE_AUTO=1`, ignore CuratedHandlers so every operator is + * exercised through the shared-CSV auto path instead. Lets us measure how + * much of the hand-written curated set the auto tier can now replace: an op + * that stays RUNNABLE/passes under force-auto no longer needs its curated + * handler. + */ + private def forceAuto: Boolean = sys.env.get("VERIFY_FORCE_AUTO").contains("1") + + /** The shared table an operator runs on in the AUTO tier. Which table an + * operator takes is its own axis (see [[SharedFixture]]); the auto tier used + * to be pinned to the whole of [[CanonicalFixture]], which is why an operator + * needing a narrower table had to be curated just to name one. sklearn cannot + * fit canonical's string columns — `X = table.drop(target)` feeds every + * remaining column to `fit` — so its families take the petal-and-label view + * of that same table. + */ + private[verify] def fixtureFor(opClass: Class[_ <: LogicalOp]): SharedFixture = + if (CuratedHandlers.sklearnNumericClasses.contains(opClass)) CanonicalFixture.sklearnNumeric + else if (opClass == classOf[HuggingFaceSpamSMSDetectionOpDesc]) CanonicalFixture.withoutScore + else CanonicalFixture + + /** Static classification — cheap (reflection only, no subprocesses), called + * at spec construction time to decide test-vs-ignore. + */ + def disposition(opClass: Class[_ <: LogicalOp]): Disposition = + knownIssues.get(opClass) match { + case Some(reason) => Flagged(s"known issue: $reason") + case None => + Try(opClass.getDeclaredConstructor().newInstance()) match { + case Failure(e) => Flagged(s"cannot instantiate: ${e.getMessage}") + case Success(op: StandaloneCodeGenerator) => + if (!op.producesDataFrame()) + if (visualizationJsonOps.contains(opClass) || visualizationHtmlOps.contains(opClass)) + Runnable("visualization") + else Flagged("visualization: no DataFrame output to compare") + else if (!forceAuto && CuratedHandlers.byClass.contains(opClass)) + Runnable("curated") + else + ConfigGenerator.generate(opClass, fixtureFor(opClass).schemasByPort) match { + case Left(reason) => Flagged(s"cannot auto-configure: $reason") + case Right(configured) => + Try(configured.operatorInfo.inputPorts.size) match { + case Failure(e) => + Flagged(s"operatorInfo failed on generated config: ${e.getMessage}") + case Success(n) if n < 1 || n > 2 => + Flagged(s"unsupported input port count: $n") + case Success(_) + if outputHasBinaryColumn(configured, fixtureFor(opClass)) && + fixtureFor(opClass) == CanonicalFixture => + // A trained-model (BINARY) output cannot be fit on the + // canonical table, whose string columns reach `fit`. The + // model itself is not byte-comparable either, but that is + // handled for every tier alike (see modelColumns in run). + // An op that names a numeric fixture is fine here. + Flagged( + "model output: emits a BINARY (trained-model) column; " + + "requires a numeric fixture, not the canonical table" + ) + case Success(_) => Runnable(s"auto${pinnedTierNote(opClass)}") + } + } + case Success(_) => + Flagged("does not implement StandaloneCodeGenerator") + } + } + + /** True if the configured operator declares a BINARY output column (e.g. a + * serialized trained model). Best-effort: only Python descriptors expose + * getOutputSchemas, and a throw (schema needs real inputs) reads as "no + * detectable BINARY column" so the op falls through to its normal tier. + */ + private def outputHasBinaryColumn(configured: LogicalOp, fixture: SharedFixture): Boolean = + configured match { + case p: PythonOperatorDescriptor => + val inputSchemas = fixture.schemasByPort.map { + case (port, schema) => PortIdentity(port) -> schema + } + Try(p.getOutputSchemas(inputSchemas)).toOption + .exists(_.values.exists(_.getAttributes.exists(_.getType == AttributeType.BINARY))) + case _ => false + } + + /** Execute both paths and assert parity on every declared output port. + * Precondition: disposition(opClass) returned Runnable. + */ + def run(opClass: Class[_ <: LogicalOp]): Unit = { + val testRoot = Files.createTempDirectory(s"verify-${opClass.getSimpleName}-") + + // Resolve the run list: each entry is (label, configured op, its inputs). + // Both tiers yield the base config PLUS one variant per enum value, so each + // enum branch (e.g. a line chart's mode = line / dots / line+dots) is + // exercised, not just the default, PLUS the `optionals` and `hostileText` + // variants. Variants of one fixture share input files; a curated handler's + // extraScenarios carry their own (structurally different) inputs. + val runs: Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + (if (forceAuto) None else CuratedHandlers.byClass.get(opClass)) match { + case Some(handler) => + val (op, in) = handler.fixture(testRoot) + // The variants are derived against the handler's OWN fixture, not the + // canonical one — a curated handler writes the table its operator needs, + // so that is what an optional column knob has to resolve against. + // + // An enum-sweep-exempt op still gets the fills: what is cross-constrained + // with a sibling field is its ENUM values, so a blind sweep produces invalid + // configs — filling an optional knob or splicing a quote does not. + // + // Fall back to the single curated config if it can't be varied at all. + val primary = + ConfigGenerator + .fullVariantsOf( + op, + schemasOf(in), + rowCountOf(in), + sweepEnums = !notRun(opClass, RunKind.EnumSweep) + ) + .fold(_ => Seq("default" -> op), identity) + // A variant this operator does not get, named in [[variantsNotRun]]. + .filterNot { case (label, _) => notRun(opClass, kindOf(label)) } + primary.map { case (label, o) => (label, o, in) } ++ + handler.extraScenarios(testRoot) ++ + handler.nullsKeepFilled.toSeq.flatMap(curatedNullsCase(opClass, op, in, testRoot, _)) + case None => + val fixture = fixtureFor(opClass) + val vs = ConfigGenerator + .generateVariants( + opClass, + fixture.schemasByPort, + fixture.port0RowCount, + knobsFor(opClass, KnobScope.Pinned), + knobsFor(opClass, KnobScope.WithOptionals) + ) + .fold( + reason => throw new IllegalStateException(s"cannot auto-configure: $reason"), + identity + ) + val inputPortCount = vs.head._2.operatorInfo.inputPorts.size + val in = fixture.writeInputs(testRoot, inputPortCount) + // A variant the operator itself cannot take, named in [[variantsNotRun]]: + // for a swept hyperparameter that is one row per parameter, so the sweep + // keeps covering the rest. + vs.filterNot { case (label, _) => notRun(opClass, kindOf(label)) } + .map { case (label, o) => (label, o, in) } ++ + nullsCase(opClass, vs.head._2, testRoot, fixture) ++ + altScenariosFor(opClass).flatMap { alt => + // Each scenario writes under its own directory: two tables in one + // testRoot would otherwise both claim input_port_0.jsonl. + val dir = testRoot.resolve(alt.label) + Files.createDirectories(dir) + ConfigGenerator + .generateVariants( + opClass, + alt.fixture.schemasByPort, + alt.fixture.port0RowCount, + pinned = alt.pinned, + switches = knobsFor(opClass, KnobScope.WithOptionals) + ) + .fold( + reason => + throw new IllegalStateException( + s"cannot auto-configure ${alt.label}: $reason" + ), + identity + ) + // The base variant carries the branch's own column knob: the pins + // are visible while the config is built, so the knob the schema + // requires under them is filled like any other required field. + .map { + case (label, o) => + (s"${alt.label}/$label", o, alt.fixture.writeInputs(dir, inputPortCount)) + } + } + } + + runs.foreach { + case (label, opDesc, inputs) => + val workDir = + if (runs.size == 1) testRoot + else testRoot.resolve(label.replaceAll("[^A-Za-z0-9]+", "_")) + Files.createDirectories(workDir) + try runVariant(opClass, opDesc, inputs, workDir) + catch { + case e: Throwable => + throw new AssertionError(s"[variant: $label] ${e.getMessage}", e) + } + } + } + + /** One extra run per operator, on `fixture` with one empty cell per column (see + * [[SharedFixture.emptyOneCellPerColumn]]). It takes the base config rather than + * crossing with the other variants: what an operator does with a null is a + * property of the operator, and multiplying it across every knob would buy more + * runtime than signal. + * + * The auto tier's form: the table is the shared one [[fixtureFor]] resolves, so + * the holes come from the fixture itself. See [[curatedNullsCase]] for the other + * tier, which has no fixture object to ask. + */ + private def nullsCase( + opClass: Class[_ <: LogicalOp], + base: LogicalOp, + testRoot: Path, + fixture: SharedFixture + ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + if (notRun(opClass, RunKind.Nulls)) Seq.empty + else { + val dir = testRoot.resolve("nulls-input") + Files.createDirectories(dir) + val in = fixture.write(dir, base.operatorInfo.inputPorts.size, withGaps = true) + Seq(("nulls", base, in)) + } + + /** [[nullsCase]] for the curated tier, where there is no fixture object to write + * a second time: the handler's own files are read back, holed, and rewritten. + * So a handler opts in by naming its load-bearing columns and nothing else, and + * [[TransformHandler.fixture]] keeps returning paths. + */ + private def curatedNullsCase( + opClass: Class[_ <: LogicalOp], + base: LogicalOp, + inputs: Map[PortIdentity, Path], + testRoot: Path, + keepFilled: Set[String] + ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + if (notRun(opClass, RunKind.Nulls)) Seq.empty + else { + val dir = testRoot.resolve("nulls-input") + Files.createDirectories(dir) + val holed = inputs.map { + case (portId, path) => + val schema = TupleIO.readSchemaSidecar(path) + val rows = TupleIO.readTuples(path, schema).toSeq + val out = dir.resolve(path.getFileName.toString) + TupleIO.writeTuples( + out, + SharedFixture.emptyOneCellPerColumn(rows, schema, keepFilled).iterator, + schema + ) + portId -> out + } + Seq(("nulls", base, holed)) + } + + /** The schema of each input file, keyed by port index — what a curated handler + * actually wrote, read back off the sidecar its writer drops. A file without one + * contributes no schema, so a column knob resolved against that port simply finds + * nothing to fill and the variant is skipped rather than built on a guess. + */ + private def schemasOf(inputs: Map[PortIdentity, Path]): Map[Int, Schema] = + inputs.flatMap { + case (portId, path) => Try(TupleIO.readSchemaSidecar(path)).toOption.map(portId.id -> _) + } + + /** How many rows port 0 holds — the hint a numeric knob's fill is scaled against + * (a `limit` worth running is one that keeps some rows and drops some). + */ + private def rowCountOf(inputs: Map[PortIdentity, Path]): Int = + inputs + .get(PortIdentity(0)) + .flatMap(path => Try(Files.readAllLines(path).asScala.count(_.trim.nonEmpty)).toOption) + .filter(_ > 0) + .getOrElse(ConfigGenerator.DefaultRowCount) + + /** Run one configured variant of `opDesc` through both paths against `inputs`, + * writing all intermediate/output files under `workDir`, and assert parity on + * every declared output port. + */ + private def runVariant( + opClass: Class[_ <: LogicalOp], + opDesc: LogicalOp, + inputs: Map[PortIdentity, Path], + workDir: Path + ): Unit = { + val outputPortCount = opDesc.operatorInfo.outputPorts.size + val actualDir = workDir.resolve("actual") + Files.createDirectories(actualDir) + + if (!opDesc.asInstanceOf[StandaloneCodeGenerator].producesDataFrame()) { + runVisualization(opClass, opDesc, inputs, outputPortCount, actualDir, workDir) + return + } + + // Path A's getPhysicalPlan/getPhysicalOp may mutate the OpDesc in place — + // AggregateOpDesc rewrites its `aggregations` to the final stage (COUNT→SUM) + // via getFinal. Run Path A on an isolated deep copy (same JSON round-trip the + // executor itself uses) so the shared instance stays pristine for Path B, + // whose generateStandaloneCode reads the original fields directly. + val opDescForPathA = + objectMapper + .readValue(objectMapper.writeValueAsString(opDesc), opClass) + .asInstanceOf[LogicalOp] + val (pathAOutputs, pathAOutputSchemas): (Map[PortIdentity, Path], Map[PortIdentity, Schema]) = + if (classOf[PythonOperatorDescriptor].isAssignableFrom(opClass)) { + val r = PyOpExecHarness.execute(opDescForPathA, inputs = inputs, outputDir = actualDir) + (r.outputs, r.outputSchemas) + } else { + val r = OpExecHarness.execute(opDescForPathA, inputs = inputs, outputDir = actualDir) + (r.outputs, r.outputSchemas) + } + + // StandaloneRunner keys inputs by 1-based port index (the inNdf convention). + val standaloneInputs: Map[Int, Path] = + inputs.toSeq + .sortBy(_._1.id) + .zipWithIndex + .map { + case ((_, path), idx) => (idx + 1) -> path + } + .toMap + + val pathB = StandaloneRunner.run( + opDesc = opDesc, + inputs = standaloneInputs, + outputPortCount = outputPortCount, + workDir = workDir + ) + + // The operator declares whether its output row order is meaningful via + // LogicalOp.orderSensitive (true only for the sort family); default unordered. + val orderSensitive = opDesc.orderSensitive + (0 until outputPortCount).foreach { port => + val actual = pathAOutputs.getOrElse( + PortIdentity(port), + throw new AssertionError(s"Texera path produced no output for port $port") + ) + val expected = pathB.outputs.getOrElse( + port + 1, + throw new AssertionError(s"standalone path produced no output for port $port") + ) + // A BINARY column holds a trained model: the two paths produce + // behaviorally-equivalent but not bit-identical models, so the comparator + // unpickles both and asserts their predictions on the training features + // (the probe) match — verifying behavior, not just completion. + val modelColumns: Seq[String] = pathAOutputSchemas + .get(PortIdentity(port)) + .map(_.getAttributes.filter(_.getType == AttributeType.BINARY).map(_.getName)) + .getOrElse(Seq.empty) + val probePath: Option[Path] = + if (modelColumns.nonEmpty) inputs.toSeq.sortBy(_._1.id).headOption.map(_._2) else None + Comparator.assertEqual( + actual, + expected, + orderSensitive = orderSensitive, + modelColumns = modelColumns, + probePath = probePath + ) + } + } + + private def runVisualization( + opClass: Class[_ <: LogicalOp], + opDesc: LogicalOp, + inputs: Map[PortIdentity, Path], + outputPortCount: Int, + actualDir: Path, + testRoot: Path + ): Unit = { + require( + visualizationJsonOps.contains(opClass) || visualizationHtmlOps.contains(opClass), + s"${opClass.getSimpleName} is not registered for visualization validation" + ) + require( + outputPortCount == 1, + "visualization JSON validation currently supports one output port" + ) + require( + classOf[PythonOperatorDescriptor].isAssignableFrom(opClass), + "visualization JSON validation currently supports Python visualization operators" + ) + + val actual = PyOpExecHarness + .execute(opDesc, inputs = inputs, outputDir = actualDir) + .outputs + .getOrElse( + PortIdentity(0), + throw new AssertionError("Texera path produced no visualization output for port 0") + ) + + val standaloneInputs: Map[Int, Path] = + inputs.toSeq + .sortBy(_._1.id) + .zipWithIndex + .map { + case ((_, path), idx) => (idx + 1) -> path + } + .toMap + + StandaloneRunner.run( + opDesc = opDesc, + inputs = standaloneInputs, + outputPortCount = outputPortCount, + workDir = testRoot + ) + + // A JSON-compared operator can still legitimately render its own error page + // instead of a figure (a non-numeric threshold, no non-null rows). There is + // then no Plotly payload to compare on either path, so compare what the user + // actually sees — the HTML. + if (visualizationJsonOps.contains(opClass) && hasPlotlyFigure(actual)) { + val expected = testRoot.resolve("output.json") + if (!Files.exists(expected)) { + throw new AssertionError(s"standalone visualization path did not produce $expected") + } + VisualizationJsonComparator.assertEqual(actual, expected) + } else { + val expected = testRoot.resolve("output.html") + if (!Files.exists(expected)) { + throw new AssertionError(s"standalone visualization path did not produce $expected") + } + VisualizationHtmlComparator.assertEqual(actual, expected) + } + } + + /** True if the runtime path's visualization output carries a Plotly figure — + * either a `json-content` payload or an `html-content` holding a + * `Plotly.newPlot(...)` call. False for an operator's own error page. + */ + private def hasPlotlyFigure(visualizationJsonl: Path): Boolean = { + val line = Files + .readAllLines(visualizationJsonl, StandardCharsets.UTF_8) + .asScala + .find(_.trim.nonEmpty) + .getOrElse(throw new AssertionError(s"$visualizationJsonl is empty")) + val node = objectMapper.readTree(line) + val json = node.get("json-content") + if (json != null && !json.isNull && json.asText().nonEmpty) true + else { + val html = node.get("html-content") + html != null && !html.isNull && html.asText().contains("Plotly.newPlot(") + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala new file mode 100644 index 00000000000..a5199b88c81 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +// This spec pins the tier-routing logic (disposition). Per-operator end-to-end +// runs are NOT duplicated here: OperatorBehaviorSpec auto-discovers every +// registered operator and runs TransformVerificationRunner.run on each, and a +// single operator can be run in isolation with e.g. +// sbt "WorkflowCompilingService/testOnly *OperatorBehaviorSpec -- -z LimitOpDesc" +// (the auto-generated test name starts with the operator's simple name). What +// disposition asserts — which tier an operator routes to — is the one thing +// OperatorBehaviorSpec does not check, so it lives here. + +import org.apache.texera.amber.operator.limit.LimitOpDesc +import org.apache.texera.amber.operator.sortPartitions.SortPartitionsOpDesc +import org.apache.texera.amber.operator.union.UnionOpDesc +import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { + import TransformVerificationRunner._ + + "disposition" should "flag knownIssues operators with the triage reason" in { + // The prediction op consumes a trained model on its input port, which a + // JVM-written JSONL fixture can't carry; triaged as a known issue, not run. + disposition(classOf[SklearnPredictionOpDesc]) match { + case Flagged(reason) => reason should include("trained-model") + case other => fail(s"expected Flagged, got $other") + } + disposition(classOf[WordCloudOpDesc]) match { + case Flagged(reason) => reason should include("known issue") + case other => fail(s"expected Flagged, got $other") + } + } + + it should "run the union now that its code names every upstream" in { + // It used to be flagged for naming exactly two, which was wrong in both + // directions: a third link was dropped and a lone link left the second + // frame unbound. The runner draws one link per port, so what runs here is + // the one-upstream case — the one the old code got wrong. + disposition(classOf[UnionOpDesc]) shouldBe Runnable("auto") + } + + it should "route auto-configurable operators to the auto tier" in { + disposition(classOf[LimitOpDesc]) shouldBe Runnable("auto") + } + + // The operator set implements the generator a family at a time, so most of it + // does not yet. That is reported rather than passed over: an operator missing + // from the run is a fact the report has to carry, and the rows for each family + // arrive with the change that gives that family its generator. + it should "flag an operator that has no standalone generator yet" in { + disposition(classOf[SortPartitionsOpDesc]) shouldBe + Flagged("does not implement StandaloneCodeGenerator") + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala new file mode 100644 index 00000000000..c9fa74cf877 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} + +object VisualizationHtmlComparator { + + /** A pandas Styler namespaces its CSS with a uuid drawn per Styler instance, so + * the same table rendered twice differs in every `id=` and every selector even + * though the markup is identical. The uuid carries no information about the + * table — it only keeps two tables on one page from colliding — so it is + * normalized away before comparing. Only the random prefix is replaced: the + * `_row0_col0` suffix that identifies the cell stays, so a genuine structural + * difference still fails. + */ + private val StylerUuid = "T_[0-9a-f]+".r + + private def normalize(html: String): String = StylerUuid.replaceAllIn(html, "T_uuid") + + def assertEqual(actualVisualizationJsonl: Path, expectedHtmlFile: Path): Unit = { + val actual = readActualHtml(actualVisualizationJsonl) + val expected = new String(Files.readAllBytes(expectedHtmlFile), StandardCharsets.UTF_8) + + if (normalize(actual) != normalize(expected)) { + throw new VisualizationHtmlMismatchException( + actual = actualVisualizationJsonl, + expected = expectedHtmlFile, + actualHtml = actual, + expectedHtml = expected + ) + } + } + + private def readActualHtml(path: Path): String = { + val line = Files + .readAllLines(path, StandardCharsets.UTF_8) + .stream() + .filter(_.trim.nonEmpty) + .findFirst() + .orElseThrow(() => new AssertionError(s"$path is empty")) + + val node = objectMapper.readTree(line) + val htmlNode = node.get("html-content") + if (htmlNode == null || htmlNode.isNull) { + throw new AssertionError(s"$path has no html-content field") + } + htmlNode.asText() + } +} + +final class VisualizationHtmlMismatchException( + val actual: Path, + val expected: Path, + val actualHtml: String, + val expectedHtml: String +) extends RuntimeException( + s"""Visualization HTML mismatch: + | actual: $actual + | expected: $expected + |--- actual html --- + |$actualHtml + |--- expected html --- + |$expectedHtml""".stripMargin + ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala new file mode 100644 index 00000000000..f559b2c39e6 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.util.python.PythonWorkerPool + +import java.nio.file.{Files, Path, StandardCopyOption} +import scala.collection.mutable.ArrayBuffer +import scala.sys.process._ + +/** + * Compares the Plotly figure the two paths render, via `compare.py`'s + * `--plotly` mode. + * + * Shares that script — and therefore [[Comparator]]'s pool — rather than + * carrying one of its own: a worker is bound to the script it was launched + * with, so a separate script would mean a separate pool of interpreters for + * what is the same job, comparing one operator's two outputs. One comparison + * pool serves both output shapes. + */ +object VisualizationJsonComparator extends LazyLogging { + + private val ScriptResourcePath = "/python/compare.py" + + def assertEqual( + actualVisualizationJsonl: Path, + expectedPlotlyJson: Path, + pythonExe: String = resolvePython() + ): Unit = { + val (exit, stdout, stderr) = + compare(actualVisualizationJsonl, expectedPlotlyJson, pythonExe) + if (exit != 0) { + throw new VisualizationJsonMismatchException( + actual = actualVisualizationJsonl, + expected = expectedPlotlyJson, + exitCode = exit, + stdout = stdout, + stderr = stderr + ) + } + } + + // Pooled worker first, one-shot CLI as the fallback and as the behavior + // selected by TEXERA_TEST_PYTHON_WORKER=0. Both run the same + // `_run_plotly_comparison`, so results are identical. + private def compare(actual: Path, expected: Path, pythonExe: String): (Int, String, String) = { + if (PythonWorkerPool.enabled) { + try { + val req = objectMapper.createObjectNode() + req.put("kind", "plotly") + req.put("actual", actual.toString) + req.put("expected", expected.toString) + val o = PythonWorkerPool.run(ScriptResourcePath, Seq("--serve"), pythonExe, req) + return (o.exit, o.stdout, o.stderr) + } catch { + case e: PythonWorkerPool.WorkerDiedException => + logger.warn( + s"Comparator worker unavailable; falling back to one-shot CLI: ${e.getMessage}" + ) + } + } + runCli(actual, expected, pythonExe) + } + + private def runCli(actual: Path, expected: Path, pythonExe: String): (Int, String, String) = { + val scriptPath = extractScript() + val outBuf = ArrayBuffer.empty[String] + val errBuf = ArrayBuffer.empty[String] + val processLogger = ProcessLogger(line => outBuf += line, line => errBuf += line) + + val exit = Process( + Seq(pythonExe, scriptPath.toString, "--plotly", actual.toString, expected.toString) + ).!(processLogger) + (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) + } + + private def extractScript(): Path = { + val stream = getClass.getResourceAsStream(ScriptResourcePath) + require(stream != null, s"compare.py not found at $ScriptResourcePath") + try { + val tmp = Files.createTempFile("compare-", ".py") + Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING) + tmp.toFile.deleteOnExit() + tmp + } finally stream.close() + } + + private def resolvePython(): String = + sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") +} + +final class VisualizationJsonMismatchException( + val actual: Path, + val expected: Path, + val exitCode: Int, + val stdout: String, + val stderr: String +) extends RuntimeException( + s"""Visualization JSON mismatch (compare.py --plotly exit $exitCode): + | actual: $actual + | expected: $expected + |--- stderr --- + |$stderr""".stripMargin + ) From 41724ff828202ce8cdf029fe3d83dbafae036ef2 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 11:18:23 -0700 Subject: [PATCH 04/33] feat(workflow-operator): export the source operators as Python Ten sources say how they read outside the engine, so an exported script starts from the same data the workflow did rather than from a variable the reader has to fill in: the CSV family, JSON Lines, Arrow, plain text, and the two that read a file named at runtime. A source is the one place where the script cannot simply repeat what the operator does. The engine resolves a dataset through Texera's storage and hands the operator a URI; a script has no such resolver, so it reads the file from its own directory by the name the URI ended with. That is what makes an exported script portable, and it is also its one precondition: the data has to sit beside the script. Two are reported as unverifiable rather than exported blind. File Scan takes its filenames from an input port at run time, which a source harness has nothing to feed; URL Fetcher reads a live URL, so no two runs are required to agree. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/fetcher/URLFetcherOpDesc.scala | 45 ++++++++- .../source/scan/arrow/ArrowSourceOpDesc.scala | 25 ++++- .../source/scan/csv/CSVScanSourceOpDesc.scala | 62 +++++++++++- .../csv/ParallelCSVScanSourceOpDesc.scala | 35 ++++++- .../scan/csvOld/CSVOldScanSourceOpDesc.scala | 46 ++++++++- .../source/scan/file/FileScanOpDesc.scala | 68 ++++++++++++- .../scan/file/FileScanSourceOpDesc.scala | 72 +++++++++++++- .../scan/json/JSONLScanSourceOpDesc.scala | 53 ++++++++++- .../scan/text/TextInputSourceOpDesc.scala | 48 +++++++++- .../source/scan/text/TextSourceOpDesc.scala | 5 + .../source/fetcher/URLFetcherOpDescSpec.scala | 35 +++++++ .../scan/csv/CSVScanSourceOpDescSpec.scala | 25 +++++ .../source/scan/file/FileScanOpDescSpec.scala | 95 +++++++++++++++++++ 13 files changed, 598 insertions(+), 16 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala index fc12c11e366..484bab3861a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala @@ -20,22 +20,33 @@ package org.apache.texera.amber.operator.source.fetcher import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} -import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{OutputPort, PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.source.SourceOperatorDescriptor import org.apache.texera.amber.util.JSONUtils.objectMapper -class URLFetcherOpDesc extends SourceOperatorDescriptor { +class URLFetcherOpDesc extends SourceOperatorDescriptor with StandaloneCodeGenerator { + // No `pattern`: the reader is `java.net.URL`, which asks only that the value carry + // a scheme its JVM has a handler for. That is not something a regex can state -- + // one excluding `www.example.com` would still pass `htp://x`, so it would advertise + // a validation the field does not have. `examples` offers a realistic value without + // claiming to constrain anything. @JsonProperty(required = true) @JsonSchemaTitle("URL") @JsonPropertyDescription( "Only accepts standard URL format" ) + @JsonSchemaInject(json = """ +{ + "examples": ["https://example.com"] +} +""") var url: String = _ @JsonProperty(required = true) @@ -87,4 +98,34 @@ class URLFetcherOpDesc extends SourceOperatorDescriptor { outputPorts = List(OutputPort()) ) + // The generated snippet uses `urllib.request`, which the translator's shared + // imports don't include. Following the per-operator convention (e.g. Split + // emits `import numpy as np`), the code block prepends its own import so the + // generated script is self-contained. + override def generateStandaloneCode(): String = { + val urlLiteral = objectMapper.writeValueAsString(url) + val isUtf8 = decodingMethod == DecodingMethod.UTF_8 + val valueExpr = if (isUtf8) """_content.decode("utf-8")""" else "_content" + val buf = scala.collection.mutable.ArrayBuffer[String]() + buf += "import http.client" + buf += "import urllib.request" + buf += s"_url = $urlLiteral" + // Catch the fetch failures, not everything: the executor guards only the fetch, so + // a value with no scheme stops it, and `except Exception` here would swallow that + // and hand back a row instead. The two are told apart by type -- a fetch raises + // OSError (URLError, HTTPError, TimeoutError) or HTTPException on a malformed + // response, a missing scheme raises ValueError. + // Still divergent, and not fixable this way: a scheme that merely is not RECOGNISED + // ("htp://x") stops the executor but reaches Python as URLError, and the two + // languages do not recognise the same schemes anyway -- Java takes `mailto:`, + // urlopen does not -- so no list of schemes is right on both sides. + buf += "try:" + buf += " with urllib.request.urlopen(_url) as _resp:" + buf += " _content = _resp.read()" + buf += "except (OSError, http.client.HTTPException):" + buf += """ _content = f"Fetch failed for URL: {_url}".encode("utf-8")""" + buf += s"""out1df = pd.DataFrame({"URL content": [$valueExpr]})""" + buf.mkString("\n") + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index ad1d7a34176..5114e3560ed 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -25,8 +25,10 @@ import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.util.ArrowUtils +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.arrow.memory.RootAllocator import org.apache.arrow.vector.ipc.ArrowFileReader @@ -38,10 +40,31 @@ import java.nio.file.{Files, StandardOpenOption} import scala.util.Using @JsonIgnoreProperties(value = Array("fileEncoding")) -class ArrowSourceOpDesc extends ScanSourceOpDesc { +class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { fileTypeName = Option("Arrow") + override def generateStandaloneCode(): String = { + val basename = sourceBasename(fileName.getOrElse("")) + val read = s"""out1df = pd.read_feather(${pyStringLiteral(basename)})""" + // A timestamp column needs nothing here. The file names UTC and holds the + // wall clock as UTC, so pd.read_feather and the executor read the same + // reading off it — no zone of the reader's own enters either side. The other + // scan sources have to name their date columns, CSV and JSONL carrying no + // types to go on, but Arrow states its own. + // + // The executor drops `offset` rows and then takes `limit` of them. Feather has + // no row-range read, so the same window is taken once the frame is in memory. + val window = (offset, limit) match { + case (Some(o), Some(l)) => Some(s"$o:${o + l}") + case (Some(o), None) => Some(s"$o:") + case (None, Some(l)) => Some(s":$l") + case _ => None + } + (read +: window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)").toSeq) + .mkString("\n") + } + @throws[IOException] override def getPhysicalOp( workflowId: WorkflowIdentity, diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index 69088abb6c9..c11216d34c9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -28,22 +28,28 @@ import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpExec +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import java.io.{IOException, InputStreamReader} import java.net.URI +import scala.util.Try -class CSVScanSourceOpDesc extends ScanSourceOpDesc { +class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // One character: every reader narrows this with charAt(0), because univocity's // setDelimiter and scala-csv's DefaultCSVFormat both take a Char. + // + // `examples` names a delimiter the fixture's rows do not contain, so the + // verification config generator does not pick one that parses them ragged. @JsonProperty(defaultValue = ",") @JsonSchemaTitle("Delimiter") @JsonPropertyDescription("single character separating the fields on each line") @JsonInclude(JsonInclude.Include.NON_ABSENT) - @JsonSchemaInject(json = """{ "maxLength": 1 }""") + @JsonSchemaInject(json = """{ "maxLength": 1, "examples": [","] }""") var customDelimiter: Option[String] = None @JsonProperty(defaultValue = "true") @@ -145,4 +151,56 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc { } + override def generateStandaloneCode(): String = { + // Strip to just the basename. The standalone script assumes the CSV + // lives in the same directory as the script (Texera's resolved URIs + // can't be used directly outside the system). + val basename = sourceBasename(fileName.getOrElse("")) + + // Resolve the delimiter the same way the parser above does — first character, empty + // means comma — and escape it. Every value the field accepts has to survive this: + // pandas reads a separator longer than one character as a REGULAR EXPRESSION, and a + // backslash spliced raw produced `sep="\"`, which is not valid Python at all. + val sep = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString + // Texera's encoding enum uses values like UTF_8; pandas expects utf-8. + val encoding = fileEncoding.toString.replace("_", "-").toLowerCase + val headerArg = if (hasHeader) "0" else "None" + + val args = scala.collection.mutable.ArrayBuffer[String]() + args += s"""filepath_or_buffer=${pyStringLiteral(basename)}""" + args += s"sep=${pyStringLiteral(sep)}" + args += s"""encoding=${pyStringLiteral(encoding)}""" + args += s"header=$headerArg" + + // A CSV carries no types, so both readers infer, and they do not infer + // alike: the schema above tries TIMESTAMP and parses what it can, while + // pd.read_csv leaves a date column as text. Name the columns this operator + // decided were timestamps so pandas parses the same ones — by position when + // there is no header, the frame's columns having no names until the rename + // below. A schema that cannot be read (an unresolved file) leaves the + // argument off rather than failing the export. + val dateColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes.zipWithIndex + .filter(_._1.getType == AttributeType.TIMESTAMP) + .map { case (a, i) => if (hasHeader) pyStringLiteral(a.getName) else i.toString } + ) + if (dateColumns.nonEmpty) args += s"parse_dates=[${dateColumns.mkString(", ")}]" + + offset.foreach { o => + // With a header, skip offset rows after row 0; without, skip offset rows from the start. + if (hasHeader) args += s"skiprows=range(1, ${o + 1})" + else args += s"skiprows=$o" + } + limit.foreach(l => args += s"nrows=$l") + + val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" + + if (hasHeader) readCall + else { + // Match Texera's fallback column naming when there's no header + s"""$readCall + |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin + } + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala index cdade62d1e1..902284ce520 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala @@ -29,20 +29,22 @@ import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import java.io.IOException import java.net.URI -class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc { +class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // One character -- see CSVScanSourceOpDesc. @JsonProperty(defaultValue = ",") @JsonSchemaTitle("Delimiter") @JsonPropertyDescription("single character separating the fields on each line") @JsonDeserialize(contentAs = classOf[java.lang.String]) - @JsonSchemaInject(json = """{ "maxLength": 1 }""") + @JsonSchemaInject(json = """{ "maxLength": 1, "examples": [","] }""") var customDelimiter: Option[String] = None @JsonProperty(defaultValue = "true") @@ -80,6 +82,35 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc { ) } + override def generateStandaloneCode(): String = { + val basename = sourceBasename(fileName.getOrElse("")) + // First character, empty means comma — the same resolution the reader below does — + // and escaped, so every value the field accepts survives being spliced into Python. + // See CSVScanSourceOpDesc for what handing pandas the raw value did. + val sep = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString + val encoding = fileEncoding.toString.replace("_", "-").toLowerCase + val headerArg = if (hasHeader) "0" else "None" + + val args = scala.collection.mutable.ArrayBuffer[String]() + args += s"""filepath_or_buffer=${pyStringLiteral(basename)}""" + args += s"sep=${pyStringLiteral(sep)}" + args += s"""encoding=${pyStringLiteral(encoding)}""" + args += s"header=$headerArg" + + offset.foreach { o => + if (hasHeader) args += s"skiprows=range(1, ${o + 1})" + else args += s"skiprows=$o" + } + limit.foreach(l => args += s"nrows=$l") + + val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" + + if (hasHeader) readCall + else + s"""$readCall + |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin + } + override def sourceSchema(): Schema = { val delimiterChar = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0) require( diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index 57dec76c556..a7b7607c6bb 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -28,19 +28,22 @@ import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import java.io.IOException import java.net.URI +import scala.util.Try -class CSVOldScanSourceOpDesc extends ScanSourceOpDesc { +class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // One character -- see CSVScanSourceOpDesc. @JsonProperty(defaultValue = ",") @JsonSchemaTitle("Delimiter") @JsonPropertyDescription("single character separating the fields on each line") - @JsonSchemaInject(json = """{ "maxLength": 1 }""") + @JsonSchemaInject(json = """{ "maxLength": 1, "examples": [","] }""") var customDelimiter: Option[String] = Some(",") @JsonProperty(defaultValue = "true") @@ -76,6 +79,45 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc { ) } + override def generateStandaloneCode(): String = { + val basename = sourceBasename(fileName.getOrElse("")) + // First character, empty means comma — the same resolution the reader below does — + // and escaped, so every value the field accepts survives being spliced into Python. + // See CSVScanSourceOpDesc for what handing pandas the raw value did. + val sep = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString + val encoding = fileEncoding.toString.replace("_", "-").toLowerCase + val headerArg = if (hasHeader) "0" else "None" + + val args = scala.collection.mutable.ArrayBuffer[String]() + args += s"""filepath_or_buffer=${pyStringLiteral(basename)}""" + args += s"sep=${pyStringLiteral(sep)}" + args += s"""encoding=${pyStringLiteral(encoding)}""" + args += s"header=$headerArg" + + // Name the columns this operator inferred as timestamps, so pandas parses + // the same ones instead of leaving them as text. See CSVScanSourceOpDesc. + val dateColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes.zipWithIndex + .filter(_._1.getType == AttributeType.TIMESTAMP) + .map { case (a, i) => if (hasHeader) pyStringLiteral(a.getName) else i.toString } + ) + if (dateColumns.nonEmpty) args += s"parse_dates=[${dateColumns.mkString(", ")}]" + + offset.foreach { o => + if (hasHeader) args += s"skiprows=range(1, ${o + 1})" + else args += s"skiprows=$o" + } + limit.foreach(l => args += s"nrows=$l") + + val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" + + if (hasHeader) readCall + else + s"""$readCall + |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin + } + override def sourceSchema(): Schema = { val delimiterChar = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0) require( diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala index f83d1d76307..f945454d7ca 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala @@ -30,13 +30,18 @@ import org.apache.texera.amber.core.workflow.{ PhysicalOp, SchemaPropagationFunc } +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.apache.texera.amber.operator.source.scan.FileDecodingMethod +import org.apache.texera.amber.operator.source.scan.{FileAttributeType, FileDecodingMethod} import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class FileScanOpDesc extends SourceOperatorDescriptor with TextSourceOpDesc { +class FileScanOpDesc + extends SourceOperatorDescriptor + with TextSourceOpDesc + with StandaloneCodeGenerator { @JsonProperty(defaultValue = "UTF_8", required = true) @JsonSchemaTitle("Encoding") var fileEncoding: FileDecodingMethod = FileDecodingMethod.UTF_8 @@ -86,4 +91,63 @@ class FileScanOpDesc extends SourceOperatorDescriptor with TextSourceOpDesc { inputPorts = List(InputPort(displayName = "Filename")), outputPorts = List(OutputPort()) ) + + override def generateStandaloneCode(): String = { + val col = attributeName + val enc = fileEncoding.toString.replace("_", "-").toLowerCase + val buf = scala.collection.mutable.ArrayBuffer[String]() + + if (extract) + buf += "# WARNING: extract=true is not supported in standalone mode; files are read as-is, not unpacked from archives." + + val isBinary = + attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY + val openArgs = + if (isBinary) """"rb"""" + else s""""r", encoding=${pyStringLiteral(enc)}""" + + buf += "_rows = []" + buf += "for _fn in in1df.iloc[:, 0]:" + buf += s" with open(_fn, $openArgs) as _f:" + + // Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line + // branch ignores outputFileName and emits only the value, so the filename + // column is added ONLY in single-value mode. + val emitFilename = outputFileName && attributeType.isSingle + + if (attributeType.isSingle) { + if (emitFilename) buf += " _rows.append((_fn, _f.read()))" + else buf += " _rows.append(_f.read())" + } else { + val castExpr = attributeType match { + case FileAttributeType.INTEGER => "int(l.rstrip())" + case FileAttributeType.LONG => "int(l.rstrip())" + case FileAttributeType.DOUBLE => "float(l.rstrip())" + case FileAttributeType.BOOLEAN => """l.rstrip().lower() == "true"""" + case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" + case _ => """l.rstrip("\n")""" + } + val hasSlice = fileScanOffset.isDefined || fileScanLimit.isDefined + if (hasSlice) { + val start = fileScanOffset.getOrElse(0) + val sliceExpr = fileScanLimit match { + case Some(l) => s"_lines[$start:${start + l}]" + case None => s"_lines[$start:]" + } + buf += s" _lines = [$castExpr for l in _f]" + buf += s" _rows.extend($sliceExpr)" + } else { + buf += s" _rows.extend($castExpr for l in _f)" + } + } + + val colLit = pyStringLiteral(col) + if (emitFilename) { + buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])""" + } else { + buf += s"""out1df = pd.DataFrame({$colLit: _rows})""" + } + + buf.mkString("\n") + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala index 82997632d14..0ebefd5023b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala @@ -29,13 +29,22 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.annotations.HideAnnotation import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc -import org.apache.texera.amber.operator.source.scan.{FileDecodingMethod, ScanSourceOpDesc} +import org.apache.texera.amber.operator.source.scan.{ + FileAttributeType, + FileDecodingMethod, + ScanSourceOpDesc +} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper @JsonIgnoreProperties(value = Array("limit", "offset", "fileEncoding")) -class FileScanSourceOpDesc extends ScanSourceOpDesc with TextSourceOpDesc { +class FileScanSourceOpDesc + extends ScanSourceOpDesc + with TextSourceOpDesc + with StandaloneCodeGenerator { @JsonProperty(defaultValue = "UTF_8", required = true) @JsonSchemaTitle("Encoding") @JsonSchemaInject( @@ -64,6 +73,65 @@ class FileScanSourceOpDesc extends ScanSourceOpDesc with TextSourceOpDesc { fileTypeName = Option("") + override def generateStandaloneCode(): String = { + val basename = sourceBasename(fileName.getOrElse("")) + val col = attributeName + val enc = encoding.toString.replace("_", "-").toLowerCase + val basenameLit = pyStringLiteral(basename) + val colLit = pyStringLiteral(col) + val encLit = pyStringLiteral(enc) + val buf = scala.collection.mutable.ArrayBuffer[String]() + + if (extract) + buf += s"""# WARNING: extract=true is not supported in standalone mode; provide the unarchived $basenameLit directly.""" + + val isBinary = + attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY + + if (attributeType.isSingle) { + val openArgs = + if (isBinary) s"""$basenameLit, "rb"""" + else s"""$basenameLit, "r", encoding=$encLit""" + val dfCols = + if (outputFileName) s"""{"filename": $basenameLit, $colLit: [_f.read()]}""" + else s"""{$colLit: [_f.read()]}""" + buf += s"""with open($openArgs) as _f:""" + buf += s""" out1df = pd.DataFrame($dfCols)""" + } else { + val castExpr = attributeType match { + case FileAttributeType.INTEGER => "int(l.rstrip())" + case FileAttributeType.LONG => "int(l.rstrip())" + case FileAttributeType.DOUBLE => "float(l.rstrip())" + case FileAttributeType.BOOLEAN => """l.rstrip().lower() == "true"""" + case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" + case _ => """l.rstrip("\n")""" + } + val hasSlice = fileScanOffset.isDefined || fileScanLimit.isDefined + if (hasSlice) { + val start = fileScanOffset.getOrElse(0) + val sliceExpr = fileScanLimit match { + case Some(l) => s"_lines[$start:${start + l}]" + case None => s"_lines[$start:]" + } + val dfCols = + if (outputFileName) s"""{"filename": $basenameLit, $colLit: $sliceExpr}""" + else s"""{$colLit: $sliceExpr}""" + buf += s"""with open($basenameLit, "r", encoding=$encLit) as _f:""" + buf += s""" _lines = [$castExpr for l in _f]""" + buf += s""" out1df = pd.DataFrame($dfCols)""" + } else { + val dfCols = + if (outputFileName) + s"""{"filename": $basenameLit, $colLit: [$castExpr for l in _f]}""" + else s"""{$colLit: [$castExpr for l in _f]}""" + buf += s"""with open($basenameLit, "r", encoding=$encLit) as _f:""" + buf += s""" out1df = pd.DataFrame($dfCols)""" + } + } + + buf.mkString("\n") + } + override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index faccc76f882..b5cb9c54f3b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -24,18 +24,21 @@ import com.fasterxml.jackson.databind.JsonNode import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows -import org.apache.texera.amber.core.tuple.{Attribute, Schema} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.{JSONToMap, objectMapper} import java.io._ import java.net.URI import scala.collection.mutable.ArrayBuffer +import scala.util.Try import scala.jdk.CollectionConverters.IteratorHasAsScala -class JSONLScanSourceOpDesc extends ScanSourceOpDesc { +class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { @JsonProperty(required = true, defaultValue = "false") @JsonPropertyDescription("flatten nested objects and arrays") @@ -43,6 +46,52 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc { fileTypeName = Option("JSONL") + override def generateStandaloneCode(): String = { + val basename = sourceBasename(fileName.getOrElse("")) + val enc = fileEncoding.toString.replace("_", "-").toLowerCase + + val readArgs = scala.collection.mutable.ArrayBuffer[String]() + readArgs += pyStringLiteral(basename) + readArgs += "lines=True" + readArgs += s"""encoding=${pyStringLiteral(enc)}""" + + // JSON has no timestamp of its own, so both readers infer from the text and + // do not infer alike: the schema below tries TIMESTAMP and parses what it + // can, while pd.read_json guesses from the COLUMN NAME (anything ending + // "_at" or "_time", anything called "date") and leaves the rest as text. + // Naming the columns this operator decided were timestamps settles both + // halves — the ones it misses and the ones it would have taken on its own. + // An unreadable schema leaves the argument off rather than failing the + // export. + val dateColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes + .filter(_.getType == AttributeType.TIMESTAMP) + .map(a => pyStringLiteral(a.getName)) + ) + readArgs += s"convert_dates=[${dateColumns.mkString(", ")}]" + + if (offset.isEmpty) limit.foreach(l => readArgs += s"nrows=$l") + + val readExpr = s"pd.read_json(${readArgs.mkString(", ")})" + val baseExpr = + if (flatten) s"pd.json_normalize($readExpr.to_dict('records'))" + else readExpr + + val lines = scala.collection.mutable.ArrayBuffer[String]() + lines += s"out1df = $baseExpr" + + (offset, limit) match { + case (Some(o), Some(l)) => + lines += s"out1df = out1df.iloc[$o:${o + l}].reset_index(drop=True)" + case (Some(o), None) => + lines += s"out1df = out1df.iloc[$o:].reset_index(drop=True)" + case _ => + } + + lines.mkString("\n") + } + @throws[IOException] override def getPhysicalOp( workflowId: WorkflowIdentity, diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala index 50de683f70a..1a49e5bff69 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala @@ -25,12 +25,18 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{OutputPort, PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.source.SourceOperatorDescriptor +import org.apache.texera.amber.operator.source.scan.FileAttributeType +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class TextInputSourceOpDesc extends SourceOperatorDescriptor with TextSourceOpDesc { +class TextInputSourceOpDesc + extends SourceOperatorDescriptor + with TextSourceOpDesc + with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("Text") @JsonSchemaInject(json = UIWidget.UIWidgetTextArea) @@ -68,4 +74,44 @@ class TextInputSourceOpDesc extends SourceOperatorDescriptor with TextSourceOpDe inputPorts = List.empty, outputPorts = List(OutputPort()) ) + + override def generateStandaloneCode(): String = { + val text = objectMapper.writeValueAsString(textInput) + val col = attributeName + val colLit = pyStringLiteral(col) + val buf = scala.collection.mutable.ArrayBuffer[String]() + + buf += s"_text = $text" + + val isBinary = + attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY + + if (attributeType.isSingle) { + val valueExpr = if (isBinary) """_text.encode("utf-8")""" else "_text" + buf += s"""out1df = pd.DataFrame({$colLit: [$valueExpr]})""" + } else { + val castExpr = attributeType match { + case FileAttributeType.INTEGER => "int(l)" + case FileAttributeType.LONG => "int(l)" + case FileAttributeType.DOUBLE => "float(l)" + case FileAttributeType.BOOLEAN => """l.lower() == "true"""" + case FileAttributeType.TIMESTAMP => "pd.Timestamp(l)" + case _ => "l" + } + val hasSlice = fileScanOffset.isDefined || fileScanLimit.isDefined + if (hasSlice) { + val start = fileScanOffset.getOrElse(0) + val sliceExpr = fileScanLimit match { + case Some(l) => s"_lines[$start:${start + l}]" + case None => s"_lines[$start:]" + } + buf += s"""_lines = [$castExpr for l in _text.splitlines()]""" + buf += s"""out1df = pd.DataFrame({$colLit: $sliceExpr})""" + } else { + buf += s"""out1df = pd.DataFrame({$colLit: [$castExpr for l in _text.splitlines()]})""" + } + } + + buf.mkString("\n") + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala index 07a773580f6..654e11c02d2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala @@ -48,6 +48,10 @@ trait TextSourceOpDesc { @JsonDeserialize(contentAs = classOf[java.lang.String]) var attributeName: String = "line" + // Named explicitly so reflection can see it. These are the row-window knobs the + // text sources actually read — the ones they inherit are ignored — but they carried + // no @JsonProperty, which leaves them invisible to anything walking the config. + @JsonProperty @JsonSchemaTitle("Limit (lines)") @JsonDeserialize(contentAs = classOf[Int]) @JsonPropertyDescription( @@ -66,6 +70,7 @@ trait TextSourceOpDesc { ) var fileScanLimit: Option[Int] = None + @JsonProperty @JsonSchemaTitle("Offset (lines)") @JsonPropertyDescription( "Number of lines to skip from the start before reading. " + diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala index 16b9821cb19..5add175ca11 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala @@ -92,6 +92,41 @@ class URLFetcherOpDescSpec extends AnyFlatSpec with Matchers { } } + "URLFetcherOpDesc.generateStandaloneCode" should + "fetch the URL and decode the body as UTF-8 text" in { + configured(DecodingMethod.UTF_8).generateStandaloneCode() shouldBe + """import http.client + |import urllib.request + |_url = "https://example.test/data" + |try: + | with urllib.request.urlopen(_url) as _resp: + | _content = _resp.read() + |except (OSError, http.client.HTTPException): + | _content = f"Fetch failed for URL: {_url}".encode("utf-8") + |out1df = pd.DataFrame({"URL content": [_content.decode("utf-8")]})""".stripMargin + } + + // The executor guards only the fetch, so a value with no scheme stops it. `Exception` + // would swallow that too and hand back a row where the platform stopped. + it should "let a value that is not a URL through, not just fetch failures" in { + val code = configured(DecodingMethod.UTF_8).generateStandaloneCode() + code should include("except (OSError, http.client.HTTPException):") + code should not include "except Exception:" + } + + it should "keep the raw bytes when decoding is not UTF-8" in { + configured(DecodingMethod.RAW_BYTES).generateStandaloneCode() should + endWith("""out1df = pd.DataFrame({"URL content": [_content]})""") + } + + // The URL is user-supplied and lands inside the generated Python source, so + // it goes through JSON string encoding rather than raw interpolation. + it should "emit the URL as an escaped Python string literal" in { + val op = configured(DecodingMethod.UTF_8) + op.url = """https://example.test/a"b\c""" + op.generateStandaloneCode() should include("""_url = "https://example.test/a\"b\\c"""") + } + it should "propagate sourceSchema onto the single output port" in { // Exercise propagateSchema.func directly so the test actually proves the // sourceSchema gets routed to the output port id, not just that an diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index fe426552a96..7a023239ef9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -211,6 +211,31 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { ) } + it should "use the csv basename in standalone code" in { + csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(csvScanSourceOpDesc.fileName.get)) + + val code = csvScanSourceOpDesc.generateStandaloneCode() + + assert(code.contains("""filepath_or_buffer="country_sales_small_multi_line.csv"""")) + assert(!code.contains("base64.b64decode")) + assert(!code.contains("io.BytesIO")) + } + + it should "use the unresolved csv basename in standalone code" in { + csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + + val code = csvScanSourceOpDesc.generateStandaloneCode() + + assert(code.contains("""filepath_or_buffer="country_sales_small_multi_line.csv"""")) + assert(!code.contains("base64.b64decode")) + assert(!code.contains("io.BytesIO")) + } + it should "use comma as the default delimiter when customDelimiter is not set for parallel CSV" in { parallelCsvScanSourceOpDesc.customDelimiter = None diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala index c5a8ffc1b7e..b992cd4dd86 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala @@ -144,4 +144,99 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { val out = physical.propagateSchema.func(Map.empty) assert(out(outPortId) == fileScanOpDesc.sourceSchema()) } + + "FileScanOpDesc.generateStandaloneCode" should + "read every line as text with the configured encoding by default" in { + assert( + fileScanOpDesc.generateStandaloneCode() == + """_rows = [] + |for _fn in in1df.iloc[:, 0]: + | with open(_fn, "r", encoding="utf-8") as _f: + | _rows.extend(l.rstrip("\n") for l in _f) + |out1df = pd.DataFrame({"line": _rows})""".stripMargin + ) + } + + // The enum name is "US_ASCII", not a Python codec name. + it should "render the encoding as a Python codec name" in { + fileScanOpDesc.fileEncoding = FileDecodingMethod.ASCII + assert(fileScanOpDesc.generateStandaloneCode().contains("""encoding="us-ascii"""")) + } + + it should "read the whole file in single-value mode, keeping the filename only when asked" in { + fileScanOpDesc.attributeType = FileAttributeType.SINGLE_STRING + + fileScanOpDesc.outputFileName = true + val withName = fileScanOpDesc.generateStandaloneCode() + assert(withName.contains(" _rows.append((_fn, _f.read()))")) + assert(withName.endsWith("""out1df = pd.DataFrame(_rows, columns=["filename", "line"])""")) + + fileScanOpDesc.outputFileName = false + val withoutName = fileScanOpDesc.generateStandaloneCode() + assert(withoutName.contains(" _rows.append(_f.read())")) + assert(withoutName.endsWith("""out1df = pd.DataFrame({"line": _rows})""")) + + // The platform's line-by-line branch (FileScanUtils.createTuplesFromFile) + // emits only the value, so line mode must drop the filename column too. + fileScanOpDesc.attributeType = FileAttributeType.STRING + fileScanOpDesc.outputFileName = true + assert(!fileScanOpDesc.generateStandaloneCode().contains("filename")) + } + + it should "open binary attribute types in binary mode" in { + Seq(FileAttributeType.BINARY, FileAttributeType.LARGE_BINARY).foreach { attrType => + fileScanOpDesc.attributeType = attrType + val code = fileScanOpDesc.generateStandaloneCode() + assert(code.contains(""" with open(_fn, "rb") as _f:""")) + assert(!code.contains("encoding=")) + assert(code.contains(" _rows.append(_f.read())")) + } + } + + it should "cast each line to the configured attribute type" in { + val castByType = Seq( + FileAttributeType.INTEGER -> "int(l.rstrip())", + FileAttributeType.LONG -> "int(l.rstrip())", + FileAttributeType.DOUBLE -> "float(l.rstrip())", + FileAttributeType.BOOLEAN -> """l.rstrip().lower() == "true"""", + FileAttributeType.TIMESTAMP -> "pd.Timestamp(l.rstrip())", + FileAttributeType.STRING -> """l.rstrip("\n")""" + ) + castByType.foreach { + case (attrType, cast) => + fileScanOpDesc.attributeType = attrType + assert( + fileScanOpDesc + .generateStandaloneCode() + .contains(s" _rows.extend($cast for l in _f)") + ) + } + } + + it should "materialize the lines and slice them when a limit or offset is set" in { + fileScanOpDesc.attributeType = FileAttributeType.INTEGER + + fileScanOpDesc.fileScanOffset = Option(3) + fileScanOpDesc.fileScanLimit = None + assert(fileScanOpDesc.generateStandaloneCode().contains(" _rows.extend(_lines[3:])")) + + fileScanOpDesc.fileScanOffset = None + fileScanOpDesc.fileScanLimit = Option(5) + assert(fileScanOpDesc.generateStandaloneCode().contains(" _rows.extend(_lines[0:5])")) + + fileScanOpDesc.fileScanOffset = Option(3) + fileScanOpDesc.fileScanLimit = Option(5) + val code = fileScanOpDesc.generateStandaloneCode() + assert(code.contains(" _lines = [int(l.rstrip()) for l in _f]")) + assert(code.contains(" _rows.extend(_lines[3:8])")) + } + + it should "warn that archive extraction is unsupported when extract is on" in { + // `extract` is a val, so it can only be set through deserialization. + val desc = objectMapper.readValue( + """{"operatorType":"FileScanOp","extract":true}""", + classOf[FileScanOpDesc] + ) + assert(desc.generateStandaloneCode().startsWith("# WARNING: extract=true is not supported")) + } } From bc8fc1791e490702df1c638ead0d8d565a55f026 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 11:32:28 -0700 Subject: [PATCH 05/33] ci: give the verify spec a job with an interpreter, and keep it out of the one without MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split this change relies on was declared but never wired. The specs carry `@IntegrationTest` and `build.sbt` reads `WCS_TEST_FILTER` to act on it, but nothing set that variable, so the filter was a no-op and the specs that fork Python ran in the job that provisions none — failing on `No module named 'pandas'` rather than on anything they were testing. The platform job now sets `skip-integration`, which excludes them. The platform-integration job sets `integration-only` and provisions what they need: Python 3.12, amber's requirements, protoc, and the generated proto bindings, which are gitignored and so have to be regenerated before a forked driver can import pyamber. Every step is guarded on the service, so the other entries in that matrix are untouched. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0107997b39..0614d54ae0a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -783,6 +783,13 @@ jobs: env: JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + # Exclude @IntegrationTest-tagged specs (workflow-compiling-service's + # OperatorBehaviorSpec forks Python, which this job does not provision). + # Those run in the platform-integration job's workflow-compiling-service + # matrix entry, which provisions Python. Read only by + # workflow-compiling-service/build.sbt; a no-op for the other services + # in this matrix. + WCS_TEST_FILTER: skip-integration services: # Each platform service transitively depends on DAO, which runs JOOQ # code generation at compile time and needs the live texera schema. @@ -972,6 +979,45 @@ jobs: - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d # v8.1.1 with: extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + # --- workflow-compiling-service only: provision Python so the verify + # spec (OperatorBehaviorSpec) can fork real interpreters. These four + # steps mirror the retired standalone workflow-compiling-service-integration + # job; they are no-ops for every other service in the matrix. + - name: Setup Python for Scala-Python verification tests + if: ${{ matrix.service == 'workflow-compiling-service' }} + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install Python dependencies + # OperatorBehaviorSpec forks Python subprocesses that import pandas / + # pyarrow / plotly and pytexera (the harness adds amber/src/main/python + # to PYTHONPATH). Install amber's runtime deps, same as amber-integration. + # dev-requirements.txt provides the betterproto plugin used by + # bin/python-proto-gen.sh in the proto-generation step below. + if: ${{ matrix.service == 'workflow-compiling-service' }} + run: | + python -m pip install uv + if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi + if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt; fi + - name: Install protoc + # Path A forks py_op_driver, which imports pyamber and hence the + # generated betterproto bindings in amber/src/main/python/proto (gitignored, + # not checked in). Pin protoc to bin/protoc-version.txt via the upstream + # release zip. Linux-only: this job runs on ubuntu-latest. + if: ${{ matrix.service == 'workflow-compiling-service' }} + run: | + PROTOC_VERSION=$(cat bin/protoc-version.txt) + curl -fsSL -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" + sudo unzip -o /tmp/protoc.zip -d /usr/local + sudo chmod +x /usr/local/bin/protoc + sudo chmod -R a+rX /usr/local/include/google + - name: Generate Python proto bindings + # Regenerate amber/src/main/python/proto so the forked py_op_driver can + # import pyamber; without this Path A fails with ImportError on proto + # symbols (e.g. ChannelIdentity). + if: ${{ matrix.service == 'workflow-compiling-service' }} + run: bash bin/python-proto-gen.sh - name: Create Databases run: | psql -h localhost -U postgres -f sql/texera_ddl.sql @@ -1053,6 +1099,25 @@ jobs: # smoke-boot's verdict is LISTEN-based, never log-scraping (#6332). TEXERA_SERVICE_LOG_LEVEL: ${{ runner.debug == '1' && 'DEBUG' || 'WARN' }} run: .github/scripts/smoke-boot.sh "/tmp/dists/${{ matrix.service }}-*/bin/${{ matrix.service }}" "${{ matrix.port }}" + - name: Run workflow-compiling-service Python e2e (integration) tests + # workflow-compiling-service only: the @IntegrationTest-tagged verify + # spec (OperatorBehaviorSpec) forks real Python processes and compares + # translator-generated standalone code against platform output, operator + # by operator. Reuses this job's compiled WCS (the dist step above) and + # postgres instead of a separate integration job. + # WCS_TEST_FILTER=integration-only keeps only @IntegrationTest specs and + # bounds ScalaTest's parallel pool to 4 threads (build.sbt). Every operator + # carrying standalone code runs: the spec's narrowing knobs + # (VERIFY_ONLY / VERIFY_SKIP) are unset here, as they are in any run that + # does not ask for less, and what stays withheld is a single variant or an + # operator that cannot run, recorded against an issue in the runner itself. + # UDF_PYTHON_PATH points the harness at the provisioned interpreter (bare + # name resolves via PATH); it auto-locates amber/src/main/python. + if: ${{ matrix.service == 'workflow-compiling-service' }} + env: + WCS_TEST_FILTER: integration-only + UDF_PYTHON_PATH: python + run: sbt "WorkflowCompilingService/test" pyamber: if: ${{ inputs.run_pyamber }} From 883ea72501d925b2314a91f516e021003f929949 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 14:00:22 -0700 Subject: [PATCH 06/33] test(verify): pin the no-generator case to an operator that can never have one The example was an operator another batch gives a generator to, so the assertion held only until that batch landed. A Python UDF holds whatever order these land in: its body is written by whoever drops the operator, so there is nothing for a generator to emit. The word cloud assertion goes for the same reason. It says the operator is withheld, which a later batch stops being true once its placement is seeded, and the prediction op alone already covers what the test is for. Co-Authored-By: Claude Opus 5 (1M context) --- .../TransformVerificationRunnerSpec.scala | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala index a5199b88c81..39aca487b69 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala @@ -29,9 +29,8 @@ package org.apache.texera.amber.translator.verify // OperatorBehaviorSpec does not check, so it lives here. import org.apache.texera.amber.operator.limit.LimitOpDesc -import org.apache.texera.amber.operator.sortPartitions.SortPartitionsOpDesc +import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 import org.apache.texera.amber.operator.union.UnionOpDesc -import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -46,10 +45,6 @@ class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { case Flagged(reason) => reason should include("trained-model") case other => fail(s"expected Flagged, got $other") } - disposition(classOf[WordCloudOpDesc]) match { - case Flagged(reason) => reason should include("known issue") - case other => fail(s"expected Flagged, got $other") - } } it should "run the union now that its code names every upstream" in { @@ -64,12 +59,11 @@ class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { disposition(classOf[LimitOpDesc]) shouldBe Runnable("auto") } - // The operator set implements the generator a family at a time, so most of it - // does not yet. That is reported rather than passed over: an operator missing - // from the run is a fact the report has to carry, and the rows for each family - // arrive with the change that gives that family its generator. - it should "flag an operator that has no standalone generator yet" in { - disposition(classOf[SortPartitionsOpDesc]) shouldBe + // A UDF's body is written by whoever drops the operator, so there is nothing + // for a generator to emit. It stands here for the shape of the report: an + // operator that cannot be exported is carried as a row, not passed over. + it should "flag an operator that has no standalone generator" in { + disposition(classOf[PythonUDFOpDescV2]) shouldBe Flagged("does not implement StandaloneCodeGenerator") } } From 5a44c3cccb9204c24add0d7e91988489a6d9cb89 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 16:10:52 -0700 Subject: [PATCH 07/33] chore: leave the harness to the change that introduces it Everything here that is not a source operator belongs to #8327 and was carried only so this branch could compile and run its own tests before that one landed. Reviewing it twice costs more than the red build does: what is left is the thirteen files this change is actually about. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 65 -- build.sbt | 1 - .../pybuilder/PythonTemplateBuilder.scala | 23 - .../PythonTemplateBuilderApiSpec.scala | 20 - .../texera/amber/operator/LogicalOp.scala | 10 - .../operator/StandaloneCodeGenerator.scala | 59 -- .../operator/distinct/DistinctOpDesc.scala | 9 +- .../filter/SpecializedFilterOpDesc.scala | 40 +- .../amber/operator/limit/LimitOpDesc.scala | 8 +- .../metadata/annotations/SampleColumn.java | 45 - .../projection/ProjectionOpDesc.scala | 35 +- .../amber/operator/union/UnionOpDesc.scala | 11 +- .../distinct/DistinctOpDescSpec.scala | 8 - .../filter/SpecializedFilterOpDescSpec.scala | 11 - .../operator/limit/LimitOpDescSpec.scala | 9 - .../projection/ProjectionOpDescSpec.scala | 13 - .../operator/union/UnionOpDescSpec.scala | 13 - workflow-compiling-service/build.sbt | 27 +- .../WorkflowToPythonTranslator.scala | 198 ---- .../service/WorkflowCompilingService.scala | 9 +- .../resource/WorkflowToPythonResource.scala | 70 -- .../verify/tags/IntegrationTest.java | 56 - .../src/test/resources/python/compare.py | 444 -------- .../src/test/resources/python/py_op_driver.py | 531 ---------- .../resources/python/standalone_worker.py | 122 --- .../resources/verify/canonical_fixture.json | 542 ---------- .../WorkflowToPythonTranslatorSpec.scala | 106 -- .../translator/verify/CanonicalFixture.scala | 249 ----- .../verify/CanonicalFixtureSpec.scala | 302 ------ .../amber/translator/verify/Comparator.scala | 189 ---- .../translator/verify/ComparatorSpec.scala | 97 -- .../verify/ConfigCoverageSpec.scala | 122 --- .../verify/ConfigGeneratorSpec.scala | 98 -- .../translator/verify/CuratedHandlers.scala | 666 ------------ .../amber/translator/verify/HarnessSpec.scala | 115 --- .../translator/verify/OpExecHarness.scala | 454 --------- .../verify/OperatorBehaviorSpec.scala | 151 --- .../translator/verify/PyOpExecHarness.scala | 406 -------- .../translator/verify/SharedFixture.scala | 178 ---- .../verify/SourceCategoryRunner.scala | 471 --------- .../verify/StandaloneEscapingCheck.scala | 156 --- .../translator/verify/StandaloneRunner.scala | 367 ------- .../verify/TransformVerificationRunner.scala | 962 ------------------ .../verify/VisualizationHtmlComparator.scala | 85 -- .../verify/VisualizationJsonComparator.scala | 123 --- 45 files changed, 10 insertions(+), 7666 deletions(-) delete mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala delete mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java delete mode 100644 workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala delete mode 100644 workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala delete mode 100644 workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java delete mode 100644 workflow-compiling-service/src/test/resources/python/compare.py delete mode 100644 workflow-compiling-service/src/test/resources/python/py_op_driver.py delete mode 100644 workflow-compiling-service/src/test/resources/python/standalone_worker.py delete mode 100644 workflow-compiling-service/src/test/resources/verify/canonical_fixture.json delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0614d54ae0a..c0107997b39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -783,13 +783,6 @@ jobs: env: JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 - # Exclude @IntegrationTest-tagged specs (workflow-compiling-service's - # OperatorBehaviorSpec forks Python, which this job does not provision). - # Those run in the platform-integration job's workflow-compiling-service - # matrix entry, which provisions Python. Read only by - # workflow-compiling-service/build.sbt; a no-op for the other services - # in this matrix. - WCS_TEST_FILTER: skip-integration services: # Each platform service transitively depends on DAO, which runs JOOQ # code generation at compile time and needs the live texera schema. @@ -979,45 +972,6 @@ jobs: - uses: coursier/cache-action@95e5b1029b6b86e7bac033ee44a0697d8a527d2d # v8.1.1 with: extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' - # --- workflow-compiling-service only: provision Python so the verify - # spec (OperatorBehaviorSpec) can fork real interpreters. These four - # steps mirror the retired standalone workflow-compiling-service-integration - # job; they are no-ops for every other service in the matrix. - - name: Setup Python for Scala-Python verification tests - if: ${{ matrix.service == 'workflow-compiling-service' }} - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - name: Install Python dependencies - # OperatorBehaviorSpec forks Python subprocesses that import pandas / - # pyarrow / plotly and pytexera (the harness adds amber/src/main/python - # to PYTHONPATH). Install amber's runtime deps, same as amber-integration. - # dev-requirements.txt provides the betterproto plugin used by - # bin/python-proto-gen.sh in the proto-generation step below. - if: ${{ matrix.service == 'workflow-compiling-service' }} - run: | - python -m pip install uv - if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi - if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi - if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt; fi - - name: Install protoc - # Path A forks py_op_driver, which imports pyamber and hence the - # generated betterproto bindings in amber/src/main/python/proto (gitignored, - # not checked in). Pin protoc to bin/protoc-version.txt via the upstream - # release zip. Linux-only: this job runs on ubuntu-latest. - if: ${{ matrix.service == 'workflow-compiling-service' }} - run: | - PROTOC_VERSION=$(cat bin/protoc-version.txt) - curl -fsSL -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" - sudo unzip -o /tmp/protoc.zip -d /usr/local - sudo chmod +x /usr/local/bin/protoc - sudo chmod -R a+rX /usr/local/include/google - - name: Generate Python proto bindings - # Regenerate amber/src/main/python/proto so the forked py_op_driver can - # import pyamber; without this Path A fails with ImportError on proto - # symbols (e.g. ChannelIdentity). - if: ${{ matrix.service == 'workflow-compiling-service' }} - run: bash bin/python-proto-gen.sh - name: Create Databases run: | psql -h localhost -U postgres -f sql/texera_ddl.sql @@ -1099,25 +1053,6 @@ jobs: # smoke-boot's verdict is LISTEN-based, never log-scraping (#6332). TEXERA_SERVICE_LOG_LEVEL: ${{ runner.debug == '1' && 'DEBUG' || 'WARN' }} run: .github/scripts/smoke-boot.sh "/tmp/dists/${{ matrix.service }}-*/bin/${{ matrix.service }}" "${{ matrix.port }}" - - name: Run workflow-compiling-service Python e2e (integration) tests - # workflow-compiling-service only: the @IntegrationTest-tagged verify - # spec (OperatorBehaviorSpec) forks real Python processes and compares - # translator-generated standalone code against platform output, operator - # by operator. Reuses this job's compiled WCS (the dist step above) and - # postgres instead of a separate integration job. - # WCS_TEST_FILTER=integration-only keeps only @IntegrationTest specs and - # bounds ScalaTest's parallel pool to 4 threads (build.sbt). Every operator - # carrying standalone code runs: the spec's narrowing knobs - # (VERIFY_ONLY / VERIFY_SKIP) are unset here, as they are in any run that - # does not ask for less, and what stays withheld is a single variant or an - # operator that cannot run, recorded against an issue in the runner itself. - # UDF_PYTHON_PATH points the harness at the provisioned interpreter (bare - # name resolves via PATH); it auto-locates amber/src/main/python. - if: ${{ matrix.service == 'workflow-compiling-service' }} - env: - WCS_TEST_FILTER: integration-only - UDF_PYTHON_PATH: python - run: sbt "WorkflowCompilingService/test" pyamber: if: ${{ inputs.run_pyamber }} diff --git a/build.sbt b/build.sbt index 94a5af5606c..035edbb68cc 100644 --- a/build.sbt +++ b/build.sbt @@ -239,7 +239,6 @@ lazy val WorkflowCompiler = (project in file("common/workflow-compiler")) .dependsOn(WorkflowOperator) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) .dependsOn(WorkflowCompiler, Auth, Config, Resource) - .dependsOn(WorkflowOperator % "test->test") // reuse PythonWorkerPool in verify tests .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( diff --git a/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala b/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala index 90b4162041b..73f4a3846dd 100644 --- a/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala +++ b/common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala @@ -209,29 +209,6 @@ object PythonTemplateBuilder { def wrapWithPythonDecoderExpr(text: String): String = s"self.decode_python_template('$text')" - /** - * Render `text` as a Python double-quoted string literal, quotes included. - * - * For generators that emit standalone Python source rather than an operator - * for the runtime: they cannot use the decode expression (it needs the - * operator's `decode_python_template`, and it is deliberately rejected inside - * quotes), so they need the value as a *literal*. Writing `"$value"` by hand - * instead lets any quote, backslash or newline in the value close the literal - * early and change — or break — the emitted program. - * - * Escapes exactly what can end a double-quoted single-line literal. - */ - def pyStringLiteral(text: String): String = { - val escaped = Option(text) - .getOrElse("") - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\r", "\\r") - .replace("\n", "\\n") - .replace("\t", "\\t") - "\"" + escaped + "\"" - } - sealed trait RenderMode extends Product with Serializable object RenderMode { case object Plain extends RenderMode diff --git a/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala b/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala index 149a1538574..acbed3031fc 100644 --- a/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala +++ b/common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderApiSpec.scala @@ -244,24 +244,4 @@ class PythonTemplateBuilderApiSpec extends AnyFunSuite { test("hasUnclosedQuote: three opening single quotes count as unclosed") { assert(PythonLexerUtils.hasUnclosedQuote("'''abc")) } - - // -------- pyStringLiteral -------- - - // Every character that can end a double-quoted single-line literal, since one that - // slips through does not fail here but changes the emitted program. - test("pyStringLiteral: quotes the value and escapes what would close the literal") { - assert(PythonTemplateBuilder.pyStringLiteral("plain") == "\"plain\"") - assert(PythonTemplateBuilder.pyStringLiteral("say \"hi\"") == "\"say \\\"hi\\\"\"") - assert(PythonTemplateBuilder.pyStringLiteral("a\\b") == "\"a\\\\b\"") - assert(PythonTemplateBuilder.pyStringLiteral("one\ntwo") == "\"one\\ntwo\"") - assert(PythonTemplateBuilder.pyStringLiteral("a\tb") == "\"a\\tb\"") - assert(PythonTemplateBuilder.pyStringLiteral("a\rb") == "\"a\\rb\"") - } - - // A column name arrives from JSON and can be absent; an empty literal is a value the - // emitted program can carry, where `null` would reach it as the four letters. - test("pyStringLiteral: renders a null as the empty literal") { - assert(PythonTemplateBuilder.pyStringLiteral(null) == "\"\"") - } - } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index d28c806ce56..efa46144180 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -459,16 +459,6 @@ abstract class LogicalOp extends PortDescriptor with Serializable { def operatorInfo: OperatorInfo - /** - * Whether the row ORDER of this operator's output is part of its contract. - * Defaults to false: the engine runs operators across parallel workers, so - * for almost every operator the output row order is an implementation-defined - * interleaving that consumers must not rely on. Only operators whose very - * purpose is to establish an order (the sort family) override this to true. - * Consumers that must not rely on a stable row order read this flag. - */ - def orderSensitive: Boolean = false - private def getOperatorVersion: String = { val path = "amber/src/main/scala/" val operatorPath = path + this.getClass.getPackage.getName.replace(".", "/") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala deleted file mode 100644 index 8ca23e55cb5..00000000000 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator - -import java.net.URLDecoder -import java.nio.charset.StandardCharsets - -trait StandaloneCodeGenerator { - - def generateStandaloneCode(): String - - /** - * The file's own name, for a script that reads it from its own directory - * rather than through Texera's resolved URI. - * - * Taken from the last path segment instead of by parsing the whole string as a - * URI: the resolver percent-encodes the file-relative segments but leaves the - * repository and version names as the user typed them, so a dataset version - * called `v3 - with long text` makes `new URI` throw on the space and no code - * is generated at all. - */ - protected def sourceBasename(rawPath: String): String = { - val segment = rawPath.split("/").lastOption.getOrElse("") - // Percent-decoding only, matching what `URI.getPath` used to return here: form - // decoding would also turn a literal `+` in a file name into a space. - URLDecoder.decode(segment.replace("+", "%2B"), StandardCharsets.UTF_8) - } - - def producesDataFrame(): Boolean = true - - /** - * Definitions this operator's standalone code depends on, emitted once near - * the top of the script rather than inline. - * - * The translator concatenates operator bodies into a single module, so an - * operator needing a helper class has nowhere to put it that another operator - * would not duplicate. Helpers returned here are collected across the whole - * plan and deduplicated by their text, so two sampling operators in one - * workflow yield one copy of the generator they share. - */ - def standaloneHelpers(): Seq[String] = Seq.empty -} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala index 17b646bfa10..9e75e648bb4 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/distinct/DistinctOpDesc.scala @@ -22,10 +22,10 @@ package org.apache.texera.amber.operator.distinct import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -class DistinctOpDesc extends LogicalOp with StandaloneCodeGenerator { +class DistinctOpDesc extends LogicalOp { override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -54,9 +54,4 @@ class DistinctOpDesc extends LogicalOp with StandaloneCodeGenerator { outputPorts = List(OutputPort(blocking = true)) ) - override def generateStandaloneCode(): String = { - // JVM op uses LinkedHashSet to preserve first-occurrence order; - // pandas drop_duplicates does the same by default. - "out1df = in1df.drop_duplicates(ignore_index=True)" - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala index cc23a553991..9e86773df7c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala @@ -23,12 +23,10 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} -import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator { +class SpecializedFilterOpDesc extends FilterOpDesc { @JsonProperty(value = "predicates", required = true) @JsonPropertyDescription("multiple predicates in OR") @@ -62,40 +60,4 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator supportReconfiguration = true ) } - - override def generateStandaloneCode(): String = { - if (predicates.isEmpty) return "out1df = in1df.copy()" - val conditions = predicates.map { p => - val colLit = pyStringLiteral(p.attribute) - p.condition match { - case ComparisonType.IS_NULL => s"""(in1df[$colLit].isna())""" - case ComparisonType.IS_NOT_NULL => s"""(in1df[$colLit].notna())""" - case other => - val op = other.getName // returns "=", ">=", "<", etc. (see ComparisonType.java) - val pyOp = if (op == "=") "==" else op - // notna mirrors FilterPredicate, which answers false for every condition - // but IS_NULL / IS_NOT_NULL once the field is null. Only `!=` needs it — - // pandas answers True there, where every other operator answers False — - // but guarding all of them keeps the one rule visible in one place. - s"""(in1df[$colLit].notna() & (in1df[$colLit] $pyOp ${coerceValue(p.value)}))""" - } - } - s"out1df = in1df[${conditions.mkString(" | ")}].reset_index(drop=True)" - } - - // Try numeric coercion so generated code compares column values against the right type. - // Strings that don't parse fall through to a quoted string literal. - private def coerceValue(raw: String): String = { - try { - raw.toInt.toString - } catch { - case _: NumberFormatException => - try { - raw.toDouble.toString - } catch { - case _: NumberFormatException => - pyStringLiteral(raw) - } - } - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala index bbfb7015036..6e1b7f37af3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/limit/LimitOpDesc.scala @@ -25,12 +25,12 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, StateTransferFunc} +import org.apache.texera.amber.operator.{LogicalOp, StateTransferFunc} import org.apache.texera.amber.util.JSONUtils.objectMapper import scala.util.{Success, Try} -class LimitOpDesc extends LogicalOp with StandaloneCodeGenerator { +class LimitOpDesc extends LogicalOp { @JsonProperty(required = true) @JsonSchemaTitle("Limit") @@ -80,8 +80,4 @@ class LimitOpDesc extends LogicalOp with StandaloneCodeGenerator { } Success(newPhysicalOp, Some(stateTransferFunc)) } - - override def generateStandaloneCode(): String = { - s"out1df = in1df.head($limit).reset_index(drop=True)" - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java deleted file mode 100644 index cb35c9aab91..00000000000 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/SampleColumn.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator.metadata.annotations; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Test-only metadata for transform verification: names the column in the shared - * verification fixture the operator runs on that should fill this - * {@code @AutofillAttributeName} field when the operator is auto-configured. - * - *

It lets a field declare a semantic sample that the column's - * {@code AttributeType} alone cannot express — e.g. a valid three-letter ISO - * country code, or a genuine OHLC price column — so the parity test exercises - * the operator on realistic input instead of a degenerate first-column pick - * (which can hide translation bugs and produce vacuous passes). - * - *

This has no effect on production: it is not a Jackson / JSON-schema - * annotation and is read only by the test-side ConfigGenerator. - */ -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD}) -public @interface SampleColumn { - String value(); -} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala index c98c61d566f..fb9258410cb 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/projection/ProjectionOpDesc.scala @@ -26,23 +26,17 @@ import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PhysicalOp.oneToOnePhysicalOp import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.map.MapOpDesc import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -class ProjectionOpDesc extends MapOpDesc with StandaloneCodeGenerator { +class ProjectionOpDesc extends MapOpDesc { @JsonProperty(required = true, defaultValue = "false") @JsonSchemaTitle("Drop Option") @JsonPropertyDescription("check to drop the selected attributes") var isDrop: Boolean = false - // Named explicitly, without `required`: the form already asks for these and must go - // on accepting an empty list, but a field carrying no annotation is invisible to - // anything reading the operator's config by reflection. - @JsonProperty var attributes: List[AttributeUnit] = List() override def getPhysicalOp( @@ -104,31 +98,4 @@ class ProjectionOpDesc extends MapOpDesc with StandaloneCodeGenerator { outputPorts = List(OutputPort()) ) } - - override def generateStandaloneCode(): String = { - val units = Option(attributes).getOrElse(List.empty) - // JVM validates non-empty at runtime via Preconditions; emit passthrough - // as best-effort so the standalone script still runs. - if (units.isEmpty) return "out1df = in1df.copy()" - - if (isDrop) { - // Drop mode ignores aliases (matches ProjectionOpExec). - val cols = units.map(u => pyStringLiteral(u.getOriginalAttribute)).mkString("[", ", ", "]") - s"out1df = in1df.drop(columns=$cols)" - } else { - val originals = - units.map(u => pyStringLiteral(u.getOriginalAttribute)).mkString("[", ", ", "]") - // AttributeUnit.getAlias returns originalAttribute when alias is blank, - // so an explicit rename is only needed when they differ. - val renames = units - .filter(u => u.getAlias != u.getOriginalAttribute) - .map(u => s"""${pyStringLiteral(u.getOriginalAttribute)}: ${pyStringLiteral(u.getAlias)}""") - if (renames.isEmpty) { - s"out1df = in1df[$originals].copy()" - } else { - val renameMap = renames.mkString("{", ", ", "}") - s"out1df = in1df[$originals].rename(columns=$renameMap)" - } - } - } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala index 460aa7990b1..82e292c8f38 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/union/UnionOpDesc.scala @@ -22,10 +22,10 @@ package org.apache.texera.amber.operator.union import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} +import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -class UnionOpDesc extends LogicalOp with StandaloneCodeGenerator { +class UnionOpDesc extends LogicalOp { override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -50,11 +50,4 @@ class UnionOpDesc extends LogicalOp with StandaloneCodeGenerator { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) - - // UNION ALL: UnionOpExec passes tuples through without dedup. The port is - // variadic, so the code names the whole list of upstreams rather than a fixed - // two — naming two dropped a third and left the second unbound when only one - // was drawn. - override def generateStandaloneCode(): String = - "out1df = pd.concat(inAlldf, ignore_index=True)" } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala index 15a9cf5d64e..2aba788acfe 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala @@ -106,12 +106,4 @@ class DistinctOpDescSpec extends AnyFlatSpec with Matchers { val b = new DistinctOpDesc a.operatorIdentifier should not equal b.operatorIdentifier } - - // The JVM operator keeps the first occurrence of a duplicate, which is what - // drop_duplicates does by default, so the emitted line says nothing about order. - "DistinctOpDesc.generateStandaloneCode" should "drop duplicates in place" in { - (new DistinctOpDesc).generateStandaloneCode() shouldBe - "out1df = in1df.drop_duplicates(ignore_index=True)" - } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala index 16b7e424ea7..84c7ec93779 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala @@ -70,15 +70,4 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { restored shouldBe a[SpecializedFilterOpDesc] restored.asInstanceOf[SpecializedFilterOpDesc].predicates shouldBe empty } - - // A null answers false for every condition but IS_NULL / IS_NOT_NULL, which pandas - // does not do on its own for `!=`, so the emitted condition carries the guard. - "SpecializedFilterOpDesc.generateStandaloneCode" should "emit one condition per predicate" in { - val d = new SpecializedFilterOpDesc - d.predicates = List(new FilterPredicate("age", ComparisonType.GREATER_THAN, "18")) - val code = d.generateStandaloneCode() - code should include("in1df[\"age\"]") - code should include("out1df") - } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala index d7314668175..f01e8f63f08 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/limit/LimitOpDescSpec.scala @@ -94,13 +94,4 @@ class LimitOpDescSpec extends AnyFlatSpec with Matchers { transfer(oldExec, newExec) newExec.count shouldBe 3 } - - // The index is reset because the operator hands its downstream a fresh table - // rather than a view of the one it read. - "LimitOpDesc.generateStandaloneCode" should "take the first N rows" in { - val d = new LimitOpDesc - d.limit = 3 - d.generateStandaloneCode() shouldBe "out1df = in1df.head(3).reset_index(drop=True)" - } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala index ee4468665d1..66e6f556afb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala @@ -216,17 +216,4 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(out == SinglePartition()) } - // Drop mode names the columns to remove; keep mode names the ones to hold on to, - // in the order the user put them in. - "ProjectionOpDesc.generateStandaloneCode" should "select or drop the named columns" in { - val keep = new ProjectionOpDesc - keep.attributes = List(new AttributeUnit("a", ""), new AttributeUnit("b", "")) - assert(keep.generateStandaloneCode().contains("""in1df[["a", "b"]]""")) - - val drop = new ProjectionOpDesc - drop.attributes = List(new AttributeUnit("a", "")) - drop.isDrop = true - assert(drop.generateStandaloneCode() == """out1df = in1df.drop(columns=["a"])""") - } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala index 16c7e027bc5..a9c58bbcda4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/union/UnionOpDescSpec.scala @@ -87,19 +87,6 @@ class UnionOpDescSpec extends AnyFlatSpec with Matchers { physical.partitionRequirement shouldBe empty } - // --------------------------------------------------------------------------- - // generateStandaloneCode - // --------------------------------------------------------------------------- - - // UNION ALL: UnionOpExec passes tuples through without dedup, so the - // generated concat must not drop duplicates either. It names the whole list - // of upstreams rather than a fixed two, because the port is variadic and any - // count the code stated would be wrong for some workflow. - "UnionOpDesc.generateStandaloneCode" should "concatenate every input without dedup" in { - (new UnionOpDesc).generateStandaloneCode() shouldBe - "out1df = pd.concat(inAlldf, ignore_index=True)" - } - // --------------------------------------------------------------------------- // Independent instances // --------------------------------------------------------------------------- diff --git a/workflow-compiling-service/build.sbt b/workflow-compiling-service/build.sbt index 94d5dedb389..2af92efc4d4 100644 --- a/workflow-compiling-service/build.sbt +++ b/workflow-compiling-service/build.sbt @@ -41,34 +41,9 @@ ThisBuild / semanticdbVersion := scalafixSemanticdb.revision // Manage dependency conflicts by always using the latest revision ThisBuild / conflictManager := ConflictManager.latestRevision -// Restrict parallel execution of tests to avoid conflicts. This caps how many -// test *suites* run concurrently; ParallelTestExecution still parallelizes the -// tests *within* a suite (e.g. OperatorBehaviorSpec) via ScalaTest's own pool. +// Restrict parallel execution of tests to avoid conflicts Global / concurrentRestrictions += Tags.limit(Tags.Test, 1) -// The fast-unit / integration test split; the selection logic itself is shared -// in project/TestFilters.scala. -Test / testOptions ++= TestFilters.integrationSplit( - envVar = "WCS_TEST_FILTER", - tag = "org.apache.texera.amber.translator.verify.tags.IntegrationTest" -) - -// -P4 bounds ScalaTest's ParallelTestExecution pool, and only this module wants -// it: OperatorBehaviorSpec forks a Python subprocess per operator, and at -// core-count concurrency (e.g. 12) resource contention caused rare flakes. A -// fixed 4 stays deterministic across machines (incl. CI runners) while still -// running ~3x faster than serial, and it matches PythonWorkerPool's own default -// worker cap so the two bounds agree rather than multiply. Unconditional, so a -// local run reproduces the concurrency CI runs at instead of a faster one that -// flakes differently; WCS_TEST_FILTER selects which tests run, which is a -// separate question from how many run at once. The fast-unit job is unaffected -// either way, since OperatorBehaviorSpec is the only spec here that -// parallelizes and that job excludes it. It lives here rather than in the -// shared helper so that helper stays identical for every module. sbt -// concatenates the ScalaTest arguments of every testOptions entry, so this -// lands in the same argument list as the -n above. -Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-P4") - ///////////////////////////////////////////////////////////////////////////// // Compiler Options ///////////////////////////////////////////////////////////////////////////// diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala deleted file mode 100644 index 85ce6f98039..00000000000 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator - -import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.core.virtualidentity.OperatorIdentity -import org.apache.texera.common.compiler.model.LogicalPlan -import org.apache.texera.amber.operator.StandaloneCodeGenerator - -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer -import scala.jdk.CollectionConverters._ - -class WorkflowToPythonTranslator extends LazyLogging { - - // Output-port-level key. An operator with N output ports gets N entries - // (e.g. Split has port 0 and port 1, each with its own assigned dfN var). - private type PortKey = (String, Int) // (opId, portIdx) - - def translate(logicalPlan: LogicalPlan): String = { - // Track downstream connections per (opId, fromPortIdx). A port is a leaf - // if it has no outgoing edges — operator-level "no outgoing links" is too - // coarse for multi-output ops (Split's port 0 may have downstream while - // port 1 doesn't, or vice versa). - val outgoingFromPort = mutable.Map[PortKey, Int]().withDefaultValue(0) - logicalPlan.links.foreach { link => - outgoingFromPort((link.fromOpId.id, link.fromPortId.id)) += 1 - } - - val outputVar = mutable.Map[PortKey, String]() - var varCounter = 1 - val script = ArrayBuffer[String]() - - script += "import pandas as pd" - script += "import plotly.express as px" - script += "import plotly.graph_objects as go" - script += "import plotly.io" - script += "" - - // getTopologicalOpIds() uses jgrapht internally — no need for a custom topo sort - val topoOrder = logicalPlan.getTopologicalOpIds.asScala.toList - - // Helper definitions the operator bodies below refer to. Collected across the - // whole plan and deduplicated by text, so a workflow holding two operators - // that share one helper still emits it once. Order follows the topological - // order, which keeps the script stable for a given plan. - val helpers = topoOrder - .map(logicalPlan.getOperator) - .collect { case gen: StandaloneCodeGenerator => gen.standaloneHelpers() } - .flatten - .distinct - if (helpers.nonEmpty) { - helpers.foreach { helper => script += helper; script += "" } - } - - for (opIdentity <- topoOrder) { - val opId = opIdentity.id - val op = logicalPlan.getOperator(opIdentity) - val displayName = op.operatorInfo.userFriendlyName - - // Resolve upstream inputs in the consuming operator's input-port order - // (link.toPortId), NOT the order links happen to appear in the plan's - // link list. This makes in1df/in2df/... deterministic and correct for - // multi-input operators (joins, set ops) where port 0 vs port 1 carries - // semantics (e.g. build vs probe side). Ties on the same toPortId keep - // link order — relevant for variadic single-port operators like Union. - // Each upstream link is resolved via (fromOpId, fromPortId) so that a - // multi-output upstream (Split) hands each downstream the correct DF. - val inVars = logicalPlan - .getUpstreamLinks(opIdentity) - .sortBy(link => (link.toPortId.id, link.toPortId.internal)) - .map(link => outputVar((link.fromOpId.id, link.fromPortId.id))) - - // Allocate one dfN per declared output port. Existing single-output - // operators have outputPorts.size == 1, so they get exactly one var and - // their behavior is identical to the previous flat scheme. - val outVars = op.operatorInfo.outputPorts.map { port => - val v = s"df$varCounter" - varCounter += 1 - outputVar((opId, port.id.id)) = v - v - } - - script += s"# [$displayName]" - - // Jackson deserializes each operator into its concrete subclass via @JsonSubTypes on LogicalOp, - // so the pattern match below will resolve to the correct descriptor (e.g. BarChartOpDesc). - op match { - case gen: StandaloneCodeGenerator => - // generateStandaloneCode() returns a code block using in{N}df / out{N}df - // placeholders; substituteVars() replaces them with the assigned vars. - script += substituteVars(gen.generateStandaloneCode(), inVars, outVars, displayName) - - case _ => - logger.warn( - s"Operator '$displayName' does not implement StandaloneCodeGenerator. Skipping." - ) - script += s"# TODO: '$displayName' is not yet supported by the translator." - outVars.zipWithIndex.foreach { - case (v, i) => script += s"# $v = " - } - } - - script += "" - } - - // Leaf detection runs at the port level: a (opId, port) pair is a leaf - // if no link consumes it. For Split with one downstream port and one - // dangling port, only the dangling port is treated as a leaf to print. - val leafPorts = outputVar.keys.toList - .sortBy { case (_, portIdx) => portIdx } - .filter(key => outgoingFromPort(key) == 0) - val dataFrameLeafPorts = leafPorts.filter { - case (opId, _) => - logicalPlan.getOperator(OperatorIdentity(opId)) match { - case gen: StandaloneCodeGenerator => gen.producesDataFrame() - case _ => false - } - } - - if (dataFrameLeafPorts.nonEmpty) { - script += "# --- Output ---" - // Print in topological order of the producing operator so multi-port - // operators print contiguously and the order matches the script flow. - val topoIndex = topoOrder.map(_.id).zipWithIndex.toMap - dataFrameLeafPorts - .sortBy { case (opId, portIdx) => (topoIndex.getOrElse(opId, Int.MaxValue), portIdx) } - .foreach { - case (opId, portIdx) => - val varName = outputVar((opId, portIdx)) - val displayName = - logicalPlan.getOperator(OperatorIdentity(opId)).operatorInfo.userFriendlyName - val portSuffix = if (outputVar.keys.count(_._1 == opId) > 1) s" port $portIdx" else "" - script += s"""print("\\n[$displayName$portSuffix] $varName:")""" - script += s"print($varName.head())" - script += "" - } - } - - script.mkString("\n") - } - - // Replaces in{N}df / out{N}df placeholders with concrete variable names. - // Substitutes in reverse index order to prevent partial matches (e.g. in1df - // inside in10df). After substitution, scans for any leftover placeholders - // and logs a warning — that signals a mismatch between an operator's - // declared port count and what its generateStandaloneCode actually emits. - private def substituteVars( - code: String, - inVars: List[String], - outVars: List[String], - displayName: String - ): String = { - var result = code - - // A variadic port takes as many upstream links as the user draws, and an - // operator reading one cannot name them: `in1df`/`in2df` state a count, and - // whichever count it states is wrong for every other workflow. This one - // placeholder becomes the whole list, so the operator writes the same line - // whether it is fed one table or five. - result = result.replaceAll("""\binAlldf\b""", inVars.mkString("[", ", ", "]")) - inVars.zipWithIndex.reverse.foreach { - case (v, idx) => result = result.replaceAll(s"\\bin${idx + 1}df\\b", v) - } - outVars.zipWithIndex.reverse.foreach { - case (v, idx) => result = result.replaceAll(s"\\bout${idx + 1}df\\b", v) - } - - val leftoverIn = """\bin\d+df\b""".r.findAllIn(result).toSet - val leftoverOut = """\bout\d+df\b""".r.findAllIn(result).toSet - if (leftoverIn.nonEmpty || leftoverOut.nonEmpty) { - logger.warn( - s"Operator '$displayName' emitted placeholders that don't match its port " + - s"count: leftover inputs=$leftoverIn, leftover outputs=$leftoverOut. " + - s"Generated script will reference unbound variables." - ) - } - - result - } -} diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala index 46649647a0a..a69ef545246 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/WorkflowCompilingService.scala @@ -27,11 +27,7 @@ import org.apache.texera.common.config.StorageConfig import org.apache.texera.amber.util.ObjectMapperUtils import org.apache.texera.auth.{AuthFeatures, RoleAnnotationEnforcer} import org.apache.texera.dao.SqlServer -import org.apache.texera.service.resource.{ - HealthCheckResource, - WorkflowCompilationResource, - WorkflowToPythonResource -} +import org.apache.texera.service.resource.{HealthCheckResource, WorkflowCompilationResource} import org.eclipse.jetty.servlet.FilterHolder import java.nio.file.Path @@ -71,9 +67,6 @@ class WorkflowCompilingService extends Application[WorkflowCompilingServiceConfi // register the compilation endpoint environment.jersey.register(classOf[WorkflowCompilationResource]) - // register the workflow-to-python endpoint - environment.jersey.register(classOf[WorkflowToPythonResource]) - RoleAnnotationEnforcer.enforce( environment.jersey.getResourceConfig, "WorkflowCompilingService" diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala deleted file mode 100644 index 18b75ef338f..00000000000 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.service.resource - -import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} -import com.typesafe.scalalogging.LazyLogging -import jakarta.annotation.security.RolesAllowed -import jakarta.ws.rs.core.MediaType -import jakarta.ws.rs.{Consumes, POST, Path, Produces} -import org.apache.texera.common.compiler.model.{LogicalPlan, LogicalPlanPojo} -import org.apache.texera.amber.translator.WorkflowToPythonTranslator - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "type" -) -@JsonSubTypes( - Array( - new JsonSubTypes.Type(value = classOf[WorkflowToPythonSuccess], name = "success"), - new JsonSubTypes.Type(value = classOf[WorkflowToPythonFailure], name = "failure") - ) -) -sealed trait WorkflowToPythonResponse - -case class WorkflowToPythonSuccess(pythonCode: String) extends WorkflowToPythonResponse - -case class WorkflowToPythonFailure(errorMessage: String) extends WorkflowToPythonResponse - -@Consumes(Array(MediaType.APPLICATION_JSON)) -@Produces(Array(MediaType.APPLICATION_JSON)) -@RolesAllowed(Array("REGULAR", "ADMIN")) -@Path("/workflow-to-python") -class WorkflowToPythonResource extends LazyLogging { - - private val translator = new WorkflowToPythonTranslator() - - @POST - @Path("") - def convertWorkflowToPython( - logicalPlanPojo: LogicalPlanPojo - ): WorkflowToPythonResponse = { - try { - val logicalPlan = LogicalPlan(logicalPlanPojo) - val pythonCode = translator.translate(logicalPlan) - WorkflowToPythonSuccess(pythonCode) - } catch { - case e: Exception => - logger.error("Failed to translate workflow to Python", e) - WorkflowToPythonFailure(e.getMessage) - } - } -} diff --git a/workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java b/workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java deleted file mode 100644 index 4da3aa9cd2d..00000000000 --- a/workflow-compiling-service/src/test/java/org/apache/texera/amber/translator/verify/tags/IntegrationTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify.tags; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import org.scalatest.TagAnnotation; - -/** - * Class-level marker tag for workflow-compiling-service ScalaTest specs that - * exercise both Scala and Python end-to-end (they fork a real Python process - * to run and compare the translator-generated code). Routing to the - * {@code workflow-compiling-service-integration} CI job is by ScalaTest tag - * filtering, controlled by the {@code WCS_TEST_FILTER} env var in - * {@code workflow-compiling-service/build.sbt}: the lighter - * {@code workflow-compiling-service} test run uses {@code skip-integration} - * (which passes {@code -l org.apache.texera.amber.translator.verify.tags.IntegrationTest} - * to ScalaTest), and the integration job uses {@code integration-only} (which - * passes {@code -n} for the same tag). - * - *

Mirrors amber's {@code org.apache.texera.amber.tags.IntegrationTest}. Only - * {@code OperatorBehaviorSpec} carries this tag today — it is the sole spec that - * spawns Python; the other verify specs only exercise pure-JVM classification - * and comparison logic. - * - *

Written in Java rather than Scala because ScalaTest detects tag - * annotations via {@code java.lang.annotation} reflection. A Scala - * {@code class extends StaticAnnotation} does not produce a JVM annotation - * interface that {@code @TagAnnotation} can attach to, so the tag would be - * invisible to ScalaTest at runtime. - */ -@TagAnnotation -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD, ElementType.TYPE}) -public @interface IntegrationTest { -} diff --git a/workflow-compiling-service/src/test/resources/python/compare.py b/workflow-compiling-service/src/test/resources/python/compare.py deleted file mode 100644 index 518b5fac901..00000000000 --- a/workflow-compiling-service/src/test/resources/python/compare.py +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Compare the two paths' outputs for one operator: JSONL DataFrames, or the Plotly -figure a visualization operator renders. - -Usage: compare.py [--unordered] [--ignore-cols c1,c2] - [--model-cols c1,c2 --probe features.jsonl] - - compare.py --plotly - - --unordered Sort both DataFrames lexicographically by all columns before - comparing, so rows match as a set/bag rather than positionally. - This is the norm: the engine runs operators across parallel - workers, so output row order is not part of the contract. - Without this flag the comparator matches rows positionally - (after reset_index(drop=True)) — used only for the sort family, - whose output order IS meaningful. - - --ignore-cols Comma-separated column names to drop from both frames before - comparing. For opaque columns whose value isn't compared. - - --model-cols Comma-separated columns holding a base64(pickle) sklearn model. - Rather than byte-compare them (two independently-trained models - are functionally equal but not bit-identical), the comparator - unpickles both sides, has each model predict on the --probe - feature set, and asserts the predictions match — verifying the - two code paths produce behaviorally-equivalent models. The raw - model columns are then dropped before the frame comparison. - - --probe JSONL feature set the --model-cols models predict on. Each - model uses its own feature_names_in_ to select columns, so the - probe may include extra columns (e.g. the training target). - - --plotly Compare Plotly figures instead of DataFrames. The actual side is - a one-row JSONL with `html-content` or `json-content`; for - `html-content` the first `Plotly.newPlot(...)` payload is - extracted. The expected side is the standalone path's - `fig.write_json(...)`. Only data and layout are compared, with - display-only `uid` fields stripped and floats matched by - tolerance. Takes none of the DataFrame flags. - -Exit 0 - Outputs equal (and model predictions match, if --model-cols) -Exit 1 - Outputs differ; detail on stderr -Exit 2 - Bad invocation - -Persistent mode: `compare.py --serve` imports pandas once and then serves many -comparisons over its lifetime, reading one JSON job per line on stdin and -writing one JSON result per line on stdout. This avoids paying the ~214 ms -pandas import on every comparison (the comparison itself is ~ms). It reuses the -exact same functions the CLI calls, so behavior is identical. - - request {"kind": "dataframe", "actual": "", "expected": "", - "unordered": false, "ignoreCols": [], "modelCols": [], - "probe": null}\n - {"kind": "plotly", "actual": "", "expected": ""}\n - response {"exit": 0|1, "stdout": "", "stderr": ""}\n - -`kind` defaults to "dataframe". Both kinds are served by the same worker so a -run needs one comparison pool rather than one per output shape; the Plotly side -needs nothing pandas does not already pull in. - -A mismatch is exit 1 with the diff on `stderr`, mirroring the CLI's nonzero -exit so the Scala side's ComparatorMismatchException path is unchanged. A -comparison error never kills the server; only closing stdin (EOF) ends it. -""" -import sys - -# pandas is imported where it is used, not here: the --plotly comparison needs -# nothing from it, and a module-level import would make that one-shot invocation -# pay ~500 ms for an interpreter that then compares two JSON documents. `serve()` -# imports it eagerly at startup instead, so a pooled worker still pays it once -# rather than once per DataFrame comparison. - - -def _compare_model_predictions(actual, expected, model_cols, probe_path) -> None: - """For each model column, unpickle both sides and assert their predictions - on the probe set match. Raises AssertionError on any divergence.""" - import base64 - import pickle - - import numpy as np - import pandas as pd - - if probe_path is None: - raise AssertionError("--model-cols requires --probe with a feature set") - probe = pd.read_json(probe_path, lines=True) - # The probe is the operator's own input table, so under the nulls scenario it - # carries the holes that scenario punched. What is under test is whether the - # two models agree, and an estimator that refuses a NaN at predict time would - # end the comparison over the probe rather than over either model. Drop those - # rows: both models are asked the same questions either way. - probe = probe.dropna() - if probe.empty: - raise AssertionError( - "probe has no complete row to predict on; the two models cannot be compared" - ) - - for col in model_cols: - if col not in actual.columns or col not in expected.columns: - continue - if len(actual) != len(expected): - raise AssertionError( - f"model column {col!r}: row count differs " - f"({len(actual)} vs {len(expected)})" - ) - for i in range(len(actual)): - m_actual = pickle.loads(base64.b64decode(actual[col].iloc[i])) - m_expected = pickle.loads(base64.b64decode(expected[col].iloc[i])) - - # A model with feature_names_in_ selects its (numeric) feature - # columns from the probe, naturally dropping the training target the - # probe may still carry. A model WITHOUT it was fitted on a 1-D input - # rather than a named frame — i.e. a text pipeline (e.g. - # CountVectorizer) trained on a single text Series — so feed the - # probe's first column as a Series, not the whole frame (predicting - # on a DataFrame would make CountVectorizer iterate column labels). - names = getattr(m_actual, "feature_names_in_", None) - x_a = probe[list(names)] if names is not None else probe.iloc[:, 0] - names_e = getattr(m_expected, "feature_names_in_", None) - x_e = probe[list(names_e)] if names_e is not None else probe.iloc[:, 0] - - pred_a = np.asarray(m_actual.predict(x_a)) - pred_e = np.asarray(m_expected.predict(x_e)) - - if pred_a.shape != pred_e.shape: - raise AssertionError( - f"model column {col!r} row {i}: prediction shape differs " - f"({pred_a.shape} vs {pred_e.shape})" - ) - numeric = np.issubdtype(pred_a.dtype, np.number) and np.issubdtype( - pred_e.dtype, np.number - ) - ok = ( - np.allclose(pred_a, pred_e, rtol=1e-5, atol=1e-8) - if numeric - else np.array_equal(pred_a, pred_e) - ) - if not ok: - raise AssertionError( - f"model column {col!r} row {i}: predictions differ\n" - f" actual: {pred_a}\n" - f" expected: {pred_e}" - ) - - -def _run_comparison( - actual_path: str, - expected_path: str, - unordered: bool, - ignore_cols: list, - model_cols: list, - probe_path, -) -> "str | None": - """Compare two JSONL DataFrames. Returns None if they match, or a human - diff string if they differ (exit-1 condition). Unexpected errors (e.g. a - bad input file) propagate to the caller. This is the single source of - comparison truth shared by the CLI and the --serve loop.""" - import pandas as pd - - actual = pd.read_json(actual_path, lines=True) - expected = pd.read_json(expected_path, lines=True) - - # Model columns: compare behavior (predictions) rather than bytes, then drop - # the raw columns so the frame comparison covers everything else exactly. - if model_cols: - try: - _compare_model_predictions(actual, expected, model_cols, probe_path) - except AssertionError as exc: - return str(exc) - actual = actual.drop(columns=model_cols, errors="ignore") - expected = expected.drop(columns=model_cols, errors="ignore") - - if ignore_cols: - actual = actual.drop(columns=ignore_cols, errors="ignore") - expected = expected.drop(columns=ignore_cols, errors="ignore") - - if unordered: - # Sort both sides by the same column key so set-equal frames collapse - # to the same row sequence. assert_frame_equal still does the actual - # value diff and respects rtol/check_dtype. Mergesort = stable, so - # rows that are tied on all columns keep their relative order — not - # strictly necessary for set equality (no ties → no duplicates after - # the op's dedup step) but cheap insurance. - cols = list(actual.columns) - if cols: - actual = actual.sort_values( - by=cols, kind="mergesort", na_position="last" - ).reset_index(drop=True) - expected = expected.sort_values( - by=cols, kind="mergesort", na_position="last" - ).reset_index(drop=True) - - try: - pd.testing.assert_frame_equal( - actual, - expected, - check_like=True, - check_dtype=False, - rtol=1e-5, - ) - except AssertionError as exc: - return str(exc) - return None - - -def _load_actual_plot(path) -> dict: - import json - - with open(path, "r", encoding="utf-8") as fh: - line = next((raw for raw in fh if raw.strip()), None) - if line is None: - raise AssertionError(f"{path} is empty") - - row = json.loads(line) - if "json-content" in row and row["json-content"]: - value = row["json-content"] - return json.loads(value) if isinstance(value, str) else value - if "html-content" in row and row["html-content"]: - return _plotly_payload_from_html(row["html-content"]) - raise AssertionError(f"{path} has neither html-content nor json-content") - - -def _plotly_payload_from_html(html: str) -> dict: - """Pull the data/layout arguments out of the first Plotly.newPlot(...) call. - - Scanned with a JSON decoder rather than a regex because the payload is - arbitrary nested JSON that no bracket-matching pattern handles reliably. - """ - import json - - marker = "Plotly.newPlot(" - start = html.find(marker) - if start < 0: - raise AssertionError("html-content does not contain Plotly.newPlot(...)") - - decoder = json.JSONDecoder() - index = start + len(marker) - args: list = [] - while len(args) < 4: - while index < len(html) and html[index] in " \t\r\n,": - index += 1 - value, consumed = decoder.raw_decode(html[index:]) - args.append(value) - index += consumed - - return {"data": args[1], "layout": args[2]} - - -def _load_expected_plot(path) -> dict: - import json - - with open(path, "r", encoding="utf-8") as fh: - value = json.load(fh) - return {"data": value.get("data", []), "layout": value.get("layout", {})} - - -def _strip_unstable(value): - """Remove display-only fields that are unrelated to chart semantics.""" - if isinstance(value, dict): - return { - key: _strip_unstable(child) - for key, child in value.items() - if key not in {"uid"} - } - if isinstance(value, list): - return [_strip_unstable(child) for child in value] - return value - - -def _plots_equal(actual, expected) -> bool: - import math - - if isinstance(actual, (int, float)) and isinstance(expected, (int, float)): - return math.isclose(float(actual), float(expected), rel_tol=1e-9, abs_tol=1e-12) - if isinstance(actual, dict) and isinstance(expected, dict): - return actual.keys() == expected.keys() and all( - _plots_equal(actual[key], expected[key]) for key in actual.keys() - ) - if isinstance(actual, list) and isinstance(expected, list): - return len(actual) == len(expected) and all( - _plots_equal(left, right) for left, right in zip(actual, expected) - ) - return actual == expected - - -def _run_plotly_comparison(actual_path, expected_path) -> "str | None": - """Compare two Plotly figures. Returns None if they match, or a human diff - string if they differ — the same contract as `_run_comparison`, so the CLI - and the --serve loop treat both kinds identically.""" - import json - - actual = _strip_unstable(_load_actual_plot(actual_path)) - expected = _strip_unstable(_load_expected_plot(expected_path)) - if _plots_equal(actual, expected): - return None - return "\n".join( - [ - "Plotly JSON mismatch", - "--- actual ---", - json.dumps(actual, indent=2, sort_keys=True), - "--- expected ---", - json.dumps(expected, indent=2, sort_keys=True), - ] - ) - - -def main() -> None: - args = sys.argv[1:] - unordered = False - ignore_cols: list = [] - model_cols: list = [] - probe_path = None - - if args and args[0] == "--plotly": - if len(args) != 3: - print( - f"usage: {sys.argv[0]} --plotly ", - file=sys.stderr, - ) - sys.exit(2) - msg = _run_plotly_comparison(args[1], args[2]) - if msg is not None: - print(msg, file=sys.stderr) - sys.exit(1) - return - - while args and args[0].startswith("--"): - if args[0] == "--unordered": - unordered = True - args = args[1:] - elif args[0] == "--ignore-cols": - if len(args) < 2: - print("--ignore-cols requires an argument", file=sys.stderr) - sys.exit(2) - ignore_cols = [c for c in args[1].split(",") if c] - args = args[2:] - elif args[0] == "--model-cols": - if len(args) < 2: - print("--model-cols requires an argument", file=sys.stderr) - sys.exit(2) - model_cols = [c for c in args[1].split(",") if c] - args = args[2:] - elif args[0] == "--probe": - if len(args) < 2: - print("--probe requires an argument", file=sys.stderr) - sys.exit(2) - probe_path = args[1] - args = args[2:] - else: - print(f"unknown flag: {args[0]}", file=sys.stderr) - sys.exit(2) - if len(args) != 2: - print( - f"usage: {sys.argv[0]} [--unordered] [--ignore-cols c1,c2] " - f"[--model-cols c1,c2 --probe features.jsonl] " - f" ", - file=sys.stderr, - ) - sys.exit(2) - - msg = _run_comparison( - args[0], args[1], unordered, ignore_cols, model_cols, probe_path - ) - if msg is not None: - print(msg, file=sys.stderr) - sys.exit(1) - - -def serve() -> None: - """Persistent comparison server. See the module docstring for the protocol. - - Each job runs the same function the CLI calls for its kind. A comparison - error is reported as exit 1 with the diff on `stderr`; only closing stdin - ends the loop. - """ - import io - import json - import traceback - from contextlib import redirect_stderr, redirect_stdout - - # Eagerly, before signalling ready: the point of a persistent worker is that - # this cost is paid once per worker instead of once per comparison, and - # `ready` should mean the worker is warm. - import pandas # noqa: F401 - - sys.stdout.write(json.dumps({"ready": True}) + "\n") - sys.stdout.flush() - - for line in sys.stdin: - line = line.strip() - if not line: - continue - out_buf, err_buf = io.StringIO(), io.StringIO() - try: - job = json.loads(line) - with redirect_stdout(out_buf), redirect_stderr(err_buf): - if job.get("kind", "dataframe") == "plotly": - msg = _run_plotly_comparison(job["actual"], job["expected"]) - else: - msg = _run_comparison( - job["actual"], - job["expected"], - job.get("unordered", False), - job.get("ignoreCols", []), - job.get("modelCols", []), - job.get("probe"), - ) - resp = { - "exit": 0 if msg is None else 1, - "stdout": out_buf.getvalue(), - "stderr": err_buf.getvalue() + ("" if msg is None else msg), - } - except BaseException: # noqa: BLE001 — a bad job must not kill the server - resp = { - "exit": 1, - "stdout": out_buf.getvalue(), - "stderr": err_buf.getvalue() + traceback.format_exc(), - } - sys.stdout.write(json.dumps(resp) + "\n") - sys.stdout.flush() - - -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == "--serve": - serve() - else: - main() diff --git a/workflow-compiling-service/src/test/resources/python/py_op_driver.py b/workflow-compiling-service/src/test/resources/python/py_op_driver.py deleted file mode 100644 index 1ff2a9000da..00000000000 --- a/workflow-compiling-service/src/test/resources/python/py_op_driver.py +++ /dev/null @@ -1,531 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Driver that runs a Texera Python-native operator without spinning up the -Pekko/Arrow worker stack. - -Symmetric to ``OpExecHarness`` on the JVM side: take an OpDesc's -``generatePythonCode()`` output (which defines a ``UDFOperatorV2`` / -``UDFTableOperator`` / ``UDFBatchOperator`` / ``UDFSourceOperator`` subclass), -load JSONL+sidecar inputs into ``Tuple`` instances, drive -``open -> process_tuple/on_finish per port -> close``, and write the emitted -tuples back as JSONL+sidecar in the same format ``TupleIO`` reads. - -The harness invokes us as:: - - python3 py_op_driver.py - -with ``PYTHONPATH`` pointing at ``amber/src/main/python`` so ``pytexera`` / -``pyamber`` import cleanly. - -Config schema (all paths absolute):: - - { - "operatorCode": "", - "isSource": false, - "portOrder": [0, 1], # input-port dependency order - "inputs": [{"portIndex": 0, "dataPath": "...", "schemaPath": "..."}], - "outputs": [{"portIndex": 0, "dataPath": "...", "schema": - {"attributes": [{"attributeName": "...", - "attributeType": "..."}]}}] - } - -Output schemas come from the JVM side (``PhysicalOp.propagateSchema``) so -this driver never has to infer them. The driver writes the schema back as a -``.jsonl.schema.json`` sidecar next to each ``dataPath``, matching -``TupleIO.writeTuples``. -""" -from __future__ import annotations - -import base64 -import inspect -import json -import pickle -import sys -import traceback -from pathlib import Path -from typing import Any, Iterable, Iterator, List, Mapping, Sequence - -import pandas as pd - -# pytexera re-exports the operator base classes and the Tuple/Table types. -# The Scala side prepends `amber/src/main/python` to PYTHONPATH so these -# resolve. If they don't, raise a clean error rather than a cryptic -# ImportError deep in user code. -try: - from pytexera import ( # noqa: F401 (used dynamically in user code's globals) - Batch, - BatchLike, - Iterator as PyIterator, # noqa: F401 - Optional as PyOptional, # noqa: F401 - Table, - TableLike, - Tuple, - TupleLike, - UDFBatchOperator, - UDFOperatorV2, - UDFSourceOperator, - UDFTableOperator, - Union as PyUnion, # noqa: F401 - logger as pytexera_logger, # noqa: F401 - overrides, # noqa: F401 - ) - from core.models.schema.schema import Schema as TexeraSchema - from core.models.schema.attribute_type import AttributeType, RAW_TYPE_MAPPING -except ImportError as exc: - sys.stderr.write( - "py_op_driver.py: failed to import pytexera/pyamber. The harness must " - "set PYTHONPATH to `amber/src/main/python` and the venv must have all " - "amber Python deps installed (see amber/requirements.txt).\n" - f"Underlying error: {exc!r}\n" - ) - raise - - -# -------------------------------------------------------------------------- -# Schema sidecar I/O. -# -------------------------------------------------------------------------- -# The JVM writes attributes using AttributeType's Jackson @JsonValue ("string", -# "integer", "long", "double", "boolean", "timestamp", "binary", -# "large_binary"). The Python Schema's RAW_TYPE_MAPPING uses uppercase keys -# ("STRING", "INTEGER", ...). Translate at the boundary; keep the rest of -# the pipeline using Python's AttributeType enum. -_SCALA_TO_PY_TYPE: Mapping[str, str] = { - "string": "STRING", - "integer": "INTEGER", - "long": "LONG", - "double": "DOUBLE", - "boolean": "BOOLEAN", - "timestamp": "TIMESTAMP", - "binary": "BINARY", - "large_binary": "LARGE_BINARY", -} - -_PY_TO_SCALA_TYPE: Mapping[AttributeType, str] = { - AttributeType.STRING: "string", - AttributeType.INT: "integer", - AttributeType.LONG: "long", - AttributeType.DOUBLE: "double", - AttributeType.BOOL: "boolean", - AttributeType.TIMESTAMP: "timestamp", - AttributeType.BINARY: "binary", - AttributeType.LARGE_BINARY: "large_binary", -} - - -def _schema_from_dict(payload: Mapping[str, Any]) -> TexeraSchema: - raw: "dict[str, str]" = {} - for attr in payload["attributes"]: - raw_name = attr["attributeName"] - raw_type = attr["attributeType"].lower() - if raw_type not in _SCALA_TO_PY_TYPE: - raise ValueError( - f"py_op_driver: unknown attributeType {attr['attributeType']!r} " - f"for attribute {raw_name!r}" - ) - raw[raw_name] = _SCALA_TO_PY_TYPE[raw_type] - return TexeraSchema(raw_schema=raw) - - -def _schema_to_dict(schema: TexeraSchema) -> "dict[str, Any]": - return { - "attributes": [ - {"attributeName": name, "attributeType": _PY_TO_SCALA_TYPE[attr_type]} - for name, attr_type in schema.as_key_value_pairs() - ] - } - - -def _read_schema_sidecar(data_path: Path) -> TexeraSchema: - sidecar = data_path.with_name(data_path.name + ".schema.json") - with sidecar.open("r", encoding="utf-8") as fh: - return _schema_from_dict(json.load(fh)) - - -def _write_schema_sidecar(data_path: Path, schema: TexeraSchema) -> None: - sidecar = data_path.with_name(data_path.name + ".schema.json") - with sidecar.open("w", encoding="utf-8") as fh: - json.dump(_schema_to_dict(schema), fh) - - -# -------------------------------------------------------------------------- -# Tuple I/O. JSONL with sidecar — same on-disk shape as TupleIO on the JVM. -# -------------------------------------------------------------------------- -def _coerce_field(raw: Any, attr_type: AttributeType) -> Any: - """Coerce a JSON-decoded field to the type the schema expects.""" - if raw is None: - return None - if attr_type == AttributeType.STRING: - return str(raw) - if attr_type == AttributeType.INT: - return int(raw) - if attr_type == AttributeType.LONG: - return int(raw) - if attr_type == AttributeType.DOUBLE: - return float(raw) - if attr_type == AttributeType.BOOL: - return bool(raw) - if attr_type == AttributeType.BINARY: - return base64.b64decode(raw) - if attr_type == AttributeType.TIMESTAMP: - # TupleIO writes java.sql.Timestamp.toString ("YYYY-MM-DD HH:MM:SS[.f]"); - # the native path's schema maps TIMESTAMP -> datetime.datetime, and - # pandas parses the JDBC form robustly. - return pd.Timestamp(raw).to_pydatetime() - # LARGE_BINARY: defer until an operator actually exercises it. Failing loud - # beats silently passing a string through. - raise NotImplementedError( - f"py_op_driver: reading attribute type {attr_type!r} from JSONL is " - f"not implemented yet" - ) - - -def _read_tuples(data_path: Path, schema: TexeraSchema) -> List[Tuple]: - rows: List[Tuple] = [] - if not data_path.exists(): - return rows - with data_path.open("r", encoding="utf-8") as fh: - for line_num, raw_line in enumerate(fh, 1): - line = raw_line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError as exc: - raise ValueError( - f"py_op_driver: invalid JSON on line {line_num} of {data_path}: {exc}" - ) from exc - field_data: "dict[str, Any]" = {} - for name, attr_type in schema.as_key_value_pairs(): - field_data[name] = _coerce_field(obj.get(name), attr_type) - tup = Tuple(field_data) - tup.finalize(schema) - rows.append(tup) - return rows - - -def _emit_as_dicts( - emitted: Iterable[Any], schema: TexeraSchema -) -> Iterator["dict[str, Any]"]: - """ - Flatten whatever the operator yields into per-row dicts keyed by the - output schema's attribute names. The operator may yield: - * pandas.DataFrame (UDFTableOperator's process_table return) - * pandas.Series (single row) - * dict / OrderedDict (e.g. BarChart yields {'html-content': html}) - * Tuple - * None (skip — matches the engine's behavior) - """ - attr_names = schema.get_attr_names() - for item in emitted: - if item is None: - continue - if isinstance(item, pd.DataFrame): - for _, row in item.iterrows(): - yield {col: row[col] for col in attr_names if col in row.index} - elif isinstance(item, pd.Series): - yield {col: item[col] for col in attr_names if col in item.index} - elif isinstance(item, Tuple): - yield {name: item[name] for name in attr_names} - elif isinstance(item, Mapping): - yield {name: item.get(name) for name in attr_names} - else: - raise TypeError( - f"py_op_driver: cannot serialize emitted value of type " - f"{type(item).__name__}: {item!r}" - ) - - -def _jsonify(value: Any, attr_type: AttributeType) -> Any: - """Convert a Python value into something json.dumps will accept.""" - if value is None: - return None - # A missing cell reaches pandas as NaN or NaT, not None, and every branch - # below assumes a real value: the timestamp one formats NaT's float - # microsecond with a "d" code and raises. Emitting null matches the - # standalone path, whose to_json writes NaN and NaT that way. Scalars only — - # an object column can hold a list or an array, where isna answers - # element-wise and the result is not a truth value. - if pd.api.types.is_scalar(value) and pd.isna(value): - return None - # pandas often hands us numpy scalars; .item() collapses them to native. - if hasattr(value, "item") and not isinstance(value, (str, bytes)): - try: - value = value.item() - except (ValueError, AttributeError): - pass - if attr_type == AttributeType.STRING: - return str(value) - if attr_type in (AttributeType.INT, AttributeType.LONG): - return int(value) - if attr_type == AttributeType.DOUBLE: - return float(value) - if attr_type == AttributeType.BOOL: - return bool(value) - if attr_type == AttributeType.BINARY: - # Trained-model / object columns: pickle then base64 so the value - # survives JSONL round-trip. Mirrors the BINARY read path in - # _coerce_field. For deterministic estimators the pickle is byte-stable - # across processes, so the two verification paths compare equal. - raw = value if isinstance(value, (bytes, bytearray)) else pickle.dumps(value) - return base64.b64encode(raw).decode("ascii") - if attr_type == AttributeType.TIMESTAMP: - # Emit the same JDBC string java.sql.Timestamp.toString produces (>=1 - # fractional digit), so a passed-through timestamp column matches the - # standalone path, which carries it as that exact string. - ts = pd.Timestamp(value) - frac = f"{ts.microsecond:06d}".rstrip("0") or "0" - return ts.strftime("%Y-%m-%d %H:%M:%S") + "." + frac - raise NotImplementedError( - f"py_op_driver: writing attribute type {attr_type!r} to JSONL is " - f"not implemented yet" - ) - - -def _write_tuples( - data_path: Path, rows: Iterable["dict[str, Any]"], schema: TexeraSchema -) -> None: - _write_schema_sidecar(data_path, schema) - with data_path.open("w", encoding="utf-8") as fh: - for row in rows: - serialized: "dict[str, Any]" = {} - for name, attr_type in schema.as_key_value_pairs(): - serialized[name] = _jsonify(row.get(name), attr_type) - fh.write(json.dumps(serialized)) - fh.write("\n") - - -# -------------------------------------------------------------------------- -# Operator discovery + lifecycle. -# -------------------------------------------------------------------------- -_OPERATOR_BASES = ( - UDFOperatorV2, - UDFTableOperator, - UDFBatchOperator, - UDFSourceOperator, -) - - -def _exec_user_code(code: str) -> "dict[str, Any]": - """ - Execute the operator code in a fresh namespace seeded with the pytexera - re-exports, the way the real Texera Python worker does it (see - ``InitializeExecutorHandler``). Returning the namespace lets us pick the - user's operator class out of it. - """ - namespace: "dict[str, Any]" = { - "__name__": "__texera_user_op__", - "__builtins__": __builtins__, - } - # pytexera does `from pyamber import *` itself, so this single import is - # equivalent to what the generated code's `from pytexera import *` brings - # into scope. - exec("from pytexera import *", namespace) - try: - exec(code, namespace) - except Exception: - sys.stderr.write("py_op_driver: error executing operator code:\n") - traceback.print_exc() - raise - return namespace - - -def _discover_operator_class(namespace: Mapping[str, Any]) -> type: - candidates: List[type] = [] - for name, obj in namespace.items(): - if not inspect.isclass(obj): - continue - if obj in _OPERATOR_BASES: - continue # the base classes themselves come in via the import - if any(issubclass(obj, base) for base in _OPERATOR_BASES): - candidates.append(obj) - if not candidates: - raise RuntimeError( - "py_op_driver: operator code did not define a subclass of " - "UDFOperatorV2 / UDFTableOperator / UDFBatchOperator / UDFSourceOperator" - ) - if len(candidates) > 1: - names = ", ".join(c.__name__ for c in candidates) - raise RuntimeError( - f"py_op_driver: operator code defined multiple UDF subclasses " - f"({names}); expected exactly one" - ) - return candidates[0] - - -def _run_operator( - op: Any, - is_source: bool, - port_order: Sequence[int], - inputs_by_port: Mapping[int, Sequence[Tuple]], -) -> List[Any]: - """ - Drive the operator's lifecycle. Returns the flat list of emitted values - (anything not-None yielded by process_tuple / on_finish, in emission - order). UDF operators don't expose multi-output ports today, so we don't - bucket by output port — same convention as ``OpExecHarness`` when port - is unset. - """ - emitted: List[Any] = [] - - op.open() - try: - if is_source: - # Source ops: SourceOperator.on_finish iterates produce() and - # yields Tuples. Single synthetic port 0 — see OpExecHarness. - for item in op.on_finish(0): - if item is not None: - emitted.append(item) - return emitted - - for port in port_order: - for tup in inputs_by_port.get(port, ()): # type: ignore[arg-type] - for item in op.process_tuple(tup, port): - if item is not None: - emitted.append(item) - for item in op.on_finish(port): - if item is not None: - emitted.append(item) - finally: - op.close() - - return emitted - - -# -------------------------------------------------------------------------- -# Entry point. -# -------------------------------------------------------------------------- -def run_config(config: Mapping[str, Any]) -> None: - """Run one operator to completion from a parsed config dict. Writes the - output JSONL+sidecar as a side effect. Raises on any failure. Shared by the - CLI (main) and the persistent server (serve) so both behave identically. - - Each call execs the user code in a FRESH namespace and constructs a FRESH - operator instance, so operators don't share Python-level state across jobs - when run through the server (the isolation the per-process CLI gave for - free).""" - # Both paths seed numpy's global RNG with the same value before running, so - # an estimator built without random_state (sklearn reads the global RNG for - # that) draws the same samples on each and the two models come out - # identical. Without it a stochastic estimator makes the parity check - # inconclusive in both directions: a difference could be the translation or - # could be the draw, and a match could be either. Per call, not per process: - # the worker pool reuses this process, so a job that inherited the previous - # job's RNG position would not line up with a fresh standalone one. Keep in - # step with StandaloneRunner.VerifySeed. - import numpy as _texera_np - - _texera_np.random.seed(20260811) - - operator_code: str = config["operatorCode"] - is_source: bool = bool(config.get("isSource", False)) - port_order: Sequence[int] = list(config.get("portOrder", [])) - - inputs_by_port: "dict[int, List[Tuple]]" = {} - for entry in config.get("inputs", []): - port = int(entry["portIndex"]) - data_path = Path(entry["dataPath"]) - schema = _read_schema_sidecar(data_path) - inputs_by_port[port] = _read_tuples(data_path, schema) - - # Default port order: sorted by index. Matches OpExecHarness's fallback - # when getInputPortDependencyPairs is empty. - if not port_order: - port_order = sorted(inputs_by_port.keys()) - - namespace = _exec_user_code(operator_code) - op_class = _discover_operator_class(namespace) - op_instance = op_class() - - emitted = _run_operator(op_instance, is_source, port_order, inputs_by_port) - - outputs = config.get("outputs", []) - if len(outputs) > 1: - raise NotImplementedError( - "py_op_driver: multi-output Python operators are not supported " - "yet (no UDF base class exposes per-port emission)" - ) - if outputs: - out_entry = outputs[0] - out_path = Path(out_entry["dataPath"]) - out_schema = _schema_from_dict(out_entry["schema"]) - rows = list(_emit_as_dicts(emitted, out_schema)) - _write_tuples(out_path, rows, out_schema) - - -def main(argv: Sequence[str]) -> int: - if len(argv) != 2: - sys.stderr.write(f"usage: {argv[0]} \n") - return 2 - config_path = Path(argv[1]) - with config_path.open("r", encoding="utf-8") as fh: - config = json.load(fh) - run_config(config) - return 0 - - -def serve() -> int: - """Persistent driver: import pyamber once, then run many operators. - - pytexera/pyamber import at module load (~300 ms) is the dominant per-call - cost; paying it once here instead of per operator is the whole point. Reads - one JSON job per line on stdin, writes one JSON result per line on stdout: - - request {"configPath": ""}\n - response {"exit": 0|1, "stdout": "...", "stderr": "..."}\n - - exit=1 with the traceback on stderr mirrors a nonzero CLI exit, so the - Scala side's PyOpDriverException path is unchanged. An operator error never - kills the server; only closing stdin (EOF) ends it. The executed script's - stdout/stderr are captured so they can't corrupt the protocol channel. - """ - import io - from contextlib import redirect_stderr, redirect_stdout - - sys.stdout.write(json.dumps({"ready": True}) + "\n") - sys.stdout.flush() - - for line in sys.stdin: - line = line.strip() - if not line: - continue - out_buf, err_buf = io.StringIO(), io.StringIO() - try: - job = json.loads(line) - with Path(job["configPath"]).open("r", encoding="utf-8") as fh: - config = json.load(fh) - with redirect_stdout(out_buf), redirect_stderr(err_buf): - run_config(config) - resp = {"exit": 0, "stdout": out_buf.getvalue(), "stderr": err_buf.getvalue()} - except BaseException: # noqa: BLE001 — a bad job must not kill the server - resp = { - "exit": 1, - "stdout": out_buf.getvalue(), - "stderr": err_buf.getvalue() + traceback.format_exc(), - } - sys.stdout.write(json.dumps(resp) + "\n") - sys.stdout.flush() - return 0 - - -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == "--serve": - sys.exit(serve()) - else: - sys.exit(main(sys.argv)) diff --git a/workflow-compiling-service/src/test/resources/python/standalone_worker.py b/workflow-compiling-service/src/test/resources/python/standalone_worker.py deleted file mode 100644 index 946fec23206..00000000000 --- a/workflow-compiling-service/src/test/resources/python/standalone_worker.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Persistent worker for the Path B (standalone) verify path. - -Motivation: forking a fresh interpreter per operator pays the pandas/plotly -import cost (~260-310 ms) on every spawn, while the operator's actual compute -on the tiny canonical fixtures is ~4 ms. Imports dominate ~96% of the per-spawn -cost. This worker imports those heavy libraries ONCE at startup, then executes -many operators' generated scripts over its lifetime — so the import cost is -paid once, not once per operator. - -It is a drop-in replacement for `python `: it runs the exact same -rendered script `StandaloneRunner` already produces (imports + prologue + body -+ epilogue). The script's own top-of-file `import pandas` becomes a ~0 ms -`sys.modules` cache hit. - -Protocol (line-delimited JSON, both directions): - - startup worker -> parent: {"ready": true} - request parent -> worker: {"scriptPath": "", "workDir": ""}\n - response worker -> parent: {"exit": 0, "stdout": "...", "stderr": "..."}\n - -`exit` is 0 on success or 1 if the script raised; on 1, `stderr` carries the -traceback — mirroring a nonzero subprocess exit so the Scala side's -StandaloneExecutionException path is unchanged. The worker keeps running after -a script error (only a hard interpreter crash ends it); parent closes stdin -(EOF) to shut it down. - -Isolation trade-off (accepted, per design discussion): all jobs share one -interpreter, so module-level state (e.g. pandas display options) can leak -between operators. Each job is exec'd in a FRESH namespace and chdir'd to its -own workDir to contain the common cases; this is weaker than the old -process-per-operator isolation. -""" -from __future__ import annotations - -import io -import json -import os -import sys -import traceback -from contextlib import redirect_stderr, redirect_stdout - -# --- Pay the heavy import cost ONCE, here, at startup. ---------------------- -# These mirror the imports StandaloneRunner injects at the top of every -# rendered script. Pre-importing them populates sys.modules, so each executed -# script's own `import pandas as pd` / `import plotly...` is a cache hit. -# numpy is intentionally NOT imported (see StandaloneRunner.renderScript: the -# production translator only provides pandas + plotly, so an operator needing -# numpy must import it itself — we must not mask that). -import pandas as pd # noqa: F401 -import plotly.express as px # noqa: F401 -import plotly.graph_objects as go # noqa: F401 -import plotly.io # noqa: F401 - - -def _run_one(script_path: str, work_dir: str) -> "dict[str, object]": - """Execute one rendered standalone script and capture its output. - - Runs in a fresh namespace with cwd = work_dir (generated code may use - relative paths, e.g. CSVScan's `pd.read_csv("sample.csv")`; absolute paths - written by the prologue/epilogue are unaffected). The script's stdout / - stderr are redirected into buffers so they never corrupt the protocol - channel on real stdout. - """ - out_buf, err_buf = io.StringIO(), io.StringIO() - # __name__ = "__main__" so scripts with a `if __name__ == "__main__"` guard - # still run their body (the translator does not emit one, but it is free - # insurance and matches `python script.py` semantics). - namespace = {"__name__": "__main__", "__file__": script_path} - try: - with open(script_path, "r", encoding="utf-8") as f: - source = f.read() - os.chdir(work_dir) - code = compile(source, script_path, "exec") - with redirect_stdout(out_buf), redirect_stderr(err_buf): - exec(code, namespace) # noqa: S102 (running generated verify code by design) - return {"exit": 0, "stdout": out_buf.getvalue(), "stderr": err_buf.getvalue()} - except BaseException: # noqa: BLE001 — a script error must NOT kill the worker - # Match a nonzero subprocess exit: traceback goes to stderr, exit = 1. - err = err_buf.getvalue() + traceback.format_exc() - return {"exit": 1, "stdout": out_buf.getvalue(), "stderr": err} - - -def main() -> None: - # Signal readiness only after the heavy imports above have completed, so the - # parent can warm a pool and attribute startup cost deterministically. - sys.stdout.write(json.dumps({"ready": True}) + "\n") - sys.stdout.flush() - - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - req = json.loads(line) - result = _run_one(req["scriptPath"], req["workDir"]) - except Exception: # malformed request — report, keep serving - result = {"exit": 1, "stdout": "", "stderr": traceback.format_exc()} - sys.stdout.write(json.dumps(result) + "\n") - sys.stdout.flush() - - -if __name__ == "__main__": - main() diff --git a/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json b/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json deleted file mode 100644 index 59dc4823c14..00000000000 --- a/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json +++ /dev/null @@ -1,542 +0,0 @@ -[ - { - "id": 7, - "name": "eve", - "score": 0.9, - "open": 7.0, - "high": 8.5, - "low": 6.0, - "close": 7.5, - "iso_country": "IND", - "trade_date": "2024-01-07", - "pvalue": 0.2597402597402597, - "log2fc": 1.4, - "comp_a": 3.0, - "comp_b": 1.0, - "comp_c": 2.0, - "uvec": -3.0, - "edge_pair": "[0, 7]", - "node_src": "n3", - "node_dst": "n4", - "start_ts": "2024-01-07 00:00:00.0", - "finish_ts": "2024-01-07 08:00:00.0", - "uniq_name": "cat_7", - "simplex_a": 35.0, - "simplex_b": 30.0, - "simplex_c": 35.0, - "short_text": "The meeting is scheduled for three o'clock tomorrow.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 3.9000000000000004, - "petal_width": 1.3, - "species": 1, - "csv_list": "a7", - "mixed_case": "abacus", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 3, - "name": "bob", - "score": 1.2, - "open": 3.0, - "high": 4.5, - "low": 2.0, - "close": 3.5, - "iso_country": "JPN", - "trade_date": "2024-01-03", - "pvalue": 0.11188811188811189, - "log2fc": -1.4, - "comp_a": 4.0, - "comp_b": 4.0, - "comp_c": 1.0, - "uvec": 0.0, - "edge_pair": "[0, 3]", - "node_src": "n3", - "node_dst": "n4", - "start_ts": "2024-01-03 00:00:00.0", - "finish_ts": "2024-01-03 04:00:00.0", - "uniq_name": "cat_3", - "simplex_a": 35.0, - "simplex_b": 25.0, - "simplex_c": 40.0, - "short_text": "The meeting is scheduled for three o'clock tomorrow.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 5.2, - "petal_width": 1.85, - "species": 1, - "csv_list": "a3,b3", - "mixed_case": "ABBEY", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 11, - "name": "1", - "score": 1.7, - "open": 11.0, - "high": 12.5, - "low": 10.0, - "close": 11.5, - "iso_country": "USA", - "trade_date": "2024-01-11", - "pvalue": 0.4075924075924076, - "log2fc": -3.5, - "comp_a": 2.0, - "comp_b": 5.0, - "comp_c": 3.0, - "uvec": 1.0, - "edge_pair": "[0, 11]", - "node_src": "n3", - "node_dst": "n4", - "start_ts": "2024-01-11 00:00:00.0", - "finish_ts": "2024-01-11 04:00:00.0", - "uniq_name": "cat_11", - "simplex_a": 35.0, - "simplex_b": 35.0, - "simplex_c": 30.0, - "short_text": "URGENT: your account needs verification, click the link immediately.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 2.6, - "petal_width": 0.75, - "species": 0, - "csv_list": "a11,b11,c11", - "mixed_case": "101", - "species_pred": 0, - "species_name": "setosa", - "species_name_pred": "setosa" - }, - { - "id": 1, - "name": "1", - "score": 0.5, - "open": 1.0, - "high": 2.5, - "low": 0.0, - "close": 1.5, - "iso_country": "USA", - "trade_date": "2024-01-01", - "pvalue": 0.03796203796203796, - "log2fc": -2.8, - "comp_a": 2.0, - "comp_b": 2.0, - "comp_c": 2.0, - "uvec": -2.0, - "edge_pair": "[0, 1]", - "node_src": "n1", - "node_dst": "n2", - "start_ts": "2024-01-01 00:00:00.0", - "finish_ts": "2024-01-01 02:00:00.0", - "uniq_name": "cat_1", - "simplex_a": 25.0, - "simplex_b": 30.0, - "simplex_c": 45.0, - "short_text": "I'm really not sure how I feel about it.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 2.6, - "petal_width": 0.75, - "species": 0, - "csv_list": "a1,b1,c1,d1", - "mixed_case": "abdomen", - "species_pred": 1, - "species_name": "setosa", - "species_name_pred": "versicolor" - }, - { - "id": 14, - "name": "carol", - "score": 4.5, - "open": 14.0, - "high": 15.5, - "low": 13.0, - "close": 14.5, - "iso_country": "DEU", - "trade_date": "2024-01-14", - "pvalue": 0.5184815184815185, - "log2fc": -1.4, - "comp_a": 5.0, - "comp_b": 1.0, - "comp_c": 3.0, - "uvec": -3.0, - "edge_pair": "[0, 14]", - "node_src": "n2", - "node_dst": "n3", - "start_ts": "2024-01-14 00:00:00.0", - "finish_ts": "2024-01-14 07:00:00.0", - "uniq_name": "cat_14", - "simplex_a": 30.0, - "simplex_b": 35.0, - "simplex_c": 35.0, - "short_text": "I absolutely love this, it completely made my day!", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 6.5, - "petal_width": 2.4000000000000004, - "species": 1, - "csv_list": "a14", - "mixed_case": "ABILITY", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 5, - "name": "dave", - "score": 2.0, - "open": 5.0, - "high": 6.5, - "low": 4.0, - "close": 5.5, - "iso_country": "GBR", - "trade_date": "2024-01-05", - "pvalue": 0.18581418581418582, - "log2fc": 0.0, - "comp_a": 1.0, - "comp_b": 6.0, - "comp_c": 3.0, - "uvec": 2.0, - "edge_pair": "[0, 5]", - "node_src": "n1", - "node_dst": "n2", - "start_ts": "2024-01-05 00:00:00.0", - "finish_ts": "2024-01-05 06:00:00.0", - "uniq_name": "cat_5", - "simplex_a": 25.0, - "simplex_b": 35.0, - "simplex_c": 40.0, - "short_text": "This is the worst experience I have ever had.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 1.3, - "petal_width": 0.2, - "species": 0, - "csv_list": "a5,b5", - "mixed_case": "202", - "species_pred": 0, - "species_name": "setosa", - "species_name_pred": "setosa" - }, - { - "id": 9, - "name": "grace", - "score": 2.4, - "open": 9.0, - "high": 10.5, - "low": 8.0, - "close": 9.5, - "iso_country": "CAN", - "trade_date": "2024-01-09", - "pvalue": 0.3336663336663337, - "log2fc": 2.8, - "comp_a": 5.0, - "comp_b": 3.0, - "comp_c": 1.0, - "uvec": -1.0, - "edge_pair": "[0, 9]", - "node_src": "n1", - "node_dst": "n2", - "start_ts": "2024-01-09 00:00:00.0", - "finish_ts": "2024-01-09 02:00:00.0", - "uniq_name": "cat_9", - "simplex_a": 25.0, - "simplex_b": 25.0, - "simplex_c": 50.0, - "short_text": "I absolutely love this, it completely made my day!", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 6.5, - "petal_width": 2.4000000000000004, - "species": 1, - "csv_list": "a9,b9,c9", - "mixed_case": "abstract", - "species_pred": 0, - "species_name": "versicolor", - "species_name_pred": "setosa" - }, - { - "id": 2, - "name": "alice", - "score": 3.1, - "open": 2.0, - "high": 3.5, - "low": 1.0, - "close": 2.5, - "iso_country": "CHN", - "trade_date": "2024-01-02", - "pvalue": 0.07492507492507493, - "log2fc": -2.0999999999999996, - "comp_a": 3.0, - "comp_b": 3.0, - "comp_c": 3.0, - "uvec": -1.0, - "edge_pair": "[0, 2]", - "node_src": "n2", - "node_dst": "n3", - "start_ts": "2024-01-02 00:00:00.0", - "finish_ts": "2024-01-02 03:00:00.0", - "uniq_name": "cat_2", - "simplex_a": 30.0, - "simplex_b": 35.0, - "simplex_c": 35.0, - "short_text": "Thank you so much, everything was absolutely perfect.", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 3.9000000000000004, - "petal_width": 1.3, - "species": 1, - "csv_list": "a2,b2,c2,d2", - "mixed_case": "ABACUS", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 13, - "name": "bob", - "score": 2.8, - "open": 13.0, - "high": 14.5, - "low": 12.0, - "close": 13.5, - "iso_country": "JPN", - "trade_date": "2024-01-13", - "pvalue": 0.48151848151848153, - "log2fc": -2.0999999999999996, - "comp_a": 4.0, - "comp_b": 7.0, - "comp_c": 2.0, - "uvec": 3.0, - "edge_pair": "[0, 13]", - "node_src": "n1", - "node_dst": "n2", - "start_ts": "2024-01-13 00:00:00.0", - "finish_ts": "2024-01-13 06:00:00.0", - "uniq_name": "cat_13", - "simplex_a": 25.0, - "simplex_b": 30.0, - "simplex_c": 45.0, - "short_text": "Thank you so much, everything was absolutely perfect.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 5.2, - "petal_width": 1.85, - "species": 1, - "csv_list": "a13", - "mixed_case": "303", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 6, - "name": "1", - "score": 5.5, - "open": 6.0, - "high": 7.5, - "low": 5.0, - "close": 6.5, - "iso_country": "FRA", - "trade_date": "2024-01-06", - "pvalue": 0.22277722277722278, - "log2fc": 0.7, - "comp_a": 2.0, - "comp_b": 7.0, - "comp_c": 1.0, - "uvec": 3.0, - "edge_pair": "[0, 6]", - "node_src": "n2", - "node_dst": "n3", - "start_ts": "2024-01-06 00:00:00.0", - "finish_ts": "2024-01-06 07:00:00.0", - "uniq_name": "cat_6", - "simplex_a": 30.0, - "simplex_b": 25.0, - "simplex_c": 45.0, - "short_text": "I'm really not sure how I feel about it.", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 2.6, - "petal_width": 0.75, - "species": 0, - "csv_list": "a6,b6", - "mixed_case": "abbey", - "species_pred": 0, - "species_name": "setosa", - "species_name_pred": "setosa" - }, - { - "id": 10, - "name": "heidi", - "score": 4.2, - "open": 10.0, - "high": 11.5, - "low": 9.0, - "close": 10.5, - "iso_country": "AUS", - "trade_date": "2024-01-10", - "pvalue": 0.3706293706293706, - "log2fc": 3.5, - "comp_a": 1.0, - "comp_b": 4.0, - "comp_c": 2.0, - "uvec": 0.0, - "edge_pair": "[0, 10]", - "node_src": "n2", - "node_dst": "n3", - "start_ts": "2024-01-10 00:00:00.0", - "finish_ts": "2024-01-10 03:00:00.0", - "uniq_name": "cat_10", - "simplex_a": 30.0, - "simplex_b": 30.0, - "simplex_c": 40.0, - "short_text": "This is the worst experience I have ever had.", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 1.3, - "petal_width": 0.2, - "species": 0, - "csv_list": "a10,b10,c10", - "mixed_case": "ABDOMEN", - "species_pred": 1, - "species_name": "setosa", - "species_name_pred": "versicolor" - }, - { - "id": 4, - "name": "carol", - "score": 4.8, - "open": 4.0, - "high": 5.5, - "low": 3.0, - "close": 4.5, - "iso_country": "DEU", - "trade_date": "2024-01-04", - "pvalue": 0.14885114885114886, - "log2fc": -0.7, - "comp_a": 5.0, - "comp_b": 5.0, - "comp_c": 2.0, - "uvec": 1.0, - "edge_pair": "[0, 4]", - "node_src": "n0", - "node_dst": "n1", - "start_ts": "2024-01-04 00:00:00.0", - "finish_ts": "2024-01-04 05:00:00.0", - "uniq_name": "cat_4", - "simplex_a": 20.0, - "simplex_b": 30.0, - "simplex_c": 50.0, - "short_text": "Congratulations! You have won a free prize, reply now to claim it.", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 6.5, - "petal_width": 2.4000000000000004, - "species": 1, - "csv_list": "a4,b4,c4,d4", - "mixed_case": "404", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 15, - "name": "dave", - "score": 3.3, - "open": 15.0, - "high": 16.5, - "low": 14.0, - "close": 15.5, - "iso_country": "GBR", - "trade_date": "2024-01-15", - "pvalue": 0.5554445554445554, - "log2fc": -0.7, - "comp_a": 1.0, - "comp_b": 2.0, - "comp_c": 1.0, - "uvec": -2.0, - "edge_pair": "[0, 15]", - "node_src": "n3", - "node_dst": "n4", - "start_ts": "2024-01-15 00:00:00.0", - "finish_ts": "2024-01-15 08:00:00.0", - "uniq_name": "cat_15", - "simplex_a": 35.0, - "simplex_b": 25.0, - "simplex_c": 40.0, - "short_text": "URGENT: your account needs verification, click the link immediately.", - "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", - "petal_length": 1.3, - "petal_width": 0.2, - "species": 0, - "csv_list": "a15", - "mixed_case": "ability", - "species_pred": 0, - "species_name": "setosa", - "species_name_pred": "setosa" - }, - { - "id": 8, - "name": "frank", - "score": 3.7, - "open": 8.0, - "high": 9.5, - "low": 7.0, - "close": 8.5, - "iso_country": "BRA", - "trade_date": "2024-01-08", - "pvalue": 0.2967032967032967, - "log2fc": 2.0999999999999996, - "comp_a": 4.0, - "comp_b": 2.0, - "comp_c": 3.0, - "uvec": -2.0, - "edge_pair": "[0, 8]", - "node_src": "n0", - "node_dst": "n1", - "start_ts": "2024-01-08 00:00:00.0", - "finish_ts": "2024-01-08 01:00:00.0", - "uniq_name": "cat_8", - "simplex_a": 20.0, - "simplex_b": 35.0, - "simplex_c": 45.0, - "short_text": "See you at lunch, save me a seat by the window.", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 5.2, - "petal_width": 1.85, - "species": 1, - "csv_list": "a8,b8", - "mixed_case": "ABSTRACT", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - }, - { - "id": 12, - "name": "alice", - "score": 5.1, - "open": 12.0, - "high": 13.5, - "low": 11.0, - "close": 12.5, - "iso_country": "CHN", - "trade_date": "2024-01-12", - "pvalue": 0.44455544455544455, - "log2fc": -2.8, - "comp_a": 3.0, - "comp_b": 6.0, - "comp_c": 1.0, - "uvec": 2.0, - "edge_pair": "[0, 12]", - "node_src": "n0", - "node_dst": "n1", - "start_ts": "2024-01-12 00:00:00.0", - "finish_ts": "2024-01-12 05:00:00.0", - "uniq_name": "cat_12", - "simplex_a": 20.0, - "simplex_b": 25.0, - "simplex_c": 55.0, - "short_text": "Congratulations! You have won a free prize, reply now to claim it.", - "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", - "petal_length": 3.9000000000000004, - "petal_width": 1.3, - "species": 1, - "csv_list": "a12,b12,c12", - "mixed_case": "505", - "species_pred": 1, - "species_name": "versicolor", - "species_name_pred": "versicolor" - } -] diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala deleted file mode 100644 index f1a37755c25..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator - -import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.distinct.DistinctOpDesc -import org.apache.texera.amber.operator.union.UnionOpDesc -import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlan} -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** The placeholder substitution, which is where an operator's generated code - * meets the variables the script actually binds. A variadic port is the case - * the numbered placeholders cannot state, so it is the case worth pinning. - */ -class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { - - private def upstream(id: String): LogicalOp = { - val op = new DistinctOpDesc - op.setOperatorId(id) - op - } - - /** `n` upstreams, all drawn into the union's single port, which is what a - * variadic port looks like in a plan. - */ - private def unionOf(n: Int): String = { - val union = new UnionOpDesc - union.setOperatorId("union") - val ups = (1 to n).map(i => upstream(s"up$i")) - val links = ups.map { up => - LogicalLink( - up.operatorIdentifier, - PortIdentity(0), - union.operatorIdentifier, - PortIdentity(0) - ) - } - new WorkflowToPythonTranslator().translate( - LogicalPlan(ups.toList :+ union, links.toList) - ) - } - - "WorkflowToPythonTranslator" should "hand a variadic port every upstream it was drawn" in { - unionOf(3) should include("pd.concat([df1, df2, df3], ignore_index=True)") - } - - it should "hand a variadic port a one-element list when only one link is drawn" in { - // The case the old fixed `[in1df, in2df]` got wrong in the other direction: - // it named a second frame the script never bound. - unionOf(1) should include("pd.concat([df1], ignore_index=True)") - } - - it should "leave no placeholder behind for a variadic port" in { - unionOf(2) should not include "inAlldf" - } - - it should "still resolve a numbered placeholder against its own upstream" in { - // The variadic form is an addition, not a replacement: a chain of ordinary - // single-input operators has to keep reading `in1df` as its predecessor. - val first = upstream("first") - val second = upstream("second") - val script = new WorkflowToPythonTranslator().translate( - LogicalPlan( - List(first, second), - List( - LogicalLink( - first.operatorIdentifier, - PortIdentity(0), - second.operatorIdentifier, - PortIdentity(0) - ) - ) - ) - ) - script should include("df2 = df1.drop_duplicates(ignore_index=True)") - } - - /** The translator's own contract when it meets an operator it cannot render: - * a comment rather than a silently wrong line. - */ - it should "leave a TODO for an operator with no standalone code generator" in { - val op = new org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 - op.setOperatorId("udf") - val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) - script should include("# TODO:") - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala deleted file mode 100644 index a3e4f6a22bd..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} - -import java.sql.Timestamp -import scala.jdk.CollectionConverters._ - -/** - * The shared input dataset for auto-configured transform verification. - * Properties are deliberate (see CanonicalFixtureSpec): enough rows and - * partial port-0/port-1 overlap to defeat hash-coincidence false passes on - * set ops and joins, and the canonical value "1" present in some-but-not-all - * rows so ConfigGenerator-filled free-form predicates match a proper subset. - */ -object CanonicalFixture extends SharedFixture { - - // Columns are semantically named and type-correct so @SampleColumn-tagged or - // type-constrained fields can be filled with realistic input (a valid OHLC - // block, real ISO country codes, real dates) instead of a degenerate - // first-column pick. Ordering is deliberate: id/name/score lead so the - // first-column fallback AND the type-rule tier ("first column of a matching - // type") are unchanged for un-annotated fields — the domain-specific columns - // that follow are only reached via an explicit @SampleColumn. - val schema: Schema = new Schema( - new Attribute("id", AttributeType.INTEGER), - new Attribute("name", AttributeType.STRING), - new Attribute("score", AttributeType.DOUBLE), - new Attribute("open", AttributeType.DOUBLE), - new Attribute("high", AttributeType.DOUBLE), - new Attribute("low", AttributeType.DOUBLE), - new Attribute("close", AttributeType.DOUBLE), - new Attribute("iso_country", AttributeType.STRING), - new Attribute("trade_date", AttributeType.STRING), - // --- Domain-specific columns (reached only via @SampleColumn) --- - new Attribute("pvalue", AttributeType.DOUBLE), // strictly in (0,1): p-values - new Attribute("log2fc", AttributeType.DOUBLE), // signed, centered on 0: fold-change - new Attribute("comp_a", AttributeType.DOUBLE), // >0 ternary simplex component - new Attribute("comp_b", AttributeType.DOUBLE), // >0 ternary simplex component - new Attribute("comp_c", AttributeType.DOUBLE), // >0 ternary simplex component - new Attribute("uvec", AttributeType.DOUBLE), // any real: a 4th numeric (Quiver u/v) - new Attribute( - "edge_pair", - AttributeType.STRING - ), // "[parent, child]" literals, single-rooted tree - new Attribute("node_src", AttributeType.STRING), // edge source id (Sankey/Network) - new Attribute("node_dst", AttributeType.STRING), // edge target id, overlaps node_src (a DAG) - new Attribute( - "start_ts", - AttributeType.TIMESTAMP - ), // real timestamp; Gantt start / TimeSeries axis - new Attribute( - "finish_ts", - AttributeType.TIMESTAMP - ), // always > start_ts; Gantt finish (bar width) - new Attribute( - "uniq_name", - AttributeType.STRING - ), // distinct per row: Pie/name-keyed ops need no duplicates - new Attribute( - "simplex_a", - AttributeType.DOUBLE - ), // >0 and simplex_a+simplex_b+simplex_c == 100 (ternary-contour) - new Attribute("simplex_b", AttributeType.DOUBLE), // >0 simplex component summing to 100 - new Attribute("simplex_c", AttributeType.DOUBLE), // >0 simplex component summing to 100 - // ── text + iris-numeric columns for Hugging Face model operators ── - new Attribute( - "short_text", - AttributeType.STRING - ), // one sentence: sentiment / spam-detection input - new Attribute( - "long_text", - AttributeType.STRING - ), // a multi-sentence paragraph: summarization input - new Attribute("petal_length", AttributeType.DOUBLE), // iris petal length in cm (~1.3–6.5) - new Attribute("petal_width", AttributeType.DOUBLE), // iris petal width in cm (~0.2–2.4) - new Attribute( - "species", - AttributeType.INTEGER - ), // the 0/1 iris class, exactly `petal_length >= 3.9`: the label the sklearn - // families fit against, separable by the two petal columns above - new Attribute( - "csv_list", - AttributeType.STRING - ), // comma-delimited, 1–4 tokens per row: split/explode ops need real fan-out - new Attribute( - "mixed_case", - AttributeType.STRING - ), // a third lower-case, a third upper, a third letterless: a case flag has to - // change WHICH rows match, and on any other column it changes nothing - new Attribute( - "species_pred", - AttributeType.INTEGER - ), // a predictor's guess at `species`: the same 0/1 domain, wrong on a few rows. - // Scoring compares a PAIR of columns, and no single label column supplies one. - // Last so the first-unused fallback reaches it only after every other column - new Attribute("species_name", AttributeType.STRING), // `species` spelled out - new Attribute( - "species_name_pred", - AttributeType.STRING - ) // `species_pred` spelled out. A scorer takes a string label as readily as a - // numeric one and names the class after it rather than after its position, so - // the pair exists a second time in text - ) - - // ── Data source ── - // The rows are NOT generated at runtime — they live in a single, checked-in, - // human-readable JSON file that IS the source of truth: - // src/test/resources/verify/canonical_fixture.json (15 rows, ids 1..15) - // Open it to see the exact table; edit it to change the data. The - // CanonicalFixtureSpec invariants guard every semantic constraint (valid OHLC - // block, pvalue ∈ (0,1), ternary parts summing to 100, finish_ts > start_ts, - // etc.), so a hand-edit that breaks one fails the build. `schema` above stays - // authoritative for column types: JSON has no TIMESTAMP, so start_ts/finish_ts - // are stored as JDBC strings ("2024-01-01 00:00:00.0") and coerced back here. - private val fixtureResource = "/verify/canonical_fixture.json" - - override val allRows: Vector[Tuple] = { - val stream = Option(getClass.getResourceAsStream(fixtureResource)) - .getOrElse(sys.error(s"canonical fixture not found on classpath: $fixtureResource")) - val root = - try new ObjectMapper().readTree(stream) - finally stream.close() - root - .elements() - .asScala - .map { node => - val b = Tuple.builder(schema) - schema.getAttributes.foreach { attr => - val cell = node.get(attr.getName) - require(cell != null, s"fixture row missing column '${attr.getName}'") - val value: AnyRef = attr.getType match { - case AttributeType.INTEGER => Int.box(cell.asInt()) - case AttributeType.LONG => Long.box(cell.asLong()) - case AttributeType.DOUBLE => Double.box(cell.asDouble()) - case AttributeType.BOOLEAN => Boolean.box(cell.asBoolean()) - case AttributeType.TIMESTAMP => Timestamp.valueOf(cell.asText()) - case _ => cell.asText() // STRING - } - b.add(attr, value) - } - b.build() - } - .toVector - } - - // Each port takes two thirds of the table, from opposite ends, so the ports - // overlap by the remaining third and no row sits outside both. The overlap is - // what stops joins and set ops passing by hash coincidence: ports holding the - // same rows make intersect and union the same answer, and disjoint ports make - // both empty, which a broken operator produces too. Two thirds rather than a - // fixed count so a row added to the JSON widens the windows instead of falling - // off the end, and a port stays well under the whole table — these windows are - // what every per-test Python run is sized by. At 15 rows this is positions 0-9 - // and 5-14. Rows sit out of id order in the file, so the windows are - // positional — not id ranges. - private def windowSize: Int = allRows.size * 2 / 3 - def port0Rows: Seq[Tuple] = allRows.take(windowSize) - def port1Rows: Seq[Tuple] = allRows.takeRight(windowSize) - - override def rowsFor(port: Int): Seq[Tuple] = if (port == 0) port0Rows else port1Rows - - /** `id` keeps every value: it is what joins and set operations match on, and - * emptying it would change which rows pair up rather than what a null does. - */ - override val keepFilled: Set[String] = Set("id") - - /** This table as the sklearn families read it: the two petal columns and the - * `species` label, and nothing else, because `X = table.drop(target, axis=1)` - * hands `fit` every column that is not the target. The two features separate - * the classes exactly, so an estimator fits them without a tie to break. - */ - val sklearnNumeric: SharedFixture = ProjectedFixture( - this, - Seq("petal_length", "petal_width", "species"), - keepFilled = Set("species") - ) - - /** [[sklearnNumeric]] plus a column an estimator cannot fit. The families that - * narrow `X` to the fittable columns drop nothing on the numeric table, so the - * narrowing runs there with nothing to do. Here it has a column to drop, and - * the two paths narrow in different places: the operator once, ahead of the - * port branch; the standalone script once per port. Each has to drop it on its - * own. - * - * The text column carries no signal about the label, so the fit is the one the - * two petal columns give on their own. - */ - /** This table minus `score`. An operator whose output column is named `score` - * by default cannot run here otherwise: it would create a column the input - * already holds, and the schema refuses the duplicate before the operator - * runs. Dropping the one column puts the DEFAULT config under test, which is - * the config a user gets, rather than a hand-written name chosen to dodge the - * clash. - */ - val withoutScore: SharedFixture = ProjectedFixture( - this, - schema.getAttributeNames.filterNot(_ == "score"), - keepFilled = keepFilled - ) - - /** This table as a scorer reads it when the labels are text: the same pair as - * `species` / `species_pred`, spelled out, and nothing else. The scenario that - * takes it names the two columns itself, since the operator's `@SampleColumn`s - * name the numeric pair this projection does not carry. - */ - val scorerTextLabels: SharedFixture = ProjectedFixture( - this, - Seq("species_name", "species_name_pred"), - keepFilled = Set.empty - ) - - val sklearnNumericWithText: SharedFixture = ProjectedFixture( - this, - Seq("petal_length", "petal_width", "short_text", "species"), - keepFilled = Set("species") - ) - - /** This table as the `countVectorizer=true` path reads it: one text column and - * the same label. `short_text` leads because the model probe feeds a text - * pipeline the frame's first column as a Series. Every row carrying a given - * sentence carries the same `species` (an invariant of the table), so the - * vectorized classes separate exactly, as the numeric pair does. - */ - val sklearnText: SharedFixture = ProjectedFixture( - this, - Seq("short_text", "long_text", "species"), - keepFilled = Set("species") - ) -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala deleted file mode 100644 index aa471e30f1d..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.workflow.PortIdentity -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.nio.file.Files - -class CanonicalFixtureSpec extends AnyFlatSpec with Matchers { - - "CanonicalFixture" should "have at least 10 rows per port with partial id overlap" in { - CanonicalFixture.port0Rows.size should be >= 10 - CanonicalFixture.port1Rows.size should be >= 10 - val ids0 = CanonicalFixture.port0Rows.map(_.getField[Integer]("id")).toSet - val ids1 = CanonicalFixture.port1Rows.map(_.getField[Integer]("id")).toSet - (ids0 intersect ids1) should not be empty - (ids0 diff ids1) should not be empty - (ids1 diff ids0) should not be empty - } - - // The windows are a rule over the table's size rather than fixed indices, so - // that a row added to the JSON is read by a port instead of falling off the - // end. Fixed indices made that silent: the row was simply never fed to - // anything, and every assertion above still held. - it should "leave no row outside both ports" in { - val onAPort = (CanonicalFixture.port0Rows ++ CanonicalFixture.port1Rows) - .map(_.getField[Integer]("id")) - .toSet - onAPort shouldBe CanonicalFixture.allRows.map(_.getField[Integer]("id")).toSet - } - - it should "contain the canonical value \"1\" in some but not all name cells" in { - val names = CanonicalFixture.port0Rows.map(_.getField[String]("name")) - names.count(_ == "1") should be > 0 - names.count(_ == "1") should be < names.size - } - - it should "expose a valid OHLC block (high >= open/close >= low) for candlestick-style ops" in { - CanonicalFixture.port0Rows.foreach { t => - val o = t.getField[java.lang.Double]("open").doubleValue - val h = t.getField[java.lang.Double]("high").doubleValue - val l = t.getField[java.lang.Double]("low").doubleValue - val c = t.getField[java.lang.Double]("close").doubleValue - h should be >= math.max(o, c) - l should be <= math.min(o, c) - } - } - - it should "keep pvalue strictly inside (0, 1) for probability-domain fields" in { - CanonicalFixture.port0Rows.foreach { t => - val p = t.getField[java.lang.Double]("pvalue").doubleValue - p should (be > 0.0 and be < 1.0) - } - } - - it should "keep log2fc signed and centered (both a negative and a positive present)" in { - val vals = CanonicalFixture.port0Rows.map(_.getField[java.lang.Double]("log2fc").doubleValue) - vals.min should be < 0.0 - vals.max should be > 0.0 - } - - it should "keep ternary components strictly positive" in { - CanonicalFixture.port0Rows.foreach { t => - t.getField[java.lang.Double]("comp_a").doubleValue should be > 0.0 - t.getField[java.lang.Double]("comp_b").doubleValue should be > 0.0 - t.getField[java.lang.Double]("comp_c").doubleValue should be > 0.0 - } - } - - it should "expose uniq_name as globally distinct so name-keyed ops have no duplicates" in { - val names = CanonicalFixture.port0Rows.map(_.getField[String]("uniq_name")) - names.distinct.size shouldBe names.size - } - - it should "expose a valid ternary simplex (positive parts summing to 100)" in { - CanonicalFixture.port0Rows.foreach { t => - val a = t.getField[java.lang.Double]("simplex_a").doubleValue - val b = t.getField[java.lang.Double]("simplex_b").doubleValue - val c = t.getField[java.lang.Double]("simplex_c").doubleValue - a should be > 0.0 - b should be > 0.0 - c should be > 0.0 - (a + b + c) shouldBe 100.0 +- 1e-9 - } - } - - it should "expose trade_date as a real ISO-8601 date (parseable, not the old day-N)" in { - CanonicalFixture.port0Rows.foreach { t => - val d = t.getField[String]("trade_date") - noException should be thrownBy java.time.LocalDate.parse(d) - } - } - - it should "expose edge_pair as single-rooted 2-element list literals" in { - // Every cell is "[0, child]" → parses to a 2-list rooted at 0, so TreePlot - // builds one connected tree instead of an error page. - CanonicalFixture.port0Rows.foreach { t => - t.getField[String]("edge_pair") should fullyMatch regex """\[0, \d+\]""" - } - } - - it should "expose overlapping node_src/node_dst so graph ops have drawable edges" in { - val src = CanonicalFixture.port0Rows.map(_.getField[String]("node_src")).toSet - val dst = CanonicalFixture.port0Rows.map(_.getField[String]("node_dst")).toSet - (src intersect dst) should not be empty - } - - it should "expose finish_ts strictly after start_ts (non-degenerate Gantt bar)" in { - CanonicalFixture.port0Rows.foreach { t => - val s = t.getField[java.sql.Timestamp]("start_ts") - val f = t.getField[java.sql.Timestamp]("finish_ts") - f.after(s) shouldBe true - } - } - - it should "round-trip TIMESTAMP columns losslessly through TupleIO (write then read)" in { - val root = Files.createTempDirectory("canonical-fixture-ts-") - val path = CanonicalFixture.writeInputs(root, inputPortCount = 1)(PortIdentity(0)) - val schema = TupleIO.readSchemaSidecar(path) - val rows = TupleIO.readTuples(path, schema).toList - rows should not be empty - val read = rows.head - val orig = CanonicalFixture.port0Rows.head - // The JDBC-string codec is the exact inverse of Timestamp.toString, so the - // value read back equals the value written — no timezone drift. - read.getField[java.sql.Timestamp]("start_ts") shouldBe orig.getField[java.sql.Timestamp]( - "start_ts" - ) - read.getField[java.sql.Timestamp]("finish_ts") shouldBe orig.getField[java.sql.Timestamp]( - "finish_ts" - ) - } - - it should "expose non-empty short_text sentences for text-classification ops" in { - CanonicalFixture.port0Rows.foreach { t => - t.getField[String]("short_text").trim should not be empty - } - } - - it should "expose long_text with several sentences so summarization is non-trivial" in { - CanonicalFixture.port0Rows.foreach { t => - val txt = t.getField[String]("long_text") - // multiple sentence-terminating periods → real content to condense - txt.count(_ == '.') should be >= 2 - } - } - - // The sklearn families fit `species` against the petal columns, so the table - // has to hold up as training data. The estimators that cross-validate pass no - // fold count and so take sklearn's default of five, and a class of fewer than - // five rows leaves a fold holding none of it — no error, just a warning and a - // fold that asks nothing. A label the features cannot separate is the other - // half: it leaves the fit breaking ties, which is where two paths drift apart. - it should "expose species as a petal-separable label with enough members to fold on" in { - val rows = CanonicalFixture.allRows - val byClass = rows.groupBy(_.getField[java.lang.Integer]("species").intValue) - byClass.keySet shouldBe Set(0, 1) - byClass.values.foreach(_.size should be >= 5) - rows.foreach { t => - val large = t.getField[java.lang.Double]("petal_length").doubleValue >= 3.9 - t.getField[java.lang.Integer]("species").intValue shouldBe (if (large) 1 else 0) - } - } - - // `species_pred` exists so the scorer has a real pair to compare. All four - // cells of the confusion matrix have to be occupied: a perfect prediction - // scores every metric at 1.0, and one that never calls a class leaves that - // class's precision undefined — either way the metrics stop telling the two - // code paths apart. - it should "hold a species_pred that is right on most rows and wrong on some" in { - val cells = CanonicalFixture.allRows - .map(t => - ( - t.getField[java.lang.Integer]("species").intValue, - t.getField[java.lang.Integer]("species_pred").intValue - ) - ) - .distinct - cells should contain theSameElementsAs Seq((0, 0), (0, 1), (1, 0), (1, 1)) - } - - // The text pair exists to run the scorer's string-label path on the same - // arrangement the numeric pair gives it. Spelling out a different prediction - // would make the two paths score differently for a reason that has nothing to - // do with the label being text. - it should "spell out the species pair without changing what it says" in { - val name = Map(0 -> "setosa", 1 -> "versicolor") - CanonicalFixture.allRows.foreach { t => - t.getField[String]("species_name") shouldBe name( - t.getField[java.lang.Integer]("species").intValue - ) - t.getField[String]("species_name_pred") shouldBe name( - t.getField[java.lang.Integer]("species_pred").intValue - ) - } - } - - // The countVectorizer=true path fits the same label on short_text alone, and a - // sentence appearing under both labels makes that set unlearnable. - it should "keep every short_text sentence inside one species" in { - CanonicalFixture.allRows - .groupBy(_.getField[String]("short_text")) - .foreach { - case (sentence, rows) => - withClue(s"$sentence: ") { - rows.map(_.getField[java.lang.Integer]("species")).distinct.size shouldBe 1 - } - } - } - - it should "expose iris petal columns in a realistic centimetre range" in { - CanonicalFixture.port0Rows.foreach { t => - val len = t.getField[java.lang.Double]("petal_length").doubleValue - val wid = t.getField[java.lang.Double]("petal_width").doubleValue - len should (be > 0.0 and be < 8.0) - wid should (be > 0.0 and be < 3.0) - } - } - - // A single-token row would make split/explode a no-op, so both windows need - // rows that fan out AND a row that doesn't — the two branches of an unnest. - it should "expose csv_list as a clean delimited list with varying token counts" in { - Seq(CanonicalFixture.port0Rows, CanonicalFixture.port1Rows).foreach { rows => - val tokenCounts = rows.map { t => - val raw = t.getField[String]("csv_list") - raw should not startWith "," - raw should not endWith "," - val tokens = raw.split(",", -1) - tokens.foreach(_.trim should not be empty) - tokens.length - } - tokenCounts.min shouldBe 1 - tokenCounts.max should be > 1 - } - } - - // A case flag is only worth sweeping where flipping it changes WHICH rows match, - // which needs rows of all three kinds in EVERY window a test reads — hence the - // per-port check rather than one over the whole table. - it should "expose mixed_case with lower, upper and letterless rows on every port" in { - Seq(CanonicalFixture.port0Rows, CanonicalFixture.port1Rows).foreach { rows => - val values = rows.map(_.getField[String]("mixed_case")) - values.count(v => v.exists(_.isLower)) should be > 0 - values.count(v => v.exists(_.isUpper) && !v.exists(_.isLower)) should be > 0 - values.count(v => !v.exists(_.isLetter)) should be > 0 - } - } - - it should "write one JSONL fixture per requested input port" in { - val root = Files.createTempDirectory("canonical-fixture-") - val inputs = CanonicalFixture.writeInputs(root, inputPortCount = 2) - inputs.keySet shouldBe Set(PortIdentity(0), PortIdentity(1)) - inputs.values.foreach(p => Files.size(p) should be > 0L) - } - - it should "reject unsupported port counts" in { - val root = Files.createTempDirectory("canonical-fixture-") - an[IllegalArgumentException] should be thrownBy - CanonicalFixture.writeInputs(root, inputPortCount = 3) - } - - it should "empty every column but id exactly once in the gapped table" in { - val rows = CanonicalFixture.emptyOneCellPerColumn(CanonicalFixture.port0Rows) - rows.size shouldBe CanonicalFixture.port0Rows.size - - CanonicalFixture.schema.getAttributes.foreach { attr => - val empties = rows.count(_.getField[AnyRef](attr.getName) == null) - // id carries the joins, so it keeps every value; everything else gets one - // hole, which is what makes the case a null case rather than an empty table. - if (attr.getName == "id") empties shouldBe 0 - else empties shouldBe 1 - } - } - - it should "leave every row in the gapped table with something in it" in { - val names = CanonicalFixture.schema.getAttributes.map(_.getName) - CanonicalFixture.emptyOneCellPerColumn(CanonicalFixture.port0Rows).foreach { t => - // A wholly empty row would test the operator's handling of an empty table - // instead, and would say nothing about a null beside a filled neighbour. - names.count(n => t.getField[AnyRef](n) != null) should be > 0 - } - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala deleted file mode 100644 index 17926e593b2..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/Comparator.scala +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.apache.texera.amber.util.python.PythonWorkerPool - -import java.nio.file.{Files, Path, StandardCopyOption} -import scala.collection.mutable.ArrayBuffer -import scala.sys.process._ - -/** - * Runs the Python comparator (`compare.py`) on two JSONL files emitted by - * [[OpExecHarness]] (actual) and [[StandaloneRunner]] (expected). The - * comparator uses `pandas.testing.assert_frame_equal` with `check_like=True` - * and `check_dtype=False` so row/column-order differences and the - * pandas-int64/float64 coercion that happens when JSONL round-trips through - * `pd.read_json` don't trigger false negatives. Float tolerance: `rtol=1e-5`. - * - * Throws [[ComparatorMismatchException]] on any non-zero exit code (the - * pandas diff is in `stderr` on the exception). Successful comparisons - * return unit. - * - * Python resolution mirrors [[StandaloneRunner.resolvePython]]: - * `UDF_PYTHON_PATH` env var first, else `python3.12` on PATH. - */ -object Comparator extends LazyLogging { - - // Resource path is absolute (leading slash) so getResourceAsStream resolves - // against the classpath root regardless of caller's package. - private val ScriptResourcePath = "/python/compare.py" - - def assertEqual( - actual: Path, - expected: Path, - orderSensitive: Boolean = true, - ignoreColumns: Seq[String] = Seq.empty, - modelColumns: Seq[String] = Seq.empty, - probePath: Option[Path] = None, - pythonExe: String = resolvePython() - ): Unit = { - val (exit, stdout, stderr) = - compare(actual, expected, orderSensitive, ignoreColumns, modelColumns, probePath, pythonExe) - if (exit != 0) { - throw new ComparatorMismatchException( - actual = actual, - expected = expected, - exitCode = exit, - stdout = stdout, - stderr = stderr - ) - } - } - - // Prefer a pooled persistent worker (imports pandas once via `compare.py - // --serve`, so the ~214 ms import isn't repaid per comparison — the diff - // itself is ~ms). A rare hard worker crash falls back to the one-shot CLI so - // behavior is never worse than the original path. Both invoke the same - // `_run_comparison`, so results are identical. - private def compare( - actual: Path, - expected: Path, - orderSensitive: Boolean, - ignoreColumns: Seq[String], - modelColumns: Seq[String], - probePath: Option[Path], - pythonExe: String - ): (Int, String, String) = { - if (PythonWorkerPool.enabled) { - try { - val req = objectMapper.createObjectNode() - req.put("actual", actual.toString) - req.put("expected", expected.toString) - req.put("unordered", !orderSensitive) - val ignoreArr = req.putArray("ignoreCols") - ignoreColumns.foreach(ignoreArr.add) - val modelArr = req.putArray("modelCols") - modelColumns.foreach(modelArr.add) - // --probe only applies with --model-cols (mirrors the CLI's guard). - probePath.filter(_ => modelColumns.nonEmpty) match { - case Some(p) => req.put("probe", p.toString) - case None => req.putNull("probe") - } - val o = PythonWorkerPool.run(ScriptResourcePath, Seq("--serve"), pythonExe, req) - return (o.exit, o.stdout, o.stderr) - } catch { - case e: PythonWorkerPool.WorkerDiedException => - logger.warn( - s"Comparator worker unavailable; falling back to one-shot CLI: ${e.getMessage}" - ) - } - } - runCli(actual, expected, orderSensitive, ignoreColumns, modelColumns, probePath, pythonExe) - } - - // Original one-subprocess-per-comparison CLI path. Retained as the fallback - // and as the behavior selected by TEXERA_TEST_PYTHON_WORKER=0. - private def runCli( - actual: Path, - expected: Path, - orderSensitive: Boolean, - ignoreColumns: Seq[String], - modelColumns: Seq[String], - probePath: Option[Path], - pythonExe: String - ): (Int, String, String) = { - val scriptPath = extractScript() - val outBuf = ArrayBuffer.empty[String] - val errBuf = ArrayBuffer.empty[String] - val procLogger = ProcessLogger(line => outBuf += line, line => errBuf += line) - // --unordered tells compare.py to lex-sort both DataFrames by all columns - // before assert_frame_equal — needed for set-semantics ops whose JVM - // emission order doesn't match the pandas equivalent. Default stays - // positional so deterministic-order ops still catch row-order regressions. - // --ignore-cols drops opaque columns whose value isn't compared. - // --model-cols + --probe compare a model column by behavior: unpickle both - // sides and assert their predictions on the probe feature set match (two - // independently-trained models are functionally equal but not bit-equal). - val baseArgs = Seq(pythonExe, scriptPath.toString) - val flagArgs = if (!orderSensitive) Seq("--unordered") else Seq.empty - val ignoreArgs = - if (ignoreColumns.nonEmpty) Seq("--ignore-cols", ignoreColumns.mkString(",")) else Seq.empty - val modelArgs = - if (modelColumns.nonEmpty) Seq("--model-cols", modelColumns.mkString(",")) else Seq.empty - val probeArgs = - probePath - .filter(_ => modelColumns.nonEmpty) - .map(p => Seq("--probe", p.toString)) - .getOrElse(Seq.empty) - val cmd = - baseArgs ++ flagArgs ++ ignoreArgs ++ modelArgs ++ probeArgs ++ Seq( - actual.toString, - expected.toString - ) - val exit = Process(cmd).!(procLogger) - (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) - } - - // Resources may live inside a jar at runtime; copy to a temp file so Python - // can exec it. deleteOnExit so test runs don't accumulate /tmp clutter. - private def extractScript(): Path = { - val stream = getClass.getResourceAsStream(ScriptResourcePath) - require( - stream != null, - s"compare.py not found on classpath at $ScriptResourcePath" - ) - try { - val tmp = Files.createTempFile("compare-", ".py") - Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING) - tmp.toFile.deleteOnExit() - tmp - } finally stream.close() - } - - private def resolvePython(): String = - sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") -} - -final class ComparatorMismatchException( - val actual: Path, - val expected: Path, - val exitCode: Int, - val stdout: String, - val stderr: String -) extends RuntimeException( - s"""DataFrame mismatch (compare.py exit $exitCode): - | actual: $actual - | expected: $expected - |--- stderr --- - |$stderr""".stripMargin - ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala deleted file mode 100644 index dc5e5fda029..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} -import org.apache.texera.amber.translator.verify.tags.IntegrationTest -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.nio.file.{Files, Path} - -// Tagged @IntegrationTest: Comparator.assertEqual shells out to compare.py, so -// this spec needs Python and must run in the Python-provisioned integration job. -@IntegrationTest -class ComparatorSpec extends AnyFlatSpec with Matchers { - - private val schema: Schema = Schema() - .add(new Attribute("id", AttributeType.INTEGER)) - .add(new Attribute("name", AttributeType.STRING)) - - private val idAttr = new Attribute("id", AttributeType.INTEGER) - private val nameAttr = new Attribute("name", AttributeType.STRING) - - private def row(id: Int, name: String): Tuple = - Tuple - .builder(schema) - .add(idAttr, Int.box(id)) - .add(nameAttr, name) - .build() - - private def writeJsonl(dir: Path, name: String, rows: Seq[Tuple]): Path = { - val p = dir.resolve(name) - TupleIO.writeTuples(p, rows.iterator, schema) - p - } - - "Comparator.assertEqual" should "pass when JSONL files contain identical rows" in { - val dir = Files.createTempDirectory("comparator-spec-equal-") - val rows = Seq(row(1, "alice"), row(2, "bob")) - val a = writeJsonl(dir, "a.jsonl", rows) - val b = writeJsonl(dir, "b.jsonl", rows) - noException should be thrownBy Comparator.assertEqual(a, b) - } - - it should "throw ComparatorMismatchException when JSONL files differ" in { - val dir = Files.createTempDirectory("comparator-spec-diff-") - val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) - val b = writeJsonl(dir, "b.jsonl", Seq(row(1, "alice"), row(2, "carol"))) - intercept[ComparatorMismatchException] { - Comparator.assertEqual(a, b) - } - } - - it should "treat row-reordered files as unequal under positional comparison" in { - val dir = Files.createTempDirectory("comparator-spec-reorder-strict-") - val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) - val b = writeJsonl(dir, "b.jsonl", Seq(row(2, "bob"), row(1, "alice"))) - intercept[ComparatorMismatchException] { - Comparator.assertEqual(a, b) - } - } - - it should "treat row-reordered files as equal under orderSensitive=false" in { - val dir = Files.createTempDirectory("comparator-spec-reorder-loose-") - val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) - val b = writeJsonl(dir, "b.jsonl", Seq(row(2, "bob"), row(1, "alice"))) - noException should be thrownBy Comparator.assertEqual(a, b, orderSensitive = false) - } - - it should "still report value mismatches under orderSensitive=false" in { - // orderSensitive=false relaxes row ORDER, not row CONTENT — a genuinely - // different cell must still fail. - val dir = Files.createTempDirectory("comparator-spec-reorder-content-diff-") - val a = writeJsonl(dir, "a.jsonl", Seq(row(1, "alice"), row(2, "bob"))) - val b = writeJsonl(dir, "b.jsonl", Seq(row(2, "bob"), row(1, "carol"))) - intercept[ComparatorMismatchException] { - Comparator.assertEqual(a, b, orderSensitive = false) - } - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala deleted file mode 100644 index a2a55e4235a..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigCoverageSpec.scala +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Reports the harness verification tier per discovered [[StandaloneCodeGenerator]] - * operator: RUNNABLE (auto) / RUNNABLE (curated) / FLAG (reason). The `info` - * table printed by this spec is the coverage artifact shown at the handoff demo. - * - * Hard-asserts the must-run set: the operators that must appear as RUNNABLE for - * the demo to be credible. Flagged operators are always reported with a reason — - * never silently passed, and neither is a single withheld KIND of run - * (see [[reportWithheldRuns]]). - */ -class ConfigCoverageSpec extends AnyFlatSpec with Matchers { - - // The ops the demo must show as runnable; extend as triage flips flags. - private val mustRun = Set( - "IntersectOpDesc", - "DifferenceOpDesc", - "SymmetricDifferenceOpDesc", - "HashJoinOpDesc", - "SpecializedFilterOpDesc", - "SortOpDesc", - "LimitOpDesc" - ) - - "the harness" should "classify every discovered operator into a tier and report coverage" in { - val operators = OperatorBehaviorSpec.discoverStandaloneOperators() - - val rows = operators.map { opClass => - val name = opClass.getSimpleName - val kind = - if (classOf[SourceOperatorDescriptor].isAssignableFrom(opClass)) "source" - else if (classOf[PythonOperatorDescriptor].isAssignableFrom(opClass)) "python-udf" - else "jvm" - val tier = - if (kind == "source") { - if (SourceCategoryRunner.canRun(opClass)) - s"RUNNABLE (${SourceCategoryRunner.tier(opClass)})" - else s"FLAG (${SourceCategoryRunner.flagReason(opClass)})" - } else - TransformVerificationRunner.disposition(opClass) match { - case TransformVerificationRunner.Runnable(t) => s"RUNNABLE ($t)" - case TransformVerificationRunner.Flagged(reason) => s"FLAG ($reason)" - } - (name, kind, tier) - } - - val runnable = rows.count(_._3.startsWith("RUNNABLE")) - info(s"Coverage: $runnable/${rows.size} operators runnable, ${rows.size - runnable} flagged") - Seq("jvm", "python-udf", "source").foreach { k => - val of = rows.filter(_._2 == k) - info(s" $k: ${of.count(_._3.startsWith("RUNNABLE"))}/${of.size} runnable") - } - rows.sortBy { case (n, k, t) => (!t.startsWith("RUNNABLE"), k, n) }.foreach { - case (name, kind, tier) => info(f" $tier%-50s [$kind%-10s] $name") - } - - reportWithheldRuns(operators) - - val failedTargets = rows.collect { - case (name, _, tier) if mustRun.contains(name) && !tier.startsWith("RUNNABLE") => - s"$name → $tier" - } - withClue(s"must-run operators not runnable: $failedTargets") { - failedTargets shouldBe empty - } - } - - /** RUNNABLE is per operator, but a runnable operator can still be missing one - * KIND of run. Report those too, or the table reads as fuller than it is. - * - * Split by whether anyone should be waiting: a pending fix is a line someone - * deletes when an issue closes, by design is an answer. Both name the operator, - * so a family entry expands to the estimators it actually covers. - */ - private def reportWithheldRuns(operators: Seq[Class[_ <: LogicalOp]]): Unit = { - import TransformVerificationRunner._ - - val withheld = for { - opClass <- operators - (kind, reason) <- withheldRunsFor(opClass) - } yield (opClass.getSimpleName, kind, reason) - - val pending = withheld.collect { case (n, k, PendingFix(issue)) => (n, k, issue) } - val byDesign = withheld.collect { case (n, k, ByDesign(why)) => (n, k, why) } - info( - s"Runs withheld: ${pending.size} pending a fix, " + - s"${byDesign.size} not applicable by design" - ) - pending.sorted.foreach { - case (name, kind, issue) => info(f" PENDING $kind%-20s $name%-45s $issue") - } - byDesign.sorted.foreach { - case (name, kind, why) => info(f" BY-DESIGN $kind%-20s $name%-45s $why") - } - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala deleted file mode 100644 index d538525266b..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} -import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc -import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc -import org.apache.texera.amber.operator.intersect.IntersectOpDesc -import org.apache.texera.amber.operator.visualization.histogram2d.Histogram2DOpDesc -import org.apache.texera.amber.operator.visualization.radarChart.RadarChartOpDesc -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Drives [[ConfigGenerator]] across the range of operator config shapes: - * no-config, flat autofill + enum, and a nested list of objects with a - * free-form value. These are the cases the reflective generator must handle to - * cover the JVM-exec operators automatically. - */ -class ConfigGeneratorSpec extends AnyFlatSpec with Matchers { - - private val schema = new Schema( - new Attribute("id", AttributeType.INTEGER), - new Attribute("name", AttributeType.STRING), - new Attribute("score", AttributeType.DOUBLE) - ) - private val twoPorts = Map(0 -> schema, 1 -> schema) - - "ConfigGenerator" should "configure an operator that has no config fields" in { - val result = ConfigGenerator.generate(classOf[IntersectOpDesc], twoPorts) - withClue(result) { result.isRight shouldBe true } - result.toOption.get shouldBe a[IntersectOpDesc] - } - - it should "fill autofill column refs from the correct port and default the enum" in { - val result = ConfigGenerator.generate(classOf[HashJoinOpDesc[Any]], twoPorts) - withClue(result) { result.isRight shouldBe true } - val op = result.toOption.get.asInstanceOf[HashJoinOpDesc[Any]] - schema.getAttributeNames should contain(op.buildAttributeName) - schema.getAttributeNames should contain(op.probeAttributeName) - op.joinType should not be null - } - - it should "build a non-empty nested predicate list with a valid column and enum" in { - val result = ConfigGenerator.generate(classOf[SpecializedFilterOpDesc], Map(0 -> schema)) - result.isRight shouldBe true - val op = result.toOption.get.asInstanceOf[SpecializedFilterOpDesc] - op.predicates should not be empty - val p = op.predicates.head - schema.getAttributeNames should contain(p.attribute) - p.condition should not be null - } - - // ── semantic column resolution (the @SampleColumn / attributeTypeRules tiers) ── - - it should "assign distinct columns to sibling autofill fields (no x = y collapse)" in { - // Histogram2D's xColumn and yColumn are both plain @AutofillAttributeName - // with no type-rule; before distinct-aware binding both resolved to the - // first column ("id"). They must now differ so the plot isn't a degenerate - // diagonal. - val result = - ConfigGenerator.generate(classOf[Histogram2DOpDesc], CanonicalFixture.schemasByPort) - withClue(result) { result.isRight shouldBe true } - val op = result.toOption.get.asInstanceOf[Histogram2DOpDesc] - op.xColumn.toString should not be empty - op.xColumn.toString should not be op.yColumn.toString - } - - it should "keep a list knob off the column a single-column sibling took" in { - // Radar Chart picks its name column first and its value columns took the whole - // table, so the name arrived inside them and the generated `required_cols`, - // which is the name followed by the values, named it twice. More than one value - // column left, or the list runs the same code a single column would. - val result = - ConfigGenerator.generate(classOf[RadarChartOpDesc], CanonicalFixture.schemasByPort) - withClue(result) { result.isRight shouldBe true } - val op = result.toOption.get.asInstanceOf[RadarChartOpDesc] - op.valueColumns.map(_.toString) should not contain op.nameColumn.toString - op.valueColumns.size should be > 1 - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala deleted file mode 100644 index 52ecf4140c8..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CuratedHandlers.scala +++ /dev/null @@ -1,666 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} -import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.distinct.DistinctOpDesc -import org.apache.texera.amber.operator.aggregate.{ - AggregateOpDesc, - AggregationFunction, - AggregationOperation -} -import org.apache.texera.amber.operator.filter.{ - ComparisonType, - FilterPredicate, - SpecializedFilterOpDesc -} -import org.apache.texera.amber.operator.hashJoin.{HashJoinOpDesc, JoinType} -import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc -import org.apache.texera.amber.operator.projection.{AttributeUnit, ProjectionOpDesc} -import org.apache.texera.amber.operator.regex.RegexOpDesc -import org.apache.texera.amber.operator.typecasting.{TypeCastingOpDesc, TypeCastingUnit} -import org.apache.texera.amber.operator.visualization.ImageViz.ImageVisualizerOpDesc - -import org.apache.texera.amber.operator.visualization.dumbbellPlot.{ - DumbbellDotConfig, - DumbbellPlotOpDesc -} -import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnLinearRegressionOpDesc -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor -import org.apache.texera.amber.operator.ifStatement.IfOpDesc -import java.nio.file.{Files, Path} -import java.util - -/** - * A curated handler ships a configured OpDesc and the input fixtures it - * needs, written once into `testRoot`. Register it in [[CuratedHandlers.all]] - * to override the auto-config tier for that operator. - */ -trait TransformHandler { - def opDescClass: Class[_ <: LogicalOp] - def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) - - /** Extra independent scenarios beyond [[fixture]], each a self-contained - * (label, configured op, its own inputs). The runner runs each as a PINNED - * config (no enum sweep), in its own work subdir. Default: none. - * - * Used where one operator needs structurally different inputs per config - * branch that a single swept fixture can't cover — e.g. the sklearn - * `countVectorizer=true` text path, whose feature column must be text and so - * is incompatible with the numeric default fixture (`X = table.drop(target)` - * would feed a string column to a numeric estimator). Each scenario must - * write its input files somewhere unique (e.g. a `testRoot` subdir) so it - * does not clobber the primary fixture's files. - */ - def extraScenarios(testRoot: Path): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = - Seq.empty - - /** Opts this fixture into the `nulls` case, naming the columns it must never - * empty because their VALUE is what the fixture was built to arrange rather - * than data under test: a join key that has to pair, a grouping key that has - * to group. Emptying one of those changes what the test asks instead of asking - * what the operator does with a null. - * - * The default is `Some(Set.empty)`: most curated tables arrange nothing that a - * hole would disturb, so taking part is the normal case and a fixture that - * cannot afford a hole says so. `None` sits the case out entirely, for a table - * whose every column is load-bearing. - */ - def nullsKeepFilled: Option[Set[String]] = Some(Set.empty) -} - -/** - * The curated override tier of the config/fixture resolution chain: an - * operator listed here is verified with its hand-written fixture instead of - * the auto-generated one. This is also the seam where Xuan's curated - * operator-field-values JSON plugs in later, as a second curated source. - */ -object CuratedHandlers { - - /** Concrete `LogicalOp` classes discovered from the `@JsonSubTypes` registry - * on [[LogicalOp]] — the same source [[ConfigGenerator]] enumerates. The - * sklearn handler families below are auto-derived from this list, so a newly - * registered sklearn estimator is picked up with zero per-operator - * boilerplate here. - */ - private val registeredOps: Seq[Class[_ <: LogicalOp]] = - Option(classOf[LogicalOp].getAnnotation(classOf[com.fasterxml.jackson.annotation.JsonSubTypes])) - .map(_.value().toSeq.map(_.value().asInstanceOf[Class[_ <: LogicalOp]])) - .getOrElse(Seq.empty) - - private def isConcrete(cls: Class[_]): Boolean = - !java.lang.reflect.Modifier.isAbstract(cls.getModifiers) - - /** The concrete leaf ops under one sklearn base, excluding the base itself. - * - * No hard-coded baseline: a new sklearn operator is picked up automatically - * the moment it is registered in LogicalOp's @JsonSubTypes — zero per-op code - * here. The test suite (ConfigCoverageSpec / TransformVerificationRunnerSpec) - * is the safety net: a mis-discovered or misbehaving op fails its own parity - * check rather than being frozen by an assertion. - */ - private def sklearnFamily(base: Class[_]): Seq[Class[_ <: LogicalOp]] = - registeredOps.filter(c => base.isAssignableFrom(c) && c != base && isConcrete(c)) - - private def trainingOps = sklearnFamily(classOf[SklearnTrainingOpDesc]) - private def classifierOps = sklearnFamily(classOf[SklearnClassifierOpDesc]) - private def advancedOps = sklearnFamily(classOf[SklearnMLOperatorDescriptor[_]]) - - /** Every sklearn op, whichever tier serves it. `X = table.drop(target)` feeds - * each remaining column to `fit`, so these take canonical's petal-and-label - * projection rather than the whole table, whose string columns end the fit. - * - * Linear Regression is named on its own because it descends from - * `PythonOperatorDescriptor` directly rather than from one of the three - * bases, so no family picks it up. - */ - val sklearnNumericClasses: Set[Class[_ <: LogicalOp]] = - (trainingOps ++ classifierOps ++ advancedOps).toSet + classOf[SklearnLinearRegressionOpDesc] - - val all: Seq[TransformHandler] = Seq( - AggregateTransformHandler, - SpecializedFilterTransformHandler, - DistinctTransformHandler, - ProjectionTransformHandler, - HashJoinTransformHandler, - TypeCastingTransformHandler, - KeywordSearchTransformHandler, - DumbbellPlotVisualizationHandler, - ImageVisualizerVisualizationHandler, - IfTransformHandler, - RegexTransformHandler - ) - - val byClass: Map[Class[_ <: LogicalOp], TransformHandler] = - all.map(h => h.opDescClass -> h).toMap - - /** Generic fixture writer: builds a JSONL file with the given typed columns - * and rows, boxing each value per its declared [[AttributeType]]. Lets a - * curated handler declare bespoke per-operator input data in one call - * instead of hand-rolling a Schema + Tuple.builder loop. - */ - def writeFixture( - path: Path, - columns: Seq[(String, AttributeType)], - rows: Seq[Seq[Any]] - ): Path = { - val schema = new Schema(columns.map { case (n, t) => new Attribute(n, t) }: _*) - val tuples = rows.map { row => - val builder = Tuple.builder(schema) - columns.zip(row).foreach { - case ((name, attrType), value) => - val boxed: AnyRef = (attrType, value) match { - case (_, null) => null - case (AttributeType.INTEGER, x: Int) => Int.box(x) - case (AttributeType.INTEGER, x: Long) => Int.box(x.toInt) - case (AttributeType.INTEGER, x: Double) => Int.box(x.toInt) - case (AttributeType.LONG, x: Long) => Long.box(x) - case (AttributeType.LONG, x: Int) => Long.box(x.toLong) - case (AttributeType.DOUBLE, x: Double) => Double.box(x) - case (AttributeType.DOUBLE, x: Int) => Double.box(x.toDouble) - case (AttributeType.DOUBLE, x: Long) => Double.box(x.toDouble) - case (AttributeType.BOOLEAN, x: Boolean) => Boolean.box(x) - case (AttributeType.STRING, x) => x.toString - case (_, x) => x.toString - } - builder.add(schema.getAttribute(name), boxed) - } - builder.build() - } - TupleIO.writeTuples(path, tuples.iterator, schema) - path - } - -} - -/** - * Handler for `SpecializedFilterOpDesc`. Curated CONFIG over the shared - * canonical fixture: the auto tier fills a free-form predicate `value` with - * the canonical "1", which pins the shape of the comparison but not its - * corners. `id > 8 OR name == "eve"` exercises numeric comparison, string - * equality (the JSON predicate `value` is always a string) and OR-combination - * in one run, and keeps 5 of port 0's 10 rows — a proper subset either way. - * - * Both JVM `SpecializedFilterOpExec` and pandas boolean indexing preserve - * input row order, so positional comparator equality holds. - */ -object SpecializedFilterTransformHandler extends TransformHandler { - - override val opDescClass: Class[_ <: LogicalOp] = classOf[SpecializedFilterOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val desc = new SpecializedFilterOpDesc() - desc.predicates = List( - new FilterPredicate("id", ComparisonType.GREATER_THAN, "8"), - new FilterPredicate("name", ComparisonType.EQUAL_TO, "eve") - ) - - (desc, CanonicalFixture.writeInputs(testRoot, 1)) - } -} - -/** Handler for `DistinctOpDesc`. The canonical auto-fixture is all-distinct - * (uniq_name is globally unique by invariant), so it never exercises dedup. - * This 5-row table repeats two rows so both paths must actually drop - * duplicates; survivors keep first-occurrence order (JVM LinkedHashSet == - * pandas drop_duplicates keep="first"), so the positional comparator holds. - */ -/** - * Curated handler for [[ProjectionOpDesc]]. Its `attributes` list is not declared - * `required`, so the auto tier starts it empty the way the UI does — and - * `getPhysicalOp` refuses an empty list. Pinning one row is all this needs; the - * runner derives the rest of the variants from it. - */ -object ProjectionTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[ProjectionOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val columns = Seq( - ("id", AttributeType.INTEGER), - ("name", AttributeType.STRING), - ("score", AttributeType.DOUBLE) - ) - val rows = Seq( - Seq[Any](1, "a", 1.5), - Seq[Any](2, "b", 2.5), - Seq[Any](3, "c", 3.5) - ) - val inputPath = - CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) - val op = new ProjectionOpDesc() - // A blank alias is the untouched state of the row the `+` button adds, and it is - // the branch where the operator keeps the original name. - op.attributes = List(new AttributeUnit("id", "")) - (op, Map(PortIdentity(0) -> inputPath)) - } -} - -object DistinctTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[DistinctOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val columns = Seq( - ("id", AttributeType.INTEGER), - ("name", AttributeType.STRING) - ) - val rows = Seq( - Seq[Any](1, "a"), - Seq[Any](2, "b"), - Seq[Any](1, "a"), // duplicate of row 0 - Seq[Any](3, "c"), - Seq[Any](2, "b") // duplicate of row 1 - ) - val inputPath = - CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) - (new DistinctOpDesc(), Map(PortIdentity(0) -> inputPath)) - } -} - -/** - * Curated handler for [[RegexOpDesc]]. The auto tier only ever feeds it the - * trivial pattern `"1"` against the first column, which never exercises real - * regex semantics. This handler pins genuine patterns so the JVM↔Python engine - * parity is actually tested: - * - * - Primary fixture: `[a-z]+` over a mixed-case `text` column. The runner - * enum-sweeps the Boolean `caseInsensitive`, so BOTH branches run against - * the same data. The two branches select DIFFERENT row sets (case-sensitive - * keeps only rows with a lowercase letter; case-insensitive also keeps the - * all-caps rows), proving the flag actually flows through to both paths. - * - `extraScenarios`: `\d+` (a backslash class — verifies the escape survives - * `toPyDoubleQuotedLiteral` into Python's engine) and `\.` (an escaped - * metachar — an escaping bug would turn it into "match any char" and change - * the result, so this pins literal-vs-metachar handling). - * - * All fixture data is ASCII, where Java `\d` / `[a-z]` / CASE_INSENSITIVE and - * Python's `re` agree exactly; each pattern yields a proper subset (never - * all/none) so the comparison is meaningful. - */ -object RegexTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[RegexOpDesc] - - private def regexOp(attribute: String, regex: String, caseInsensitive: Boolean): RegexOpDesc = { - val op = new RegexOpDesc() - op.attribute = attribute - op.regex = regex - op.caseInsensitive = caseInsensitive - op - } - - // Rows chosen so `[a-z]+` differs by case flag: "ABC"/"XY9" have no lowercase - // (dropped when case-sensitive) but are all-letter (kept when insensitive). - private val textColumn = Seq(("text", AttributeType.STRING)) - private val caseRows: Seq[Seq[Any]] = - Seq(Seq[Any]("abc"), Seq[Any]("ABC"), Seq[Any]("123"), Seq[Any]("a1B"), Seq[Any]("XY9")) - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val inputPath = - CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), textColumn, caseRows) - (regexOp("text", "[a-z]+", caseInsensitive = false), Map(PortIdentity(0) -> inputPath)) - } - - override def extraScenarios( - testRoot: Path - ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = { - // `\d+`: rows where a digit is present form a proper subset. - val digitDir = testRoot.resolve("digits") - Files.createDirectories(digitDir) - val digitRows: Seq[Seq[Any]] = - Seq(Seq[Any]("abc"), Seq[Any]("a1B"), Seq[Any]("XY9"), Seq[Any]("123"), Seq[Any]("ab")) - val digitInput = - CuratedHandlers.writeFixture(digitDir.resolve("input_port_0.jsonl"), textColumn, digitRows) - - // `\.`: only rows with a literal dot match. If the backslash were lost, the - // pattern would become bare `.` (match any char) and select every row. - val dotDir = testRoot.resolve("dot") - Files.createDirectories(dotDir) - val dotRows: Seq[Seq[Any]] = - Seq(Seq[Any]("a.b"), Seq[Any]("abc"), Seq[Any]("x.y.z"), Seq[Any]("no")) - val dotInput = - CuratedHandlers.writeFixture(dotDir.resolve("input_port_0.jsonl"), textColumn, dotRows) - - Seq( - ( - "regex=\\d+", - regexOp("text", "\\d+", caseInsensitive = false), - Map(PortIdentity(0) -> digitInput) - ), - ( - "regex=\\.", - regexOp("text", "\\.", caseInsensitive = false), - Map(PortIdentity(0) -> dotInput) - ) - ) - } -} - -/** HashJoin INNER on `id`. Build (port 0) and probe (port 1) intentionally - * arrive in different id orders so any probe-major / left-major mismatch - * between the JVM emit and `pd.merge` shows up. HashJoin inherits the - * unordered `LogicalOp.orderSensitive` default, so rows compare as a set. - */ -object HashJoinTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[HashJoinOpDesc[_]] - - /** `id` is what the two sides pair on: empty it and the rows stop matching, so - * the run would be asking about an inner join that finds nothing rather than - * about a null. The payload columns carry no arrangement and take the holes. - */ - override def nullsKeepFilled: Option[Set[String]] = Some(Set("id")) - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val buildSchema = new Schema( - new Attribute("id", AttributeType.INTEGER), - new Attribute("name", AttributeType.STRING) - ) - val probeSchema = new Schema( - new Attribute("id", AttributeType.INTEGER), - new Attribute("score", AttributeType.INTEGER) - ) - - def buildTup(id: Int, name: String): Tuple = { - val b = Tuple.builder(buildSchema) - b.add(buildSchema.getAttribute("id"), Int.box(id)) - b.add(buildSchema.getAttribute("name"), name) - b.build() - } - def probeTup(id: Int, score: Int): Tuple = { - val b = Tuple.builder(probeSchema) - b.add(probeSchema.getAttribute("id"), Int.box(id)) - b.add(probeSchema.getAttribute("score"), Int.box(score)) - b.build() - } - - val buildRows = Seq( - buildTup(3, "carol"), - buildTup(1, "alice"), - buildTup(5, "eve"), - buildTup(2, "bob"), - buildTup(4, "dave") - ) - val probeRows = Seq( - probeTup(1, 95), - probeTup(2, 80), - probeTup(3, 88), - probeTup(4, 72), - probeTup(5, 91) - ) - val buildPath = testRoot.resolve("input_port_0.jsonl") - val probePath = testRoot.resolve("input_port_1.jsonl") - TupleIO.writeTuples(buildPath, buildRows.iterator, buildSchema) - TupleIO.writeTuples(probePath, probeRows.iterator, probeSchema) - - val desc = new HashJoinOpDesc[Integer]() - desc.buildAttributeName = "id" - desc.probeAttributeName = "id" - desc.joinType = JoinType.INNER - - (desc, Map(PortIdentity(0) -> buildPath, PortIdentity(1) -> probePath)) - } -} - -/** - * Handler for `TypeCastingOpDesc`. The auto tier points `attribute` at the - * canonical fixture's first column (`id`, INTEGER) and then sweeps `resultType` - * across ALL `AttributeType` values — but `TypeCastingUnit`'s attributeTypeRules - * only permit certain source types per target (e.g. `timestamp` accepts only - * string/long), and the native `TypeCastingOpExec` throws on an illegal cast - * (INTEGER → Timestamp). So the auto variant `resultType=timestamp` crashes - * Path A before any comparison. - * - * This fixture gives each cast a type-compatible source column and a value that - * round-trips identically on both paths (JVM `AttributeTypeUtils` vs the - * generated pandas), covering the value-comparable branches of - * `generateStandaloneCode`'s `resultType` match: STRING, INTEGER, LONG, DOUBLE, - * BOOLEAN. The op has an `enumSweep` row in - * [[TransformVerificationRunner.variantsNotRun]], suppressing the blind - * one-enum-at-a-time sweep that would re-pair each fixed column with every target - * type; the units below already exercise each branch. Map op: both paths keep - * input row order, so strict positional equality holds. - * - * TIMESTAMP is intentionally omitted: the two runtimes serialize a Timestamp - * differently to JSONL (native emits an ISO string `"2024-01-01 09:00:00.0"`, - * pandas emits epoch millis `1704099600000`), so the dataframe comparator flags - * a representation mismatch even though the instant is identical — a harness-wide - * timestamp-serialization gap, not a TypeCasting translation defect. - */ -object TypeCastingTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[TypeCastingOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - // One dedicated source column per target so the casts don't chain. - val columns = Seq( - ("str_to_int", AttributeType.STRING), // numeric string → INTEGER - ("int_to_dbl", AttributeType.INTEGER), // integer → DOUBLE - ("int_to_str", AttributeType.INTEGER), // integer → STRING - ("int_to_lng", AttributeType.INTEGER), // integer → LONG - ("int_to_bool", AttributeType.INTEGER) // 1/0 → BOOLEAN - ) - val rows = Seq( - Seq[Any]("10", 1, 6, 11, 1), - Seq[Any]("20", 2, 7, 12, 0), - Seq[Any]("30", 3, 8, 13, 1), - Seq[Any]("40", 4, 9, 14, 0), - Seq[Any]("50", 5, 10, 15, 1) - ) - val inputPath = - CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) - - def unit(attr: String, t: AttributeType): TypeCastingUnit = { - val u = new TypeCastingUnit() - u.attribute = attr - u.resultType = t - u - } - val desc = new TypeCastingOpDesc() - desc.typeCastingUnits = List( - unit("str_to_int", AttributeType.INTEGER), - unit("int_to_dbl", AttributeType.DOUBLE), - unit("int_to_str", AttributeType.STRING), - unit("int_to_lng", AttributeType.LONG), - unit("int_to_bool", AttributeType.BOOLEAN) - ) - - (desc, Map(PortIdentity(0) -> inputPath)) - } -} - -/** - * Handler for `KeywordSearchOpDesc`. The auto tier points `attribute` at the - * canonical fixture's first column (`id`) and fills `keyword` with the canonical - * "1", so the search runs against numeric ids and never touches a real text - * column. This fixture searches a genuine free-text column with a two-term - * query, exercising the standalone regex's meaningful branches — multi-term OR, - * whole-word boundaries — that both the JVM Lucene path and the pandas path - * agree on. Query "love day" keeps rows 1 and 2 (contain the whole words - * love/day); row 3 has neither; row 4's "lovely"/"today" are different tokens, - * so the shared word-boundary rule drops it. 4 rows → 2 kept. - * - * The rows are intentionally punctuation-free. The `isCaseSensitive` enum is - * swept (true and false), and the case-sensitive path uses `CaseSensitiveAnalyzer` - * (a `WhitespaceTokenizer` that leaves punctuation attached, e.g. "perfect."), - * which diverges from the standalone regex's `\b`-boundary matching on any - * punctuated word — and the standalone does NOT honor case at all. Clean - * whitespace-delimited words keep both tokenizers (and both case modes) in - * agreement; this is why the canonical fixture's punctuated `short_text` column - * cannot be reused here. Lucene phrase/boolean/wildcard syntax is likewise - * avoided — the regex approximation cannot reproduce it. - */ -object KeywordSearchTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[KeywordSearchOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val columns = Seq(("txt", AttributeType.STRING)) - val rows = Seq( - Seq[Any]("i love this product"), - Seq[Any]("what a great day"), - Seq[Any]("terrible experience"), - Seq[Any]("lovely weather today") - ) - val inputPath = - CuratedHandlers.writeFixture(testRoot.resolve("input_port_0.jsonl"), columns, rows) - - val desc = new KeywordSearchOpDesc() - desc.attribute = "txt" - desc.keyword = "love day" - desc.isCaseSensitive = false - - (desc, Map(PortIdentity(0) -> inputPath)) - } -} - -/** DumbbellPlot: curated CONFIG over the shared canonical fixture. A dumbbell is - * one line per entity between the entity's value in two categories, so the two - * category values have to be values the entity actually has — which the auto tier - * cannot know: it fills both with the canonical string, leaving start == end and - * every line a point. - * - * `node_src` = n3 / n1 is the pair that works on this fixture: `bob` holds both - * (score 1.2 → 2.8) and so does `1` (1.7 → 0.5), giving two real dumbbells, while - * eve, dave and grace hold one each and stay single points — both branches drawn - * at once. `comparedColumnName` is a STRING column on purpose: plotly's trace - * `name` rejects a numpy number, so a numeric column raises there instead of - * plotting (reported upstream, not worked around here). - */ -object DumbbellPlotVisualizationHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[DumbbellPlotOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val dots = new util.ArrayList[DumbbellDotConfig]() - val dot = new DumbbellDotConfig() - dot.dotValue = "open" - dots.add(dot) - - val desc = new DumbbellPlotOpDesc() - desc.categoryColumnName = "node_src" - desc.dumbbellStartValue = "n3" - desc.dumbbellEndValue = "n1" - desc.measurementColumnName = "score" - desc.comparedColumnName = "name" - desc.dots = dots - - (desc, CanonicalFixture.writeInputs(testRoot, 1)) - } -} - -/** ImageVisualizer fixture. Uses deterministic binary payloads; the operator - * base64-encodes them into img tags. - */ -object ImageVisualizerVisualizationHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[ImageVisualizerOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val schema = new Schema(new Attribute("image_bytes", AttributeType.BINARY)) - - def tup(bytes: Array[Byte]): Tuple = { - val builder = Tuple.builder(schema) - builder.add(schema.getAttribute("image_bytes"), bytes) - builder.build() - } - - val rows = Seq( - tup(Array[Byte](1, 2, 3, 4)), - tup(Array[Byte](10, 20, 30, 40)) - ) - val inputPath = testRoot.resolve("input_port_0.jsonl") - TupleIO.writeTuples(inputPath, rows.iterator, schema) - - val desc = new ImageVisualizerOpDesc() - desc.binaryContent = "image_bytes" - - (desc, Map(PortIdentity(0) -> inputPath)) - } -} - -/** If operator: routes the data port (port 1) to the True (port 1) or False - * (port 0) output. We feed an EMPTY Condition port (port 0) so IfOpExec - * forwards no condition rows; with no State message it keeps its default - * active output (True), matching the standalone's default-True branch — so - * the True output gets all data rows and the False output is empty on both - * paths. - */ -/** Aggregate fixture exercising every aggregation function in one op, including - * COUNT(*) (empty attribute). Auto-config can't build this: upstream #5896 made - * `AggregationOperation.attribute` optional (required only for non-count via a - * conditional JSON-schema rule), so ConfigGenerator skips the optional autofill - * field and leaves it null — invalid for a non-count function, which NPEs in - * AggregateOpExec. This pins valid (function, column) pairs. Enum-sweep-exempt - * (see [[TransformVerificationRunner.variantsNotRun]]): the sweep flips each - * element's function in isolation and would re-pair, e.g., concat with a numeric - * column; the fixture already covers each function with a type-compatible column. - * Aggregate inherits the unordered `orderSensitive` default, so - * the comparator lex-sorts rows before comparing. - */ -object AggregateTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[AggregateOpDesc] - - private def agg( - fn: AggregationFunction, - attr: String, - result: String - ): AggregationOperation = { - val a = new AggregationOperation() - a.aggFunction = fn - a.attribute = attr - a.resultAttribute = result - a - } - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val desc = new AggregateOpDesc() - desc.groupByKeys = List("name") - desc.aggregations = List( - agg(AggregationFunction.SUM, "score", "sum_score"), - agg(AggregationFunction.COUNT, "", "count_all"), // empty attribute => COUNT(*) - agg(AggregationFunction.COUNT, "score", "count_score"), - agg(AggregationFunction.AVERAGE, "score", "avg_score"), - agg(AggregationFunction.MIN, "score", "min_score"), - agg(AggregationFunction.MAX, "score", "max_score"), - agg(AggregationFunction.CONCAT, "iso_country", "cat_country") - ) - (desc, CanonicalFixture.writeInputs(testRoot, 1)) - } -} - -object IfTransformHandler extends TransformHandler { - override val opDescClass: Class[_ <: LogicalOp] = classOf[IfOpDesc] - - override def fixture(testRoot: Path): (LogicalOp, Map[PortIdentity, Path]) = { - val cols = Seq("id" -> AttributeType.INTEGER, "name" -> AttributeType.STRING) - val condition = - CuratedHandlers.writeFixture( - testRoot.resolve("input_port_0.jsonl"), - cols, - Seq.empty[Seq[Any]] - ) - val data = CuratedHandlers.writeFixture( - testRoot.resolve("input_port_1.jsonl"), - cols, - Seq(Seq(1, "a"), Seq(2, "b"), Seq(3, "c")) - ) - val desc = new IfOpDesc() - desc.conditionName = "cond" - (desc, Map(PortIdentity(0) -> condition, PortIdentity(1) -> data)) - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala deleted file mode 100644 index 685b6f056db..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/HarnessSpec.scala +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} -import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.distinct.DistinctOpDesc -import org.scalatest.Tag -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.nio.file.{Files, Path} - -/** The two ways of running one operator, and the file format they meet in. - * - * `Distinct` is the operator under test throughout, because what is being - * tested is the harness rather than the operator: it takes one input, needs no - * configuration, and its answer is short enough to state in full. - */ -class HarnessSpec extends AnyFlatSpec with Matchers { - - /** Only the standalone run needs an interpreter, so only it is held back from - * the job that provisions none. The other two are JVM-side and run there. - */ - private val NeedsPython = - Tag("org.apache.texera.amber.translator.verify.tags.IntegrationTest") - - private val schema = new Schema( - new Attribute("id", AttributeType.INTEGER), - new Attribute("name", AttributeType.STRING) - ) - - private def tuple(id: Int, name: String): Tuple = { - val b = Tuple.builder(schema) - b.add(schema.getAttribute("id"), Int.box(id)) - b.add(schema.getAttribute("name"), name) - b.build() - } - - /** Four rows, the last a repeat of the second. */ - private val rows = Seq(tuple(1, "a"), tuple(2, "b"), tuple(3, "c"), tuple(2, "b")) - - private def withInput(test: (Path, Path) => Unit): Unit = { - val dir = Files.createTempDirectory("harness-spec-") - val input = dir.resolve("input_port_0.jsonl") - TupleIO.writeTuples(input, rows.iterator, schema) - test(dir, input) - } - - "TupleIO" should "read back the rows and the schema it wrote" in { - withInput { (_, input) => - // The schema travels in a sidecar rather than in the JSONL, which carries - // values alone and so cannot say a column is INTEGER rather than a number. - TupleIO.readSchemaSidecar(input) shouldBe schema - val read = TupleIO.readTuples(input, schema).toSeq - read should have length 4 - read.map(_.getField[Integer]("id").intValue) shouldBe Seq(1, 2, 3, 2) - } - } - - "OpExecHarness" should "run an operator and write one file per output port" in { - withInput { (dir, input) => - val out = dir.resolve("actual") - val result = - OpExecHarness.execute(new DistinctOpDesc, Map(PortIdentity(0) -> input), out) - - result.outputs should have size 1 - val produced = result.outputs(PortIdentity(0)) - Files.exists(produced) shouldBe true - - val written = TupleIO.readTuples(produced, result.outputSchemas(PortIdentity(0))).toSeq - written.map(_.getField[Integer]("id").intValue) shouldBe Seq(1, 2, 3) - } - } - - "StandaloneRunner" should "run the generated script and reach the same answer" taggedAs NeedsPython in { - withInput { (dir, input) => - val work = dir.resolve("standalone") - Files.createDirectories(work) - val result = StandaloneRunner.run( - opDesc = new DistinctOpDesc, - inputs = Map(1 -> input), - outputPortCount = 1, - workDir = work - ) - - // The script is kept where it ran, so a failing operator can be opened as - // generated rather than described second-hand. - Files.exists(work.resolve("script.py")) shouldBe true - - val produced = result.outputs(1) - val lines = Files.readAllLines(produced) - lines should have size 3 - lines.get(0) should include("\"id\":1") - lines.get(2) should include("\"id\":3") - } - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala deleted file mode 100644 index 2bb04515646..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OpExecHarness.scala +++ /dev/null @@ -1,454 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.databind.node.ObjectNode -import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.core.executor.{ExecFactory, OpExecWithClassName, OperatorExecutor} -import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple, TupleLike} -import org.apache.texera.amber.core.virtualidentity.{ - ExecutionIdentity, - PhysicalOpIdentity, - WorkflowIdentity -} -import org.apache.texera.amber.core.workflow.{PhysicalOp, PhysicalPlan, PortIdentity} -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.util.JSONUtils.objectMapper - -import java.nio.file.{Files, Path} -import java.sql.Timestamp -import java.util.Base64 -import scala.collection.mutable -import scala.jdk.CollectionConverters._ - -/** - * Generic harness that drives an OpDesc's OpExec(s) directly, bypassing the - * Pekko/actor runtime. - * - * Works uniformly for: single-OpExec ops (filter, sort, projection), multi- - * OpExec ops (hash join build+probe), multi-output ops (split), and source - * ops (empty input map). All wiring info — number of OpExecs, internal links, - * input-port dependency order — is derived from `opDesc.getPhysicalPlan(...)`, - * so adding a new operator requires no harness changes. - * - * I/O is JSON Lines with sidecar schemas. Each input/output `*.jsonl` file - * has a companion `*.jsonl.schema.json` describing its [[Schema]]. - * - * Limitations (intentional for MVP): - * - Only `OpExecWithClassName` is supported; Python UDFs (`OpExecWithCode`) - * are out of scope because driving them needs a real Python worker. - * - Single worker only (idx=0, workerCount=1). Multi-worker partitioning - * would require coordinating partitioners across executors. - * - JSONL types: STRING / INTEGER / LONG / DOUBLE / BOOLEAN / BINARY / - * TIMESTAMP (the latter two via explicit base64 / JDBC-string codecs). - */ -object OpExecHarness extends LazyLogging { - - // Test-only workflow / execution IDs. The values don't matter — the harness - // never persists state under them — but the PhysicalOp factory needs *some* - // IDs to embed in PhysicalOpIdentity. - private val TestWorkflowId = WorkflowIdentity(0L) - private val TestExecutionId = ExecutionIdentity(0L) - - /** - * @param outputs external output port → JSONL file path - * @param outputSchemas same keys as outputs, gives each port's [[Schema]] - */ - final case class Result( - outputs: Map[PortIdentity, Path], - outputSchemas: Map[PortIdentity, Schema] - ) - - /** - * Run `opDesc` against the given inputs and write the outputs to `outputDir`. - * - * @param inputs map keyed by the *external* input port identifier (the one - * the user-visible LogicalOp exposes). Each path points to a - * `.jsonl` file with a sibling `.schema.json`. - * @param outputDir destination directory; created if missing. Output files - * are named `output_port_.jsonl` per external output. - */ - def execute( - opDesc: LogicalOp, - inputs: Map[PortIdentity, Path], - outputDir: Path - ): Result = { - Files.createDirectories(outputDir) - - // 1. Compile OpDesc → PhysicalPlan. For most ops this is a single-PhysicalOp - // plan; HashJoin and other multi-stage ops return multiple PhysicalOps - // plus internal PhysicalLinks (e.g. build.out → probe.in0). - val plan = opDesc.getPhysicalPlan(TestWorkflowId, TestExecutionId) - - // 2. Identify external input ports. A PhysicalOp port is "external" iff no - // PhysicalLink in this plan terminates at it. The user's `inputs` map - // must cover exactly these (matched by PortIdentity). - val externalInputs: Set[(PhysicalOpIdentity, PortIdentity)] = - plan.operators.flatMap { phOp => - phOp.inputPorts.keys.collect { - case portId if !plan.links.exists(l => l.toOpId == phOp.id && l.toPortId == portId) => - (phOp.id, portId) - } - } - validateInputCoverage(externalInputs, inputs.keySet) - - // 3. Load each input file as (Schema, Iterator[Tuple]). We read schemas - // eagerly but tuples lazily (saves memory on large fixtures). - val inputSchemas: Map[PortIdentity, Schema] = - inputs.map { case (portId, path) => portId -> TupleIO.readSchemaSidecar(path) } - val inputTuples: Map[PortIdentity, () => Iterator[Tuple]] = - inputs.map { - case (portId, path) => - val schema = inputSchemas(portId) - portId -> (() => TupleIO.readTuples(path, schema)) - } - - // 4. Propagate schemas. We CAN'T use `plan.propagateSchema(inputSchemas)` - // directly because it indexes by PortIdentity globally — for HashJoin - // probe's internal port 0 would collide with build's external port 0. - // Instead, set schemas only on the external (phOpId, portId) pairs and - // let `addLink` propagate the internal links' schemas naturally. - val planWithSchemas = - propagateExternalSchemas(plan, externalInputs, inputSchemas) - - // 5. Identify external output ports symmetrically: no outgoing PhysicalLink. - val externalOutputs: Set[(PhysicalOpIdentity, PortIdentity)] = - planWithSchemas.operators.flatMap { phOp => - phOp.outputPorts.keys.collect { - case portId - if !planWithSchemas.links - .exists(l => l.fromOpId == phOp.id && l.fromPortId == portId) => - (phOp.id, portId) - } - } - - // 6. Instantiate OpExec per PhysicalOp. We only support OpExecWithClassName; - // fail loudly otherwise so test authors know to mock Python UDFs out. - val opExecs: Map[PhysicalOpIdentity, OperatorExecutor] = - planWithSchemas.operators.map { phOp => - phOp.opExecInitInfo match { - case OpExecWithClassName(className, descString) => - phOp.id -> ExecFactory.newExecFromJavaClassName( - className, - descString, - idx = 0, - workerCount = 1 - ) - case other => - throw new UnsupportedOperationException( - s"OpExecHarness only supports OpExecWithClassName, got: $other" - ) - } - }.toMap - - // 7. Drive each PhysicalOp in topological order. Buffer outputs in memory - // keyed by (producer phOpId, output port). Downstream PhysicalOps then - // consume from these buffers via the plan's internal links. - val producedBuffer = - mutable.Map.empty[(PhysicalOpIdentity, PortIdentity), mutable.ArrayBuffer[Tuple]] - - planWithSchemas.topologicalIterator().foreach { phOpId => - val phOp = planWithSchemas.getOperator(phOpId) - val opExec = opExecs(phOpId) - runOneOp( - phOp, - opExec, - externalInputProvider = portId => inputTuples.get(portId).map(_.apply()), - upstreamBuffer = producedBuffer, - plan = planWithSchemas, - produced = producedBuffer - ) - } - - // 8. Materialize external outputs to JSONL with their propagated schemas. - val outputPaths = mutable.Map.empty[PortIdentity, Path] - val outputSchemas = mutable.Map.empty[PortIdentity, Schema] - externalOutputs.foreach { - case (phOpId, portId) => - val schema = planWithSchemas - .getOperator(phOpId) - .outputPorts(portId) - ._3 - .toOption - .getOrElse( - throw new IllegalStateException( - s"Output schema for ($phOpId, $portId) was not propagated" - ) - ) - val tuples = - producedBuffer.getOrElse((phOpId, portId), mutable.ArrayBuffer.empty[Tuple]) - val file = outputDir.resolve(s"output_port_${portId.id}.jsonl") - TupleIO.writeTuples(file, tuples.iterator, schema) - outputPaths(portId) = file - outputSchemas(portId) = schema - } - - Result(outputPaths.toMap, outputSchemas.toMap) - } - - /** - * Drives one PhysicalOp's lifecycle: open → input ports in dependency order - * (processTupleMultiPort + onFinishMultiPort per port) → close. Source ops - * (no input ports) get a single onFinishMultiPort(0) call which gives their - * `produceTuple()`-backed implementation a chance to emit. - * - * Outputs are bucketed by output PortIdentity. `processTupleMultiPort`'s - * `Option[PortIdentity]` return: None means port 0 (the default single- - * output convention used by the trait's fallback). Multi-output ops like - * Split set it explicitly. - */ - private def runOneOp( - phOp: PhysicalOp, - opExec: OperatorExecutor, - externalInputProvider: PortIdentity => Option[Iterator[Tuple]], - upstreamBuffer: mutable.Map[ - (PhysicalOpIdentity, PortIdentity), - mutable.ArrayBuffer[Tuple] - ], - plan: PhysicalPlan, - produced: mutable.Map[ - (PhysicalOpIdentity, PortIdentity), - mutable.ArrayBuffer[Tuple] - ] - ): Unit = { - opExec.open() - try { - def bucket(emitted: Iterator[(TupleLike, Option[PortIdentity])]): Unit = { - emitted.foreach { - case (tupleLike, portOpt) => - // Default: the op's single output port. Most operators have one - // output and use the trait's default port-0 wrapping, but - // multi-stage plans (e.g. HashJoin build) put their internal - // output on PortIdentity(0, internal = true) — a hardcoded - // PortIdentity(0, false) would NoSuchElementException here. - val outPortId = portOpt.getOrElse { - if (phOp.outputPorts.size == 1) phOp.outputPorts.keys.head - else PortIdentity(0) - } - val outSchema = phOp - .outputPorts(outPortId) - ._3 - .toOption - .getOrElse( - throw new IllegalStateException( - s"Op ${phOp.id} emitted to port $outPortId before its output schema was propagated" - ) - ) - val tuple = tupleLike - .asInstanceOf[org.apache.texera.amber.core.tuple.SeqTupleLike] - .enforceSchema(outSchema) - produced - .getOrElseUpdate((phOp.id, outPortId), mutable.ArrayBuffer.empty[Tuple]) += tuple - } - } - - // Process each input port in declared dependency order (e.g. HashJoin - // probe's build-side port must finish before the data-side port starts). - val portOrder = - if (phOp.getInputPortDependencyPairs.nonEmpty) - phOp.getInputPortDependencyPairs - else phOp.inputPorts.keys.toList.sortBy(_.id) - - portOrder.foreach { portId => - val tuples: Iterator[Tuple] = - if (externalInputProvider(portId).isDefined) { - externalInputProvider(portId).get - } else { - // Internal port: stitch upstream PhysicalLinks' buffers together - val upstream = plan.links - .filter(l => l.toOpId == phOp.id && l.toPortId == portId) - .toList - .sortBy(l => (l.fromOpId.toString, l.fromPortId.id)) - upstream.iterator - .flatMap(l => - upstreamBuffer - .getOrElse( - (l.fromOpId, l.fromPortId), - mutable.ArrayBuffer.empty[Tuple] - ) - .iterator - ) - } - - tuples.foreach { t => - bucket(opExec.processTupleMultiPort(t, portId.id)) - } - bucket(opExec.onFinishMultiPort(portId.id)) - } - - // Source operator: no input ports. Trigger production via onFinishMultiPort - // on a synthetic port 0 — SourceOperatorExecutor.onFinish ignores the port - // and emits everything from produceTuple(). - if (phOp.inputPorts.isEmpty) { - bucket(opExec.onFinishMultiPort(0)) - } - } finally { - opExec.close() - } - } - - // Walks the plan in topo order, propagating schemas only at the truly external - // input ports. Internal ports get their schema via `addLink` (PhysicalPlan - // re-applies the source's output schema to the destination port). This avoids - // a collision when multiple PhysicalOps share a PortIdentity (e.g. HashJoin - // probe.in0 internal vs build.in0 external both have PortIdentity(0)). - private def propagateExternalSchemas( - plan: PhysicalPlan, - externalPorts: Set[(PhysicalOpIdentity, PortIdentity)], - schemas: Map[PortIdentity, Schema] - ): PhysicalPlan = { - var acc = PhysicalPlan(operators = Set.empty, links = Set.empty) - plan.topologicalIterator().map(plan.getOperator).foreach { phOp => - val updated = phOp.inputPorts.keys.foldLeft(phOp) { (op, portId) => - if (externalPorts.contains((phOp.id, portId)) && schemas.contains(portId)) { - op.propagateSchema(Some((portId, schemas(portId)))) - } else op - } - // .propagateSchema() with no arg re-fires output derivation if all inputs - // are now resolved (source ops trigger immediately since inputPorts empty). - acc = acc.addOperator(updated.propagateSchema()) - plan.getUpstreamPhysicalLinks(phOp.id).foreach { link => - acc = acc.addLink(link) - } - } - acc - } - - private def validateInputCoverage( - external: Set[(PhysicalOpIdentity, PortIdentity)], - provided: Set[PortIdentity] - ): Unit = { - val expected = external.map(_._2) - val missing = expected -- provided - val extra = provided -- expected - require( - missing.isEmpty, - s"Missing input fixtures for external ports: $missing (expected $expected)" - ) - if (extra.nonEmpty) { - logger.warn(s"Input fixtures provided for non-external ports (ignored): $extra") - } - } -} - -/** - * JSON Lines I/O for Tuples. Each `.jsonl` file is paired with a - * `.jsonl.schema.json` sidecar listing [[Attribute]]s in column order. - * - * Format example: - * - * records.jsonl: - * {"id":1,"name":"alice"} - * {"id":2,"name":"bob"} - * - * records.jsonl.schema.json: - * {"attributes":[{"attributeName":"id","attributeType":"integer"}, - * {"attributeName":"name","attributeType":"string"}]} - * - * pandas symmetry: `pd.read_json(path, lines=True)` and - * `df.to_json(path, orient='records', lines=True)` round-trip cleanly for the - * supported types (STRING / INTEGER / LONG / DOUBLE / BOOLEAN). - */ -object TupleIO { - - private def sidecar(path: Path): Path = - path.resolveSibling(path.getFileName.toString + ".schema.json") - - def readSchemaSidecar(path: Path): Schema = { - val text = new String(Files.readAllBytes(sidecar(path))) - objectMapper.readValue(text, classOf[Schema]) - } - - def readTuples(path: Path, schema: Schema): Iterator[Tuple] = { - // readAllLines closes the underlying handle; safer than Files.lines for - // test-scale fixtures where memory cost is negligible. - val lines = Files.readAllLines(path).asScala - lines.iterator.filter(_.trim.nonEmpty).map { line => - val node = objectMapper.readTree(line) - val builder = Tuple.builder(schema) - schema.getAttributes.foreach { attr => - val fieldNode = node.get(attr.getName) - val v: Any = - if (fieldNode == null || fieldNode.isNull) null - else - attr.getType match { - case AttributeType.STRING => fieldNode.asText() - case AttributeType.INTEGER => Int.box(fieldNode.asInt()) - case AttributeType.LONG => Long.box(fieldNode.asLong()) - case AttributeType.DOUBLE => Double.box(fieldNode.asDouble()) - case AttributeType.BOOLEAN => Boolean.box(fieldNode.asBoolean()) - case AttributeType.BINARY => - Base64.getDecoder.decode(fieldNode.asText()) - // Timestamps round-trip through the JDBC string form - // ("yyyy-mm-dd hh:mm:ss[.f]"), the exact inverse of Timestamp.toString - // below — timezone-free, so no shift across write/read. The Python - // side reads this column with convert_dates=False (see - // StandaloneRunner) and treats it as an opaque string, so both paths - // agree on pass-through. - case AttributeType.TIMESTAMP => - Timestamp.valueOf(fieldNode.asText()) - case other => - throw new UnsupportedOperationException( - s"TupleIO MVP doesn't support $other yet" - ) - } - builder.add(attr, v) - } - builder.build() - } - } - - def writeTuples(path: Path, tuples: Iterator[Tuple], schema: Schema): Unit = { - // Sidecar first so a partial main-file write still has a recoverable schema. - Files.write(sidecar(path), objectMapper.writeValueAsBytes(schema)) - val writer = Files.newBufferedWriter(path) - try { - tuples.foreach { t => - val node: ObjectNode = objectMapper.createObjectNode() - schema.getAttributes.zipWithIndex.foreach { - case (attr, idx) => - val v = t.getField[Any](idx) - if (v == null) node.putNull(attr.getName) - else - attr.getType match { - case AttributeType.STRING => node.put(attr.getName, v.toString) - case AttributeType.INTEGER => node.put(attr.getName, v.asInstanceOf[Int]) - case AttributeType.LONG => node.put(attr.getName, v.asInstanceOf[Long]) - case AttributeType.DOUBLE => node.put(attr.getName, v.asInstanceOf[Double]) - case AttributeType.BOOLEAN => node.put(attr.getName, v.asInstanceOf[Boolean]) - case AttributeType.BINARY => - node.put( - attr.getName, - Base64.getEncoder.encodeToString(v.asInstanceOf[Array[Byte]]) - ) - case AttributeType.TIMESTAMP => - node.put(attr.getName, v.asInstanceOf[Timestamp].toString) - case other => - throw new UnsupportedOperationException( - s"TupleIO MVP doesn't support $other yet" - ) - } - } - writer.write(objectMapper.writeValueAsString(node)) - writer.newLine() - } - } finally writer.close() - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala deleted file mode 100644 index f6d8b6759be..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.annotation.JsonSubTypes -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.apache.texera.amber.translator.verify.tags.IntegrationTest -import org.scalatest.ParallelTestExecution -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Auto-discovered behavioral-parity tests: for every operator registered - * with [[LogicalOp]]'s `@JsonSubTypes` that implements - * [[StandaloneCodeGenerator]], emit a test that runs both Path A (Texera - * exec) and Path B (translator-generated Python via [[StandaloneRunner]]) - * and asserts their outputs are equivalent. - * - * Dispatch is auto-first: [[TransformVerificationRunner]] classifies each - * non-source transform as `Runnable("auto")` (auto-configured fixture), - * `Runnable("curated")` (hand-written fixture from [[CuratedHandlers]]), - * or `Flagged(reason)` (shown as ignored with the reason in the test name). - * Sources route to [[SourceCategoryRunner]] unchanged. - * - * No edits to this spec are needed when a new operator is added — reflection - * discovers it automatically via `@JsonSubTypes`. The tier label appears in - * the test name so the report shows which path exercised each operator. - * - * Requires Python 3 with pandas on the [[Comparator]] / [[StandaloneRunner]] - * resolution chain (`UDF_PYTHON_PATH` env var, then `python3.12`). - */ -// Tagged @IntegrationTest: this is the only verify spec that forks a real -// Python process end-to-end, so CI routes it to the Python-provisioned -// integration job (see workflow-compiling-service/build.sbt WCS_TEST_FILTER). -@IntegrationTest -class OperatorBehaviorSpec extends AnyFlatSpec with Matchers with ParallelTestExecution { - - // Build the test list at class construction. Each branch below registers - // one test (`in` for runnable, `ignore` for skipped) so the test report - // shows every translator-eligible operator and why it did or didn't run. - OperatorBehaviorSpec.discoverStandaloneOperators().foreach { opClass => - val name = opClass.getSimpleName - - if (!OperatorBehaviorSpec.isSelected(name)) { - // Narrowed out by VERIFY_ONLY / VERIFY_SKIP, which only a local run sets. - // Still registered, as an `ignore`, so the report lists every operator - // rather than reading as though the narrowed-out ones do not exist. - name should "NARROWED OUT — outside this run's VERIFY_ONLY / VERIFY_SKIP" ignore {} - } else if (classOf[SourceOperatorDescriptor].isAssignableFrom(opClass)) { - // Sources keep their handler-per-source design: each needs a real file - // in its specific format, which a generic fixture can't supply. - if (SourceCategoryRunner.canRun(opClass)) { - name should "produce equivalent output in Texera and standalone Python (source)" in { - SourceCategoryRunner.run(opClass) - } - } else { - name should s"FLAGGED — ${SourceCategoryRunner.flagReason(opClass)}" ignore {} - } - } else { - TransformVerificationRunner.disposition(opClass) match { - case TransformVerificationRunner.Runnable(tier) => - name should s"produce equivalent output in Texera and standalone Python ($tier)" in { - TransformVerificationRunner.run(opClass) - } - case TransformVerificationRunner.Flagged(reason) => - name should s"FLAGGED — $reason" ignore { - // Reason is in the test name so the report carries it; the - // coverage table in ConfigCoverageSpec aggregates these. - } - } - } - } - - // Not one test per operator like the rest of this spec: it is one assertion - // over all of them, and it deliberately ignores the selection knobs above so a - // VERIFY_ONLY run still cannot hide a broken splice site. - "Generated standalone code" should "stay parseable when the column names are hostile" in { - StandaloneEscapingCheck.run() shouldBe empty - } -} - -object OperatorBehaviorSpec { - - // Narrowing knobs for a local run, both unset by default, so the default run - // is every operator: VERIFY_ONLY names the only ones to run, VERIFY_SKIP the - // ones to leave out. Case-sensitive substrings against the operator's simple - // name, comma-separated. Neither is set in CI, which therefore runs the lot. - // - // There is deliberately no third list withholding operators by default. What - // stays withheld is narrower than an operator and lives where it can say why: - // a single variant in [[TransformVerificationRunner.variantsNotRun]], or an - // operator that cannot be run at all in its `knownIssues`, each against an - // issue or a reason. A name here would withdraw an operator's every variant - // and record nothing about what is wrong with it. - private def patterns(envVar: String): Seq[String] = - sys.env.getOrElse(envVar, "").split(",").iterator.map(_.trim).filter(_.nonEmpty).toSeq - - private lazy val onlyPatterns: Seq[String] = patterns("VERIFY_ONLY") - private lazy val skipPatterns: Seq[String] = patterns("VERIFY_SKIP") - - /** True if `name` should run: in VERIFY_ONLY when that is set, and not in - * VERIFY_SKIP. True for everything when neither is set. - */ - def isSelected(name: String): Boolean = { - val included = onlyPatterns.isEmpty || onlyPatterns.exists(name.contains) - val excluded = skipPatterns.exists(name.contains) - included && !excluded - } - - /** - * Enumerates every concrete subclass of [[LogicalOp]] declared in its - * `@JsonSubTypes` annotation, filters to those implementing - * [[StandaloneCodeGenerator]], and returns them sorted by simple name - * (stable test report order). - * - * Uses the same registry Jackson uses to deserialize operators — no - * separate discovery mechanism needed. Adding an operator to - * `LogicalOp.@JsonSubTypes` makes it visible here automatically. - */ - def discoverStandaloneOperators(): Seq[Class[_ <: LogicalOp]] = { - val annotation = classOf[LogicalOp].getAnnotation(classOf[JsonSubTypes]) - if (annotation == null) Seq.empty - else - annotation - .value() - .toSeq - .map(_.value()) - .filter(classOf[StandaloneCodeGenerator].isAssignableFrom) - .map(_.asInstanceOf[Class[_ <: LogicalOp]]) - .distinct - .sortBy(_.getSimpleName) - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala deleted file mode 100644 index a6ae7b3b373..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/PyOpExecHarness.scala +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} -import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.core.executor.OpExecWithCode -import org.apache.texera.amber.core.tuple.Schema -import org.apache.texera.amber.core.virtualidentity.{ - ExecutionIdentity, - PhysicalOpIdentity, - WorkflowIdentity -} -import org.apache.texera.amber.core.workflow.{PhysicalPlan, PortIdentity} -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.apache.texera.amber.util.python.PythonWorkerPool - -import java.io.File -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path, Paths, StandardCopyOption} -import scala.collection.mutable.ArrayBuffer -import scala.sys.process._ - -/** - * Counterpart to [[OpExecHarness]] for Python-native operators - * ([[OpExecWithCode]] with language="python"). Drives the operator's - * generatePythonCode() output through a thin subprocess driver rather than - * spinning up the Pekko/Arrow worker stack. - * - * Same Result(outputs, outputSchemas) shape as OpExecHarness so the rest of - * the verify pipeline (Comparator, category runners) is harness-agnostic. - * - * Scope (MVP, mirrors OpExecHarness's MVP): - * - Single-PhysicalOp plans only. PythonOperatorDescriptor only emits - * either a `sourcePhysicalOp` or a `oneToOnePhysicalOp`, so multi-op - * plans don't exist for Python-native ops today. If that changes, add - * topo-order driving here the way OpExecHarness does. - * - Single output port. UDFOperatorV2 / UDFTableOperator / UDFBatchOperator - * / UDFSourceOperator all yield TupleLike without specifying a port — - * same convention OpExecHarness uses when port is unset. - * - JSONL types: STRING / INTEGER / LONG / DOUBLE / BOOLEAN. TIMESTAMP / - * BINARY / LARGE_BINARY require explicit codecs in both [[TupleIO]] and - * the driver — add when the first operator needs them. - */ -object PyOpExecHarness extends LazyLogging { - - private val TestWorkflowId = WorkflowIdentity(0L) - private val TestExecutionId = ExecutionIdentity(0L) - - // Same Result shape as OpExecHarness so callers can swap harnesses - // transparently. - final case class Result( - outputs: Map[PortIdentity, Path], - outputSchemas: Map[PortIdentity, Schema] - ) - - // Driver script lives on the test classpath at /python/py_op_driver.py - // (sibling to compare.py). Extracted to a temp file at runtime so it works - // whether the test resources are loose files or sealed in a jar. - private val DriverResourcePath = "/python/py_op_driver.py" - - def execute( - opDesc: LogicalOp, - inputs: Map[PortIdentity, Path], - outputDir: Path, - pythonExe: String = resolvePython(), - amberPythonHome: Path = resolveAmberPythonHome() - ): Result = { - Files.createDirectories(outputDir) - - val plan = opDesc.getPhysicalPlan(TestWorkflowId, TestExecutionId) - - // PythonOperatorDescriptor builds single-op plans; bail loudly if some - // future Python op produces a multi-stage plan (need to extend the driver - // and the per-PhysicalOp config the way OpExecHarness does). - require( - plan.operators.size == 1, - s"PyOpExecHarness only supports single-PhysicalOp plans for now, got " + - s"${plan.operators.size} PhysicalOps" - ) - val phOp = plan.operators.head - - val (pythonCode, language) = phOp.opExecInitInfo match { - case OpExecWithCode(code, lang) => (code, lang) - case other => - throw new UnsupportedOperationException( - s"PyOpExecHarness only supports OpExecWithCode; got ${other.getClass.getSimpleName}. " + - "For OpExecWithClassName, use OpExecHarness." - ) - } - require( - language == "python", - s"""PyOpExecHarness only supports language="python", got "$language".""" - ) - - // External input ports = same definition as OpExecHarness. For a - // single-op plan that's just every input port the op declares. - val externalInputs: Set[(PhysicalOpIdentity, PortIdentity)] = - phOp.inputPorts.keys.map(portId => (phOp.id, portId)).toSet - validateInputCoverage(externalInputs, inputs.keySet) - - val inputSchemas: Map[PortIdentity, Schema] = - inputs.map { case (portId, path) => portId -> TupleIO.readSchemaSidecar(path) } - - val planWithSchemas = propagateExternalSchemas(plan, externalInputs, inputSchemas) - val phOpWithSchemas = planWithSchemas.operators.head - - // Output port schemas come from PhysicalPlan.propagateSchema — same - // ground truth OpExecHarness writes to its own outputs. - val outputPortSchemas: Map[PortIdentity, Schema] = - phOpWithSchemas.outputPorts.map { - case (portId, (_, _, schemaOrErr)) => - portId -> schemaOrErr.toOption.getOrElse( - throw new IllegalStateException( - s"Output schema for ($portId) was not propagated" - ) - ) - } - - require( - outputPortSchemas.size == 1, - s"PyOpExecHarness only supports single-output-port operators, got " + - s"${outputPortSchemas.size} output ports" - ) - val (outputPortId, outputSchema) = outputPortSchemas.head - val outputPath = outputDir.resolve(s"output_port_${outputPortId.id}.jsonl") - - // Port ordering for multi-input ops: respect declared dependencies - // (matches OpExecHarness — e.g. HashJoin probe processes build-side - // first). Default = sorted by port id when no dependencies declared. - val portOrder: Seq[Int] = - if (phOpWithSchemas.getInputPortDependencyPairs.nonEmpty) - phOpWithSchemas.getInputPortDependencyPairs.map(_.id) - else phOpWithSchemas.inputPorts.keys.toList.map(_.id).sorted - - val config = buildConfig( - pythonCode = pythonCode, - isSource = phOpWithSchemas.isSourceOperator, - portOrder = portOrder, - inputs = inputs, - outputPath = outputPath, - outputSchema = outputSchema - ) - - val configPath = outputDir.resolve("py_op_driver_config.json") - Files.write(configPath, config.getBytes(StandardCharsets.UTF_8)) - - val driverPath = extractDriverScript() - runDriver(driverPath, configPath, outputDir, pythonExe, amberPythonHome) - - Result( - outputs = Map(outputPortId -> outputPath), - outputSchemas = Map(outputPortId -> outputSchema) - ) - } - - // -------------------------------------------------------------------------- - // Config serialization. Matches the driver's expected schema (see - // py_op_driver.py's module docstring). - // -------------------------------------------------------------------------- - private def buildConfig( - pythonCode: String, - isSource: Boolean, - portOrder: Seq[Int], - inputs: Map[PortIdentity, Path], - outputPath: Path, - outputSchema: Schema - ): String = { - val root: ObjectNode = objectMapper.createObjectNode() - root.put("operatorCode", pythonCode) - root.put("isSource", isSource) - - val portOrderArr: ArrayNode = root.putArray("portOrder") - portOrder.foreach(portOrderArr.add) - - val inputsArr: ArrayNode = root.putArray("inputs") - inputs.toSeq.sortBy(_._1.id).foreach { - case (portId, dataPath) => - val entry: ObjectNode = inputsArr.addObject() - entry.put("portIndex", portId.id) - entry.put("dataPath", dataPath.toAbsolutePath.toString) - // schemaPath is implicit (data_path + ".schema.json") — the driver - // resolves it the same way TupleIO does. - } - - val outputsArr: ArrayNode = root.putArray("outputs") - val outEntry: ObjectNode = outputsArr.addObject() - outEntry.put("dataPath", outputPath.toAbsolutePath.toString) - // Embed the schema directly. We can't just write the sidecar ahead of - // time and have the driver read it, because writing a sidecar before - // outputs exist would leave a stale sidecar on partial failures. - outEntry.set[ObjectNode]( - "schema", - objectMapper.valueToTree[ObjectNode](outputSchema) - ) - - objectMapper.writeValueAsString(root) - } - - // -------------------------------------------------------------------------- - // Subprocess invocation. - // -------------------------------------------------------------------------- - private def runDriver( - driverPath: Path, - configPath: Path, - cwd: Path, - pythonExe: String, - amberPythonHome: Path - ): Unit = { - // Prepend amber's Python source to PYTHONPATH so `import pytexera` - // resolves. Existing PYTHONPATH (if any) is preserved as the lower- - // priority suffix. - val existing = sys.env.getOrElse("PYTHONPATH", "") - val newPyPath = - if (existing.isEmpty) amberPythonHome.toAbsolutePath.toString - else s"${amberPythonHome.toAbsolutePath}${File.pathSeparator}$existing" - - val (exit, stdout, stderr) = execDriver(driverPath, configPath, cwd, pythonExe, newPyPath) - if (exit != 0) { - throw new PyOpDriverException( - exitCode = exit, - driverPath = driverPath, - configPath = configPath, - stdout = stdout, - stderr = stderr - ) - } - } - - // Prefer a pooled persistent worker (imports pytexera/pyamber once via - // `py_op_driver.py --serve`, the ~300 ms cost that dominates a per-Python-op - // run — see PythonWorkerPool). A rare hard worker crash falls back to a - // one-shot subprocess so behavior is never worse than the original path. Both - // paths use absolute config paths, so cwd only matters to the subprocess - // form; the worker constructs a fresh operator per job for isolation. - private def execDriver( - driverPath: Path, - configPath: Path, - cwd: Path, - pythonExe: String, - pythonPath: String - ): (Int, String, String) = { - if (PythonWorkerPool.enabled) { - try { - val req = objectMapper.createObjectNode() - req.put("configPath", configPath.toAbsolutePath.toString) - val o = PythonWorkerPool.run( - DriverResourcePath, - Seq("--serve"), - pythonExe, - req, - env = Map("PYTHONPATH" -> pythonPath) - ) - return (o.exit, o.stdout, o.stderr) - } catch { - case e: PythonWorkerPool.WorkerDiedException => - logger.warn( - s"py_op_driver worker unavailable; falling back to one-shot subprocess " + - s"for $configPath: ${e.getMessage}" - ) - } - } - runDriverSubprocess(driverPath, configPath, cwd, pythonExe, pythonPath) - } - - // Original one-process-per-operator path. Retained as the fallback and as the - // behavior selected by TEXERA_TEST_PYTHON_WORKER=0. - private def runDriverSubprocess( - driverPath: Path, - configPath: Path, - cwd: Path, - pythonExe: String, - pythonPath: String - ): (Int, String, String) = { - val outBuf = ArrayBuffer.empty[String] - val errBuf = ArrayBuffer.empty[String] - val procLogger = ProcessLogger(line => outBuf += line, line => errBuf += line) - val exit = Process( - Seq(pythonExe, driverPath.toString, configPath.toString), - Some(cwd.toFile), - "PYTHONPATH" -> pythonPath - ).!(procLogger) - (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) - } - - // -------------------------------------------------------------------------- - // Resolution helpers. - // -------------------------------------------------------------------------- - private def resolvePython(): String = - sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") - - /** - * Locate `amber/src/main/python`. Resolution chain: - * 1. Env var TEXERA_AMBER_PYTHON_HOME (set by CI / dev shell). - * 2. Walk up from cwd looking for `amber/src/main/python`. - * sbt runs tests with cwd = the subproject dir (`workflow-compiling-service/`), - * so the walk-up is two levels at most for the normal layout. - */ - private def resolveAmberPythonHome(): Path = { - sys.env.get("TEXERA_AMBER_PYTHON_HOME").filter(_.nonEmpty).map(Paths.get(_)).getOrElse { - val cwd = Paths.get(".").toAbsolutePath.normalize() - val maxDepth = 5 - var current: Path = cwd - var depth = 0 - while (current != null && depth <= maxDepth) { - val candidate = current.resolve("amber/src/main/python") - if (Files.isDirectory(candidate)) return candidate.toAbsolutePath - current = current.getParent - depth += 1 - } - throw new RuntimeException( - s"PyOpExecHarness: could not locate amber/src/main/python from cwd $cwd. " + - "Set TEXERA_AMBER_PYTHON_HOME to the absolute path." - ) - } - } - - private def extractDriverScript(): Path = { - val stream = getClass.getResourceAsStream(DriverResourcePath) - require( - stream != null, - s"py_op_driver.py not found on classpath at $DriverResourcePath" - ) - try { - val tmp = Files.createTempFile("py_op_driver-", ".py") - Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING) - tmp.toFile.deleteOnExit() - tmp - } finally stream.close() - } - - // -------------------------------------------------------------------------- - // Schema propagation — mirrors OpExecHarness.propagateExternalSchemas. - // Kept inline (rather than shared) so the two harnesses stay independently - // readable; consolidate if a third harness shows up. - // -------------------------------------------------------------------------- - private def propagateExternalSchemas( - plan: PhysicalPlan, - externalPorts: Set[(PhysicalOpIdentity, PortIdentity)], - schemas: Map[PortIdentity, Schema] - ): PhysicalPlan = { - var acc = PhysicalPlan(operators = Set.empty, links = Set.empty) - plan.topologicalIterator().map(plan.getOperator).foreach { phOp => - val updated = phOp.inputPorts.keys.foldLeft(phOp) { (op, portId) => - if (externalPorts.contains((phOp.id, portId)) && schemas.contains(portId)) { - op.propagateSchema(Some((portId, schemas(portId)))) - } else op - } - acc = acc.addOperator(updated.propagateSchema()) - plan.getUpstreamPhysicalLinks(phOp.id).foreach { link => - acc = acc.addLink(link) - } - } - acc - } - - private def validateInputCoverage( - external: Set[(PhysicalOpIdentity, PortIdentity)], - provided: Set[PortIdentity] - ): Unit = { - val expected = external.map(_._2) - val missing = expected -- provided - val extra = provided -- expected - require( - missing.isEmpty, - s"Missing input fixtures for external ports: $missing (expected $expected)" - ) - if (extra.nonEmpty) { - logger.warn(s"Input fixtures provided for non-external ports (ignored): $extra") - } - } -} - -final class PyOpDriverException( - val exitCode: Int, - val driverPath: Path, - val configPath: Path, - val stdout: String, - val stderr: String -) extends RuntimeException( - s"""py_op_driver.py exited with code $exitCode. - |Driver: $driverPath - |Config: $configPath - |--- stdout --- - |$stdout - |--- stderr --- - |$stderr""".stripMargin - ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala deleted file mode 100644 index f42c1ca6897..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.tuple.{Schema, Tuple} -import org.apache.texera.amber.core.workflow.PortIdentity - -import java.nio.file.Path - -/** - * A checked-in table a whole FAMILY of operators runs on, as opposed to one - * written for a single operator. - * - * Which table an operator runs on is its own axis, separate from who writes its - * config. [[CanonicalFixture]] is the wide mixed-type table every operator takes; - * the sklearn families take [[ProjectedFixture]] views of it, since - * `X = table.drop(target)` feeds every remaining column to `fit`, which a string - * or timestamp column ends. - */ -trait SharedFixture { - - def schema: Schema - - /** The rows port `port` gets. Ports may take different windows of the table - * (canonical overlaps them partially, to defeat hash-coincidence passes on - * joins) or the same rows twice. - */ - def rowsFor(port: Int): Seq[Tuple] - - /** Every row of the table, ports aside — what a [[ProjectedFixture]] of it - * narrows. A table whose ports read windows says so by overriding; by default - * a port already sees the whole of it. - */ - def allRows: Seq[Tuple] = rowsFor(0) - - /** Columns [[write]] never empties, because their VALUE is what the table was - * built to arrange rather than data under test: canonical's `id` is what joins - * and set operations pair rows on, and a sklearn table's label is what its - * estimator fits against. Emptying one of those changes what the test asks - * instead of asking what an operator does with a null. - */ - def keepFilled: Set[String] - - /** Write one JSONL file per 0-based input port under `dir`. At most 2 ports. */ - final def write( - dir: Path, - inputPortCount: Int, - withGaps: Boolean - ): Map[PortIdentity, Path] = { - require( - inputPortCount >= 1 && inputPortCount <= 2, - s"unsupported input port count: $inputPortCount" - ) - (0 until inputPortCount).map { port => - val rows = rowsFor(port) - val path = dir.resolve(s"input_port_$port.jsonl") - TupleIO.writeTuples( - path, - (if (withGaps) emptyOneCellPerColumn(rows) else rows).iterator, - schema - ) - PortIdentity(port) -> path - }.toMap - } - - /** Schemas ConfigGenerator resolves @AutofillAttributeName fields against. - * Every port sees the same columns: a fixture's ports differ in which ROWS - * they get, not in shape. - */ - final def schemasByPort: Map[Int, Schema] = Map(0 -> schema, 1 -> schema) - - /** Write one JSONL fixture per 0-based input port, every cell filled. */ - final def writeInputs(dir: Path, inputPortCount: Int): Map[PortIdentity, Path] = - write(dir, inputPortCount, withGaps = false) - - /** How many rows port 0 gets — what a row-count-sensitive knob (`limit`, - * `offset`) is sized against so its value keeps some rows and drops some. - */ - final def port0RowCount: Int = rowsFor(0).size - - /** This table's rows with [[SharedFixture.emptyOneCellPerColumn]] applied. */ - private[verify] def emptyOneCellPerColumn(rows: Seq[Tuple]): Seq[Tuple] = - SharedFixture.emptyOneCellPerColumn(rows, schema, keepFilled) -} - -/** - * A column subset of another table: the same rows in the same order, keeping - * only the named columns, in the order named. - * - * The sklearn families need one. Their generated code is - * `X = table.drop(target, axis=1)`, so every column that is not the target - * reaches `fit`, and a string or a timestamp ends it. A projection hands them a - * table an estimator can fit without a second dataset to keep in step: the rows - * are still [[CanonicalFixture]]'s, only narrower. - */ -final case class ProjectedFixture( - source: SharedFixture, - columns: Seq[String], - keepFilled: Set[String] -) extends SharedFixture { - - val schema: Schema = new Schema(columns.map(c => source.schema.getAttribute(c)): _*) - - private val rows: Vector[Tuple] = source.allRows.map { t => - val b = Tuple.builder(schema) - schema.getAttributes.foreach(a => b.add(a, t.getField[AnyRef](a.getName))) - b.build() - }.toVector - - /** Every port gets the whole table. An estimator pair trains on port 0 and - * tests on port 1, and the point of the pair is the two ports rather than two - * datasets: what the comparison sees is the fitted model, which port 1 has no - * hand in, so giving the ports different rows buys nothing. - * - * The whole table rather than the source's ten-row window, because the - * estimators that cross-validate pass no fold count and so take sklearn's - * default of five: the window would leave the smaller class at four, and one - * fold holding none of a class is a fold that asks nothing (sklearn warns and - * splits anyway rather than refusing). - */ - override def rowsFor(port: Int): Seq[Tuple] = rows -} - -object SharedFixture { - - /** One empty cell per column, spread across rows so no row is wholly empty — an - * operator that reads two columns should still meet a row where one is filled - * and the other is not. Placement is by column position, so it is the same on - * every run. - * - * Free-standing rather than a member, because a curated handler's table has no - * [[SharedFixture]] behind it: the runner reads back the rows the handler wrote - * and punches the holes here. - */ - def emptyOneCellPerColumn( - rows: Seq[Tuple], - schema: Schema, - keepFilled: Set[String] - ): Seq[Tuple] = { - if (rows.isEmpty) return rows - val holes: Map[Int, Set[String]] = schema.getAttributes.zipWithIndex - .filterNot { case (attr, _) => keepFilled.contains(attr.getName) } - .map { case (attr, i) => (i % rows.size) -> attr.getName } - .groupBy(_._1) - .map { case (row, pairs) => row -> pairs.map(_._2).toSet } - rows.zipWithIndex.map { - case (t, rowIdx) => - val emptied = holes.getOrElse(rowIdx, Set.empty) - if (emptied.isEmpty) t - else { - val b = Tuple.builder(schema) - schema.getAttributes.foreach { attr => - val v: AnyRef = - if (emptied.contains(attr.getName)) null else t.getField[AnyRef](attr.getName) - b.add(attr, v) - } - b.build() - } - } - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala deleted file mode 100644 index 01c13bf777a..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala +++ /dev/null @@ -1,471 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.core.tuple.{Schema, Tuple} -import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.source.fetcher.URLFetcherOpDesc -import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc -import org.apache.texera.amber.operator.source.scan.file.{FileScanOpDesc, FileScanSourceOpDesc} -import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc -import com.fasterxml.jackson.databind.node.ObjectNode -import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.ipc.ArrowFileWriter -import org.apache.arrow.vector.VectorSchemaRoot -import org.apache.texera.amber.util.ArrowUtils - -import java.nio.channels.FileChannel -import java.nio.charset.{Charset, StandardCharsets} -import java.nio.file.{Files, Path, StandardOpenOption} -import scala.collection.mutable -import scala.util.{Try, Using} - -/** - * Per-category test runner for source operators (operators with no input - * ports — they read from an external resource and emit tuples). - * - * Dispatch is auto-first: - * - Auto tier: a scan source declares the file format it reads via - * [[ScanSourceOpDesc.fileTypeName]]. If that tag is in [[encoderByFileType]], - * the operator is fixtured with zero per-operator code — the shared - * [[CanonicalSourceFixture]] is encoded into that format and `fileName` - * points at it. A newly added file-scan source in a known format - * (CSV/JSONL/Arrow/…) is verified the moment it is registered in - * [[LogicalOp]]'s `@JsonSubTypes`, no edit here. - * - Curated tier: sources that can't take the shared table (text-family - * single-`line` output, or inline-config data) keep a hand-written - * [[SourceHandler]] in [[curatedHandlersByClass]]. - * - Otherwise the test is flagged (a [[knownIssues]] reason, an unsupported - * declared format, or no match) — never silently skipped. - * - * The runner itself is operator-agnostic: it builds an OpDesc, drives - * [[OpExecHarness]] (Path A) and [[StandaloneRunner]] (Path B), compares via - * [[Comparator]]. Sources have no input ports so `inputs = Map.empty` for both. - */ -object SourceCategoryRunner { - - /** - * The curated tier: sources that keep a hand-written handler because they - * can't go through the shared-fixture + encoder (auto) path — their output - * isn't the shared 3-column table (text-family, single `line` column) or - * their data is inline config rather than a file. Mirrors the transform - * side's [[CuratedHandlers]] (hand-written vs auto-generated fixture). - */ - private val curatedHandlersByClass: Map[Class[_ <: LogicalOp], SourceHandler] = - Seq[SourceHandler](TextInputHandler, FileScanSourceHandler) - .map(h => h.opDescClass -> h) - .toMap - - /** - * The auto tier. A scan source declares the file format it reads via - * [[ScanSourceOpDesc.fileTypeName]] ("CSV", "JSONL", "Arrow", …). Map that - * tag to the [[CanonicalSourceFixture]] encoder that writes a file in that - * format. Any source whose `fileTypeName` is a key here runs with zero - * per-operator code, so a newly added file-scan source in a known format is - * verified the moment it is registered in `@JsonSubTypes` — no handler, no - * edit here. (ParallelCSV also declares "CSV" and would be covered for free, - * but it is currently commented out of `@JsonSubTypes`, so the suite doesn't - * enumerate it.) - */ - private val encoderByFileType: Map[String, (Path, Charset) => Path] = Map( - "CSV" -> CanonicalSourceFixture.writeCsv, - "CSVOld" -> CanonicalSourceFixture.writeCsv, - "JSONL" -> CanonicalSourceFixture.writeJsonl, - // Arrow is binary and its descriptor declares fileEncoding ignored, so the - // charset a variant asks for has nothing to apply to. - "Arrow" -> ((dir, _) => CanonicalSourceFixture.writeArrow(dir)) - ) - - /** - * Sources this runner cannot verify, with the honest reason. Mirrors - * `TransformVerificationRunner.knownIssues`: the reason surfaces in the - * ignored test's name and the coverage table. - */ - private val knownIssues: Map[Class[_ <: LogicalOp], String] = Map( - classOf[FileScanOpDesc] -> - ("input-driven source: filenames arrive on an input port at runtime, but this runner " + - "feeds sources no inputs — Path B's generated code references an undefined in1df"), - classOf[URLFetcherOpDesc] -> - ("live-network source: the operator fetches a real URL over the network, so its " + - "output is non-deterministic and depends on external connectivity — it cannot be " + - "verified against a fixed fixture in isolation") - ) - - /** The format tag a source declares, or `None` if it isn't an instantiable - * ScanSourceOpDesc (non-scan sources, or ones that fail to construct). - */ - private def declaredFileType(opDescClass: Class[_ <: LogicalOp]): Option[String] = - Try(opDescClass.getDeclaredConstructor().newInstance()).toOption.collect { - case scan: ScanSourceOpDesc => scan.fileTypeName - }.flatten - - def canRun(opDescClass: Class[_ <: LogicalOp]): Boolean = - curatedHandlersByClass.contains(opDescClass) || - declaredFileType(opDescClass).exists(encoderByFileType.contains) - - /** - * Tier label for a runnable source, mirroring the transform side's - * auto/curated distinction: `"curated source"` when a hand-written - * [[SourceHandler]] serves it, else `"auto source"` (a declared-format scan - * source fixtured by an [[encoderByFileType]] encoder with zero per-op code). - */ - def tier(opDescClass: Class[_ <: LogicalOp]): String = - if (curatedHandlersByClass.contains(opDescClass)) "curated source" else "auto source" - - /** Why a non-runnable source is flagged: a specific known issue, an - * unsupported declared format, or no handler/format match at all. - */ - def flagReason(opDescClass: Class[_ <: LogicalOp]): String = - knownIssues.getOrElse( - opDescClass, - declaredFileType(opDescClass) match { - case Some(fileType) => - s"unsupported source format '$fileType' — no encoder registered in SourceCategoryRunner" - case None => "no source handler registered yet" - } - ) - - private def newScanSource(opDescClass: Class[_ <: LogicalOp]): ScanSourceOpDesc = - opDescClass.getDeclaredConstructor().newInstance() match { - case s: ScanSourceOpDesc => s - case other => - throw new IllegalArgumentException( - s"${opDescClass.getSimpleName} has no curated handler and is not a " + - s"ScanSourceOpDesc (${other.getClass.getName})" - ) - } - - /** - * Every configuration of one source worth running, as (label, op, its own - * directory). - * - * Each variant gets a directory of its own holding its OWN copy of the fixture, - * because the generated script reads the file by bare name (`pd.read_csv( - * "sample.csv")`) out of the directory it runs in. Two variants wanting two - * different `sample.csv` files cannot share one. - */ - private def variantsFor( - opDescClass: Class[_ <: LogicalOp], - testRoot: Path - ): Seq[(String, LogicalOp, Path)] = { - // Punctuation collapses to '_', so two labels differing only in punctuation - // would name the same directory and share one fixture and one output dir. Fail - // loudly instead of letting a variant quietly run someone else's file. - val taken = mutable.Set.empty[String] - def dirFor(label: String): Path = { - val name = label.replaceAll("[^A-Za-z0-9]+", "_") - require(taken.add(name), s"two variants of $opDescClass both map to the directory '$name'") - Files.createDirectories(testRoot.resolve(name)) - } - - curatedHandlersByClass.get(opDescClass) match { - case Some(handler) => - val baseDir = dirFor("default") - val base = handler.makeOpDesc(baseDir) - // Every variant calls the handler AGAIN rather than reusing `base`: the handler - // writes its fixture into the directory it is given, and a second op carrying - // the first one's `fileName` would read a file outside the directory it runs in. - // No enum sweep: both curated sources are the text family, whose `attributeType` - // says how to PARSE the fixture (`alice` is not an integer) and whose - // `fileEncoding` describes its BYTES — flipping either without rewriting the - // fixture compares nothing but how the two paths fail. The auto branch below - // rewrites its fixture per variant and does sweep them. - ConfigGenerator - .fullVariantEditsOf(base, Map.empty, handler.rowCount, sweepEnums = false) - .fold( - reason => - throw new IllegalStateException( - s"cannot vary ${opDescClass.getSimpleName}: $reason" - ), - identity - ) - .map { variant => - if (variant.at.isEmpty) ("default", base, baseDir) - else { - val dir = dirFor(variant.label) - val op = ConfigGenerator - .applyVariant(handler.makeOpDesc(dir), variant) - .fold( - reason => - throw new IllegalStateException( - s"cannot build ${opDescClass.getSimpleName} variant '${variant.label}': $reason" - ), - identity - ) - (variant.label, op, dir) - } - } - case None => - val fileType = declaredFileType(opDescClass).getOrElse("") - val encoder = encoderByFileType.getOrElse( - fileType, - throw new IllegalArgumentException( - s"No encoder for ${opDescClass.getSimpleName} (fileTypeName='$fileType')" - ) - ) - val base = { - val dir = dirFor("default") - val op = newScanSource(opDescClass) - op.fileName = Some(encoder(dir, op.fileEncoding.getCharset).toUri.toString) - ("default", op: LogicalOp, dir) - } - base +: generatedVariants(opDescClass, encoder, dirFor) - } - } - - /** - * The variants the shared [[ConfigGenerator]] derives from the operator's own - * fields — the base config with every knob filled, plus one per enum branch - * (`hasHeader`, JSONL's `flatten`). Nothing to register per operator: a knob - * added to a source is swept the day it is added. - * - * `fileEncoding` is swept like any other enum, and the fixture FOLLOWS it: each - * variant's file is written in the charset that variant declares. Encoding is a - * statement about the bytes, so a UTF_16 config over a file left in UTF-8 would - * only compare how each path fails. - * - * Variants that serialize identically are dropped — an operator that ignores - * `fileEncoding` (Arrow declares `@JsonIgnoreProperties`) would otherwise run the - * same config three times. - */ - private def generatedVariants( - opDescClass: Class[_ <: LogicalOp], - encoder: (Path, Charset) => Path, - dirFor: String => Path - ): Seq[(String, LogicalOp, Path)] = { - val seen = mutable.Set.empty[String] - ConfigGenerator - .generateVariants(opDescClass, Map.empty, CanonicalSourceFixture.rows.size) - .fold( - reason => - throw new IllegalStateException( - s"cannot auto-configure ${opDescClass.getSimpleName}: $reason" - ), - identity - ) - .flatMap { - case (label, op) => - val scan = op.asInstanceOf[ScanSourceOpDesc] - val shape = objectMapper.valueToTree[ObjectNode](scan) - shape.remove("fileName") // every variant reads its own copy of the file - if (!seen.add(shape.toString)) None - else { - // "default" is already the bare newInstance config above; this one is - // the generator's, which additionally fills limit and offset. - val name = if (label == "default") "auto-base" else label - val dir = dirFor(name) - scan.fileName = Some(encoder(dir, scan.fileEncoding.getCharset).toUri.toString) - Some((name, scan: LogicalOp, dir)) - } - } - } - - /** Runs the parity test for the operator, once per variant. Throws on mismatch. */ - def run(opDescClass: Class[_ <: LogicalOp]): Unit = { - val testRoot = Files.createTempDirectory(s"op-behavior-${opDescClass.getSimpleName}-") - variantsFor(opDescClass, testRoot).foreach { - case (label, opDesc, workDir) => - try runVariant(opDesc, workDir) - catch { - case e: Throwable => - throw new AssertionError(s"[variant: $label] ${e.getMessage}", e) - } - } - } - - /** Drive one configured source through both paths inside `workDir`, which holds - * that variant's fixture, and assert the two tables match. - */ - private def runVariant(opDesc: LogicalOp, workDir: Path): Unit = { - val actualDir = workDir.resolve("actual") - Files.createDirectories(actualDir) - - val pathA = OpExecHarness.execute(opDesc, inputs = Map.empty, outputDir = actualDir) - val pathB = StandaloneRunner.run( - opDesc = opDesc, - inputs = Map.empty, - outputPortCount = 1, - workDir = workDir - ) - - val actual = pathA.outputs(PortIdentity(0)) - val expected = pathB.outputs(1) - Comparator.assertEqual(actual, expected) - } -} - -/** - * A hand-written recipe for one source that can't use the auto tier - * (fileTypeName + [[CanonicalSourceFixture]] encoder): which OpDesc class it - * handles and how to fixture a working instance. Used for the text-family - * sources ([[TextInputHandler]], [[FileScanSourceHandler]]). - */ -trait SourceHandler { - - /** The concrete OpDesc class this handler tests. */ - def opDescClass: Class[_ <: LogicalOp] - - /** - * Generate the fixture file inside `testRoot` and return a configured - * OpDesc instance whose `fileName` (or analogous URI field) points at it. - */ - def makeOpDesc(testRoot: Path): LogicalOp - - /** How many rows the fixture holds. Only the handler knows — it writes its own, - * rather than the shared [[CanonicalSourceFixture]]. A row-window knob the - * variants fill (`limit`, `offset`) is sized against this, so that the value they - * take keeps some rows and drops some instead of landing past the end. - */ - def rowCount: Int -} - -/** - * The rows every structured-file source reads: [[CanonicalFixture]]'s, whole. - * - * A source has no input port, so the fixture is delivered not as an input JSONL - * but as a file the operator opens itself. Each `writeXxx` encodes these rows - * into one on-disk format (CSV / JSONL / Arrow); a source handler picks the - * encoder its operator understands and points `fileName` at the result. So CSV, - * CSVOld, JSONL and Arrow all verify that the operator reconstructs one shared - * table, instead of each asserting against its own ad-hoc sample. - * - * It reads the canonical table rather than a narrow one of its own. A source - * fixture picked for the types that survive a round trip would be choosing not - * to ask the question this suite exists to ask: these files carry no types, both - * readers infer, and where they infer differently is exactly what should show. A - * date column does part them, and [[StandaloneRunner.sourceCasts]] is where that - * is settled — on Path B's reading, not by leaving the column out. - */ -object CanonicalSourceFixture { - - val schema: Schema = CanonicalFixture.schema - - val rows: Vector[Tuple] = CanonicalFixture.allRows - - /** Write the rows as a header-first, comma-delimited CSV encoded in `charset`. - * - * The charset is a parameter because it describes the BYTES, not the config: a - * variant declaring `fileEncoding = UTF_16` over a file left in UTF-8 would - * compare nothing but how each path fails. - */ - def writeCsv(dir: Path, charset: Charset): Path = { - val path = dir.resolve("sample.csv") - val header = schema.getAttributes.map(a => csvField(a.getName)).mkString(",") - val body = rows.map { t => - schema.getAttributes - .map(a => csvField(Option(t.getField[AnyRef](a.getName)).map(_.toString).orNull)) - .mkString(",") - } - Files.write(path, ((header +: body).mkString("\n") + "\n").getBytes(charset)) - path - } - - /** One CSV field, quoted per RFC 4180. - * - * The table carries commas inside values — a bracketed edge pair, a - * comma-delimited list, an ordinary English sentence — and writing those raw - * shifts every column after them. What the two paths then disagree about is a - * broken file rather than anything either of them does. - */ - private def csvField(value: String): String = - if (value == null) "" - else if (value.exists(c => c == ',' || c == '"' || c == '\n' || c == '\r')) - "\"" + value.replace("\"", "\"\"") + "\"" - else value - - /** Write the rows as JSON Lines (one object per line, keys in schema order). - * Reuses [[TupleIO.writeTuples]] — the same writer the transform fixtures - * use; it also drops a `.schema.json` sidecar the source ignores. - * - * That writer is shared and always writes UTF-8, so a variant asking for another - * charset gets the bytes transcoded afterwards rather than a second writer. - */ - def writeJsonl(dir: Path, charset: Charset): Path = { - val path = dir.resolve("sample.jsonl") - TupleIO.writeTuples(path, rows.iterator, schema) - if (charset != StandardCharsets.UTF_8) { - val text = new String(Files.readAllBytes(path), StandardCharsets.UTF_8) - Files.write(path, text.getBytes(charset)) - } - path - } - - /** Write the rows as an uncompressed Arrow IPC ("file" format) stream — the - * format both `ArrowFileReader` (Path A) and `pd.read_feather` (Path B) - * read. - */ - def writeArrow(dir: Path): Path = { - val path = dir.resolve("sample.arrow") - // Texera's own Schema-to-Arrow mapping and tuple writer, so the file carries - // exactly the types `ArrowUtils.toTexeraSchema` reads back on the other side. - // Hand-listing the fields is what let the table outgrow them unnoticed: the - // columns past the list were simply not written, and both paths went on - // agreeing about the few that were. - val arrowSchema = ArrowUtils.fromTexeraSchema(schema) - Using.Manager { use => - val allocator = use(new RootAllocator()) - val root = use(VectorSchemaRoot.create(arrowSchema, allocator)) - root.allocateNew() - rows.zipWithIndex.foreach { case (t, i) => ArrowUtils.setTexeraTuple(t, i, root) } - root.setRowCount(rows.size) - val channel = use( - FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE) - ) - val writer = use(new ArrowFileWriter(root, null, channel)) - writer.start() - writer.writeBatch() - writer.end() - }.get - path - } -} - -/** Handler for `TextInputSourceOpDesc`. The text lives in the config — no fixture file. */ -object TextInputHandler extends SourceHandler { - - override val opDescClass: Class[_ <: LogicalOp] = classOf[TextInputSourceOpDesc] - - override val rowCount: Int = 3 - - override def makeOpDesc(testRoot: Path): LogicalOp = { - val desc = new TextInputSourceOpDesc() - desc.textInput = "alice\nbob\ncarol" - desc // defaults: attributeType STRING (one row per line), attributeName "line" - } -} - -/** Handler for `FileScanSourceOpDesc`. Plain text file read in default line mode. */ -object FileScanSourceHandler extends SourceHandler { - - override val opDescClass: Class[_ <: LogicalOp] = classOf[FileScanSourceOpDesc] - - override val rowCount: Int = 3 - - override def makeOpDesc(testRoot: Path): LogicalOp = { - val txtPath = testRoot.resolve("sample.txt") - Files.write(txtPath, "alice\nbob\ncarol\n".getBytes(StandardCharsets.UTF_8)) - - val desc = new FileScanSourceOpDesc() - desc.fileName = Some(txtPath.toUri.toString) - desc // defaults: attributeType STRING (one row per line), attributeName "line" - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala deleted file mode 100644 index 5280b4cf1e7..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneEscapingCheck.scala +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} -import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -import org.apache.texera.amber.util.JSONUtils.objectMapper - -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Path} -import scala.jdk.CollectionConverters._ -import scala.sys.process.{Process, ProcessLogger} -import scala.util.Try - -/** - * Configures every operator against hostile column names and parses what - * `generateStandaloneCode` produces. A value spliced in without - * `pyStringLiteral` is silent until someone names a column `a"b`, and then the - * exported script will not compile. - * - * The check has to be behavioural. A generator that builds its quoted literal - * inside a helper shows no quotes in its template, which is how RadarPlot and - * Aggregate survived a source-level sweep that reported zero remaining sites. - */ -object StandaloneEscapingCheck { - - private val schemas = CanonicalFixture.schemasByPort - private val columns = CanonicalFixture.schema.getAttributes.map(_.getName).toSet - - /** Every problem found, empty when the suite is clean. An operator whose config - * cannot be built lands here too: dropping it would check less while still - * passing. - */ - def run(): Seq[String] = { - // Sources have no input schema and so no column knobs. Their free-text knobs - // already get a hostile variant in the normal verify run. - val operators = OperatorBehaviorSpec - .discoverStandaloneOperators() - .filterNot(classOf[SourceOperatorDescriptor].isAssignableFrom) - val dir = Files.createTempDirectory("standalone-escaping-") - dir.toFile.deleteOnExit() - - val (unconfigurable, code) = operators - .map(op => op.getSimpleName -> codeFor(op, dir)) - .partitionMap { - case (name, Left(why)) => Left(s"$name: $why") - case (name, Right(variants)) => - Right(variants.map { case (label, src) => s"$name/$label" -> src }) - } - unconfigurable ++ parse(code.flatten, dir) - } - - /** Holds the four characters that end a Python literal, and carries the column - * it replaces so that two knobs never collide on one name. - */ - private def hostile(column: String): String = "a\"b'c\\d\ne_" + column - - private def hostilize(node: JsonNode): Unit = - node match { - case obj: ObjectNode => - obj.fields().asScala.toSeq.foreach { e => - if (isColumn(e.getValue)) obj.put(e.getKey, hostile(e.getValue.asText)) - else hostilize(e.getValue) - } - case arr: ArrayNode => - (0 until arr.size).foreach { i => - if (isColumn(arr.get(i))) - arr.set(i, objectMapper.getNodeFactory.textNode(hostile(arr.get(i).asText))) - else hostilize(arr.get(i)) - } - case _ => () - } - - private def isColumn(n: JsonNode): Boolean = n.isTextual && columns.contains(n.asText) - - /** Split the way the runner splits: a curated operator's config is hand-written - * because the generator cannot derive one. Auto operators contribute every - * variant, since a knob only `optionals` fills is a site the base never reaches. - */ - private def codeFor( - opClass: Class[_ <: LogicalOp], - dir: Path - ): Either[String, Seq[(String, String)]] = { - val configs = CuratedHandlers.byClass.get(opClass) match { - // The handler's config, not its enum sweep: sweeping moves an enum value, - // never a column name, and would need schemas only the runner holds. - case Some(h) => - Try(Seq("curated" -> h.fixture(dir)._1)).toEither.left.map(e => s"curated: ${e.getMessage}") - case None => ConfigGenerator.generateVariants(opClass, schemas) - } - configs.flatMap { variants => - val results = variants.map { - case (label, op) => - val node = objectMapper.valueToTree[ObjectNode](op) - hostilize(node) - Try( - objectMapper - .treeToValue(node, opClass) - .asInstanceOf[StandaloneCodeGenerator] - .generateStandaloneCode() - ).toEither.left.map(e => s"$label: $e").map(label -> _) - } - results.collectFirst { case Left(why) => why }.toLeft(results.collect { case Right(r) => r }) - } - } - - /** One Python process for all of them; the cost is startup, not parsing. The - * snippets stay in `dir` so a reported operator can be opened as generated. - */ - private def parse(snippets: Seq[(String, String)], dir: Path): Seq[String] = { - val payload = objectMapper.createObjectNode() - snippets.foreach { case (name, src) => payload.put(name, src) } - val input = write(dir, "snippets.json", payload.toString) - val script = write( - dir, - "parse_all.py", - """import ast, json, sys - |for name, code in json.load(open(sys.argv[1])).items(): - | try: - | ast.parse(code) - | except SyntaxError as e: - | print(f"{name}: line {e.lineno}: {e.msg}") - |""".stripMargin - ) - val out = Seq.newBuilder[String] - val err = Seq.newBuilder[String] - val python = sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") - val exit = Process(Seq(python, script.toString, input.toString)) - .!(ProcessLogger(out += _, err += _)) - // Without this a parser that never ran reads as "nothing failed". - require(exit == 0, s"parse_all.py exited $exit: ${err.result().mkString("\n")}") - out.result() - } - - private def write(dir: Path, name: String, content: String): Path = - Files.write(dir.resolve(name), content.getBytes(UTF_8)) -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala deleted file mode 100644 index 359bad51e5a..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/StandaloneRunner.scala +++ /dev/null @@ -1,367 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -import org.apache.texera.amber.util.python.PythonWorkerPool - -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} -import scala.collection.mutable.ArrayBuffer -import scala.sys.process._ - -/** - * Executes the Python code an OpDesc's [[StandaloneCodeGenerator]] emits and - * captures its DataFrame outputs as JSONL files (compatible with - * [[TupleIO]]'s sidecar-schema format on the comparison side). - * - * Wraps the operator's raw generated code with: - * - * ── prologue ────────────────────────────────────────────── - * in1df = pd.read_json("input_port_0.jsonl", lines=True) - * in2df = pd.read_json("input_port_1.jsonl", lines=True) - * ... - * inAlldf = [in1df, in2df] - * ── operator body (verbatim from generateStandaloneCode) ── - * out1df = in1df[in1df["age"] > 18] - * ── epilogue ───────────────────────────────────────────── - * out1df.to_json("output_port_0.jsonl", orient='records', lines=True) - * ... - * - * Port indexing matches the placeholder convention used by the translator: - * `inNdf`/`outNdf` is 1-based and corresponds to the operator's N-th external - * input/output port in declaration order. The harness key (a 1-based Int) is - * what the placeholder uses; the caller is responsible for ordering inputs - * the same way the operator's `generateStandaloneCode()` expects. - * - * The subprocess inherits the caller's environment so the Python interpreter - * picks up whatever pandas/plotly the test fixture installed. - */ -object StandaloneRunner extends LazyLogging { - - /** The value both paths seed numpy's global RNG with. Any fixed number does; - * what matters is that the two agree, so it is declared once here and - * referenced by name from py_op_driver's comment. - */ - private[verify] val VerifySeed: Int = 20260811 - - /** - * @param outputs paths to the per-port output JSONL files. Empty map iff - * the operator's `producesDataFrame()` returned false - * (visualizations, etc.) — caller handles those separately. - * @param stdout raw subprocess stdout (useful for failure diagnostics) - * @param stderr raw subprocess stderr - */ - final case class Result(outputs: Map[Int, Path], stdout: String, stderr: String) - - /** - * Generate, write, and execute the standalone Python script for `opDesc`. - * - * @param opDesc must mix in [[StandaloneCodeGenerator]]; otherwise we throw - * since there's nothing to test. - * @param inputs map from 1-based port index → JSONL fixture path. The - * script reads each into `inNdf`. - * @param outputPortCount how many `outNdf` variables the operator declares. - * Caller derives this from the OpDesc's output ports. - * @param workDir directory used for the generated `script.py` and output - * JSONL files. Created if missing. - * @param pythonExe path to the Python 3.12 interpreter. Defaults to - * the env var `UDF_PYTHON_PATH`, then `python3.12`, then - * `python3`. The same fallback chain used by the rest of - * the Texera test suite for Python-backed operators. - */ - def run( - opDesc: LogicalOp, - inputs: Map[Int, Path], - outputPortCount: Int, - workDir: Path, - pythonExe: String = resolvePython() - ): Result = { - val gen = opDesc match { - case g: StandaloneCodeGenerator => g - case other => - throw new IllegalArgumentException( - s"OpDesc ${other.getClass.getSimpleName} does not implement " + - s"StandaloneCodeGenerator; nothing to verify" - ) - } - - Files.createDirectories(workDir) - val scriptPath = workDir.resolve("script.py") - val outputPaths: Map[Int, Path] = - if (gen.producesDataFrame()) - (1 to outputPortCount).map(i => i -> workDir.resolve(s"output_port_${i - 1}.jsonl")).toMap - else Map.empty - - val source = - renderScript(gen.generateStandaloneCode(), inputs, outputPaths, gen.standaloneHelpers()) - Files.write(scriptPath, source.getBytes(StandardCharsets.UTF_8)) - - val (exit, stdout, stderr) = execute(scriptPath, workDir, pythonExe) - if (exit != 0) { - throw new StandaloneExecutionException(exit, scriptPath, source, stdout, stderr) - } - Result(outputPaths, stdout, stderr) - } - - private val WorkerResourcePath = "/python/standalone_worker.py" - - // Run the rendered script and return (exitCode, stdout, stderr). Prefers a - // pooled persistent worker (imports pandas/plotly once, ~18x faster per op — - // see PythonWorkerPool); a rare hard worker crash falls back to a one-shot - // subprocess so behavior is never worse than the original path. Both paths - // run with cwd = workDir and read results from files, so they are - // interchangeable — the executed script is byte-identical. - private def execute(scriptPath: Path, workDir: Path, pythonExe: String): (Int, String, String) = { - if (PythonWorkerPool.enabled) { - try { - val req = org.apache.texera.amber.util.JSONUtils.objectMapper.createObjectNode() - req.put("scriptPath", scriptPath.toString) - req.put("workDir", workDir.toString) - val o = PythonWorkerPool.run(WorkerResourcePath, Seq.empty, pythonExe, req) - return (o.exit, o.stdout, o.stderr) - } catch { - case e: PythonWorkerPool.WorkerDiedException => - logger.warn( - s"Standalone worker unavailable; falling back to one-shot subprocess " + - s"for $scriptPath: ${e.getMessage}" - ) - } - } - runSubprocess(scriptPath, workDir, pythonExe) - } - - // Original one-process-per-operator path. Retained as the fallback and as the - // behavior selected by TEXERA_TEST_PYTHON_WORKER=0. - private def runSubprocess( - scriptPath: Path, - workDir: Path, - pythonExe: String - ): (Int, String, String) = { - // Capture stdout/stderr separately. ProcessLogger's append is called from - // the subprocess's I/O thread, so we collect into ArrayBuffer (thread-safe - // append is fine for this serial use) and join at the end. - val outBuf = ArrayBuffer.empty[String] - val errBuf = ArrayBuffer.empty[String] - val logger = ProcessLogger(line => outBuf += line, line => errBuf += line) - // cwd = workDir so generated code using *relative* paths (e.g. CSVScan's - // basename-stripped `pd.read_csv("sample.csv")`) resolves against workDir. - // Absolute paths written by the prologue/epilogue are unaffected. - val exit = Process(Seq(pythonExe, scriptPath.toString), Some(workDir.toFile)).!(logger) - (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) - } - - // Builds the full Python source: imports + prologue + verbatim operator body - // + epilogue. We intentionally do NOT substitute the inNdf/outNdf placeholders - // — the body keeps them so the var-bindings the prologue/epilogue introduce - // (also named inNdf/outNdf) reference the same names. - private def renderScript( - body: String, - inputs: Map[Int, Path], - outputs: Map[Int, Path], - helpers: Seq[String] - ): String = { - val sb = new StringBuilder - - sb.append("# Auto-generated by StandaloneRunner. Do not commit.\n") - sb.append("import json\n") - sb.append("import sys\n") - sb.append("import base64\n") - sb.append("import pickle\n") - // NOTE: numpy is intentionally NOT injected here. The production translator - // (WorkflowToPythonTranslator) only provides pandas + plotly to standalone - // scripts, so any operator whose standalone code needs numpy must import it - // itself. Injecting numpy here would mask that class of bug in verify tests. - sb.append("import pandas as pd\n") - sb.append("import plotly.express as px\n") - sb.append("import plotly.graph_objects as go\n") - sb.append("import plotly.io\n") - // Same seed as py_op_driver's run_config, for the reason given there. Bound - // under a private name and deleted so the note above still holds: a script - // that wants numpy has to import it, and this does not hand it one. - sb.append(s"import numpy as _texera_np; _texera_np.random.seed($VerifySeed); del _texera_np\n") - sb.append("\n") - - // Object columns holding non-primitive values (e.g. a trained sklearn model - // in a BINARY output column) can't go through to_json. Pickle+base64 them so - // the JSONL matches py_op_driver's BINARY write path exactly. Primitives - // (str/int/float/bool/None) pass through unchanged, so ordinary DataFrame - // outputs are unaffected. - sb.append("def _texera_encode_obj_cols(df):\n") - sb.append(" for _c in df.columns:\n") - sb.append(" if df[_c].dtype == object:\n") - sb.append( - " df[_c] = df[_c].map(lambda _v: base64.b64encode(pickle.dumps(_v)).decode('ascii') " + - "if not isinstance(_v, (str, int, float, bool, type(None))) else _v)\n" - ) - sb.append(" return df\n") - sb.append("\n") - - // TIMESTAMP columns are handed to the operator as datetime64 (see the - // prologue below) to match the schema-typed runtime path, but the runtime - // path serializes a TIMESTAMP back out with java.sql.Timestamp.toString — - // "yyyy-mm-dd hh:mm:ss.f", trailing zeros trimmed to at least one digit — - // whereas pandas' to_json would emit epoch millis. Convert datetime columns - // back to that exact form before writing so both paths' JSONL agree. - sb.append("def _texera_ts_str(_v):\n") - sb.append(" if pd.isna(_v):\n") - sb.append(" return None\n") - sb.append(" _s = _v.strftime('%Y-%m-%d %H:%M:%S.%f').rstrip('0')\n") - sb.append(" return _s + '0' if _s.endswith('.') else _s\n") - sb.append("\n") - sb.append("def _texera_encode_ts_cols(df):\n") - sb.append(" for _c in df.columns:\n") - sb.append(" if pd.api.types.is_datetime64_any_dtype(df[_c]):\n") - sb.append(" df[_c] = df[_c].map(_texera_ts_str)\n") - sb.append(" return df\n") - sb.append("\n") - - // Prologue: load each external input into in{N}df. Note: pd.read_json with - // lines=True correctly handles empty files (returns empty DataFrame). - // convert_dates=False: pd.read_json otherwise auto-coerces ISO-ish strings - // and columns named like dates ("date", "*_at", …) to datetime64, which the - // schema-typed runtime path (STRING) does not do — that divergence would - // make a plain date string column serialize as "...T00:00:00" on only one - // side. Operators that genuinely need datetimes convert explicitly, so both - // paths stay in sync. - // precise_float=True: pd.read_json's default (ujson) fast double parser is - // lossy in the last few ULPs, so a DOUBLE column would load slightly - // different values than the schema-typed runtime path (which parses doubles - // exactly). Operators that stringify raw cell values (e.g. Radar hover text) - // then diverge; precise_float=True keeps both paths bit-identical. - // The blanket convert_dates=False also leaves genuine TIMESTAMP columns as - // strings, which the runtime path delivers as datetime64 — a divergence for - // any operator that renders or computes on them. The fixture's schema - // sidecar says which columns those are, so cast exactly those back. - inputs.toSeq.sortBy(_._1).foreach { - case (n, path) => - sb.append( - s"in${n}df = pd.read_json(${py(path.toString)}, lines=True, convert_dates=False, precise_float=True)\n" - ) - timestampColumns(path).foreach { col => - sb.append(s"if ${py(col)} in in${n}df.columns:\n") - sb.append(s" in${n}df[${py(col)}] = pd.to_datetime(in${n}df[${py(col)}])\n") - } - doubleColumns(path).foreach { col => - sb.append(s"if ${py(col)} in in${n}df.columns:\n") - sb.append(s" in${n}df[${py(col)}] = in${n}df[${py(col)}].astype('float64')\n") - } - } - // The variadic placeholder, bound here for the same reason the numbered ones - // are: this script leaves the body's placeholders alone and defines names to - // match them, so an operator reading a variadic port finds its list here the - // way the translator would have written one out. - if (inputs.nonEmpty) { - sb.append( - inputs.keys.toSeq.sorted.map(n => s"in${n}df").mkString("inAlldf = [", ", ", "]\n") - ) - } - sb.append("\n") - - // Body verbatim — placeholders left in place. - // Emitted ahead of the body the way the translator does, so an operator that - // declares a helper is exercised here exactly as it runs in a real script. - helpers.foreach { helper => - sb.append(helper) - if (!helper.endsWith("\n")) sb.append('\n') - sb.append('\n') - } - - sb.append("# ── operator body ──\n") - sb.append(body) - if (!body.endsWith("\n")) sb.append('\n') - sb.append("\n") - - // Epilogue: dump each out{N}df to JSONL. When producesDataFrame() is false - // (visualization ops), `outputs` is empty and this block is a no-op — the - // caller is expected to verify viz outputs by other means. - outputs.toSeq.sortBy(_._1).foreach { - case (n, path) => - sb.append( - s"_texera_encode_obj_cols(_texera_encode_ts_cols(out${n}df))" + - s".to_json(${py(path.toString)}, orient='records', lines=True)\n" - ) - } - - sb.toString - } - - // TIMESTAMP-typed column names from a fixture's `.jsonl.schema.json` sidecar. - // A missing or unreadable sidecar means no casts — the prologue then behaves - // exactly as before. - private def timestampColumns(input: Path): Seq[String] = - columnsOfType(input, AttributeType.TIMESTAMP) - - // DOUBLE-typed column names. pd.read_json narrows a float column whose values - // are all integral to int64, while the runtime path keeps the schema's DOUBLE, - // so a column like 7.0 stringifies as "7" on one side and "7.0" on the other — - // invisible to numeric comparison, visible the moment an operator uses the - // column as a label (a trace name, a legend entry, hover text). - private def doubleColumns(input: Path): Seq[String] = - columnsOfType(input, AttributeType.DOUBLE) - - private def columnsOfType(input: Path, attributeType: AttributeType): Seq[String] = - scala.util - .Try(TupleIO.readSchemaSidecar(input)) - .toOption - .toSeq - .flatMap( - _.getAttributes.filter(_.getType == attributeType).map(_.getName) - ) - - // Python string literal, single-quoted with backslashes escaped. We - // deliberately don't use repr() in Scala (no such thing) — JSON.toString - // would also work but introduces double-quote escaping when the path has - // spaces. - private def py(s: String): String = - "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'" - - // Resolution chain mirrors the rest of the Texera test infra: env var first - // (set by CI / the shared-venv setup), then conventional names. - private def resolvePython(): String = { - val fromEnv = sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty) - fromEnv.getOrElse { - // We don't try to probe `which` here — if neither env var nor a literal - // `python3.12` is on PATH, the subprocess invocation will fail and the - // error path below surfaces it. - "python3.12" - } - } -} - -final class StandaloneExecutionException( - val exitCode: Int, - val scriptPath: Path, - val source: String, - val stdout: String, - val stderr: String -) extends RuntimeException( - // The script path goes first in the message so a failing CI log makes it - // immediately obvious which file to open. stderr ends the message because - // the Python traceback (if any) is the most actionable signal. - s"""Standalone Python script exited with code $exitCode. - |Script: $scriptPath - |--- stdout --- - |$stdout - |--- stderr --- - |$stderr""".stripMargin - ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala deleted file mode 100644 index 590e0c361ce..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ /dev/null @@ -1,962 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.{BooleanNode, IntNode, TextNode} -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.apache.texera.amber.operator.{ - LogicalOp, - PythonOperatorDescriptor, - StandaloneCodeGenerator -} -import org.apache.texera.amber.operator.aggregate.AggregateOpDesc -import org.apache.texera.amber.operator.dummy.DummyOpDesc -import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc -import org.apache.texera.amber.operator.sleep.SleepOpDesc -import org.apache.texera.amber.operator.split.SplitOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnGaussianNaiveBayesOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnLinearRegressionOpDesc -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor -import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import org.apache.texera.amber.operator.huggingFace.HuggingFaceSpamSMSDetectionOpDesc -import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc -import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingGaussianNaiveBayesOpDesc -import org.apache.texera.amber.operator.regex.RegexOpDesc -import org.apache.texera.amber.operator.sklearn.testing.SklearnTestingOpDesc -import org.apache.texera.amber.operator.typecasting.TypeCastingOpDesc -import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc -import org.apache.texera.amber.operator.visualization.DotPlot.DotPlotOpDesc -import org.apache.texera.amber.operator.visualization.barChart.BarChartOpDesc -import org.apache.texera.amber.operator.visualization.boxViolinPlot.BoxViolinPlotOpDesc -import org.apache.texera.amber.operator.visualization.ImageViz.ImageVisualizerOpDesc -import org.apache.texera.amber.operator.visualization.IcicleChart.IcicleChartOpDesc -import org.apache.texera.amber.operator.visualization.bubbleChart.BubbleChartOpDesc -import org.apache.texera.amber.operator.visualization.bulletChart.BulletChartOpDesc -import org.apache.texera.amber.operator.visualization.candlestickChart.CandlestickChartOpDesc -import org.apache.texera.amber.operator.visualization.carpetPlot.CarpetPlotOpDesc -import org.apache.texera.amber.operator.visualization.choroplethMap.ChoroplethMapOpDesc -import org.apache.texera.amber.operator.visualization.continuousErrorBands.ContinuousErrorBandsOpDesc -import org.apache.texera.amber.operator.visualization.contourPlot.ContourPlotOpDesc -import org.apache.texera.amber.operator.visualization.dendrogram.DendrogramOpDesc -import org.apache.texera.amber.operator.visualization.dumbbellPlot.DumbbellPlotOpDesc -import org.apache.texera.amber.operator.visualization.ecdfPlot.ECDFPlotOpDesc -import org.apache.texera.amber.operator.visualization.figureFactoryTable.FigureFactoryTableOpDesc -import org.apache.texera.amber.operator.visualization.filledAreaPlot.FilledAreaPlotOpDesc -import org.apache.texera.amber.operator.visualization.funnelPlot.FunnelPlotOpDesc -import org.apache.texera.amber.operator.visualization.ganttChart.GanttChartOpDesc -import org.apache.texera.amber.operator.visualization.gaugeChart.GaugeChartOpDesc -import org.apache.texera.amber.operator.visualization.ScatterMatrixChart.ScatterMatrixChartOpDesc - -import org.apache.texera.amber.operator.visualization.heatMap.HeatMapOpDesc -import org.apache.texera.amber.operator.visualization.hierarchychart.HierarchyChartOpDesc -import org.apache.texera.amber.operator.visualization.histogram2d.Histogram2DOpDesc -import org.apache.texera.amber.operator.visualization.histogram.HistogramChartOpDesc -import org.apache.texera.amber.operator.visualization.lineChart.LineChartOpDesc -import org.apache.texera.amber.operator.visualization.nestedTable.NestedTableOpDesc -import org.apache.texera.amber.operator.visualization.networkGraph.NetworkGraphOpDesc -import org.apache.texera.amber.operator.visualization.parallelCoordinatesPlot.ParallelCoordinatesPlotOpDesc -import org.apache.texera.amber.operator.visualization.pieChart.PieChartOpDesc -import org.apache.texera.amber.operator.visualization.polarChart.PolarChartOpDesc -import org.apache.texera.amber.operator.visualization.quiverPlot.QuiverPlotOpDesc -import org.apache.texera.amber.operator.visualization.radarChart.RadarChartOpDesc -import org.apache.texera.amber.operator.visualization.radarPlot.RadarPlotOpDesc -import org.apache.texera.amber.operator.visualization.rangeSlider.RangeSliderOpDesc -import org.apache.texera.amber.operator.visualization.sankeyDiagram.SankeyDiagramOpDesc -import org.apache.texera.amber.operator.visualization.scatter3DChart.Scatter3dChartOpDesc -import org.apache.texera.amber.operator.visualization.scatterplot.ScatterplotOpDesc -import org.apache.texera.amber.operator.visualization.stripChart.StripChartOpDesc -import org.apache.texera.amber.operator.visualization.tablesChart.TablesPlotOpDesc -import org.apache.texera.amber.operator.visualization.ternaryContour.TernaryContourOpDesc -import org.apache.texera.amber.operator.visualization.ternaryPlot.TernaryPlotOpDesc -import org.apache.texera.amber.operator.visualization.timeSeriesplot.TimeSeriesOpDesc -import org.apache.texera.amber.operator.visualization.treeplot.TreePlotOpDesc -import org.apache.texera.amber.operator.visualization.volcanoPlot.VolcanoPlotOpDesc -import org.apache.texera.amber.operator.visualization.waterfallChart.WaterfallChartOpDesc -import org.apache.texera.amber.operator.visualization.windRoseChart.WindRoseChartOpDesc -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} -import scala.jdk.CollectionConverters._ -import scala.util.{Failure, Success, Try} - -/** - * Unified verification runner for non-source operators implementing - * [[StandaloneCodeGenerator]]. Resolves, per operator: - * - Path A engine: [[PyOpExecHarness]] for PythonOperatorDescriptor, - * [[OpExecHarness]] otherwise; Path B is always [[StandaloneRunner]]. - * - Config + fixture: curated handler ([[CuratedHandlers]]) if registered, - * else [[ConfigGenerator]] against the [[CanonicalFixture]] schemas. - * - Comparison: order-insensitive by default (parallel output order isn't a - * contract); strict positional only when the operator declares - * `orderSensitive = true` (the sort family). All output ports are compared. - * Operators that can't be run are Flagged with a reason — never silently - * skipped. - */ -object TransformVerificationRunner { - - /** - * Per-operator knob handling: a value this operator's variants must carry, - * and where it applies. Two needs, one table, because both answer the same - * question — what does the generator have to be told about this operator's - * knobs that its metadata does not say. - * - * `Pinned` holds a knob at one value and keeps it out of the sweep, for a - * knob whose other value selects non-determinism rather than a different - * behavior to check. Split's "Auto-Generate Seed" is the case: with it on the - * executor seeds from the clock, so that run agrees with nothing — its own - * previous run included — and there is no output for a script to reproduce. - * Everything else about the operator is deterministic, so pinning covers the - * partition rather than abandoning the operator over one switch. The value - * reaches the test name via [[pinnedTierNote]], so the run does not read as - * full coverage. - * - * `WithOptionals` sets a knob inside the `optionals` variant, for a branch - * that needs a switch AND the field it governs. Ternary Plot colours its - * points only when `colorEnabled` is on and `colorDataField` is set, and the - * two belong to different mechanisms: the sweep turns the switch on with the - * column empty, the optional fill supplies the column with the switch off, so - * neither variant generated the coloured branch. Naming the switch here puts - * it in the variant that fills the column. - * - * Named per operator rather than applied wholesale, because switches are not - * generally independent: turning every Boolean on in that variant paired - * Sklearn's `countVectorizer` with `tfidfTransformer` (mutually exclusive - * text pipelines), asked File Scan to extract an archive from a plain file, - * and re-enabled the very auto-seed switch the first scope holds off. - * - * Distinct from an `enumSweep` row in [[variantsNotRun]], which is about an - * operator's enums as a whole rather than one named knob. - */ - sealed trait KnobScope - object KnobScope { - case object Pinned extends KnobScope - case object WithOptionals extends KnobScope - } - - final case class Knob(field: String, value: JsonNode, scope: KnobScope) - - val knobOverrides: Map[Class[_], Seq[Knob]] = Map( - classOf[SplitOpDesc] -> Seq(Knob("random", BooleanNode.FALSE, KnobScope.Pinned)), - // `SleepOpExec` sleeps this many seconds per tuple, and the generator fills a - // required Int with half the row count, so the fixture would spend tens of - // seconds asleep for nothing: the delay never reaches the output being - // compared, and the standalone translation is a passthrough by design. - classOf[SleepOpDesc] -> Seq(Knob("sleepTime", IntNode.valueOf(0), KnobScope.Pinned)), - classOf[TernaryPlotOpDesc] -> Seq( - Knob("colorEnabled", BooleanNode.TRUE, KnobScope.WithOptionals) - ), - // Both text switches are pinned off on the numeric table: the pipeline they - // build reads a column that table does not have, and `tfidfTransformer` has - // no meaning at all outside a CountVectorizer pipeline (the schema hides it - // when the vectorizer is off). Their branches are generated against the text - // table by the [[AltScenario]]s instead. - classOf[SklearnClassifierOpDesc] -> Seq( - Knob("countVectorizer", BooleanNode.FALSE, KnobScope.Pinned), - Knob("tfidfTransformer", BooleanNode.FALSE, KnobScope.Pinned) - ), - classOf[SklearnTrainingOpDesc] -> Seq( - Knob("countVectorizer", BooleanNode.FALSE, KnobScope.Pinned), - Knob("tfidfTransformer", BooleanNode.FALSE, KnobScope.Pinned) - ) - ) - - /** This operator's overrides for one scope, as the generator takes them. - * Exact class first, then base class, so a family can be named once instead - * of per estimator. - */ - private def knobsFor(opClass: Class[_ <: LogicalOp], scope: KnobScope): Map[String, JsonNode] = - knobOverrides - .get(opClass) - .orElse(knobOverrides.collectFirst { - case (base, knobs) if base.isAssignableFrom(opClass) => knobs - }) - .getOrElse(Seq.empty) - .filter(_.scope == scope) - .map(k => k.field -> k.value) - .toMap - - /** A second generation pass for a branch the base config cannot reach. Usually - * that is a branch needing a DIFFERENT table: a swept variant cannot switch - * tables, since [[ConfigGenerator]] resolves every column picker against ONE - * schema, so the branch is generated separately against the table it needs - * with its switch pinned on. - * - * It also reaches a branch whose own knobs the base config leaves empty. The - * sweep offers the values a config already holds, so a list filled only on the - * far side of a switch has nothing to offer until the switch is pinned — and - * then the second pass may well take the same table as the first. - * - * The auto-tier twin of [[TransformHandler.extraScenarios]]: it names only the - * table and the pins, and the generator writes the config. - */ - final case class AltScenario( - label: String, - fixture: SharedFixture, - pinned: Map[String, JsonNode] - ) - - /** Every kind of run a [[variantsNotRun]] row can name: the derived variants, - * plus the [[AltScenario]] labels (an alt scenario IS one kind of run). Named - * so a row cannot misspell one and silently stop suppressing anything. - */ - object RunKind { - val Nulls = "nulls" - val EnumSweep = "enumSweep" - val HostileText = "hostileText" - val CountVectorizerText = "countVectorizer_text" - val TfidfText = "tfidf_text" - val NonFeatureColumn = "nonFeatureColumn" - val TextLabels = "textLabels" - val RegressionBranch = "regressionBranch" - - /** One swept hyperparameter of an advanced trainer, which the sweep labels by the - * pointer it flips. Built here rather than spelled out at each row, since a row - * that misspelled the pointer would suppress nothing and say nothing. - */ - def hyperParameter(name: String): String = s"paraList/0/parameter=$name" - } - - /** The [[RunKind]] a generated variant's label names. A `merged` variant labels - * itself `kind(fields…)`, and the fields it happened to move are not part of - * what is being withheld. - */ - private def kindOf(label: String): String = label.takeWhile(_ != '(') - - /** Keyed by BASE class, not by concrete operator: a newly registered sklearn - * estimator is covered with no entry of its own, matching how the families - * themselves are discovered. - */ - val altFixtureScenarios: Map[Class[_], Seq[AltScenario]] = { - // One scenario per text pipeline rather than a sweep inside one: which - // pipeline is built is the branch under test, and the two are alternatives, - // not a knob crossed with everything else. - val countVectorizerText = AltScenario( - label = RunKind.CountVectorizerText, - fixture = CanonicalFixture.sklearnText, - pinned = Map( - "countVectorizer" -> BooleanNode.TRUE, - "tfidfTransformer" -> BooleanNode.FALSE - ) - ) - val tfidfText = countVectorizerText.copy( - label = RunKind.TfidfText, - pinned = Map( - "countVectorizer" -> BooleanNode.TRUE, - "tfidfTransformer" -> BooleanNode.TRUE - ) - ) - // The vectorizer stays off: what this scenario covers is the branch that - // narrows `X` for an estimator, and Count Vectorizer replaces it rather than - // feeding it, naming the text columns the narrowing would otherwise drop. - val nonFeatureColumn = AltScenario( - label = RunKind.NonFeatureColumn, - fixture = CanonicalFixture.sklearnNumericWithText, - pinned = Map( - "countVectorizer" -> BooleanNode.FALSE, - "tfidfTransformer" -> BooleanNode.FALSE - ) - ) - // The advanced trainers are the ones left out: they name the feature columns - // themselves rather than taking every column but the target, so a column an - // estimator cannot fit is not reachable for them. - Map( - classOf[SklearnClassifierOpDesc] -> Seq(countVectorizerText, tfidfText, nonFeatureColumn), - classOf[SklearnTrainingOpDesc] -> Seq(countVectorizerText, tfidfText, nonFeatureColumn), - // No text scenarios, and nothing pinned: this operator declares neither - // switch, and a pin is set on the config whether or not the field exists, - // so pinning one here would hand it a property it cannot read back. - classOf[SklearnLinearRegressionOpDesc] -> Seq(nonFeatureColumn.copy(pinned = Map.empty)), - // A scorer reads a text label as readily as a numeric one, and names the - // class after the label rather than after its position. Regression is - // pinned off rather than swept: a regression metric puts both columns - // through `float()`, so on this table the sweep would generate the one - // configuration the operator is right to refuse. The two columns are - // pinned because the operator's `@SampleColumn`s name the numeric pair, - // and an annotation naming a column the table does not hold ends the run - // rather than falling back — which is what catches a misspelling. - classOf[MachineLearningScorerOpDesc] -> Seq( - AltScenario( - label = RunKind.TextLabels, - fixture = CanonicalFixture.scorerTextLabels, - pinned = Map( - "isRegression" -> BooleanNode.FALSE, - "actualValueColumn" -> TextNode.valueOf("species_name"), - "predictValueColumn" -> TextNode.valueOf("species_name_pred") - ) - ), - // The regression metrics are unreachable from the base config: the sweep - // reads the sites the config already holds, and on the classification - // branch the regression list is empty, so it offers none. Pinned on, the - // list is filled before the sweep looks, and the other three metrics - // become variants like any other enum. Same table as the default runs — - // this scenario is here for the branch, not for a different set of rows. - AltScenario( - label = RunKind.RegressionBranch, - fixture = CanonicalFixture, - pinned = Map("isRegression" -> BooleanNode.TRUE) - ) - ) - ) - } - - /** The alternate-table scenarios this operator takes, resolved by family and - * minus any [[variantsNotRun]] names. - */ - private def altScenariosFor(opClass: Class[_ <: LogicalOp]): Seq[AltScenario] = - altFixtureScenarios - .collectFirst { case (base, scenarios) if base.isAssignableFrom(opClass) => scenarios } - .getOrElse(Seq.empty) - .filterNot(alt => notRun(opClass, alt.label)) - - /** How a pinned operator's tier reads in the report, e.g. `auto, random=false`. */ - private def pinnedTierNote(opClass: Class[_ <: LogicalOp]): String = { - val pinned = knobsFor(opClass, KnobScope.Pinned) - if (pinned.isEmpty) "" - else pinned.map { case (field, value) => s"$field=${value.asText}" }.mkString(", ", ", ", "") - } - - /** Why one kind of run is left out for an operator. The distinction is whether - * anyone should be waiting for it: [[PendingFix]] is a debt someone closes, - * [[ByDesign]] is an answer that will not change. - */ - sealed trait NotRunReason - final case class PendingFix(issue: String) extends NotRunReason - final case class ByDesign(why: String) extends NotRunReason - - final case class NotRun(op: Class[_], kind: String, reason: NotRunReason) - - /** The runs an operator does not get, and why. - * - * One table rather than one per kind. Every row makes the same statement, so - * the coverage report can print them together, and the next exemption has an - * obvious home instead of arriving as another set somewhere else. - * - * `op` matches its subclasses, so one row covers a family. `kind` is a - * [[RunKind]]. - * - * A curated handler's own [[TransformHandler.unfillableVariants]] stays where - * it is: those describe the table that handler wrote rather than the operator, - * and change when the fixture is rewritten. - */ - val variantsNotRun: Seq[NotRun] = { - // The platform raises on an empty cell, so the two paths cannot be compared - // on one until it stops. - val emptyCellRaises: Seq[(Class[_], String)] = Seq( - // Regex alone: apache/texera#7566 answers the empty cell in Substring Search - // and Unnest String, and this one was not part of it. - classOf[RegexOpDesc] -> "apache/texera#7548" - ) - - // The operator refuses the text pipeline in `getOutputSchemas`, so there is no - // configuration to compare: neither path is generated. An invalid configuration - // rather than a translation gap. - // - // Not sklearn raising, which is what the estimator's own limitation would look - // like. This fixture's word counts repeat enough that `ColumnTransformer` stays - // above its 0.3 sparse threshold and hands over a dense array, which GaussianNB - // fits without complaint. Only a wider vocabulary would reach the limitation - // the operator is guarding against. - val dense = ByDesign("the operator refuses Count Vectorizer at compile time") - val denseOnly = for { - op <- Seq( - classOf[SklearnGaussianNaiveBayesOpDesc], - classOf[SklearnTrainingGaussianNaiveBayesOpDesc] - ) - label <- Seq(RunKind.CountVectorizerText, RunKind.TfidfText) - } yield NotRun(op, label, dense) - - emptyCellRaises.map { - case (op, issue) => NotRun(op, RunKind.Nulls, PendingFix(issue)) - } ++ Seq( - // An enum whose legal values depend on a sibling field: flipping it alone - // builds a config the curated fixture already covers properly. - NotRun( - classOf[TypeCastingOpDesc], - RunKind.EnumSweep, - ByDesign( - "resultType is legal only for certain source column types, and the native " + - "executor throws on an illegal cast; the fixture pairs each type with a " + - "compatible column already" - ) - ), - NotRun( - classOf[AggregateOpDesc], - RunKind.EnumSweep, - ByDesign( - "aggFunction is cross-constrained with its attribute's type and with " + - "COUNT(*)'s empty attribute; the fixture pairs each function with a " + - "compatible column already" - ) - ), - // Stated about the operator's own fixture rather than about the operator: a - // predicate over a string column takes the hostile value fine, so this row goes - // the day that fixture filters on one. - NotRun( - classOf[SpecializedFilterOpDesc], - RunKind.HostileText, - ByDesign( - "`id > 8` compares against an INTEGER column, and the platform parses the " + - "predicate value as that column's type, so the number parser refuses the " + - "hostile string before any escaping could matter" - ) - ), - NotRun( - classOf[SklearnMLOperatorDescriptor[_]], - RunKind.HostileText, - ByDesign( - "what a hyperparameter's value may hold is decided by the parameter beside " + - "it, and every one of those is a number or a word from a fixed set, so a " + - "spliced a\"b fails at the conversion rather than at any escaping" - ) - ) - ) ++ denseOnly - } - - /** Every kind of run withheld from this operator, with why. This is the whole of - * what the coverage report needs, so it never walks the table itself. One entry - * per kind: a family row and an operator row for the same kind are the same - * statement twice, and the first one wins. - */ - def withheldRunsFor(opClass: Class[_ <: LogicalOp]): Seq[(String, NotRunReason)] = - variantsNotRun - .collect { case NotRun(op, kind, reason) if op.isAssignableFrom(opClass) => kind -> reason } - .distinctBy(_._1) - - private def notRun(opClass: Class[_ <: LogicalOp], kind: String): Boolean = - withheldRunsFor(opClass).exists(_._1 == kind) - - /** Visualization operators with deterministic Plotly JSON validation. */ - val visualizationJsonOps: Set[Class[_]] = Set( - classOf[RangeSliderOpDesc], - classOf[HeatMapOpDesc], - classOf[HierarchyChartOpDesc], - classOf[HistogramChartOpDesc], - classOf[Histogram2DOpDesc], - classOf[LineChartOpDesc], - classOf[ParallelCoordinatesPlotOpDesc], - classOf[PieChartOpDesc], - classOf[PolarChartOpDesc], - classOf[QuiverPlotOpDesc], - classOf[RadarChartOpDesc], - classOf[RadarPlotOpDesc], - classOf[SankeyDiagramOpDesc], - classOf[Scatter3dChartOpDesc], - classOf[ScatterplotOpDesc], - classOf[StripChartOpDesc], - classOf[TablesPlotOpDesc], - classOf[TernaryContourOpDesc], - classOf[TernaryPlotOpDesc], - classOf[TimeSeriesOpDesc], - classOf[TreePlotOpDesc], - classOf[VolcanoPlotOpDesc], - classOf[WaterfallChartOpDesc], - classOf[WindRoseChartOpDesc], - classOf[BarChartOpDesc], - classOf[BulletChartOpDesc], - classOf[CandlestickChartOpDesc], - classOf[CarpetPlotOpDesc], - classOf[ChoroplethMapOpDesc], - classOf[ContinuousErrorBandsOpDesc], - classOf[ContourPlotOpDesc], - classOf[DendrogramOpDesc], - classOf[DumbbellPlotOpDesc], - classOf[ECDFPlotOpDesc], - classOf[FigureFactoryTableOpDesc], - classOf[FilledAreaPlotOpDesc], - classOf[FunnelPlotOpDesc], - classOf[GanttChartOpDesc], - classOf[GaugeChartOpDesc], - classOf[DotPlotOpDesc], - classOf[IcicleChartOpDesc], - classOf[BubbleChartOpDesc], - classOf[ScatterMatrixChartOpDesc], - classOf[BoxViolinPlotOpDesc] - ) - - /** Visualization operators with deterministic HTML validation. */ - val visualizationHtmlOps: Set[Class[_]] = Set( - classOf[ImageVisualizerOpDesc], - classOf[NestedTableOpDesc] - ) - - /** Triaged, explicitly-not-run operators: class → honest reason, shown in - * the test report and coverage table. - */ - val knownIssues: Map[Class[_], String] = Map( - classOf[DummyOpDesc] -> - ("harness gap: placeholder operator with no physical execution — " + - "LogicalOp.getPhysicalOp throws NotImplementedError"), - classOf[SklearnPredictionOpDesc] -> - ("trained-model input: the operator consumes a fitted sklearn model on " + - "its model port; a JSONL fixture written from the JVM cannot carry a " + - "live model object, so the operator cannot be run in isolation here"), - classOf[SklearnTestingOpDesc] -> - ("trained-model input: scores a fitted sklearn model read from its model " + - "port; a JVM-written JSONL fixture cannot carry a live model object, so " + - "the operator cannot be run in isolation here"), - classOf[WordCloudOpDesc] -> - ("non-deterministic image: emits a base64 PNG from the wordcloud library " + - "whose word placement is randomized (no seed), so the two paths' images " + - "never match byte-for-byte"), - classOf[NetworkGraphOpDesc] -> - ("non-deterministic layout: the native path calls nx.spring_layout with no " + - "seed, so node coordinates are random per run and differ from the seeded " + - "standalone path, and the two paths' Plotly figures never match numerically") - ) - - sealed trait Disposition - final case class Runnable(tier: String) extends Disposition // "auto" | "curated" - final case class Flagged(reason: String) extends Disposition - - /** When `VERIFY_FORCE_AUTO=1`, ignore CuratedHandlers so every operator is - * exercised through the shared-CSV auto path instead. Lets us measure how - * much of the hand-written curated set the auto tier can now replace: an op - * that stays RUNNABLE/passes under force-auto no longer needs its curated - * handler. - */ - private def forceAuto: Boolean = sys.env.get("VERIFY_FORCE_AUTO").contains("1") - - /** The shared table an operator runs on in the AUTO tier. Which table an - * operator takes is its own axis (see [[SharedFixture]]); the auto tier used - * to be pinned to the whole of [[CanonicalFixture]], which is why an operator - * needing a narrower table had to be curated just to name one. sklearn cannot - * fit canonical's string columns — `X = table.drop(target)` feeds every - * remaining column to `fit` — so its families take the petal-and-label view - * of that same table. - */ - private[verify] def fixtureFor(opClass: Class[_ <: LogicalOp]): SharedFixture = - if (CuratedHandlers.sklearnNumericClasses.contains(opClass)) CanonicalFixture.sklearnNumeric - else if (opClass == classOf[HuggingFaceSpamSMSDetectionOpDesc]) CanonicalFixture.withoutScore - else CanonicalFixture - - /** Static classification — cheap (reflection only, no subprocesses), called - * at spec construction time to decide test-vs-ignore. - */ - def disposition(opClass: Class[_ <: LogicalOp]): Disposition = - knownIssues.get(opClass) match { - case Some(reason) => Flagged(s"known issue: $reason") - case None => - Try(opClass.getDeclaredConstructor().newInstance()) match { - case Failure(e) => Flagged(s"cannot instantiate: ${e.getMessage}") - case Success(op: StandaloneCodeGenerator) => - if (!op.producesDataFrame()) - if (visualizationJsonOps.contains(opClass) || visualizationHtmlOps.contains(opClass)) - Runnable("visualization") - else Flagged("visualization: no DataFrame output to compare") - else if (!forceAuto && CuratedHandlers.byClass.contains(opClass)) - Runnable("curated") - else - ConfigGenerator.generate(opClass, fixtureFor(opClass).schemasByPort) match { - case Left(reason) => Flagged(s"cannot auto-configure: $reason") - case Right(configured) => - Try(configured.operatorInfo.inputPorts.size) match { - case Failure(e) => - Flagged(s"operatorInfo failed on generated config: ${e.getMessage}") - case Success(n) if n < 1 || n > 2 => - Flagged(s"unsupported input port count: $n") - case Success(_) - if outputHasBinaryColumn(configured, fixtureFor(opClass)) && - fixtureFor(opClass) == CanonicalFixture => - // A trained-model (BINARY) output cannot be fit on the - // canonical table, whose string columns reach `fit`. The - // model itself is not byte-comparable either, but that is - // handled for every tier alike (see modelColumns in run). - // An op that names a numeric fixture is fine here. - Flagged( - "model output: emits a BINARY (trained-model) column; " + - "requires a numeric fixture, not the canonical table" - ) - case Success(_) => Runnable(s"auto${pinnedTierNote(opClass)}") - } - } - case Success(_) => - Flagged("does not implement StandaloneCodeGenerator") - } - } - - /** True if the configured operator declares a BINARY output column (e.g. a - * serialized trained model). Best-effort: only Python descriptors expose - * getOutputSchemas, and a throw (schema needs real inputs) reads as "no - * detectable BINARY column" so the op falls through to its normal tier. - */ - private def outputHasBinaryColumn(configured: LogicalOp, fixture: SharedFixture): Boolean = - configured match { - case p: PythonOperatorDescriptor => - val inputSchemas = fixture.schemasByPort.map { - case (port, schema) => PortIdentity(port) -> schema - } - Try(p.getOutputSchemas(inputSchemas)).toOption - .exists(_.values.exists(_.getAttributes.exists(_.getType == AttributeType.BINARY))) - case _ => false - } - - /** Execute both paths and assert parity on every declared output port. - * Precondition: disposition(opClass) returned Runnable. - */ - def run(opClass: Class[_ <: LogicalOp]): Unit = { - val testRoot = Files.createTempDirectory(s"verify-${opClass.getSimpleName}-") - - // Resolve the run list: each entry is (label, configured op, its inputs). - // Both tiers yield the base config PLUS one variant per enum value, so each - // enum branch (e.g. a line chart's mode = line / dots / line+dots) is - // exercised, not just the default, PLUS the `optionals` and `hostileText` - // variants. Variants of one fixture share input files; a curated handler's - // extraScenarios carry their own (structurally different) inputs. - val runs: Seq[(String, LogicalOp, Map[PortIdentity, Path])] = - (if (forceAuto) None else CuratedHandlers.byClass.get(opClass)) match { - case Some(handler) => - val (op, in) = handler.fixture(testRoot) - // The variants are derived against the handler's OWN fixture, not the - // canonical one — a curated handler writes the table its operator needs, - // so that is what an optional column knob has to resolve against. - // - // An enum-sweep-exempt op still gets the fills: what is cross-constrained - // with a sibling field is its ENUM values, so a blind sweep produces invalid - // configs — filling an optional knob or splicing a quote does not. - // - // Fall back to the single curated config if it can't be varied at all. - val primary = - ConfigGenerator - .fullVariantsOf( - op, - schemasOf(in), - rowCountOf(in), - sweepEnums = !notRun(opClass, RunKind.EnumSweep) - ) - .fold(_ => Seq("default" -> op), identity) - // A variant this operator does not get, named in [[variantsNotRun]]. - .filterNot { case (label, _) => notRun(opClass, kindOf(label)) } - primary.map { case (label, o) => (label, o, in) } ++ - handler.extraScenarios(testRoot) ++ - handler.nullsKeepFilled.toSeq.flatMap(curatedNullsCase(opClass, op, in, testRoot, _)) - case None => - val fixture = fixtureFor(opClass) - val vs = ConfigGenerator - .generateVariants( - opClass, - fixture.schemasByPort, - fixture.port0RowCount, - knobsFor(opClass, KnobScope.Pinned), - knobsFor(opClass, KnobScope.WithOptionals) - ) - .fold( - reason => throw new IllegalStateException(s"cannot auto-configure: $reason"), - identity - ) - val inputPortCount = vs.head._2.operatorInfo.inputPorts.size - val in = fixture.writeInputs(testRoot, inputPortCount) - // A variant the operator itself cannot take, named in [[variantsNotRun]]: - // for a swept hyperparameter that is one row per parameter, so the sweep - // keeps covering the rest. - vs.filterNot { case (label, _) => notRun(opClass, kindOf(label)) } - .map { case (label, o) => (label, o, in) } ++ - nullsCase(opClass, vs.head._2, testRoot, fixture) ++ - altScenariosFor(opClass).flatMap { alt => - // Each scenario writes under its own directory: two tables in one - // testRoot would otherwise both claim input_port_0.jsonl. - val dir = testRoot.resolve(alt.label) - Files.createDirectories(dir) - ConfigGenerator - .generateVariants( - opClass, - alt.fixture.schemasByPort, - alt.fixture.port0RowCount, - pinned = alt.pinned, - switches = knobsFor(opClass, KnobScope.WithOptionals) - ) - .fold( - reason => - throw new IllegalStateException( - s"cannot auto-configure ${alt.label}: $reason" - ), - identity - ) - // The base variant carries the branch's own column knob: the pins - // are visible while the config is built, so the knob the schema - // requires under them is filled like any other required field. - .map { - case (label, o) => - (s"${alt.label}/$label", o, alt.fixture.writeInputs(dir, inputPortCount)) - } - } - } - - runs.foreach { - case (label, opDesc, inputs) => - val workDir = - if (runs.size == 1) testRoot - else testRoot.resolve(label.replaceAll("[^A-Za-z0-9]+", "_")) - Files.createDirectories(workDir) - try runVariant(opClass, opDesc, inputs, workDir) - catch { - case e: Throwable => - throw new AssertionError(s"[variant: $label] ${e.getMessage}", e) - } - } - } - - /** One extra run per operator, on `fixture` with one empty cell per column (see - * [[SharedFixture.emptyOneCellPerColumn]]). It takes the base config rather than - * crossing with the other variants: what an operator does with a null is a - * property of the operator, and multiplying it across every knob would buy more - * runtime than signal. - * - * The auto tier's form: the table is the shared one [[fixtureFor]] resolves, so - * the holes come from the fixture itself. See [[curatedNullsCase]] for the other - * tier, which has no fixture object to ask. - */ - private def nullsCase( - opClass: Class[_ <: LogicalOp], - base: LogicalOp, - testRoot: Path, - fixture: SharedFixture - ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = - if (notRun(opClass, RunKind.Nulls)) Seq.empty - else { - val dir = testRoot.resolve("nulls-input") - Files.createDirectories(dir) - val in = fixture.write(dir, base.operatorInfo.inputPorts.size, withGaps = true) - Seq(("nulls", base, in)) - } - - /** [[nullsCase]] for the curated tier, where there is no fixture object to write - * a second time: the handler's own files are read back, holed, and rewritten. - * So a handler opts in by naming its load-bearing columns and nothing else, and - * [[TransformHandler.fixture]] keeps returning paths. - */ - private def curatedNullsCase( - opClass: Class[_ <: LogicalOp], - base: LogicalOp, - inputs: Map[PortIdentity, Path], - testRoot: Path, - keepFilled: Set[String] - ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = - if (notRun(opClass, RunKind.Nulls)) Seq.empty - else { - val dir = testRoot.resolve("nulls-input") - Files.createDirectories(dir) - val holed = inputs.map { - case (portId, path) => - val schema = TupleIO.readSchemaSidecar(path) - val rows = TupleIO.readTuples(path, schema).toSeq - val out = dir.resolve(path.getFileName.toString) - TupleIO.writeTuples( - out, - SharedFixture.emptyOneCellPerColumn(rows, schema, keepFilled).iterator, - schema - ) - portId -> out - } - Seq(("nulls", base, holed)) - } - - /** The schema of each input file, keyed by port index — what a curated handler - * actually wrote, read back off the sidecar its writer drops. A file without one - * contributes no schema, so a column knob resolved against that port simply finds - * nothing to fill and the variant is skipped rather than built on a guess. - */ - private def schemasOf(inputs: Map[PortIdentity, Path]): Map[Int, Schema] = - inputs.flatMap { - case (portId, path) => Try(TupleIO.readSchemaSidecar(path)).toOption.map(portId.id -> _) - } - - /** How many rows port 0 holds — the hint a numeric knob's fill is scaled against - * (a `limit` worth running is one that keeps some rows and drops some). - */ - private def rowCountOf(inputs: Map[PortIdentity, Path]): Int = - inputs - .get(PortIdentity(0)) - .flatMap(path => Try(Files.readAllLines(path).asScala.count(_.trim.nonEmpty)).toOption) - .filter(_ > 0) - .getOrElse(ConfigGenerator.DefaultRowCount) - - /** Run one configured variant of `opDesc` through both paths against `inputs`, - * writing all intermediate/output files under `workDir`, and assert parity on - * every declared output port. - */ - private def runVariant( - opClass: Class[_ <: LogicalOp], - opDesc: LogicalOp, - inputs: Map[PortIdentity, Path], - workDir: Path - ): Unit = { - val outputPortCount = opDesc.operatorInfo.outputPorts.size - val actualDir = workDir.resolve("actual") - Files.createDirectories(actualDir) - - if (!opDesc.asInstanceOf[StandaloneCodeGenerator].producesDataFrame()) { - runVisualization(opClass, opDesc, inputs, outputPortCount, actualDir, workDir) - return - } - - // Path A's getPhysicalPlan/getPhysicalOp may mutate the OpDesc in place — - // AggregateOpDesc rewrites its `aggregations` to the final stage (COUNT→SUM) - // via getFinal. Run Path A on an isolated deep copy (same JSON round-trip the - // executor itself uses) so the shared instance stays pristine for Path B, - // whose generateStandaloneCode reads the original fields directly. - val opDescForPathA = - objectMapper - .readValue(objectMapper.writeValueAsString(opDesc), opClass) - .asInstanceOf[LogicalOp] - val (pathAOutputs, pathAOutputSchemas): (Map[PortIdentity, Path], Map[PortIdentity, Schema]) = - if (classOf[PythonOperatorDescriptor].isAssignableFrom(opClass)) { - val r = PyOpExecHarness.execute(opDescForPathA, inputs = inputs, outputDir = actualDir) - (r.outputs, r.outputSchemas) - } else { - val r = OpExecHarness.execute(opDescForPathA, inputs = inputs, outputDir = actualDir) - (r.outputs, r.outputSchemas) - } - - // StandaloneRunner keys inputs by 1-based port index (the inNdf convention). - val standaloneInputs: Map[Int, Path] = - inputs.toSeq - .sortBy(_._1.id) - .zipWithIndex - .map { - case ((_, path), idx) => (idx + 1) -> path - } - .toMap - - val pathB = StandaloneRunner.run( - opDesc = opDesc, - inputs = standaloneInputs, - outputPortCount = outputPortCount, - workDir = workDir - ) - - // The operator declares whether its output row order is meaningful via - // LogicalOp.orderSensitive (true only for the sort family); default unordered. - val orderSensitive = opDesc.orderSensitive - (0 until outputPortCount).foreach { port => - val actual = pathAOutputs.getOrElse( - PortIdentity(port), - throw new AssertionError(s"Texera path produced no output for port $port") - ) - val expected = pathB.outputs.getOrElse( - port + 1, - throw new AssertionError(s"standalone path produced no output for port $port") - ) - // A BINARY column holds a trained model: the two paths produce - // behaviorally-equivalent but not bit-identical models, so the comparator - // unpickles both and asserts their predictions on the training features - // (the probe) match — verifying behavior, not just completion. - val modelColumns: Seq[String] = pathAOutputSchemas - .get(PortIdentity(port)) - .map(_.getAttributes.filter(_.getType == AttributeType.BINARY).map(_.getName)) - .getOrElse(Seq.empty) - val probePath: Option[Path] = - if (modelColumns.nonEmpty) inputs.toSeq.sortBy(_._1.id).headOption.map(_._2) else None - Comparator.assertEqual( - actual, - expected, - orderSensitive = orderSensitive, - modelColumns = modelColumns, - probePath = probePath - ) - } - } - - private def runVisualization( - opClass: Class[_ <: LogicalOp], - opDesc: LogicalOp, - inputs: Map[PortIdentity, Path], - outputPortCount: Int, - actualDir: Path, - testRoot: Path - ): Unit = { - require( - visualizationJsonOps.contains(opClass) || visualizationHtmlOps.contains(opClass), - s"${opClass.getSimpleName} is not registered for visualization validation" - ) - require( - outputPortCount == 1, - "visualization JSON validation currently supports one output port" - ) - require( - classOf[PythonOperatorDescriptor].isAssignableFrom(opClass), - "visualization JSON validation currently supports Python visualization operators" - ) - - val actual = PyOpExecHarness - .execute(opDesc, inputs = inputs, outputDir = actualDir) - .outputs - .getOrElse( - PortIdentity(0), - throw new AssertionError("Texera path produced no visualization output for port 0") - ) - - val standaloneInputs: Map[Int, Path] = - inputs.toSeq - .sortBy(_._1.id) - .zipWithIndex - .map { - case ((_, path), idx) => (idx + 1) -> path - } - .toMap - - StandaloneRunner.run( - opDesc = opDesc, - inputs = standaloneInputs, - outputPortCount = outputPortCount, - workDir = testRoot - ) - - // A JSON-compared operator can still legitimately render its own error page - // instead of a figure (a non-numeric threshold, no non-null rows). There is - // then no Plotly payload to compare on either path, so compare what the user - // actually sees — the HTML. - if (visualizationJsonOps.contains(opClass) && hasPlotlyFigure(actual)) { - val expected = testRoot.resolve("output.json") - if (!Files.exists(expected)) { - throw new AssertionError(s"standalone visualization path did not produce $expected") - } - VisualizationJsonComparator.assertEqual(actual, expected) - } else { - val expected = testRoot.resolve("output.html") - if (!Files.exists(expected)) { - throw new AssertionError(s"standalone visualization path did not produce $expected") - } - VisualizationHtmlComparator.assertEqual(actual, expected) - } - } - - /** True if the runtime path's visualization output carries a Plotly figure — - * either a `json-content` payload or an `html-content` holding a - * `Plotly.newPlot(...)` call. False for an operator's own error page. - */ - private def hasPlotlyFigure(visualizationJsonl: Path): Boolean = { - val line = Files - .readAllLines(visualizationJsonl, StandardCharsets.UTF_8) - .asScala - .find(_.trim.nonEmpty) - .getOrElse(throw new AssertionError(s"$visualizationJsonl is empty")) - val node = objectMapper.readTree(line) - val json = node.get("json-content") - if (json != null && !json.isNull && json.asText().nonEmpty) true - else { - val html = node.get("html-content") - html != null && !html.isNull && html.asText().contains("Plotly.newPlot(") - } - } -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala deleted file mode 100644 index c9fa74cf877..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import org.apache.texera.amber.util.JSONUtils.objectMapper - -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} - -object VisualizationHtmlComparator { - - /** A pandas Styler namespaces its CSS with a uuid drawn per Styler instance, so - * the same table rendered twice differs in every `id=` and every selector even - * though the markup is identical. The uuid carries no information about the - * table — it only keeps two tables on one page from colliding — so it is - * normalized away before comparing. Only the random prefix is replaced: the - * `_row0_col0` suffix that identifies the cell stays, so a genuine structural - * difference still fails. - */ - private val StylerUuid = "T_[0-9a-f]+".r - - private def normalize(html: String): String = StylerUuid.replaceAllIn(html, "T_uuid") - - def assertEqual(actualVisualizationJsonl: Path, expectedHtmlFile: Path): Unit = { - val actual = readActualHtml(actualVisualizationJsonl) - val expected = new String(Files.readAllBytes(expectedHtmlFile), StandardCharsets.UTF_8) - - if (normalize(actual) != normalize(expected)) { - throw new VisualizationHtmlMismatchException( - actual = actualVisualizationJsonl, - expected = expectedHtmlFile, - actualHtml = actual, - expectedHtml = expected - ) - } - } - - private def readActualHtml(path: Path): String = { - val line = Files - .readAllLines(path, StandardCharsets.UTF_8) - .stream() - .filter(_.trim.nonEmpty) - .findFirst() - .orElseThrow(() => new AssertionError(s"$path is empty")) - - val node = objectMapper.readTree(line) - val htmlNode = node.get("html-content") - if (htmlNode == null || htmlNode.isNull) { - throw new AssertionError(s"$path has no html-content field") - } - htmlNode.asText() - } -} - -final class VisualizationHtmlMismatchException( - val actual: Path, - val expected: Path, - val actualHtml: String, - val expectedHtml: String -) extends RuntimeException( - s"""Visualization HTML mismatch: - | actual: $actual - | expected: $expected - |--- actual html --- - |$actualHtml - |--- expected html --- - |$expectedHtml""".stripMargin - ) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala deleted file mode 100644 index f559b2c39e6..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.typesafe.scalalogging.LazyLogging -import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.apache.texera.amber.util.python.PythonWorkerPool - -import java.nio.file.{Files, Path, StandardCopyOption} -import scala.collection.mutable.ArrayBuffer -import scala.sys.process._ - -/** - * Compares the Plotly figure the two paths render, via `compare.py`'s - * `--plotly` mode. - * - * Shares that script — and therefore [[Comparator]]'s pool — rather than - * carrying one of its own: a worker is bound to the script it was launched - * with, so a separate script would mean a separate pool of interpreters for - * what is the same job, comparing one operator's two outputs. One comparison - * pool serves both output shapes. - */ -object VisualizationJsonComparator extends LazyLogging { - - private val ScriptResourcePath = "/python/compare.py" - - def assertEqual( - actualVisualizationJsonl: Path, - expectedPlotlyJson: Path, - pythonExe: String = resolvePython() - ): Unit = { - val (exit, stdout, stderr) = - compare(actualVisualizationJsonl, expectedPlotlyJson, pythonExe) - if (exit != 0) { - throw new VisualizationJsonMismatchException( - actual = actualVisualizationJsonl, - expected = expectedPlotlyJson, - exitCode = exit, - stdout = stdout, - stderr = stderr - ) - } - } - - // Pooled worker first, one-shot CLI as the fallback and as the behavior - // selected by TEXERA_TEST_PYTHON_WORKER=0. Both run the same - // `_run_plotly_comparison`, so results are identical. - private def compare(actual: Path, expected: Path, pythonExe: String): (Int, String, String) = { - if (PythonWorkerPool.enabled) { - try { - val req = objectMapper.createObjectNode() - req.put("kind", "plotly") - req.put("actual", actual.toString) - req.put("expected", expected.toString) - val o = PythonWorkerPool.run(ScriptResourcePath, Seq("--serve"), pythonExe, req) - return (o.exit, o.stdout, o.stderr) - } catch { - case e: PythonWorkerPool.WorkerDiedException => - logger.warn( - s"Comparator worker unavailable; falling back to one-shot CLI: ${e.getMessage}" - ) - } - } - runCli(actual, expected, pythonExe) - } - - private def runCli(actual: Path, expected: Path, pythonExe: String): (Int, String, String) = { - val scriptPath = extractScript() - val outBuf = ArrayBuffer.empty[String] - val errBuf = ArrayBuffer.empty[String] - val processLogger = ProcessLogger(line => outBuf += line, line => errBuf += line) - - val exit = Process( - Seq(pythonExe, scriptPath.toString, "--plotly", actual.toString, expected.toString) - ).!(processLogger) - (exit, outBuf.mkString("\n"), errBuf.mkString("\n")) - } - - private def extractScript(): Path = { - val stream = getClass.getResourceAsStream(ScriptResourcePath) - require(stream != null, s"compare.py not found at $ScriptResourcePath") - try { - val tmp = Files.createTempFile("compare-", ".py") - Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING) - tmp.toFile.deleteOnExit() - tmp - } finally stream.close() - } - - private def resolvePython(): String = - sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12") -} - -final class VisualizationJsonMismatchException( - val actual: Path, - val expected: Path, - val exitCode: Int, - val stdout: String, - val stderr: String -) extends RuntimeException( - s"""Visualization JSON mismatch (compare.py --plotly exit $exitCode): - | actual: $actual - | expected: $expected - |--- stderr --- - |$stderr""".stripMargin - ) From df10bd870674f685372ac9f602f0a8df19875cb0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 16:11:06 -0700 Subject: [PATCH 08/33] chore: leave the two verify files to #8327 as well This branch held an older copy of both and changes neither. Co-Authored-By: Claude Opus 5 (1M context) --- .../translator/verify/ConfigGenerator.scala | 2015 ----------------- .../TransformVerificationRunnerSpec.scala | 69 - 2 files changed, 2084 deletions(-) delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala deleted file mode 100644 index 853c41c2534..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala +++ /dev/null @@ -1,2015 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.annotation.{ - JsonIgnore, - JsonIgnoreProperties, - JsonProperty, - JsonSubTypes -} -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.annotation.JsonDeserialize -import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} -import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator -import org.apache.texera.amber.operator.metadata.annotations.{ - AutofillAttributeName, - AutofillAttributeNameList, - AutofillAttributeNameOnPort1, - CommonOpDescAnnotation, - HideAnnotation, - SampleColumn -} -import org.apache.texera.amber.util.JSONUtils.objectMapper - -import java.lang.reflect.{Field, Modifier, ParameterizedType, Type, TypeVariable} -import javax.validation.constraints.{DecimalMin, Min} -import scala.collection.mutable -import scala.jdk.CollectionConverters._ -import scala.util.Try - -/** - * Produces a valid configuration for an operator automatically, from the - * metadata the operator already carries — field defaults, enums, and the - * `@AutofillAttributeName` annotation family (which marks a field as "a column - * name from input port N"). This is the baseline layer of the combined - * config-generation plan: every registered operator gets a runnable config with - * no per-operator handler. - * - * We only need a *valid* config, not a *meaningful* one — both verification - * paths get the identical OpDesc and are compared to each other, so a - * degenerate-but-valid config still tests translation fidelity. Free-form value - * fields are filled with a canonical value (see [[CanonicalString]]) that the - * synthetic dataset is built to contain, so the operator actually does - * something rather than matching nothing. - * - * Strategy: reflect over the operator's config fields (those carrying - * `@JsonProperty` or an autofill annotation), build a JSON object of - * field → value, and let Jackson deserialize it into the OpDesc. Using the - * same `objectMapper` Texera uses everywhere means enums (`@JsonValue`), - * `Option`, and `@JsonCreator` nested objects are handled by existing, - * battle-tested deserialization rather than bespoke reflection. - */ -object ConfigGenerator { - - /** Canonical literal for free-form STRING fields; present in the synthetic - * dataset so filters/comparisons actually match rows. - */ - private val CanonicalString = "1" - - /** Row count used to size the numeric "middle of the range" fallback when a - * caller doesn't supply one (real verification callers pass the fixture's - * actual row count). See [[numericFill]]. - */ - val DefaultRowCount = 10 - - /** - * @param opClass the operator descriptor class to configure. - * @param inputSchemas schema present at each 0-based input port; supplies the - * column names that `@AutofillAttributeName*` fields draw - * from. - * @return Right(configured opDesc), or Left(reason) if a required field can't - * be filled from the available metadata (the operator is then - * reported as uncovered rather than silently passed). - */ - def generate( - opClass: Class[_ <: LogicalOp], - inputSchemas: Map[Int, Schema], - rowCount: Int = DefaultRowCount - ): Either[String, LogicalOp] = { - buildObject(opClass, inputSchemas, rowCount).flatMap { node => - // LogicalOp is polymorphic (@JsonTypeInfo on `operatorType`); Jackson needs - // the registered type id to deserialize the concrete subtype. - typeNameByClass.get(opClass) match { - case Some(typeName) => node.put("operatorType", typeName) - case None => - return Left(s"${opClass.getSimpleName} not registered in LogicalOp @JsonSubTypes") - } - Try(objectMapper.treeToValue(node, opClass)).toEither.left - .map(e => s"deserialization failed: ${e.getMessage}") - } - } - - /** - * Like [[generate]], but also sweeps every enum field: returns the base - * config plus one variant per non-default enum value (one enum flipped at a - * time — linear, NOT the combinatorial product). Lets the runner exercise - * each enum branch (e.g. LineChart's line mode = line / dots / line+dots) - * instead of only the default. The label identifies the flipped value. - */ - def generateVariants( - opClass: Class[_ <: LogicalOp], - inputSchemas: Map[Int, Schema], - rowCount: Int = DefaultRowCount, - pinned: Map[String, JsonNode] = Map.empty, - switches: Map[String, JsonNode] = Map.empty - ): Either[String, Seq[(String, LogicalOp)]] = - typeNameByClass.get(opClass) match { - case None => Left(s"${opClass.getSimpleName} not registered in LogicalOp @JsonSubTypes") - case Some(typeName) => - val used = mutable.Set.empty[(Int, String)] - buildObject(opClass, inputSchemas, used, rowCount, pinned = pinned).flatMap { baseNode => - baseNode.put("operatorType", typeName) - pinned.foreach { case (field, value) => baseNode.set[JsonNode](field, value) } - applyAll( - opClass, - baseNode, - None, - allVariants( - opClass, - baseNode, - inputSchemas, - used, - rowCount, - pinned = pinned.keySet, - switches = switches - ) - ) - } - } - - /** - * Enum-sweep an ALREADY-configured op (e.g. a curated handler's OpDesc): - * serialize it to JSON, then return the base op plus one variant per - * non-default enum value found anywhere in it (including inside lists with - * more than one element). Lets curated fixtures cover every enum branch too, - * not just the single value the handler hard-coded. - */ - def variantsOf(opDesc: LogicalOp): Either[String, Seq[(String, LogicalOp)]] = { - val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] - // base = the original op (preserve the curated config exactly); variants are - // deserialized from the JSON with one enum flipped. - nodeOf(opDesc).flatMap(node => - applyAll(opClass, node, Some(opDesc), Variant.Base +: enumVariants(opClass, node)) - ) - } - - /** - * [[variantsOf]] plus the two multi-knob variants [[generateVariants]] gives an - * auto-configured op: `optionals` and `hostileText` (see [[extraVariants]]). - * - * A separate entry point rather than a widening of [[variantsOf]] so a caller - * states which it wants, and a failure points at one of them. Curated fixtures - * are the reason it exists: a hand-written config is the ONLY config its - * operator ever runs, so without this its optional knobs stay at their defaults - * and nothing ever splices a quote into the code it generates. - * - * `inputSchemas` describes the op's OWN inputs — a curated handler writes its - * own fixture, so this is not necessarily the canonical one. - */ - def fullVariantsOf( - opDesc: LogicalOp, - inputSchemas: Map[Int, Schema], - rowCount: Int = DefaultRowCount, - sweepEnums: Boolean = true - ): Either[String, Seq[(String, LogicalOp)]] = { - val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] - for { - node <- nodeOf(opDesc) - variants <- fullVariantEditsOf(opDesc, inputSchemas, rowCount, sweepEnums) - ops <- applyAll(opClass, node, Some(opDesc), variants) - } yield ops - } - - /** - * What [[fullVariantsOf]] runs, as the edits themselves rather than the finished - * ops — for a caller that has to REBUILD its fixture per variant and so must - * apply them to a FRESH op. A source is that caller: its exported script reads - * the file by bare name out of the directory it runs in, so every variant needs - * its own directory and its own copy, produced by calling the handler again. - * - * `sweepEnums = false` keeps the fills but drops the enum sweep, for a fixture - * whose enums are cross-constrained with the data it holds — flipping one then - * describes a table the fixture is not. - */ - def fullVariantEditsOf( - opDesc: LogicalOp, - inputSchemas: Map[Int, Schema], - rowCount: Int = DefaultRowCount, - sweepEnums: Boolean = true - ): Either[String, Seq[Variant]] = { - val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] - nodeOf(opDesc).map { node => - val used = occupiedColumns(opClass, node, inputSchemas) - allVariants(opClass, node, inputSchemas, used, rowCount, sweepEnums) - } - } - - /** `opDesc` with `variant`'s edits applied. The base variant carries no edits and - * hands the instance straight back, so a curated config is never round-tripped - * through JSON just to be left unchanged. - */ - def applyVariant(opDesc: LogicalOp, variant: Variant): Either[String, LogicalOp] = - if (variant.at.isEmpty) Right(opDesc) - else - nodeOf(opDesc).flatMap { node => - variant.at.foreach { case (pointer, value) => setAtPointer(node, pointer, value) } - deserialize(node, opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]]) - } - - /** One named configuration, as the pointer → value edits that turn a base config - * into it. Applied to one clone of that base. - */ - final case class Variant(label: String, at: Seq[(String, JsonNode)]) - - object Variant { - - /** The base config itself — no edits, so [[applyVariant]] returns it unchanged. */ - val Base: Variant = Variant("default", Seq.empty) - } - - /** Every variant of `baseNode`: the config itself, one per non-default enum value, - * then the two multi-knob fills. - */ - private def allVariants( - opClass: Class[_ <: LogicalOp], - baseNode: ObjectNode, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - sweepEnums: Boolean = true, - pinned: Set[String] = Set.empty, - switches: Map[String, JsonNode] = Map.empty - ): Seq[Variant] = - Variant.Base +: ((if (sweepEnums) - enumVariants( - opClass, - baseNode, - pinned, - FillContext(schemas, mutable.Set.empty ++ used, rowCount) - ) - else Seq.empty) ++ - extraVariants(opClass, baseNode, schemas, used, rowCount, switches)) - - /** The two multi-knob variants, so called because each moves every knob of its kind - * at once. An operator's knobs are worth exercising, but bisecting a rare failure - * by hand costs less than a run per field: all optional knobs are filled together, - * and all free-text knobs take the hostile value together. - * - * A row from the UI's `+` button is one of those optional knobs, not a variant of - * its own: for a list the base leaves empty it IS the "now it is set" case, exactly - * like a scalar going from unset to set. - */ - private def extraVariants( - opClass: Class[_ <: LogicalOp], - baseNode: ObjectNode, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - switches: Map[String, JsonNode] = Map.empty - ): Seq[Variant] = - Seq( - merged( - "optionals", { - // One counter and one `used` set for the whole variant: the three walks below - // land in ONE config, so a column taken by any of them is taken for all, and - // restarting the counter per walk gave two rows the same value. A copy, so the - // base pass's own set is left alone. - val ordinal = new Ordinal - val taken = mutable.Set.empty[(Int, String)] ++ used - optionalColumnFills(opClass, schemas, taken, baseNode) ++ - optionalScalarFills(opClass, baseNode, "", schemas, taken, rowCount, ordinal) ++ - extraRowFills(opClass, baseNode, schemas, taken, rowCount, ordinal) ++ - switches.toSeq.sortBy(_._1).map { - case (field, value) => Variant(s"$field=${value.asText}", Seq((s"/$field", value))) - } - } - ), - merged("hostileText", numbered(hostileTextFills(opClass, baseNode, ""))) - ).flatten - - /** Apply each variant to its own clone of `baseNode` and read the result back as an - * op. `base` is the instance to hand back for the unedited variant, when the caller - * has one whose exact state matters (a curated fixture); `None` deserializes it - * from `baseNode` like any other. - */ - private def applyAll( - opClass: Class[_ <: LogicalOp], - baseNode: ObjectNode, - base: Option[LogicalOp], - variants: Seq[Variant] - ): Either[String, Seq[(String, LogicalOp)]] = { - val results = variants.map { variant => - base.filter(_ => variant.at.isEmpty) match { - case Some(op) => Right((variant.label, op)) - case None => - val clone = baseNode.deepCopy() - variant.at.foreach { case (pointer, value) => setAtPointer(clone, pointer, value) } - deserialize(clone, opClass).map((variant.label, _)) - } - } - results.collectFirst { case Left(err) => err }.toLeft(results.collect { case Right(ok) => ok }) - } - - /** An already-configured op as the JSON this generator edits: its serialized form, - * carrying the polymorphic type id Jackson needs to read the concrete subtype back. - */ - private def nodeOf(opDesc: LogicalOp): Either[String, ObjectNode] = { - val opClass = opDesc.getClass.asInstanceOf[Class[_ <: LogicalOp]] - objectMapper.valueToTree[JsonNode](opDesc) match { - case node: ObjectNode => - if (!node.has("operatorType")) - typeNameByClass.get(opClass).foreach(node.put("operatorType", _)) - Right(node) - case _ => Left(s"${opClass.getSimpleName} did not serialize to a JSON object") - } - } - - /** The (port, column) pairs an already-configured op's column pickers already hold, - * as the `used` set the optional-knob fill resolves against — so a knob it fills - * lands on a column the fixture is not using yet, the same rule the base pass keeps - * for sibling pickers. Without it a curated x/y and a filled-in optional colour all - * collapse onto one column. - * - * Walks the fixture's nested rows too, not just its top-level fields: the picker an - * appended row has to differ from usually lives in the rows ALREADY there (a - * projection's column list), and re-picking one of those asks the operator for the - * same output column twice. - */ - private def occupiedColumns( - clazz: Class[_], - node: JsonNode, - schemas: Map[Int, Schema], - path: String = "" - ): mutable.Set[(Int, String)] = { - val used = mutable.Set.empty[(Int, String)] - configFields(clazz).foreach { f => - val childPath = pointerOf(f, path) - rowType(f) match { - case Some(row) => - rowPaths(f, node.at(childPath), childPath) - .foreach(rowPath => used ++= occupiedColumns(row, node, schemas, rowPath)) - case None if hasAutofill(f) => - val port = autofillSpec(f).map(_.port).getOrElse(0) - val columns = - schemas.get(port).map(_.getAttributes.map(_.getName).toSet).getOrElse(Set.empty) - val held = node.at(childPath) - val values = if (held.isArray) held.elements().asScala.toSeq else Seq(held) - values.filter(_.isTextual).map(_.asText).filter(columns).foreach(c => used += ((port, c))) - case None => () - } - } - used - } - - /** One fill per OPTIONAL column knob, which [[decide]] leaves unset. Unset is the - * right base config — it is what most workflows carry — but it also means the - * branch each generator emits for a knob that IS set never runs on either path, - * so the two hand-written branches are never compared. - * - * Resolved against the `used` set the whole variant shares, so the column a knob - * takes differs from what the config already reads. A list knob takes a SINGLE - * column: the "every matching column" fill suits a required axes list, not an - * optional narrowing one — all thirty columns as group-by keys would make every row - * its own group. - * - * A knob the config ALREADY points at a column is left alone. That never happens - * to the auto base — [[decide]] skips every optional picker, so each one still - * holds the value a fresh instance has — but a curated config picks its columns - * deliberately, and overwriting one would discard the fixture's whole point. - * - * Only the operator's OWN fields: a picker inside a nested row is filled by - * [[rowFills]], on the row walk that knows which row it belongs to. - */ - private def optionalColumnFills( - clazz: Class[_], - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - baseNode: JsonNode - ): Seq[Variant] = - configFields(clazz).filter(hasAutofill).filterNot(hiddenBySibling(_, baseNode)).flatMap { f => - columnFill(clazz, f, baseNode, pointerOf(f, ""), schemas, used, baseNode).map { - case (pointer, value) => - // A list knob holds its one column in an array; name the column either way. - val col = if (value.isArray) value.path(0).asText else value.asText - Variant(s"${pointer.stripPrefix("/")}=$col", Seq((pointer, value))) - } - } - - /** One fill per `+`-row list, appending ONE MORE row than the base carries: the first - * row for an optional list (empty, as the UI starts it), a second for a required one. - * - * For an optional list that is the point — its rows are otherwise never populated. - * For a required one it reaches only the code BETWEEN rows (the separator each path - * joins them with, whatever an operator does with several at once); NOT a mis-indexed - * value, since both generators read every value off the loop variable. - * - * The row is built by the same pass as the first, against the `used` set the whole - * variant shares, so its column knobs land on columns nothing else is reading. - */ - private def extraRowFills( - clazz: Class[_], - baseNode: JsonNode, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - ordinal: Ordinal - ): Seq[Variant] = { - // The appended row is built outside [[buildObject]]'s walk, so what that walk - // carries down to a row has to be handed over here as well: without them the new - // row's type-variable fields and schema-ruled fields come back empty, and the - // variant then fails on a value this generator left out rather than on anything - // the operator does. - val ownBindings = typeBindingsOf(clazz) - val ownScope = SchemaScope.of(clazz) - configFields(clazz).flatMap { f => - val childPath = pointerOf(f, "") - val rows = baseNode.at(childPath) - for { - row <- if (isList(f.getType)) elementType(f).toOption.filter(isNestedObject) else None - if rows.isArray - next <- buildObject( - row, - schemas, - used, - rowCount, - elementBindings(f, ownBindings), - ownScope.descend(jsonNameOf(f)) - ).toOption - } yield { - // Fill the new row's own optional knobs too — the `optionals` variant is - // computed against the BASE config, where this row does not exist yet, so - // otherwise the row arrives with every free-value knob at its default, a step - // whose bounds are both empty is dropped by the operator, and an optional column - // picker (which [[decide]] skips) stays null. - rowFills(row, next, "", schemas, used, rowCount, ordinal).foreach { - case (pointer, value) => setAtPointer(next, pointer, value) - } - distinguish( - row, - next, - rows, - elementBindings(f, ownBindings), - ownScope.descend(jsonNameOf(f)), - FillContext(schemas, used, rowCount) - ) - Variant(childPath, Seq((s"$childPath/${rows.size()}", next))) - } - } - } - - /** Move the appended row off a value a row beside it already holds, where holding - * the same one makes the two rows the same row. Every row is built the same way and - * so comes back with the same values, which for a second hyperparameter row means - * `C` twice: not two settings but one written twice, which the emitted Python - * rejects outright as a repeated keyword argument. - * - * Only the field the rest of the row is stated in terms of, which is the one a - * `valueRules` condition reads. A row's other choices are its own business, and two - * lines of a chart drawn in the same style are still two lines. That a knob has to - * differ is something only the schema can say, and this is where it says it. - */ - private def distinguish( - rowClass: Class[_], - next: ObjectNode, - siblings: JsonNode, - bindings: TypeBindings, - scope: SchemaScope, - fill: FillContext - ): Unit = { - val deciding = decidingFields(rowClass, scope) - enumSites(rowClass, next, "", bindings, scope, fill) - .filter(site => deciding.contains(site.pointer.stripPrefix("/"))) - .foreach { site => - val taken = siblings.elements().asScala.map(_.at(site.pointer)).toSet - if (taken.contains(next.at(site.pointer))) - site.values.find(v => !taken.contains(v)).foreach { v => - setAtPointer(next, site.pointer, v) - site.companions(v).foreach { - case (pointer, value) => setAtPointer(next, pointer, value) - } - } - } - } - - /** The fields of `clazz` that some other field's `valueRules` is stated in terms of. - * A hyperparameter row has one, `parameter`, and most objects have none. - */ - private def decidingFields(clazz: Class[_], scope: SchemaScope): Set[String] = - configFields(clazz).iterator - .flatMap(f => scope.child(jsonNameOf(f)).path("valueRules").path("allOf").elements().asScala) - .flatMap(_.path("if").fieldNames().asScala) - .toSet - - /** One variant out of many fills, labelled with the fields it sets. `None` when - * there is nothing to fill, so an operator without such knobs gains no variant. - */ - private def merged(kind: String, fills: Seq[Variant]): Option[Variant] = { - val at = fills.flatMap(_.at) - if (at.isEmpty) None - else { - val names = at.map(_._1.stripPrefix("/")).distinct - val shown = names.mkString(",") - val label = if (shown.length <= 60) shown else s"${names.size} fields" - Some(Variant(s"$kind($label)", at)) - } - } - - /** Extra variants for the OPTIONAL free-value scalar knobs — a number or a - * string the user types in, as opposed to a column picker or a dropdown. - * [[decide]] leaves these unset for the same reason [[optionalColumnFills]]'s - * knobs are unset, and they need the same treatment: the branch each generator - * emits for a knob that IS set (a gauge's delta arrow, a step row's range) - * never runs on either path, so the two hand-written branches are never - * compared. - * - * Every knob found here ends up in ONE variant (see [[merged]]), the row ones - * included: a row is what the UI's `+` button adds, and its fields are read as a - * unit anyway (a step's start AND end make one range). - * - * "Unset" is read off `baseNode` rather than re-derived, so a knob the base - * pass DID fill — one carrying a `defaultValue` or a declared enum — is left - * alone. - */ - private def optionalScalarFills( - clazz: Class[_], - baseNode: JsonNode, - path: String, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - ordinal: Ordinal - ): Seq[Variant] = - configFields(clazz).filterNot(hiddenBySibling(_, baseNode.at(path))).flatMap { f => - val childPath = pointerOf(f, path) - rowType(f) match { - case Some(row) => - // Recurse into containers whatever their own required-ness: an optional - // knob often sits inside a required list of rows. - rowPaths(f, baseNode.at(childPath), childPath).flatMap { rowPath => - val fills = rowFills(row, baseNode, rowPath, schemas, used, rowCount, ordinal) - if (fills.isEmpty) None - else Some(Variant(s"${rowPath.stripPrefix("/")}=filled", fills)) - } - case None => - leafFill(clazz, f, baseNode, childPath, schemas, rowCount) - .map(fill => Variant(s"${fill._1.stripPrefix("/")}=${fill._2.asText}", Seq(fill))) - .toSeq - } - } - - /** Value for the hostile variant of a knob that takes arbitrary text. Legal — - * a user can type it into any text box — but it ends a Python string literal, - * which is what a generator splicing it unescaped gets wrong. - */ - private val HostileString = "a\"b" - - /** Every knob that accepts ARBITRARY TEXT, to carry [[HostileString]] — all of - * them in one variant (see [[merged]]). This is the escaping check, and it is - * generic on purpose: a new operator is covered the day it is verified, with - * nothing to register. - * - * "Arbitrary text" excludes every string whose value is constrained, because - * there the hostile value would be rejected before any escaping mattered: a - * column picker, a declared enum, a CSS color, and a number-in-a-string (which - * declares bounds). Unlike [[optionalScalarFills]] this does not care whether - * the base pass filled the knob — a label carrying a default is spliced just the - * same — so the variant replaces whatever value is there. - */ - private def hostileTextFills(clazz: Class[_], baseNode: JsonNode, path: String): Seq[Variant] = - configFields(clazz).filterNot(hiddenBySibling(_, baseNode.at(path))).flatMap { f => - val childPath = pointerOf(f, path) - rowType(f) match { - case Some(row) => - rowPaths(f, baseNode.at(childPath), childPath).flatMap { rowPath => - val fills = hostileTextFills(row, baseNode, rowPath).flatMap(_.at) - if (fills.isEmpty) None - else Some(Variant(s"${rowPath.stripPrefix("/")}=hostileText", fills)) - } - case None => - hostileLeaf(f, childPath) - .map(fill => Variant(s"${fill._1.stripPrefix("/")}=hostileText", Seq(fill))) - .toSeq - } - } - - private def hostileLeaf(f: Field, childPath: String): Option[(String, JsonNode)] = - if ( - hasAutofill(f) || f.getType != classOf[String] || declaredEnumValues(f).nonEmpty || - !patternAccepts(f, HostileString) || declaredRange(f) != Bounds(None, None) - ) None - else Some((childPath, objectMapper.getNodeFactory.textNode(HostileString))) - - /** Number the knobs of the hostile variant so no two carry the same text: the first - * keeps [[HostileString]], the n-th reads `a"b2`, `a"b3`, … Every one still holds - * the quote, so the escaping this variant exists for is unchanged. - * - * Needed because the knobs land in ONE variant (see [[merged]]). Where they are the - * names of columns the operator CREATES, one shared value asks for several columns - * of the same name and the schema rejects the config outright — the run then fails - * on something this generator invented rather than on a divergence. Numbering also - * says which knob a surviving value came from. - * - * Applied here rather than inside [[hostileTextFills]] because that walk recurses - * into nested rows, and the count has to span the whole variant, not restart per - * row the way [[rowFills]]'s ordinal does. - */ - private def numbered(fills: Seq[Variant]): Seq[Variant] = { - var n = 0 - fills.map(f => - Variant( - f.label, - f.at.map { - case (pointer, _) => - n += 1 - val text = if (n == 1) HostileString else s"$HostileString$n" - (pointer, objectMapper.getNodeFactory.textNode(text)) - } - ) - ) - } - - /** Whether a field's declared `pattern` accepts `value` — the field's own answer to - * "can this be typed here", so the declaration decides rather than this generator. - * A field that declares nothing accepts anything. - * - * The point of asking instead of skipping every field that HAS a pattern: a pattern - * exists to exclude what the consumer would reject, which for many fields is nothing - * at all. Such a field still needs the escaping check — and the escaping bugs this - * variant found were in exactly that kind of knob. - * - * `matches` is a full-string match, which is what the property editor applies too - * (`Validators.pattern` wraps a string pattern in `^(?:…)$`). - */ - private def patternAccepts(f: Field, value: String): Boolean = - schemaKey(f, "pattern").filter(_.isTextual).map(_.asText) match { - case Some(p) => Try(value.matches(p)).getOrElse(false) - case None => true - } - - /** A running position shared by every row filled into one variant, so no two of - * those knobs are handed the same value. One counter rather than one per row: - * rows collide with each other as readily as knobs within a row do, and where the - * knob is an output column NAME — Projection's `alias` — two rows carrying the - * same one is a config the operator refuses outright. - */ - private final class Ordinal { - private var n = 0 - - /** The position a knob would take. Advances only once one actually does, so a - * field that yields no fill leaves no gap in the numbering. - */ - def peek: Int = n - def taken(): Unit = n += 1 - } - - /** Every optional knob under one nested row — a column picker as well as a scalar — - * as pointer → value. - * - * The scalar knobs get DISTINCT values, ascending: a row is often a pair that has to - * differ to mean anything — a step's start and end, where the operator drops the - * step unless `start < end` — and one shared value would collapse it. The first - * knob keeps the value it would have had on its own, so a lone knob is unaffected. - * - * The column pickers are resolved against the same `used` set as the top-level ones, - * so a row's column differs from what the rest of the config already reads. The row - * itself is the sibling context: whether a picker is type-constrained can depend on - * another knob of the SAME row (an aggregation's function decides whether its column - * must be numeric), so the rule is evaluated against the row, not the operator. - */ - private def rowFills( - clazz: Class[_], - baseNode: JsonNode, - path: String, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - ordinal: Ordinal - ): Seq[(String, JsonNode)] = - configFields(clazz).flatMap { f => - val childPath = pointerOf(f, path) - rowType(f) match { - case Some(row) => - rowPaths(f, baseNode.at(childPath), childPath) - .flatMap(rowPath => rowFills(row, baseNode, rowPath, schemas, used, rowCount, ordinal)) - case None if hasAutofill(f) => - columnFill(clazz, f, baseNode, childPath, schemas, used, baseNode.at(path)).toSeq - case None => - val fill = leafFill(clazz, f, baseNode, childPath, schemas, rowCount, ordinal.peek) - if (fill.nonEmpty) ordinal.taken() - fill.toSeq - } - } - - /** The fill for ONE optional column knob, or `None` when it is required (the base - * pass filled it), already points at a column, or no column resolves. - * - * Shared by the top-level pass and the row pass so both obey the same rule: an - * optional picker takes the first unused column that fits its declared type. - * - * `owner` is the object the field belongs to, NOT the class that declares it: a - * knob a family shares is declared on the abstract base, which has no instance to - * read a default off, and every such knob then read as one the config had already - * set and was skipped. - */ - private def columnFill( - owner: Class[_], - f: Field, - baseNode: JsonNode, - childPath: String, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - siblings: JsonNode - ): Option[(String, JsonNode)] = { - val required = Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) - val untouched = defaultsOf(owner).path(jsonNameOf(f)) - if (required || baseNode.at(childPath) != untouched) None - else { - val spec = autofillSpec(f).getOrElse(AutofillSpec(port = 0, holdsList = false)) - // A list knob takes every matching column here exactly as it does in the - // base config: `required` decides WHETHER a field is filled, not how many - // values go in once it is. Filled with one, a list knob runs the same code - // as a scalar — the per-element numbering and the joining between elements, - // which is the whole of what a list does differently, is never reached. - if (spec.holdsList) - listColumnFill(f, schemas, spec.port, used, siblings).toOption.map(childPath -> _) - else - resolveColumn(f, schemas, spec.port, used, siblings).toOption - .map(col => (childPath, objectMapper.getNodeFactory.textNode(col): JsonNode)) - } - } - - /** Every column at `port` a list knob may hold: the ones its `attributeTypeRules` - * admits, or all of them when the rule matches nothing (or there is no rule), - * minus the ones a single-column knob beside it already took. - * The same fill for a required and an optional field, so the two cannot drift. - * - * Subtracting `used` is what [[resolveColumn]] already does for a single-column - * knob, and the two knobs answer to the same rule: a column means something - * different to each field that names it, so handing one column to two of them - * writes a config nobody would. Radar Chart's name column arrived inside its own - * value columns that way, and sklearn's label inside the features it is fitted - * against. Not marked used in turn, since a list knob wants every column its rule - * admits and marking them would leave a later single-column knob nothing to take. - */ - private def listColumnFill( - f: Field, - schemas: Map[Int, Schema], - port: Int, - used: collection.Set[(Int, String)], - siblings: JsonNode - ): Either[String, JsonNode] = - columnNames(schemas, port).map { names => - val filtered = allowedTypes(f, siblings) match { - case Some(types) => - val matching = schemas - .get(port) - .map(_.getAttributes.filter(a => types.contains(a.getType)).map(_.getName)) - .getOrElse(Seq.empty) - if (matching.nonEmpty) matching else names - case None => names - } - val free = filtered.filterNot(name => used.contains((port, name))) - val arr = objectMapper.createArrayNode() - (if (free.nonEmpty) free else filtered).foreach(arr.add) - arr - } - - /** A field's JSON Pointer, under the pointer of the object that holds it. */ - private def pointerOf(f: Field, path: String): String = s"$path/${jsonNameOf(f)}" - - /** The key a field carries in the config JSON. */ - private def jsonNameOf(f: Field): String = - Option(f.getAnnotation(classOf[JsonProperty])) - .map(_.value) - .filter(_.nonEmpty) - .getOrElse(f.getName) - - /** Whether the config has to carry a value for this field. Two sources say so and - * both count: the annotation, and a schema branch the siblings have selected. A - * field required only under a branch carries no annotation, so reading the - * annotation alone leaves it unfilled in exactly the configuration that needs it. - */ - private def isRequired(f: Field, scope: SchemaScope, siblings: JsonNode): Boolean = - Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) || - requiredUnder(scope, siblings).contains(jsonNameOf(f)) - - /** The nested-row type a field holds — its `List[Row]` / `Option[Row]` element - * type, or its own type when the field IS the row. `None` for a scalar field. - */ - private def rowType(f: Field): Option[Class[_]] = { - val t = f.getType - if (isList(t) || isOption(t)) elementType(f).toOption.filter(isNestedObject) - else if (isNestedObject(t)) Some(t) - else None - } - - /** The pointer of each row present at `childPath` — one per array element, or - * the node itself when the field holds a single row. Empty when nothing is - * there to fill (an absent `Option`, a scalar list). - */ - private def rowPaths(f: Field, child: JsonNode, childPath: String): Seq[String] = - if (isList(f.getType) || isOption(f.getType)) - if (child.isArray) (0 until child.size()).map(i => s"$childPath/$i") - else if (child.isObject) Seq(childPath) - else Seq.empty - else if (child.isObject) Seq(childPath) - else Seq.empty - - /** The fill for one optional free-value scalar knob, or `None` if this field - * isn't one (a column picker, a required field, or a knob the base pass filled). - * - * `ordinal` is the knob's position among the ones filled in the same row (0 for a - * top-level knob, which has no siblings to differ from): it offsets the value so - * the knobs of one row do not collide — see [[rowFills]]. - */ - private def leafFill( - owner: Class[_], - f: Field, - baseNode: JsonNode, - childPath: String, - schemas: Map[Int, Schema], - rowCount: Int, - ordinal: Int = 0 - ): Option[(String, JsonNode)] = { - val required = Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) - // "Unset" means the base pass did not fill it: the key still carries the value a - // fresh instance has (see [[defaultsOf]] — every key is present, as the UI sends - // them, so absence alone no longer tells us anything). - val current = baseNode.at(childPath) - val unset = current.isMissingNode || - current == defaultsOf(owner).path(jsonNameOf(f)) - // A knob whose values the field DECLARES is left to its declaration: the enum - // sweep covers a declared value list, and a knob offering an `examples` value - // takes that one. Reading `examples` on its own, rather than only alongside a - // `pattern`, is the point: a field can state a realistic value ("https:// - // example.com" for a URL) without having to invent a constraint to hang it on, - // and inventing one to steer this generator would reject values the platform - // accepts. - // An optional knob is typed by what its Option holds, so `start`/`end` declared - // as Option[Double] are swept like the bare numbers they are. - val scalarType = effectiveScalarType(f) - if ( - hasAutofill(f) || required || !unset || - declaredEnumValues(f).size > 1 || !isFreeScalar(scalarType) - ) None - else if (declaredExample(f).isDefined) declaredExample(f).map(v => (childPath, v)) - else if (scalarType == classOf[String]) - // The canonical string is "1", so the n-th knob reads as "1", "2", … — distinct - // and ascending, so the knobs filled in one row do not collide. - Some((childPath, objectMapper.getNodeFactory.textNode((ordinal + 1).toString))) - else - scalarNode( - scalarType, - None, - schemas, - mutable.Set.empty, - NumHint(declaredRange(f), rowCount) - ).toOption - .map { v => - // Step away from the value rather than scaling it: the n-th knob lands next - // to the first instead of at n times it, so a pair stays inside the span the - // fixture actually holds — doubling walked `end` past the last row. - val stepped = - if (ordinal == 0) v - else objectMapper.getNodeFactory.numberNode(v.asDouble() + ordinal) - (childPath, stepped) - } - } - - /** The first value a field offers under `examples` — a legal sample the operator - * states itself, so nothing here has to invent one. - */ - private def declaredExample(f: Field): Option[JsonNode] = - schemaKey(f, "examples").filter(_.isArray).flatMap(_.elements().asScala.toSeq.headOption) - - /** One key out of a field's own `@JsonSchemaInject` JSON. */ - private def schemaKey(f: Field, key: String): Option[JsonNode] = - Option(f.getAnnotation(classOf[JsonSchemaInject])) - .map(_.json) - .filter(_.nonEmpty) - .flatMap(js => Try(objectMapper.readTree(js)).toOption) - .map(_.path(key)) - .filterNot(_.isMissingNode) - - /** A type whose value the user types in freely — the fills of - * [[optionalScalarFills]]. Boolean is excluded: the enum sweep already covers - * both of its values. - */ - private def isFreeScalar(t: Class[_]): Boolean = - t == classOf[String] || t == classOf[Int] || t == classOf[java.lang.Integer] || - t == classOf[Short] || t == classOf[Long] || t == classOf[java.lang.Long] || - t == classOf[Double] || t == classOf[java.lang.Double] || t == classOf[Float] - - private def deserialize( - node: ObjectNode, - opClass: Class[_ <: LogicalOp] - ): Either[String, LogicalOp] = - Try(objectMapper.treeToValue(node, opClass)).toEither.left - .map(e => s"deserialization failed: ${e.getMessage}") - - /** One variant per non-default enum value reachable in `baseNode`. One enum - * flipped at a time — linear, NOT the combinatorial product. - */ - private def enumVariants( - opClass: Class[_ <: LogicalOp], - baseNode: ObjectNode, - pinned: Set[String] = Set.empty, - fill: FillContext = FillContext() - ): Seq[Variant] = - enumSites(opClass, baseNode, "", Map.empty, SchemaScope.of(opClass), fill) - .filterNot(site => pinned.contains(site.pointer.stripPrefix("/"))) - .flatMap { site => - val baseVal = baseNode.at(site.pointer) - site.values.filterNot(_ == baseVal).map { v => - Variant( - s"${site.pointer.stripPrefix("/")}=${v.asText}", - (site.pointer, v) +: site.companions(v) - ) - } - } - - /** An enum-typed position in the config JSON: its JSON Pointer plus every - * possible JSON value (each enum constant serialized via its `@JsonValue`). - * - * `companions` names the edits a value has to arrive with. A hyperparameter's - * `parameter` needs them: the `value` beside it holds something the PREVIOUS - * parameter accepted, and a flip that left it there would ask the operator to - * put a kernel name through `int()`. - * - * A choice the operator offers and no value runs is NOT dropped here: it is - * generated, it fails, and it is withheld by name in - * [[TransformVerificationRunner.variantsNotRun]], where a reader sees it and the row - * goes away when the operator is fixed. - */ - private final case class EnumSite( - pointer: String, - values: Seq[JsonNode], - companions: JsonNode => Seq[(String, JsonNode)] = _ => Seq.empty - ) - - /** Collect every enum-typed leaf reachable in `node`. Walks the operator's - * fields for type info but the ACTUAL JSON for structure, so it honours real - * list lengths (a curated fixture may hold >1 element) and skipped optionals. - * `path` is the JSON Pointer of the sub-node currently typed by `clazz`. - * - * `bindings` and `scope` are what [[buildObject]] filled the config with, and - * are needed here for the same two reasons: a field declared as a type variable - * reports `Object` from [[Field.getType]] and so hides the enum it really holds, - * and a field whose values are stated in the schema rather than on itself has - * none to sweep as far as reflection can see. - */ - private def enumSites( - clazz: Class[_], - node: JsonNode, - path: String, - bindings: TypeBindings, - scope: SchemaScope, - fill: FillContext - ): Seq[EnumSite] = { - val bound = bindings ++ typeBindingsOf(clazz) - val row = node.at(path) - val sites = configFields(clazz).filterNot(hiddenBySibling(_, row)).flatMap { f => - if (hasAutofill(f)) Seq.empty - else { - val jsonName = jsonNameOf(f) - val childPath = s"$path/$jsonName" - val child = node.at(childPath) - if (child.isMissingNode || child.isNull) Seq.empty - else { - val t = f.getType - val declared = declaredEnumValues(f) - val nested = scope.descend(jsonName) - if (declared.size > 1) Seq(EnumSite(childPath, declared)) - else if (isList(t)) - elementType(f).toOption.toSeq.flatMap { elem => - if (child.isArray) - (0 until child.size()).flatMap(i => - enumSiteFor(elem, node, s"$childPath/$i", elementBindings(f, bound), nested, fill) - ) - else Seq.empty - } - else if (isOption(t)) - elementType(f).toOption.toSeq - .flatMap(elem => enumSiteFor(elem, node, childPath, bound, nested, fill)) - else - ruledEnumSite(f, row, childPath, scope) - .map(Seq(_)) - .getOrElse(enumSiteFor(boundType(f, bound), node, childPath, bound, nested, fill)) - } - } - } - withCompanions(clazz, row, path, bound, scope, fill, sites) - } - - private def enumSiteFor( - t: Class[_], - node: JsonNode, - path: String, - bindings: TypeBindings, - scope: SchemaScope, - fill: FillContext - ): Seq[EnumSite] = - if (t.isEnum) { - val vals = t.getEnumConstants.toSeq.map(c => objectMapper.valueToTree[JsonNode](c)) - if (vals.size > 1) Seq(EnumSite(path, vals)) else Seq.empty - } else if (t == classOf[Boolean] || t == classOf[java.lang.Boolean]) { - // A Boolean is a 2-value "enum": sweep both true and false. - val nf = objectMapper.getNodeFactory - Seq(EnumSite(path, Seq(nf.booleanNode(true), nf.booleanNode(false)))) - } else if (isNestedObject(t)) enumSites(t, node, path, bindings, scope, fill) - else Seq.empty - - /** The site a `valueRules` branch gives a field whose own type names no values: - * the set the branch holding for `row` accepts. `None` where that branch names - * none, a numeric hyperparameter's `value` being one value out of a range rather - * than a choice between named ones. - */ - private def ruledEnumSite( - f: Field, - row: JsonNode, - childPath: String, - scope: SchemaScope - ): Option[EnumSite] = - schemaValueRule(f, scope, row) - .map(_.path("enum")) - .filter(e => e.isArray && e.size() > 1) - .map(e => EnumSite(childPath, e.elements().asScala.toSeq)) - - /** Every site paired with the fields that move with it, in the two ways a schema - * says one field's content depends on another's. - * - * The first is `valueRules`: what such a field may hold is decided by the sibling - * the rule reads. The pairing is derived from the rules themselves rather than - * named here, so an operator stating a rule over some other sibling gets the same - * treatment. The paired field is REFILLED rather than left as it was, since what - * sits there belongs to the PREVIOUS choice, and it is refilled the way any field - * is: by the rule where the rule names a value, and by the field's own type where - * it does not. A branch naming nothing is the operator saying it knows of no value - * worth offering, which is not the same as there being none, and a choice this - * generator cannot fill is a choice that has to fail loudly and be withheld by name - * in [[TransformVerificationRunner.variantsNotRun]] rather than disappear here. - * - * The second is a conditional `required`, which is how an object says that exactly - * one of two fields applies and therefore that neither can be marked required on - * its own. A hyperparameter row is written that way: `value` is required while its - * switch is off and `attribute` once it is on. The base pass filled the one the - * base config needs, so flipping the switch has to fill the other, which until then - * was rightly left empty. - */ - private def withCompanions( - clazz: Class[_], - row: JsonNode, - path: String, - bindings: TypeBindings, - scope: SchemaScope, - fill: FillContext, - sites: Seq[EnumSite] - ): Seq[EnumSite] = { - val ruled = configFields(clazz) - .map(f => (f, scope.child(jsonNameOf(f)).path("valueRules").path("allOf"))) - .filter { case (_, branches) => branches.isArray } - val conditional = conditionallyRequiredFields(scope) - if ((ruled.isEmpty && conditional.isEmpty) || !row.isObject) sites - else - sites.map { site => - val sibling = site.pointer.stripPrefix(s"$path/") - val paired = ruled.filter { - case (_, branches) => - branches.elements().asScala.exists(_.path("if").has(sibling)) - } - if (paired.isEmpty && conditional.isEmpty) site - else - site.copy(companions = v => { - val hypothetical = row.deepCopy[ObjectNode]() - hypothetical.set[JsonNode](sibling, v) - val ruleFills = paired.map { - case (f, _) => companionFill(f, path, hypothetical, bindings, scope, fill) - } - // Only what this value newly asks for and the row does not already - // carry: a config that set the field by hand keeps what it set. - val revealed = requiredUnder(scope, hypothetical) - .diff(requiredUnder(scope, row)) - .flatMap(name => configFields(clazz).find(jsonNameOf(_) == name)) - .filter(f => isBlank(hypothetical.path(jsonNameOf(f)))) - .map(f => companionFill(f, path, hypothetical, bindings, scope, fill)) - ruleFills ++ revealed - }) - } - } - - /** One companion edit, through [[valueFor]], which reads the rule first and falls back - * to the field's own type: the order the base pass filled it in. - * - * A field this generator cannot fill ENDS the run rather than quietly costing the - * choice its variant, the auto tier already ending it when a whole config cannot be - * built. Both are the same thing said about a smaller piece, and a generator that - * came up empty is a gap here rather than a statement about the operator. - */ - private def companionFill( - f: Field, - path: String, - row: JsonNode, - bindings: TypeBindings, - scope: SchemaScope, - fill: FillContext - ): (String, JsonNode) = - valueFor(f, fill.schemas, fill.used, fill.rowCount, row, bindings, scope) match { - case Right(value) => (s"$path/${jsonNameOf(f)}", value) - case Left(reason) => - throw new IllegalStateException(s"cannot fill ${jsonNameOf(f)} beside it: $reason") - } - - /** Set `value` at a JSON Pointer inside `root` — used to clone the base config - * and flip one enum leaf. Handles object fields and array indices. - */ - private def setAtPointer(root: ObjectNode, pointer: String, value: JsonNode): Unit = { - val tokens = pointer.stripPrefix("/").split("/").toList - var cur: JsonNode = root - tokens.dropRight(1).foreach { tk => - cur = if (cur.isArray) cur.get(tk.toInt) else cur.get(tk) - } - (cur, tokens.last) match { - case (o: ObjectNode, name) => o.set[JsonNode](name, value) - // One past the end appends — the `+`-row fill adds a row rather than - // replacing one. - case (a: ArrayNode, idx) if idx.toInt == a.size() => a.add(value); () - case (a: ArrayNode, idx) => a.set(idx.toInt, value); () - case _ => () - } - } - - /** Maps each registered operator class to its `operatorType` discriminator, - * read from [[LogicalOp]]'s `@JsonSubTypes` (the same registry Jackson uses). - */ - private val typeNameByClass: Map[Class[_], String] = { - Option(classOf[LogicalOp].getAnnotation(classOf[JsonSubTypes])) - .map(_.value().toSeq.map(t => (t.value(): Class[_]) -> t.name()).toMap) - .getOrElse(Map.empty) - } - - // ── object assembly ────────────────────────────────────────────────────── - - /** Build a JSON object for `clazz` by filling each of its config fields. - * `rowCount` sizes the numeric fallback for range-less fields (e.g. Limit). - */ - private def buildObject( - clazz: Class[_], - schemas: Map[Int, Schema], - rowCount: Int - ): Either[String, ObjectNode] = - buildObject(clazz, schemas, mutable.Set.empty[(Int, String)], rowCount) - - /** `used` tracks (port, column) already assigned within THIS operator, so that - * sibling autofill fields resolve to DISTINCT columns (e.g. a scatter's x and - * y don't both collapse onto the first numeric column, which would be a - * degenerate diagonal). Shared across the operator, nested objects included. - * An explicit `@SampleColumn` always wins even if the column is already taken; - * only the type-match and first-column tiers avoid reuse. - */ - private def buildObject( - clazz: Class[_], - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - bindings: TypeBindings = Map.empty, - scope: SchemaScope = SchemaScope.empty, - pinned: Map[String, JsonNode] = Map.empty - ): Either[String, ObjectNode] = { - // What `clazz` itself supplies is added to what its caller passed in: an operator - // names the arguments for its own supertypes, a row class receives them from the - // field that holds it. - val bound = bindings ++ typeBindingsOf(clazz) - // An operator carries its own schema, so it is derived here rather than at each - // entry point: a caller that forgot would lose every rule the schema states and - // get a config that merely looks filled. A nested class has no schema of its own - // and uses the scope the field holding it handed down. - val doc = if (classOf[LogicalOp].isAssignableFrom(clazz)) SchemaScope.of(clazz) else scope - val node = defaultsOf(clazz) - // Pins go in BEFORE the fields are decided, not after: `node` is the sibling - // context below, so a knob pinned on decides what its dependents do. Set - // afterwards, a pin cannot reach the field it was pinned to steer. - pinned.foreach { case (name, value) => node.set[JsonNode](name, value) } - configFields(clazz).foreach { f => - // A pinned knob keeps the value it was pinned to. Deciding it again would - // refill it from its default and undo the pin before the fields that read it - // are reached. - if (!pinned.contains(jsonNameOf(f))) { - // `node` doubles as the sibling context: a field whose rule depends on another - // field of the same object reads it here, so declaration order decides what is - // visible — the knob a rule branches on is declared before the column it binds. - decide(f, schemas, used, rowCount, node, bound, doc) match { - case Fill(name, value) => node.set[JsonNode](name, value) - case Skip => () - case Fail(reason) => return Left(s"${clazz.getSimpleName}.${f.getName}: $reason") - } - } - } - Right(node) - } - - /** A fresh instance's own values, as the starting JSON — what the UI submits for a - * form nobody touched, where every key is present carrying the operator's default. - * - * Leaving a skipped knob's key OUT instead produces a shape the UI cannot: a - * config object built through a `@JsonCreator` constructor then receives `null` - * for the missing keys, overwriting the field initializers, and a generator that - * reads them crashes on a value no user can enter (BulletChart's step bounds). - * Empty when the class has no usable no-arg constructor. - */ - private def defaultsOf(clazz: Class[_]): ObjectNode = - Try(clazz.getDeclaredConstructor()) - .flatMap { ctor => - ctor.setAccessible(true) - Try(objectMapper.valueToTree[JsonNode](ctor.newInstance())) - } - .toOption - .collect { case o: ObjectNode => o } - .getOrElse(objectMapper.createObjectNode()) - - private sealed trait Decision - private case class Fill(jsonName: String, value: JsonNode) extends Decision - private case object Skip extends Decision - private case class Fail(reason: String) extends Decision - - /** Decide whether/how to fill one field, applying required-vs-optional policy: - * required (or required autofill) fields that can't be filled fail the whole - * operator; optional fields without a meaningful value are skipped (left at the - * operator's default). - */ - private def decide( - f: Field, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - siblings: JsonNode = noSiblings, - bindings: TypeBindings = Map.empty, - scope: SchemaScope = SchemaScope.empty - ): Decision = { - val jp = Option(f.getAnnotation(classOf[JsonProperty])) - val jsonName = jp.map(_.value).filter(_.nonEmpty).getOrElse(f.getName) - val required = isRequired(f, scope, siblings) - val autofill = hasAutofill(f) - // An optional knob is judged by what it WRAPS: `Option[Double]` is a number the - // user may leave blank, not a thing the base config has to carry. - val held = effectiveScalarType(f, bindings) - val isBoolean = held == classOf[Boolean] || held == classOf[java.lang.Boolean] - - // An OPTIONAL column-name field (`@AutofillAttributeName*` with required=false) - // is left at its operator default rather than force-filled. These are the - // "No Selection" grouping/pattern knobs (e.g. BarChart's categoryColumn / - // pattern); forcing a real column into one produces a degenerate config (one - // trace per row) that the native and generated paths disagree on. - if (hiddenBySibling(f, siblings)) Skip - else if (autofill && !required) Skip - else { - // A field declaring its values in the annotation counts as meaningful just as - // an enum-TYPED one does: the sweep flips it from the base config, so it has - // to BE in the base config (a `defaultValue = ""` alone would skip it). So does - // one whose schema states a rule for it: an untyped hyperparameter `value` is - // an optional plain string, which alone would be skipped, but the operator does - // read it and the rule says what it should hold. - val meaningful = required || autofill || held.isEnum || isBoolean || isList(f.getType) || - isNestedObject(held) || declaredEnumValues(f).size > 1 || - schemaValueRule(f, scope, siblings).isDefined || jp - .map(_.defaultValue) - .exists(_.nonEmpty) - - valueFor(f, schemas, used, rowCount, siblings, bindings, scope) match { - case Right(v) if meaningful => Fill(jsonName, v) - case Right(_) => Skip // optional plain scalar w/o default — leave operator default - case Left(reason) if required || autofill => Fail(reason) - case Left(_) => Skip - } - } - } - - // ── value resolution ───────────────────────────────────────────────────── - - /** Resolve a JSON value node for a field: autofill column refs first, then by - * declared type (list / option / scalar / nested object). - */ - private def valueFor( - f: Field, - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - siblings: JsonNode = noSiblings, - bindings: TypeBindings = Map.empty, - scope: SchemaScope = SchemaScope.empty - ): Either[String, JsonNode] = { - val ruled = schemaValueRule(f, scope, siblings).flatMap(ruleFill) - val nested = scope.descend(jsonNameOf(f)) - autofillSpec(f) match { - case Some(spec) if spec.holdsList => - listColumnFill(f, schemas, spec.port, used, siblings) - case Some(spec) => - resolveColumn(f, schemas, spec.port, used, siblings) - .map(objectMapper.getNodeFactory.textNode) - // A rule stated in the schema wins over the type-driven fill below: it names a - // value this field may hold given the sibling chosen beside it, which the type - // alone — a bare `String` — cannot narrow. - case None if ruled.isDefined => Right(ruled.get) - case None => - val t = boundType(f, bindings) - if (isList(t)) - // An OPTIONAL list starts EMPTY, the way the UI does: its `+` button adds the - // first row, so a config nobody touched has none, and the branch an operator - // takes for "no rows at all" is only reached this way. A REQUIRED list gets - // one row — its operator asserts the list is non-empty, so zero is not a - // config it can run. Either way the extra row comes from [[extraRowFills]]. - // - // Required counts the schema's conditional form too: a list the operator - // needs only on one branch is empty-by-annotation, and reading the - // annotation alone hands that branch the empty list it cannot run. - if (!isRequired(f, scope, siblings)) - Right(objectMapper.createArrayNode()) - else - elementType(f) - .flatMap( - scalarOrNested(_, schemas, used, rowCount, elementBindings(f, bindings), nested) - ) - .map { e => - val arr: ArrayNode = objectMapper.createArrayNode(); arr.add(e); arr - } - else if (isOption(t)) - // An optional scalar is filled like the bare type: the `defaultValue` and any - // declared range sit on the field, not on the element, so a Grid Size that - // declares 10 is still filled with 10 rather than a generic number. - elementType(f).flatMap { elem => - if (isNestedObject(elem)) - scalarOrNested(elem, schemas, used, rowCount, elementBindings(f, bindings), nested) - else - scalarNode(elem, baseValueOf(f), schemas, used, NumHint(declaredRange(f), rowCount)) - } - else if (declaredEnumValues(f).size > 1) Right(declaredEnumDefault(f)) - else - scalarNode( - t, - baseValueOf(f), - schemas, - used, - NumHint(declaredRange(f), rowCount), - Map.empty, - nested - ) - } - } - - /** The base value for a field whose values are declared in its annotation: the - * `default` the annotation names, else its first value. Never the canonical - * string — for such a field that is a value the operator does not accept. - */ - private def declaredEnumDefault(f: Field): JsonNode = { - val declared = declaredEnumValues(f) - Option(f.getAnnotation(classOf[JsonSchemaInject])) - .map(_.json) - .filter(_.nonEmpty) - .flatMap(js => Try(objectMapper.readTree(js).path("default")).toOption) - .filterNot(_.isMissingNode) - .filter(declared.contains) - .getOrElse(declared.head) - } - - /** What the base config should carry for a scalar field, before this generator - * invents anything: the operator's own `defaultValue` if it has one, else the - * value it offers under `examples`. - * - * `examples` matters most on a REQUIRED field, which [[leafFill]] never reaches — - * a required knob with no default would otherwise take the canonical "1", and "1" - * is not a URL, a regex or a delimiter. A field can now say what a realistic value - * looks like without declaring a constraint it does not have. - */ - private def baseValueOf(f: Field): Option[String] = - defaultOf(f).orElse(declaredExample(f).filter(_.isTextual).map(_.asText)) - - /** A node for a list element or Option inner type — no field-level default or - * range annotation (those live on the field, not the element type). - */ - private def scalarOrNested( - clazz: Class[_], - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - rowCount: Int, - bindings: TypeBindings = Map.empty, - scope: SchemaScope = SchemaScope.empty - ): Either[String, JsonNode] = - scalarNode(clazz, None, schemas, used, NumHint(Bounds(None, None), rowCount), bindings, scope) - - /** How to fill a numeric field: `@JsonProperty(defaultValue)` if present, else the - * middle of a declared `[min, max]` (an opacity's 0.0–1.0 → 0.5), else twice a - * lower bound declared on its own, else half the row count (the middle of - * `[0, rowCount]`, e.g. Limit). - * - * Twice, because a field that declares `>= 30` usually also defaults to 30, so - * filling the bound itself would just re-run the base config; `max mid` keeps a - * `>= 0` knob off zero. Doubling can only overshoot a ceiling the field does not - * declare, and a field with a ceiling is supposed to declare it — which is why the - * `[min, max]` case must stay: RadarChart's and Scatterplot's opacity declare one, - * and doubling their floor of 0 would hand them 5. - * - * An upper bound declared ALONE is not handled: no field does that, so there would - * be no way to tell whether the code was right. - */ - private final case class NumHint(bounds: Bounds, rowCount: Int) - - private final case class Bounds(min: Option[Double], max: Option[Double]) - - private def numericFill(default: Option[String], hint: NumHint): Double = - default.flatMap(s => Try(s.trim.toDouble).toOption) match { - case Some(d) => d - case None => - val mid = hint.rowCount / 2.0 - hint.bounds match { - case Bounds(Some(mn), Some(mx)) => (mn + mx) / 2.0 - case Bounds(Some(mn), None) => (mn * 2) max mid - case _ => mid - } - } - - /** A node for a concrete (non-list, non-option) type. Numeric fields follow - * [[numericFill]]; enums/strings honor an optional `@JsonProperty(defaultValue)`. - */ - private def scalarNode( - t: Class[_], - default: Option[String], - schemas: Map[Int, Schema], - used: mutable.Set[(Int, String)], - hint: NumHint, - bindings: TypeBindings = Map.empty, - scope: SchemaScope = SchemaScope.empty - ): Either[String, JsonNode] = { - val nf = objectMapper.getNodeFactory - if (t.isEnum) - Right( - default - .map(nf.textNode) - .getOrElse(objectMapper.valueToTree[JsonNode](t.getEnumConstants.head)) - ) - else if (t == classOf[Boolean] || t == classOf[java.lang.Boolean]) - Right(nf.booleanNode(default.map(_.trim.toBoolean).getOrElse(false))) - else if (t == classOf[Int] || t == classOf[java.lang.Integer] || t == classOf[Short]) - Right(nf.numberNode(numericFill(default, hint).round.toInt)) - else if (t == classOf[Long] || t == classOf[java.lang.Long]) - Right(nf.numberNode(numericFill(default, hint).round)) - else if (t == classOf[Double] || t == classOf[java.lang.Double] || t == classOf[Float]) - Right(nf.numberNode(numericFill(default, hint))) - else if (t == classOf[String]) - Right(nf.textNode(default.getOrElse(CanonicalString))) - else if (isNestedObject(t)) - buildObject(t, schemas, used, hint.rowCount, bindings, scope) - else Left(s"unhandled type ${t.getName}") - } - - /** The values a field declares via its own `@JsonSchemaInject(json = ...)` - * `enum` array — a String field the UI renders as a dropdown (e.g. an ECDF's - * cdfMode = standard / reversed / complementary). To the JVM these are plain - * Strings, so [[enumSiteFor]]'s `isEnum` check can't see them, yet each value - * takes a different branch in the generated code exactly as a real enum does. - * Empty unless the annotation carries an array (TimeSeries declares - * `"enum": "autofill"`, a UI directive rather than a value list). - */ - private def declaredEnumValues(f: Field): Seq[JsonNode] = - Option(f.getAnnotation(classOf[JsonSchemaInject])) - .map(_.json) - .filter(_.nonEmpty) - .toSeq - .flatMap { js => - Try(objectMapper.readTree(js).path("enum")).toOption.toSeq - .filter(_.isArray) - .flatMap(_.elements().asScala.toSeq) - } - - /** The bounds a field declares, from either of the two places an operator states - * them: `@JsonSchemaInject`'s `minimum`/`maximum` (an opacity's 0.0–1.0), which the - * UI reads, and javax validation's `@DecimalMin`/`@Min` (a row height's floor of 30), - * which the compiler's validation pass reads. Either bound may be absent. - */ - private def declaredRange(f: Field): Bounds = { - val schema = Option(f.getAnnotation(classOf[JsonSchemaInject])) - .map(_.json) - .filter(_.nonEmpty) - .flatMap(js => Try(objectMapper.readTree(js)).toOption) - def fromSchema(key: String): Option[Double] = - schema.map(_.path(key)).filter(_.isNumber).map(_.asDouble()) - Bounds( - fromSchema("minimum") - .orElse(Option(f.getAnnotation(classOf[DecimalMin])).flatMap(a => asDouble(a.value))) - .orElse(Option(f.getAnnotation(classOf[Min])).map(_.value.toDouble)), - fromSchema("maximum") - ) - } - - private def asDouble(s: String): Option[Double] = Try(s.trim.toDouble).toOption - - // ── reflection helpers ─────────────────────────────────────────────────── - - /** Config fields declared on `clazz` and its superclasses up to (not - * including) [[LogicalOp]] — i.e. the operator's own knobs, not the - * framework's bookkeeping. A field counts if it carries `@JsonProperty` or an - * autofill annotation. - */ - private def configFields(clazz: Class[_]): Seq[Field] = { - val ignored = ignoredProperties(clazz) - val out = mutable.LinkedHashMap.empty[String, Field] // de-dup by name, keep most-derived - var c: Class[_] = clazz - while (c != null && c != classOf[LogicalOp] && c != classOf[Object]) { - c.getDeclaredFields - .filterNot(f => Modifier.isStatic(f.getModifiers)) - .filter(isConfigField) - .filterNot(f => ignored.contains(jsonNameOf(f))) - .foreach(f => out.getOrElseUpdate(f.getName, { f.setAccessible(true); f })) - c = c.getSuperclass - } - out.values.toSeq - } - - /** The properties an operator declares it does NOT carry, via `@JsonIgnoreProperties`. - * - * An operator that inherits a knob it does not read says so this way — FileScanSource - * over `ScanSourceOpDesc`'s `limit`/`offset` — and the annotation sits on the operator - * while the field sits on the parent, so the field alone cannot be judged. Jackson - * drops these on the way back in, so filling one yields the config it started from: - * the variant built from it would run a second time over the same config and report - * the two paths agreeing about nothing. - */ - private def ignoredProperties(clazz: Class[_]): Set[String] = { - val names = mutable.Set.empty[String] - var c: Class[_] = clazz - while (c != null && c != classOf[Object]) { - Option(c.getAnnotation(classOf[JsonIgnoreProperties])).foreach(names ++= _.value) - c = c.getSuperclass - } - names.toSet - } - - private def isConfigField(f: Field): Boolean = - // `@JsonIgnore` is the field's own way of saying the same thing - // [[ignoredProperties]] handles for the class: not part of the config. - !f.isAnnotationPresent(classOf[JsonIgnore]) && - (f.isAnnotationPresent(classOf[JsonProperty]) || hasAutofill(f)) - - private def hasAutofill(f: Field): Boolean = autofillSpec(f).isDefined - - /** How a field says "fill me with a column name from input port N", and whether - * it holds one name or a list of them. - * - * Two spellings mean the same thing: the `@AutofillAttributeName` family, or - * the `@JsonSchemaInject` that family is defined as, which `SklearnModelOpDesc.text` - * writes out so its `hide*` keys sit in one annotation. They emit identical - * schema keys, so reading only the annotations left such a field out of the - * config entirely — which read as the operator having no such knob. - */ - private def autofillSpec(f: Field): Option[AutofillSpec] = - if (f.isAnnotationPresent(classOf[AutofillAttributeNameList])) - Some(AutofillSpec(port = 0, holdsList = true)) - else if (f.isAnnotationPresent(classOf[AutofillAttributeNameOnPort1])) - Some(AutofillSpec(port = 1, holdsList = false)) - else if (f.isAnnotationPresent(classOf[AutofillAttributeName])) - Some(AutofillSpec(port = 0, holdsList = false)) - else injectedAutofill(f) - - private final case class AutofillSpec(port: Int, holdsList: Boolean) - - /** The `@JsonSchemaInject` spelling: an `autofill` string key naming one of - * the two autofill kinds, plus an optional port. Anything else in the - * annotation (titles, `hide*`) is ignored here. - */ - private def injectedAutofill(f: Field): Option[AutofillSpec] = - for { - inject <- Option(f.getAnnotation(classOf[JsonSchemaInject])) - kind <- inject.strings.find(_.path == CommonOpDescAnnotation.autofill).map(_.value) - holdsList <- - if (kind == CommonOpDescAnnotation.attributeNameList) Some(true) - else if (kind == CommonOpDescAnnotation.attributeName) Some(false) - else None - } yield AutofillSpec( - port = inject.ints - .find(_.path == CommonOpDescAnnotation.autofillAttributeOnPort) - .map(_.value) - .getOrElse(0), - holdsList = holdsList - ) - - private def defaultOf(f: Field): Option[String] = - Option(f.getAnnotation(classOf[JsonProperty])).map(_.defaultValue).filter(_.nonEmpty) - - /** Whether the UI hides this field, given what its siblings currently hold. - * - * A `hide*` triple says "hide me when THAT field holds THIS value", and the UI - * honours it, so a config that fills a hidden field is one no user can submit. - * Filling one was harmless where nothing read it and misleading where something - * did: sklearn's `text` was filled off the numeric projection with the - * vectorizer off, a form the UI never shows. - * - * The sibling's value is read from the node being built, which starts as the - * operator's own defaults, so the target is present whatever the declaration - * order. - */ - private def hiddenBySibling(f: Field, siblings: JsonNode): Boolean = - Option(f.getAnnotation(classOf[JsonSchemaInject])).exists { inject => - val by = inject.strings.find(_.path == HideAnnotation.hideTarget).map(_.value) - val expected = inject.strings.find(_.path == HideAnnotation.hideExpectedValue).map(_.value) - val kind = inject.strings - .find(_.path == HideAnnotation.hideType) - .map(_.value) - .getOrElse(HideAnnotation.Type.equals) - (by, expected) match { - case (Some(target), Some(want)) => - val actual = Option(siblings.get(target)).map(_.asText).getOrElse("") - if (kind == HideAnnotation.Type.regex) Try(actual.matches(want)).getOrElse(false) - else actual == want - case _ => false - } - } - - private def isList(t: Class[_]): Boolean = - classOf[scala.collection.Seq[_]].isAssignableFrom(t) || - classOf[java.util.List[_]].isAssignableFrom(t) - - private def isOption(t: Class[_]): Boolean = classOf[Option[_]].isAssignableFrom(t) - - /** The element class of a `List[X]` / `Option[X]` field, from its generic - * signature. - */ - private def elementType(f: Field): Either[String, Class[_]] = - contentAs(f) match { - case Some(c) => Right(c) - case None => - f.getGenericType match { - case p: ParameterizedType => - p.getActualTypeArguments.headOption match { - case Some(c: Class[_]) => Right(c) - case Some(pt: ParameterizedType) => Right(pt.getRawType.asInstanceOf[Class[_]]) - case _ => Left(s"cannot resolve element type of ${f.getName}") - } - case _ => Left(s"${f.getName} has no generic element type") - } - } - - /** What a field holds as a scalar: an `Option`'s element type, else the field type - * itself. Everything that reasons about a knob's type goes through this, so an - * optional knob is treated exactly like the bare value it wraps. - */ - private def effectiveScalarType(f: Field, bindings: TypeBindings = Map.empty): Class[_] = - if (isOption(f.getType)) elementType(f).getOrElse(f.getType) else boundType(f, bindings) - - /** The concrete classes standing in for the type variables in scope, keyed by the - * variable itself so that two classes declaring a `T` cannot be confused. - * - * Needed because a field declared as a type variable — a trainer's hyperparameter - * row holds `var parameter: T` — reports `Object` from [[Field.getType]], which is - * not a type anything can be filled with. The operator does name the class it means, - * one level up in `SklearnMLOperatorDescriptor[SklearnAdvancedKNNParameters]`, and - * these carry that down to the field. - */ - private type TypeBindings = Map[TypeVariable[_], Class[_]] - - /** What `clazz` supplies for the variables its generic supertypes declare, walking up - * the chain so an argument stated several levels above still arrives. An argument - * that is itself a variable is followed through what the subclass already bound, - * which is why the walk goes downward-first. - */ - private def typeBindingsOf(clazz: Class[_]): TypeBindings = { - val acc = mutable.Map.empty[TypeVariable[_], Class[_]] - var t: Type = clazz.getGenericSuperclass - while (t != null) t match { - case p: ParameterizedType => - val raw = p.getRawType.asInstanceOf[Class[_]] - raw.getTypeParameters.zip(p.getActualTypeArguments).foreach { - case (declared, arg: Class[_]) => acc(declared) = arg - case (declared, arg: TypeVariable[_]) => acc.get(arg).foreach(acc(declared) = _) - case _ => () - } - t = raw.getGenericSuperclass - case c: Class[_] => t = c.getGenericSuperclass - case _ => t = null - } - acc.toMap - } - - /** `f`'s type with a type variable resolved against the bindings in scope. Falls back - * to [[Field.getType]], i.e. to `Object`, so an unresolvable variable still reaches - * [[scalarNode]] and is reported there rather than silently mis-filled. - */ - private def boundType(f: Field, bindings: TypeBindings): Class[_] = - f.getGenericType match { - case tv: TypeVariable[_] => bindings.getOrElse(tv, f.getType) - case _ => f.getType - } - - /** What a `List[Row[T]]` field passes down to its row class: `Row`'s own variables - * bound to the arguments the field names. Those arguments are usually the enclosing - * operator's variables rather than classes, so they are resolved against the bindings - * already in scope before being handed on. - */ - private def elementBindings(f: Field, bindings: TypeBindings): TypeBindings = - f.getGenericType match { - case p: ParameterizedType => - p.getActualTypeArguments.headOption match { - case Some(row: ParameterizedType) => - val raw = row.getRawType.asInstanceOf[Class[_]] - raw.getTypeParameters - .zip(row.getActualTypeArguments) - .flatMap { - case (declared, arg: Class[_]) => Some(declared -> arg) - case (declared, arg: TypeVariable[_]) => bindings.get(arg).map(declared -> _) - case _ => None - } - .toMap - case _ => Map.empty - } - case _ => Map.empty - } - - /** The element class `@JsonDeserialize(contentAs = ...)` names, and the only place - * a Scala `Option[Double]`'s element type survives: the generic signature erases - * it to Object, which is why Jackson needs the annotation too. Checked before the - * signature so an operator that carries it is read the way Jackson reads it. - */ - private def contentAs(f: Field): Option[Class[_]] = - Option(f.getAnnotation(classOf[JsonDeserialize])) - .map(_.contentAs()) - .filterNot(c => c == classOf[java.lang.Void] || c == classOf[Void]) - - /** A type we should recurse into and build as a nested JSON object: not a - * primitive/boxed/String/enum/collection, and it actually declares config - * fields or a creator. - */ - private def isNestedObject(t: Class[_]): Boolean = { - val excluded = t.isPrimitive || t.isEnum || t == classOf[String] || - isList(t) || isOption(t) || t.getName.startsWith("java.lang.") - !excluded && (configFields(t).nonEmpty || t.getDeclaredConstructors.exists( - _.getParameterCount > 0 - )) - } - - private def columnNames(schemas: Map[Int, Schema], port: Int): Either[String, Seq[String]] = - schemas.get(port).map(_.getAttributeNames).filter(_.nonEmpty) match { - case Some(names) => Right(names) - case None => Left(s"no input columns at port $port") - } - - /** First column at `port` not yet claimed by a sibling field of the same - * operator (so two un-annotated / same-type fields don't collapse onto the - * same column); the first column if every column is already taken. Marks the - * pick in `used`. - */ - private def firstUnused( - schemas: Map[Int, Schema], - port: Int, - used: mutable.Set[(Int, String)] - ): Either[String, String] = - columnNames(schemas, port).map { names => - val col = names.find(c => !used.contains((port, c))).getOrElse(names.head) - used += ((port, col)); col - } - - /** Pick which input column fills an `@AutofillAttributeName*` field, in - * priority order: - * 1. `@SampleColumn("x")` — an explicit semantic pick (e.g. a valid ISO - * country code or a real OHLC column) that the column's type can't - * express; always honored, even if already used; - * 2. the first *unused* column whose [[AttributeType]] satisfies the field's - * `attributeTypeRules` (falling back to the first matching column if all - * are taken); - * 3. the first unused column (the original first-column behavior, made - * distinct-aware). - * Tiers 1–2 keep the parity test on realistic, type-correct input; the - * distinct-column preference stops sibling fields (x/y, source/target) from - * collapsing onto one column and producing a degenerate result. - */ - private def resolveColumn( - f: Field, - schemas: Map[Int, Schema], - port: Int, - used: mutable.Set[(Int, String)], - siblings: JsonNode = noSiblings - ): Either[String, String] = { - def take(col: String): String = { used += ((port, col)); col } - Option(f.getAnnotation(classOf[SampleColumn])).map(_.value) match { - case Some(col) => - columnNames(schemas, port).flatMap { names => - if (names.contains(col)) Right(take(col)) - else - Left( - s"""@SampleColumn("$col") not present at port $port (have: ${names.mkString(", ")})""" - ) - } - case None => - allowedTypes(f, siblings) match { - case Some(types) => - schemas - .get(port) - .map(_.getAttributes.filter(a => types.contains(a.getType)).map(_.getName)) match { - case Some(cols) if cols.nonEmpty => - Right(take(cols.find(c => !used.contains((port, c))).getOrElse(cols.head))) - case _ => firstUnused(schemas, port, used) // no type-matching column; fall back - } - case None => firstUnused(schemas, port, used) - } - } - } - - /** [[AttributeType]]s permitted for `f` by its declaring class's - * `@JsonSchemaInject(json = ...)` `attributeTypeRules`, keyed by the field's - * JSON name. `None` when the field is unconstrained. - * - * A rule may be CONDITIONAL — an `allOf` of `if`/`then` branches naming a sibling - * field, which is how an operator says "what this column may hold depends on that - * knob" (an aggregation's `attribute` is numeric for sum/min/max, string for - * concat). `siblings` is the JSON object holding `f`, against which each branch's - * condition is tested; branches that do not apply contribute nothing, and `allOf` - * means the ones that do all bind, so their sets intersect. - */ - private def allowedTypes(f: Field, siblings: JsonNode): Option[Set[AttributeType]] = - Option(f.getDeclaringClass.getAnnotation(classOf[JsonSchemaInject])) - .map(_.json) - .filter(_.nonEmpty) - .flatMap(js => Try(objectMapper.readTree(js)).toOption) - .map(_.path("attributeTypeRules").path(jsonNameOf(f))) - .flatMap { rule => - val branches = - if (rule.path("allOf").isArray) rule.path("allOf").elements().asScala.toSeq - else Seq.empty - val bound = typeSet(rule.path("enum")).toSeq ++ branches - .filter(branch => conditionHolds(branch.path("if"), siblings)) - .flatMap(branch => typeSet(branch.path("then").path("enum"))) - bound.reduceOption(_ intersect _).filter(_.nonEmpty) - } - - /** The [[AttributeType]]s an `enum` array names, or `None` if it names none. */ - private def typeSet(enumNode: JsonNode): Option[Set[AttributeType]] = - if (!enumNode.isArray) None - else { - val set = enumNode.elements().asScala.flatMap(n => typeFromString(n.asText())).toSet - if (set.nonEmpty) Some(set) else None - } - - /** Whether every `sibling: { valEnum: [...] }` clause of a rule's `if` holds for the - * object the field sits in. An empty condition holds vacuously; a clause naming a - * sibling the object has not set does not. - */ - private def conditionHolds(cond: JsonNode, siblings: JsonNode): Boolean = - cond.isObject && cond.fields().asScala.forall { clause => - val permitted = clause.getValue.path("valEnum") - permitted.isArray && - permitted.elements().asScala.exists(_.asText == siblings.path(clause.getKey).asText) - } - - /** The empty object, for a caller with no sibling context: only the unconditional - * part of a rule can bind. - */ - private def noSiblings: JsonNode = objectMapper.getNodeFactory.objectNode() - - /** Where a field's constraints are read from when its own annotation cannot carry - * them: the operator's finished JSON schema, and the node within it describing the - * object currently being built. - * - * An operator implementing `JsonSchemaCustomizer` writes rules into that document - * after the annotations have been read. A hyperparameter row's `value` is stated only - * there, because what it may hold depends on the `parameter` chosen beside it and so - * cannot be annotated on a field every parameter shares. Reflection alone does not - * see those, which is why the document travels alongside the walk. - */ - private final case class SchemaScope(root: JsonNode, node: JsonNode) { - - /** The node describing one field of this object. */ - def child(jsonName: String): JsonNode = node.path("properties").path(jsonName) - - /** The scope a nested object or list element is built under, following the `$ref` - * Jackson emits in place of a class it has already defined. - */ - def descend(jsonName: String): SchemaScope = { - val field = child(jsonName) - val target = if (field.path("items").isObject) field.path("items") else field - val ref = target.path("$ref").asText("") - SchemaScope( - root, - if (ref.isEmpty) target - else root.path("definitions").path(ref.stripPrefix("#/definitions/")) - ) - } - } - - private object SchemaScope { - val empty: SchemaScope = { - val nothing = objectMapper.getNodeFactory.objectNode() - SchemaScope(nothing, nothing) - } - - /** An operator's finished schema, or [[empty]] where one cannot be produced — such a - * class is then read from its annotations alone, as every operator was before. - */ - def of(clazz: Class[_]): SchemaScope = - Try( - OperatorMetadataGenerator - .generateOperatorJsonSchema(clazz.asInstanceOf[Class[_ <: LogicalOp]]) - ).toOption.map(s => SchemaScope(s, s)).getOrElse(empty) - } - - /** What filling a field needs beyond the field itself: the input schemas a column - * picker resolves against, the columns already spoken for, and the row count a - * range-less number is sized from. Carried into the enum walk so that a value which - * makes a field apply can fill it the way the base pass would have. - * - * Empty for a caller sweeping an already-configured operator: such a config states - * both sides of a conditional itself, so nothing there is left to fill. - */ - private final case class FillContext( - schemas: Map[Int, Schema] = Map.empty, - used: mutable.Set[(Int, String)] = mutable.Set.empty, - rowCount: Int = DefaultRowCount - ) - - /** Whether a config holds nothing at this key: absent, null, the empty string a - * field initialised to `""` starts as, or the empty list a `List()` field starts - * as. All four mean the same thing to an operator reading it, so a conditional - * `required` is unmet by any of them. - */ - private def isBlank(node: JsonNode): Boolean = - node.isMissingNode || node.isNull || (node.isTextual && node.asText.isEmpty) || - (node.isArray && node.isEmpty) - - /** The fields an object's schema requires only under some condition, named by every - * `required` its `allOf` states in either branch. Empty for an object whose - * requirements are all unconditional, which is every one but a hyperparameter row. - */ - private def conditionallyRequiredFields(scope: SchemaScope): Set[String] = - scope.node - .path("allOf") - .elements() - .asScala - .flatMap(branch => Seq(branch.path("then"), branch.path("else"))) - .flatMap(_.path("required").elements().asScala) - .map(_.asText) - .toSet - - /** The fields an object's conditional `allOf` requires of `row` as it stands. The - * condition is JSON Schema's own `properties`/`const`, not the `valEnum` form a - * Texera rule uses, because this one is read by the validator rather than by the - * form. - */ - private def requiredUnder(scope: SchemaScope, row: JsonNode): Set[String] = - scope.node - .path("allOf") - .elements() - .asScala - .flatMap { branch => - val holds = branch.path("if").path("properties").fields().asScala.forall { clause => - clause.getValue.path("const") == row.path(clause.getKey) - } - val outcome = if (holds) branch.path("then") else branch.path("else") - outcome.path("required").elements().asScala.map(_.asText) - } - .toSet - - /** What a field's `valueRules` call for, given the object it sits in: the one branch - * whose condition holds. `None` for a field declaring no such rule, which is every - * field but a trainer's hyperparameter `value`. - * - * One branch at most: each names a single parameter, so unlike `attributeTypeRules` - * there is nothing to intersect. - */ - private def schemaValueRule( - f: Field, - scope: SchemaScope, - siblings: JsonNode - ): Option[JsonNode] = { - val branches = scope.child(jsonNameOf(f)).path("valueRules").path("allOf") - if (!branches.isArray) None - else - branches - .elements() - .asScala - .find(branch => conditionHolds(branch.path("if"), siblings)) - .map(_.path("then")) - } - - /** The value a `valueRules` branch calls for: the example it offers, else the head of - * its accepted set, which the branch states default-first. Both arrive as text and - * the field they fill is a `String` — the branch's `type` says how the OPERATOR will - * convert that text, not how the config carries it. - */ - private def ruleFill(rule: JsonNode): Option[JsonNode] = - rule - .path("examples") - .elements() - .asScala - .toSeq - .headOption - .orElse(rule.path("enum").elements().asScala.toSeq.headOption) - - private def typeFromString(s: String): Option[AttributeType] = - AttributeType.values().find(_.name.equalsIgnoreCase(s)) -} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala deleted file mode 100644 index 39aca487b69..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -// This spec pins the tier-routing logic (disposition). Per-operator end-to-end -// runs are NOT duplicated here: OperatorBehaviorSpec auto-discovers every -// registered operator and runs TransformVerificationRunner.run on each, and a -// single operator can be run in isolation with e.g. -// sbt "WorkflowCompilingService/testOnly *OperatorBehaviorSpec -- -z LimitOpDesc" -// (the auto-generated test name starts with the operator's simple name). What -// disposition asserts — which tier an operator routes to — is the one thing -// OperatorBehaviorSpec does not check, so it lives here. - -import org.apache.texera.amber.operator.limit.LimitOpDesc -import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 -import org.apache.texera.amber.operator.union.UnionOpDesc -import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { - import TransformVerificationRunner._ - - "disposition" should "flag knownIssues operators with the triage reason" in { - // The prediction op consumes a trained model on its input port, which a - // JVM-written JSONL fixture can't carry; triaged as a known issue, not run. - disposition(classOf[SklearnPredictionOpDesc]) match { - case Flagged(reason) => reason should include("trained-model") - case other => fail(s"expected Flagged, got $other") - } - } - - it should "run the union now that its code names every upstream" in { - // It used to be flagged for naming exactly two, which was wrong in both - // directions: a third link was dropped and a lone link left the second - // frame unbound. The runner draws one link per port, so what runs here is - // the one-upstream case — the one the old code got wrong. - disposition(classOf[UnionOpDesc]) shouldBe Runnable("auto") - } - - it should "route auto-configurable operators to the auto tier" in { - disposition(classOf[LimitOpDesc]) shouldBe Runnable("auto") - } - - // A UDF's body is written by whoever drops the operator, so there is nothing - // for a generator to emit. It stands here for the shape of the report: an - // operator that cannot be exported is carried as a row, not passed over. - it should "flag an operator that has no standalone generator" in { - disposition(classOf[PythonUDFOpDescV2]) shouldBe - Flagged("does not implement StandaloneCodeGenerator") - } -} From 73375b95f26a9fea1e8318b1f829d1fdd31fc39f Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 9 Sep 2026 16:17:38 -0700 Subject: [PATCH 09/33] fix(operator): slice the raw lines before converting them The engine drops and takes before it parses, so a line outside the configured window is never converted and an unparseable one there costs nothing. The export converted first, which raised on a line the run would never have looked at. TextInputSourceOpDesc had the same ordering. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/file/FileScanOpDesc.scala | 22 ++++++------ .../scan/file/FileScanSourceOpDesc.scala | 36 +++++++++---------- .../scan/text/TextInputSourceOpDesc.scala | 20 +++++------ .../source/scan/file/FileScanOpDescSpec.scala | 22 ++++++++---- .../scan/file/FileScanSourceOpDescSpec.scala | 29 +++++++++++++++ .../scan/text/TextInputSourceOpDescSpec.scala | 30 ++++++++++++++++ 6 files changed, 110 insertions(+), 49 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala index f945454d7ca..9de8f371e64 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala @@ -127,18 +127,18 @@ class FileScanOpDesc case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" case _ => """l.rstrip("\n")""" } - val hasSlice = fileScanOffset.isDefined || fileScanLimit.isDefined - if (hasSlice) { - val start = fileScanOffset.getOrElse(0) - val sliceExpr = fileScanLimit match { - case Some(l) => s"_lines[$start:${start + l}]" - case None => s"_lines[$start:]" + // The slice applies to the raw lines, as the engine drops and takes + // before parsing: a line outside the window is never converted, so an + // unparseable one there costs nothing. Taking after dropping also keeps + // a large limit from overflowing the end index. + val linesExpr = + if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) "_f" + else { + val dropped = + fileScanOffset.filter(_ > 0).fold("_f.readlines()")(o => s"_f.readlines()[$o:]") + fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") } - buf += s" _lines = [$castExpr for l in _f]" - buf += s" _rows.extend($sliceExpr)" - } else { - buf += s" _rows.extend($castExpr for l in _f)" - } + buf += s" _rows.extend($castExpr for l in $linesExpr)" } val colLit = pyStringLiteral(col) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala index 0ebefd5023b..e5a82a1a418 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala @@ -106,27 +106,23 @@ class FileScanSourceOpDesc case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" case _ => """l.rstrip("\n")""" } - val hasSlice = fileScanOffset.isDefined || fileScanLimit.isDefined - if (hasSlice) { - val start = fileScanOffset.getOrElse(0) - val sliceExpr = fileScanLimit match { - case Some(l) => s"_lines[$start:${start + l}]" - case None => s"_lines[$start:]" + // The slice applies to the raw lines, as the engine drops and takes + // before parsing: a line outside the window is never converted, so an + // unparseable one there costs nothing. Taking after dropping also keeps + // a large limit from overflowing the end index. + val linesExpr = + if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) "_f" + else { + val dropped = + fileScanOffset.filter(_ > 0).fold("_f.readlines()")(o => s"_f.readlines()[$o:]") + fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") } - val dfCols = - if (outputFileName) s"""{"filename": $basenameLit, $colLit: $sliceExpr}""" - else s"""{$colLit: $sliceExpr}""" - buf += s"""with open($basenameLit, "r", encoding=$encLit) as _f:""" - buf += s""" _lines = [$castExpr for l in _f]""" - buf += s""" out1df = pd.DataFrame($dfCols)""" - } else { - val dfCols = - if (outputFileName) - s"""{"filename": $basenameLit, $colLit: [$castExpr for l in _f]}""" - else s"""{$colLit: [$castExpr for l in _f]}""" - buf += s"""with open($basenameLit, "r", encoding=$encLit) as _f:""" - buf += s""" out1df = pd.DataFrame($dfCols)""" - } + val dfCols = + if (outputFileName) + s"""{"filename": $basenameLit, $colLit: [$castExpr for l in $linesExpr]}""" + else s"""{$colLit: [$castExpr for l in $linesExpr]}""" + buf += s"""with open($basenameLit, "r", encoding=$encLit) as _f:""" + buf += s""" out1df = pd.DataFrame($dfCols)""" } buf.mkString("\n") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala index 1a49e5bff69..b9ecfa89d1c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala @@ -98,18 +98,14 @@ class TextInputSourceOpDesc case FileAttributeType.TIMESTAMP => "pd.Timestamp(l)" case _ => "l" } - val hasSlice = fileScanOffset.isDefined || fileScanLimit.isDefined - if (hasSlice) { - val start = fileScanOffset.getOrElse(0) - val sliceExpr = fileScanLimit match { - case Some(l) => s"_lines[$start:${start + l}]" - case None => s"_lines[$start:]" - } - buf += s"""_lines = [$castExpr for l in _text.splitlines()]""" - buf += s"""out1df = pd.DataFrame({$colLit: $sliceExpr})""" - } else { - buf += s"""out1df = pd.DataFrame({$colLit: [$castExpr for l in _text.splitlines()]})""" - } + // The slice applies to the raw lines, as the engine drops and takes + // before parsing: a line outside the window is never converted, so an + // unparseable one there costs nothing. Taking after dropping also keeps + // a large limit from overflowing the end index. + val dropped = + fileScanOffset.filter(_ > 0).fold("_text.splitlines()")(o => s"_text.splitlines()[$o:]") + val linesExpr = fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") + buf += s"""out1df = pd.DataFrame({$colLit: [$castExpr for l in $linesExpr]})""" } buf.mkString("\n") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala index b992cd4dd86..c009a389859 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala @@ -213,22 +213,32 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { } } - it should "materialize the lines and slice them when a limit or offset is set" in { + it should "slice the raw lines before converting them when a limit or offset is set" in { fileScanOpDesc.attributeType = FileAttributeType.INTEGER fileScanOpDesc.fileScanOffset = Option(3) fileScanOpDesc.fileScanLimit = None - assert(fileScanOpDesc.generateStandaloneCode().contains(" _rows.extend(_lines[3:])")) + assert( + fileScanOpDesc + .generateStandaloneCode() + .contains(" _rows.extend(int(l.rstrip()) for l in _f.readlines()[3:])") + ) fileScanOpDesc.fileScanOffset = None fileScanOpDesc.fileScanLimit = Option(5) - assert(fileScanOpDesc.generateStandaloneCode().contains(" _rows.extend(_lines[0:5])")) + assert( + fileScanOpDesc + .generateStandaloneCode() + .contains(" _rows.extend(int(l.rstrip()) for l in _f.readlines()[:5])") + ) fileScanOpDesc.fileScanOffset = Option(3) fileScanOpDesc.fileScanLimit = Option(5) - val code = fileScanOpDesc.generateStandaloneCode() - assert(code.contains(" _lines = [int(l.rstrip()) for l in _f]")) - assert(code.contains(" _rows.extend(_lines[3:8])")) + assert( + fileScanOpDesc + .generateStandaloneCode() + .contains(" _rows.extend(int(l.rstrip()) for l in _f.readlines()[3:][:5])") + ) } it should "warn that archive extraction is unsupported when extract is on" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala index fafb696f131..2f3d76af9de 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala @@ -209,6 +209,35 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { FileScanSourceOpExec.close() } + "FileScanSourceOpDesc.generateStandaloneCode" should + "slice the raw lines before converting them" in { + fileScanSourceOpDesc.attributeType = FileAttributeType.INTEGER + + fileScanSourceOpDesc.fileScanOffset = Option(3) + fileScanSourceOpDesc.fileScanLimit = None + assert( + fileScanSourceOpDesc + .generateStandaloneCode() + .contains("""{"line": [int(l.rstrip()) for l in _f.readlines()[3:]]}""") + ) + + fileScanSourceOpDesc.fileScanOffset = None + fileScanSourceOpDesc.fileScanLimit = Option(5) + assert( + fileScanSourceOpDesc + .generateStandaloneCode() + .contains("""{"line": [int(l.rstrip()) for l in _f.readlines()[:5]]}""") + ) + + fileScanSourceOpDesc.fileScanOffset = Option(3) + fileScanSourceOpDesc.fileScanLimit = Option(5) + assert( + fileScanSourceOpDesc + .generateStandaloneCode() + .contains("""{"line": [int(l.rstrip()) for l in _f.readlines()[3:][:5]]}""") + ) + } + "FileScanSourceOpDesc.getPhysicalOp" should "wire the FileScanSourceOpExec class as a source op and propagate its schema" in { val physical = diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala index ef8a824849a..dcede030447 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala @@ -276,6 +276,36 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { new String(Files.readAllBytes(path), StandardCharsets.UTF_8) } + "TextInputSourceOpDesc.generateStandaloneCode" should + "slice the raw lines before converting them" in { + textInputSourceOpDesc.attributeType = FileAttributeType.INTEGER + textInputSourceOpDesc.textInput = "1\n2\n3" + + textInputSourceOpDesc.fileScanOffset = Option(3) + textInputSourceOpDesc.fileScanLimit = None + assert( + textInputSourceOpDesc + .generateStandaloneCode() + .endsWith("""{"line": [int(l) for l in _text.splitlines()[3:]]})""") + ) + + textInputSourceOpDesc.fileScanOffset = None + textInputSourceOpDesc.fileScanLimit = Option(5) + assert( + textInputSourceOpDesc + .generateStandaloneCode() + .endsWith("""{"line": [int(l) for l in _text.splitlines()[:5]]})""") + ) + + textInputSourceOpDesc.fileScanOffset = Option(3) + textInputSourceOpDesc.fileScanLimit = Option(5) + assert( + textInputSourceOpDesc + .generateStandaloneCode() + .endsWith("""{"line": [int(l) for l in _text.splitlines()[3:][:5]]})""") + ) + } + "TextInputSourceOpDesc.getPhysicalOp" should "wire the TextInputSourceOpExec class as a source op with one output port" in { val physical = From cb23c1e30d1f8634378f75a42dda7845be24d7c0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 9 Sep 2026 17:49:35 -0700 Subject: [PATCH 10/33] fix(operator): read a CSV the way the parser does, not the way pandas does Two divergences in one reader. The parser sets no null value, so only an empty field is null, while pandas reads a list of words as missing by default and turned the country code NA into one. And a blank header position is named by both, differently: the schema calls it column-N, pandas calls it "Unnamed: N", so a downstream operator asked for a column the frame did not have. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/csv/CSVScanSourceOpDesc.scala | 22 ++++++++++++++-- .../scan/csv/CSVScanSourceOpDescSpec.scala | 25 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index c11216d34c9..5cdfc645194 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -172,6 +172,14 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator args += s"""encoding=${pyStringLiteral(encoding)}""" args += s"header=$headerArg" + // The parser above sets no null value, so only an empty field is null and every other + // text stands for itself. pandas instead reads a list of words as missing by default, + // "NA" and "null" among them, which turned a column holding the country code NA into + // nulls. Both halves are needed: dropping the default list stops the words, and naming + // the empty string keeps the blank cell null. + args += "keep_default_na=False" + args += """na_values=[""]""" + // A CSV carries no types, so both readers infer, and they do not infer // alike: the schema above tries TIMESTAMP and parses what it can, while // pd.read_csv leaves a date column as text. Name the columns this operator @@ -196,8 +204,18 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - if (hasHeader) readCall - else { + if (hasHeader) { + // A blank header position is named by both readers, differently: the schema above calls + // it column-N, pandas calls it "Unnamed: N". A downstream operator names the column the + // schema gave it, so the frame has to carry that name. Only a position pandas actually + // filled in is renamed, which is why the placeholder is matched against the index rather + // than by its prefix: a column genuinely called "Unnamed: 3" keeps its name anywhere but + // position 3. + s"""$readCall + |out1df.columns = [ + | f"column-{i + 1}" if c == f"Unnamed: {i}" else c for i, c in enumerate(out1df.columns) + |]""".stripMargin + } else { // Match Texera's fallback column naming when there's no header s"""$readCall |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index 7a023239ef9..a1bc37e85ab 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -236,6 +236,31 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(!code.contains("io.BytesIO")) } + // The parser sets no null value, so only an empty field is null. pandas reads a list of + // words as missing by default, which turned the country code NA into a null. + it should "read only an empty field as null, the way the parser does" in { + csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + + val code = csvScanSourceOpDesc.generateStandaloneCode() + + assert(code.contains("keep_default_na=False")) + assert(code.contains("""na_values=[""]""")) + } + + // sourceSchema names a blank header column-N; pandas names it "Unnamed: N". A downstream + // operator asks for the name the schema gave, so the frame has to carry that one. + it should "give a blank header the name the schema gives it" in { + csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + + val code = csvScanSourceOpDesc.generateStandaloneCode() + + assert(code.contains("""f"column-{i + 1}" if c == f"Unnamed: {i}" else c""")) + } + it should "use comma as the default delimiter when customDelimiter is not set for parallel CSV" in { parallelCsvScanSourceOpDesc.customDelimiter = None From c2e2eb32fd177dd0fbbce4f867133aa6b654a642 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 9 Sep 2026 17:49:35 -0700 Subject: [PATCH 11/33] fix(operator): take the row's first string field as the file to scan FileScanOpExec takes the first String field of the tuple, not the first column. A row carrying an id ahead of the path reads the file on the platform, while the export opened the id. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/file/FileScanOpDesc.scala | 12 +++++++++++- .../source/scan/file/FileScanOpDescSpec.scala | 18 ++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala index 9de8f371e64..3948298d16e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala @@ -106,8 +106,18 @@ class FileScanOpDesc if (isBinary) """"rb"""" else s""""r", encoding=${pyStringLiteral(enc)}""" + // The executor takes the row's first String field, not its first column, so a row that + // carries an id ahead of the path still finds the path. Reading column 0 opened the id. + // A row with no string at all makes the executor's `.get` throw, so this raises too + // rather than quietly skipping the row. + buf += "def _texera_file_name(row):" + buf += " for _v in row:" + buf += " if isinstance(_v, str):" + buf += " return _v" + buf += """ raise ValueError(f"no file name in row: {row!r}")""" + buf += "" buf += "_rows = []" - buf += "for _fn in in1df.iloc[:, 0]:" + buf += "for _fn in (_texera_file_name(r) for r in in1df.itertuples(index=False)):" buf += s" with open(_fn, $openArgs) as _f:" // Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala index c009a389859..26ac6372b4a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala @@ -149,14 +149,28 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { "read every line as text with the configured encoding by default" in { assert( fileScanOpDesc.generateStandaloneCode() == - """_rows = [] - |for _fn in in1df.iloc[:, 0]: + """def _texera_file_name(row): + | for _v in row: + | if isinstance(_v, str): + | return _v + | raise ValueError(f"no file name in row: {row!r}") + | + |_rows = [] + |for _fn in (_texera_file_name(r) for r in in1df.itertuples(index=False)): | with open(_fn, "r", encoding="utf-8") as _f: | _rows.extend(l.rstrip("\n") for l in _f) |out1df = pd.DataFrame({"line": _rows})""".stripMargin ) } + // FileScanOpExec takes `tuple.getFields.collectFirst { case s: String => s }`, so a row + // carrying an id ahead of the path still finds the path. Reading column 0 opened the id. + it should "take the row's first string field as the file name, not its first column" in { + val code = fileScanOpDesc.generateStandaloneCode() + assert(code.contains("isinstance(_v, str)")) + assert(!code.contains("in1df.iloc[:, 0]")) + } + // The enum name is "US_ASCII", not a Python codec name. it should "render the encoding as a Python codec name" in { fileScanOpDesc.fileEncoding = FileDecodingMethod.ASCII From 1c0405fc90addc0b47860e22fdf8871a12636b81 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 9 Sep 2026 17:49:35 -0700 Subject: [PATCH 12/33] fix(operator): decode a fetched body by replacement, as the executor does IOUtils.toString substitutes U+FFFD for a malformed byte, so the executor hands back a string for any response. A strict decode raised instead, failing the whole script on a body the executor reads. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/source/fetcher/URLFetcherOpDesc.scala | 6 +++++- .../operator/source/fetcher/URLFetcherOpDescSpec.scala | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala index 484bab3861a..9b15623bfd9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDesc.scala @@ -105,7 +105,11 @@ class URLFetcherOpDesc extends SourceOperatorDescriptor with StandaloneCodeGener override def generateStandaloneCode(): String = { val urlLiteral = objectMapper.writeValueAsString(url) val isUtf8 = decodingMethod == DecodingMethod.UTF_8 - val valueExpr = if (isUtf8) """_content.decode("utf-8")""" else "_content" + // IOUtils.toString decodes through a reader that substitutes U+FFFD for a malformed + // byte, so the executor returns a string for any body at all. Python's decode raises + // instead, which turned a response the executor reads into a failed export. + val valueExpr = + if (isUtf8) """_content.decode("utf-8", errors="replace")""" else "_content" val buf = scala.collection.mutable.ArrayBuffer[String]() buf += "import http.client" buf += "import urllib.request" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala index 5add175ca11..c28dbac9fe5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/fetcher/URLFetcherOpDescSpec.scala @@ -103,7 +103,15 @@ class URLFetcherOpDescSpec extends AnyFlatSpec with Matchers { | _content = _resp.read() |except (OSError, http.client.HTTPException): | _content = f"Fetch failed for URL: {_url}".encode("utf-8") - |out1df = pd.DataFrame({"URL content": [_content.decode("utf-8")]})""".stripMargin + |out1df = pd.DataFrame({"URL content": [_content.decode("utf-8", errors="replace")]})""".stripMargin + } + + // IOUtils.toString substitutes U+FFFD for a malformed byte, so the executor returns a + // string for any body. A strict decode raised instead, failing the whole script on a + // response the executor reads. + it should "decode a malformed body the way the executor does, by replacement" in { + configured(DecodingMethod.UTF_8).generateStandaloneCode() should + include("""_content.decode("utf-8", errors="replace")""") } // The executor guards only the fetch, so a value with no scheme stops it. `Exception` From 208816d4d57408d859e8efc6e6b4e46ce304f7c4 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 10 Sep 2026 13:03:13 -0700 Subject: [PATCH 13/33] fix(operator): slice a JSONL file's lines before parsing them The source drops and takes on raw lines, before any of them is read as JSON, so a line outside the window costs nothing however malformed it is. The export read the whole file and sliced the frame afterwards, which ended it on a line the workflow skipped. Co-Authored-By: Claude Opus 5 (1M context) --- .../scan/json/JSONLScanSourceOpDesc.scala | 34 +++++--- .../scan/json/JSONLScanSourceOpDescSpec.scala | 79 +++++++++++++++++++ 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index b5cb9c54f3b..944a8588b4e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -50,10 +50,23 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato val basename = sourceBasename(fileName.getOrElse("")) val enc = fileEncoding.toString.replace("_", "-").toLowerCase + // The executor drops and takes on the RAW lines, before any of them is + // parsed, so a line outside the window is never read as JSON and an + // unparseable one there costs nothing. Reading the whole file first and + // slicing the frame would end the export on a line the workflow skipped. + val windowed = offset.exists(_ > 0) || limit.isDefined + val source = + if (!windowed) pyStringLiteral(basename) + else { + val dropped = offset.filter(_ > 0).fold("_lines")(o => s"_lines[$o:]") + val taken = limit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") + s"""io.StringIO("".join($taken))""" + } + val readArgs = scala.collection.mutable.ArrayBuffer[String]() - readArgs += pyStringLiteral(basename) + readArgs += source readArgs += "lines=True" - readArgs += s"""encoding=${pyStringLiteral(enc)}""" + if (!windowed) readArgs += s"""encoding=${pyStringLiteral(enc)}""" // JSON has no timestamp of its own, so both readers infer from the text and // do not infer alike: the schema below tries TIMESTAMP and parses what it @@ -71,23 +84,20 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato ) readArgs += s"convert_dates=[${dateColumns.mkString(", ")}]" - if (offset.isEmpty) limit.foreach(l => readArgs += s"nrows=$l") - val readExpr = s"pd.read_json(${readArgs.mkString(", ")})" val baseExpr = if (flatten) s"pd.json_normalize($readExpr.to_dict('records'))" else readExpr val lines = scala.collection.mutable.ArrayBuffer[String]() - lines += s"out1df = $baseExpr" - - (offset, limit) match { - case (Some(o), Some(l)) => - lines += s"out1df = out1df.iloc[$o:${o + l}].reset_index(drop=True)" - case (Some(o), None) => - lines += s"out1df = out1df.iloc[$o:].reset_index(drop=True)" - case _ => + if (windowed) { + lines += "import io" + lines += s"""with open(${pyStringLiteral(basename)}, "r", encoding=${pyStringLiteral( + enc + )}) as _f:""" + lines += " _lines = _f.readlines()" } + lines += s"out1df = $baseExpr" lines.mkString("\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index c0d8cf2e93c..17cc308354f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.source.scan.json +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp @@ -28,6 +29,12 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -86,4 +93,76 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { r.limit shouldBe Some(10) r.offset shouldBe Some(5) } + + // The executor drops and takes on the raw lines, so a line the window skips is + // never read as JSON. Reading the file whole and slicing the frame afterwards + // ends the export on a line the workflow never looked at. + "JSONLScanSourceOpDesc.generateStandaloneCode" should + "skip a line the window excludes without parsing it" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-window-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + Files.write( + data, + "not json at all\n{\"id\":1}\n{\"id\":2}\n".getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.offset = Some(1) + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.generateStandaloneCode()} + |print(list(out1df["id"])) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + out.trim should endWith("[1, 2]") + } + } + + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path + // (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption + .exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From 566aa156ef457eff43c32f1dc1efdabbc44131d0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 10 Sep 2026 13:30:10 -0700 Subject: [PATCH 14/33] fix(operator): take a CSV's column names from the schema Matching the placeholder against the index still renames a header the user really did spell `Unnamed: 1`, when it sits at position 1: there the two cases are the same string in the same place. The schema knows which is which, because a blank header is the only one it replaces, and its names are what every downstream operator was configured against. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/csv/CSVScanSourceOpDesc.scala | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index 5cdfc645194..1e35cef8523 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -204,19 +204,22 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - if (hasHeader) { - // A blank header position is named by both readers, differently: the schema above calls - // it column-N, pandas calls it "Unnamed: N". A downstream operator names the column the - // schema gave it, so the frame has to carry that name. Only a position pandas actually - // filled in is renamed, which is why the placeholder is matched against the index rather - // than by its prefix: a column genuinely called "Unnamed: 3" keeps its name anywhere but - // position 3. + // The schema's own names, which every downstream operator was configured + // against. They differ from pandas' in both directions: a blank header is + // `column-2` here and `Unnamed: 1` there, and a header the user really did + // spell `Unnamed: 1` is kept. Matching the placeholder against the index + // cannot tell those two apart when they coincide; taking the names by + // position can. + val schemaNames: Seq[String] = + Try(sourceSchema()).toOption.toSeq + .flatMap(_.getAttributes.map(a => pyStringLiteral(a.getName))) + + if (schemaNames.nonEmpty) s"""$readCall - |out1df.columns = [ - | f"column-{i + 1}" if c == f"Unnamed: {i}" else c for i, c in enumerate(out1df.columns) - |]""".stripMargin - } else { - // Match Texera's fallback column naming when there's no header + |out1df.columns = [${schemaNames.mkString(", ")}]""".stripMargin + else if (hasHeader) readCall + else { + // Unresolved file: fall back to Texera's headerless naming. s"""$readCall |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin } From 3d12fae2535d19e3fa73851a1592fac903490322 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 10 Sep 2026 15:51:23 -0700 Subject: [PATCH 15/33] test(operator): assert the column names the CSV reader now writes The reader takes the schema's names by position; this still asserted the placeholder-matching form it emitted before, which no longer appears in the generated code at all. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/csv/CSVScanSourceOpDescSpec.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index a1bc37e85ab..5fe9b31a919 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -250,15 +250,19 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { } // sourceSchema names a blank header column-N; pandas names it "Unnamed: N". A downstream - // operator asks for the name the schema gave, so the frame has to carry that one. - it should "give a blank header the name the schema gives it" in { - csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + // operator asks for the name the schema gave, so the frame has to carry that one, and by + // position rather than by matching the placeholder: a header the user really did spell + // "Unnamed: 1" is kept, and matching cannot tell the two apart where they coincide. + it should "give the frame the names the schema gives it" in { + val path = writeCsvWithEmptyHeader() + csvScanSourceOpDesc.fileName = Some(path) csvScanSourceOpDesc.customDelimiter = Some(",") csvScanSourceOpDesc.hasHeader = true + csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) val code = csvScanSourceOpDesc.generateStandaloneCode() - assert(code.contains("""f"column-{i + 1}" if c == f"Unnamed: {i}" else c""")) + assert(code.contains("""out1df.columns = ["id", "name", "column-3", "age"]""")) } it should "use comma as the default delimiter when customDelimiter is not set for parallel CSV" in { From edf46a47a307b5d4481a46cbbcc34093d4a88ab1 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:52:16 -0700 Subject: [PATCH 16/33] fix(operator): read an Arrow file into the dtypes that keep its nulls Arrow says of every value whether it is there. pandas' numpy dtypes have nowhere to put that: a missing double and a stored NaN both land on NaN, and a missing integer costs the column its integer type. The engine reads the file's own answer, so a set operation keeps a NaN row that a null row does not match, and a holed integer column is still made of integers. Only Arrow among these sources carries the distinction. CSV, JSON Lines and the text sources cannot express it, so they are left as they are. Co-Authored-By: Claude Opus 5 (1M context) --- .../amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index 5114e3560ed..f65dc449f6e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -46,7 +46,10 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { override def generateStandaloneCode(): String = { val basename = sourceBasename(fileName.getOrElse("")) - val read = s"""out1df = pd.read_feather(${pyStringLiteral(basename)})""" + // Arrow says of every value whether it is there, and a numpy column has + // nowhere to put that: a missing double and a stored NaN both land on NaN. + val read = + s"""out1df = pd.read_feather(${pyStringLiteral(basename)}, dtype_backend="numpy_nullable")""" // A timestamp column needs nothing here. The file names UTC and holds the // wall clock as UTC, so pd.read_feather and the executor read the same // reading off it — no zone of the reader's own enters either side. The other From fe5e8c0b5ac2c5f8b3173e33d56d976b60b1fed5 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 12:55:45 -0700 Subject: [PATCH 17/33] test(operator): pin that an Arrow read keeps a missing value apart from a NaN Co-Authored-By: Claude Opus 5 (1M context) --- .../scan/arrow/ArrowSourceOpDescSpec.scala | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 54dd677913e..534aa75a122 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.source.scan.arrow +import com.typesafe.config.ConfigFactory import org.apache.arrow.memory.RootAllocator import org.apache.arrow.vector.VectorSchemaRoot import org.apache.arrow.vector.ipc.ArrowFileWriter @@ -35,7 +36,11 @@ import org.scalatest.matchers.should.Matchers import java.io.{File, FileOutputStream} import java.nio.channels.Channels +import java.nio.charset.StandardCharsets import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { @@ -153,6 +158,61 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { d.inferSchema() shouldBe schema } + // A missing double and a stored NaN are two different values to the engine, + // and a numpy column has one slot for both. A holed integer column loses its + // type the same way. + "ArrowSourceOpDesc.generateStandaloneCode" should + "keep a missing value apart from a NaN, and an integer integral" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val schema = Schema( + List(new Attribute("d", AttributeType.DOUBLE), new Attribute("i", AttributeType.INTEGER)) + ) + val file = writeArrowFile( + schema, + Seq( + Array[Any](null, null), + Array[Any](Double.box(Double.NaN), Int.box(7)) + ) + ) + + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + // The block reads the file from the script's own directory, so run there. + val workDir = Files.createTempDirectory("arrow-standalone-") + workDir.toFile.deleteOnExit() + val beside = workDir.resolve(file.getName) + Files.copy(file.toPath, beside) + + val script = workDir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${d.generateStandaloneCode()} + |print(str(out1df["d"].dtype), str(out1df["i"].dtype)) + |print(repr(out1df["d"].iloc[0]), repr(out1df["d"].iloc[1])) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(workDir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "Float64 Int32" + // The first is the value that was missing, the second the NaN that was + // stored. Under numpy dtypes both printed as nan. + lines(1) should include("") + lines(1) should include("nan") + } + } + it should "throw a friendly error when the file is not a valid Arrow file" in { val bogus = File.createTempFile("not-arrow-", ".arrow") bogus.deleteOnExit() @@ -162,4 +222,30 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val ex = intercept[RuntimeException](d.inferSchema()) ex.getMessage shouldBe "Failed to read the .arrow file. Please ensure it is a valid Arrow file." } + + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } From 32173ef6f59956372b4fe6d3ef28ad84c05948d4 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 13:19:22 -0700 Subject: [PATCH 18/33] fix(workflow-operator): follow the native readers in the exported sources Five places where a source's exported Python answered differently from the operator the workflow ran. A boolean line is read by parseField, which takes "true" and "false" in any case and then an integer that is true only at 1, and refuses anything else. Comparing the lowercased line to "true" called 1 false and passed text the engine refuses off as a row of false. The three line readers now share one parser, emitted once per script. Parallel CSV inherits a limit and an offset its executor never reads, so slicing in the export handed back fewer rows than the workflow produced. The window is dropped, and said to be dropped where it was set. Every source named its file by the last segment of its path, so two sources reading different files both opened one of them and nothing said so. A source now writes a placeholder and declares the path it reads; the translator hands each distinct path a name, reusing the last segment when it is free and numbering it when it is not, the way a chart's output file is numbered. With extract on, the engine reads the files inside the archive. The export printed a warning and opened the archive itself. It now walks the zip entries the same way, skipping the ones macOS adds and taking the entry's own name for the filename column. Parallel CSV and Old CSV read a literal NA as a null and a blank header as an Unnamed column, where their readers keep both. They keep them now, by the same route the main CSV source took, except that Old CSV names no missing value at all: scala-csv hands a blank back as "", not as a null. A nullable long lost precision in four sources, for three different reasons. The main CSV and Arrow sources widen it through a float, so 9007199254740993 came back as ...992; they ask for the nullable integer instead. read_json rounds before any dtype can apply, so the JSONL source rebuilds those columns from an exact parse of the same lines. Parallel CSV types such a column STRING and needed the text, not an integer. Co-Authored-By: Claude Opus 5 (1M context) --- .../operator/StandaloneCodeGenerator.scala | 40 +++- .../source/scan/arrow/ArrowSourceOpDesc.scala | 32 +++- .../source/scan/csv/CSVScanSourceOpDesc.scala | 27 ++- .../csv/ParallelCSVScanSourceOpDesc.scala | 76 ++++++-- .../scan/csvOld/CSVOldScanSourceOpDesc.scala | 43 ++++- .../source/scan/file/FileScanOpDesc.scala | 61 +++++-- .../scan/file/FileScanSourceOpDesc.scala | 112 ++++++++---- .../scan/json/JSONLScanSourceOpDesc.scala | 86 +++++++-- .../scan/text/TextInputSourceOpDesc.scala | 6 +- .../source/scan/text/TextSourceOpDesc.scala | 24 +++ .../scan/arrow/ArrowSourceOpDescSpec.scala | 31 ++++ .../scan/csv/CSVScanSourceOpDescSpec.scala | 167 ++++++++++++++++- .../csvOld/CSVOldScanSourceOpDescSpec.scala | 63 +++++++ .../source/scan/file/FileScanOpDescSpec.scala | 45 ++++- .../scan/file/FileScanSourceOpDescSpec.scala | 172 +++++++++++++++++- .../scan/json/JSONLScanSourceOpDescSpec.scala | 74 +++++++- .../scan/text/TextInputSourceOpDescSpec.scala | 134 +++++++++++++- .../WorkflowToPythonTranslator.scala | 44 +++++ .../WorkflowToPythonTranslatorSpec.scala | 41 +++++ 19 files changed, 1172 insertions(+), 106 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala index 3853d1cdd67..73a9021da0e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala @@ -67,8 +67,20 @@ trait StandaloneCodeGenerator { } /** - * The file's own name, for a script that reads it from its own directory - * rather than through Texera's resolved URI. + * The file this operator reads, as Texera resolved it, or None for one that + * reads no file. + * + * The script cannot open a resolved URI, so the body writes + * [[StandaloneCodeGenerator.SourceFilePlaceholder]] where the file should be + * named and the translator puts a name there. The name has to come from the + * plan: two sources reading different files whose paths end in the same + * segment both asked for `data.csv`, and the script read one of them twice. + */ + def standaloneSourcePath(): Option[String] = None + + /** + * The name to offer for [[standaloneSourcePath]], before the plan has had a + * chance to say whether another source already wants it. * * Taken from the last path segment instead of by parsing the whole string as a * URI: the resolver percent-encodes the file-relative segments but leaves the @@ -76,12 +88,13 @@ trait StandaloneCodeGenerator { * called `v3 - with long text` makes `new URI` throw on the space and no code * is generated at all. */ - protected def sourceBasename(rawPath: String): String = { - val segment = rawPath.split("/").lastOption.getOrElse("") - // Percent-decoding only, matching what `URI.getPath` used to return here: form - // decoding would also turn a literal `+` in a file name into a space. - URLDecoder.decode(segment.replace("+", "%2B"), StandardCharsets.UTF_8) - } + final def standaloneSourceName(): Option[String] = + standaloneSourcePath().map { rawPath => + val segment = rawPath.split("/").lastOption.getOrElse("") + // Percent-decoding only, matching what `URI.getPath` used to return here: form + // decoding would also turn a literal `+` in a file name into a space. + URLDecoder.decode(segment.replace("+", "%2B"), StandardCharsets.UTF_8) + } def producesDataFrame(): Boolean = true @@ -110,3 +123,14 @@ trait StandaloneCodeGenerator { */ def standaloneImports(): Seq[String] = Seq.empty } + +object StandaloneCodeGenerator { + + /** + * What a source writes where the file it reads should be named. + * + * A bare identifier rather than a string literal, because the translator only + * rewrites the code parts of a body and leaves literals and comments alone. + */ + val SourceFilePlaceholder: String = "sourceFile" +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index 5114e3560ed..c83bb9f42b9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -22,10 +22,11 @@ package org.apache.texera.amber.operator.source.scan.arrow import com.fasterxml.jackson.annotation.JsonIgnoreProperties import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.storage.DocumentFactory -import org.apache.texera.amber.core.tuple.Schema +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.util.ArrowUtils import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral @@ -37,16 +38,17 @@ import org.apache.arrow.vector.types.pojo.{Schema => ArrowSchema} import java.io.IOException import java.net.URI import java.nio.file.{Files, StandardOpenOption} -import scala.util.Using +import scala.util.{Try, Using} @JsonIgnoreProperties(value = Array("fileEncoding")) class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { fileTypeName = Option("Arrow") + override def standaloneSourcePath(): Option[String] = fileName + override def generateStandaloneCode(): String = { - val basename = sourceBasename(fileName.getOrElse("")) - val read = s"""out1df = pd.read_feather(${pyStringLiteral(basename)})""" + val read = s"""out1df = pd.read_feather($SourceFilePlaceholder)""" // A timestamp column needs nothing here. The file names UTC and holds the // wall clock as UTC, so pd.read_feather and the executor read the same // reading off it — no zone of the reader's own enters either side. The other @@ -61,7 +63,27 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { case (None, Some(l)) => Some(s":$l") case _ => None } - (read +: window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)").toSeq) + + // A LONG column holding a null comes back through a float, which rounds every + // value past 2^53: the file's 9007199254740993 reads as ...992, where the + // executor hands the exact value on. Re-reading just those columns as the + // nullable integer leaves every other column's type as it was. + // inferSchema, not sourceSchema: this operator reads its types out of the + // file and leaves the base's sourceSchema returning null. A file that cannot + // be read leaves the re-read off rather than failing the export. + val longColumns: Seq[String] = + Try(inferSchema()).toOption.toSeq.flatMap( + _.getAttributes + .filter(_.getType == AttributeType.LONG) + .map(a => pyStringLiteral(a.getName)) + ) + val exactLongs = longColumns.map { name => + s"out1df[$name] = pd.read_feather($SourceFilePlaceholder, columns=[$name], " + + s"""dtype_backend="numpy_nullable")[$name]""" + } + + // The re-read is of the whole file, so it happens before the window is taken. + ((read +: exactLongs) ++ window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)")) .mkString("\n") } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index 1e35cef8523..7089b5ac922 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -29,6 +29,7 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpExec import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral @@ -151,12 +152,9 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator } - override def generateStandaloneCode(): String = { - // Strip to just the basename. The standalone script assumes the CSV - // lives in the same directory as the script (Texera's resolved URIs - // can't be used directly outside the system). - val basename = sourceBasename(fileName.getOrElse("")) + override def standaloneSourcePath(): Option[String] = fileName + override def generateStandaloneCode(): String = { // Resolve the delimiter the same way the parser above does — first character, empty // means comma — and escape it. Every value the field accepts has to survive this: // pandas reads a separator longer than one character as a REGULAR EXPRESSION, and a @@ -167,7 +165,7 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator val headerArg = if (hasHeader) "0" else "None" val args = scala.collection.mutable.ArrayBuffer[String]() - args += s"""filepath_or_buffer=${pyStringLiteral(basename)}""" + args += s"filepath_or_buffer=$SourceFilePlaceholder" args += s"sep=${pyStringLiteral(sep)}" args += s"""encoding=${pyStringLiteral(encoding)}""" args += s"header=$headerArg" @@ -195,6 +193,23 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator ) if (dateColumns.nonEmpty) args += s"parse_dates=[${dateColumns.mkString(", ")}]" + // A LONG column holding a null has to be asked for by name, or pandas widens + // it through a float to carry the hole: 9007199254740993 comes back as + // ...992, a value the file never held and the executor never produced. + // Int64 is the nullable integer, so the hole costs the column nothing. + // INTEGER needs none of this — every int32 is exact in a float64. + val longColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes.zipWithIndex + .filter(_._1.getType == AttributeType.LONG) + .map { + case (a, i) => + val key = if (hasHeader) pyStringLiteral(a.getName) else i.toString + s"""$key: "Int64"""" + } + ) + if (longColumns.nonEmpty) args += s"dtype={${longColumns.mkString(", ")}}" + offset.foreach { o => // With a header, skip offset rows after row 0; without, skip offset rows from the start. if (hasHeader) args += s"skiprows=range(1, ${o + 1})" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala index 902284ce520..2d2740f06ff 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala @@ -30,12 +30,14 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import java.io.IOException import java.net.URI +import scala.util.Try class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { @@ -82,8 +84,9 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe ) } + override def standaloneSourcePath(): Option[String] = fileName + override def generateStandaloneCode(): String = { - val basename = sourceBasename(fileName.getOrElse("")) // First character, empty means comma — the same resolution the reader below does — // and escaped, so every value the field accepts survives being spliced into Python. // See CSVScanSourceOpDesc for what handing pandas the raw value did. @@ -92,23 +95,74 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe val headerArg = if (hasHeader) "0" else "None" val args = scala.collection.mutable.ArrayBuffer[String]() - args += s"""filepath_or_buffer=${pyStringLiteral(basename)}""" + args += s"filepath_or_buffer=$SourceFilePlaceholder" args += s"sep=${pyStringLiteral(sep)}" args += s"""encoding=${pyStringLiteral(encoding)}""" args += s"header=$headerArg" - offset.foreach { o => - if (hasHeader) args += s"skiprows=range(1, ${o + 1})" - else args += s"skiprows=$o" - } - limit.foreach(l => args += s"nrows=$l") + // The block reader nulls an omitted field and leaves every other text alone, + // so only an empty field is missing here. pandas reads a list of words as + // missing by default, "NA" and "null" among them, which turned a column + // holding the country code NA into nulls. See CSVScanSourceOpDesc: both + // halves are needed, one to stop the words and one to keep the blank null. + args += "keep_default_na=False" + args += """na_values=[""]""" + + // Ask for the schema's own type wherever pandas would infer another one. + // The two halves of this operator disagree about a blank: sourceSchema reads + // with scala-csv, where a blank is "" and types its column STRING, while the + // executor nulls it and parses the rest as that STRING. pandas infers a number + // instead, so a column of ids came back as floats, one past 2^53 rounded: + // 9007199254740993 as ...992. A LONG needs the nullable integer for the same + // reason. See CSVScanSourceOpDesc. + val dtypes: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes.zipWithIndex + .flatMap { + case (a, i) => + val pandasType = a.getType match { + case AttributeType.LONG => Some("Int64") + case AttributeType.STRING => Some("string") + case _ => None + } + val key = if (hasHeader) pyStringLiteral(a.getName) else i.toString + pandasType.map(t => s"""$key: "$t"""") + } + ) + if (dtypes.nonEmpty) args += s"dtype={${dtypes.mkString(", ")}}" + + // Limit and offset are inherited fields the parallel reader never reads: + // ParallelCSVScanSourceOpExec.open carves the file into byte ranges and + // leaves both as TODOs. Slicing here gave the export fewer rows than the + // workflow produced, so the window is dropped and said to be dropped. + val ignoredWindow = + if (offset.isEmpty && limit.isEmpty) Seq.empty + else + Seq( + "# NOTE: this operator's limit and offset are ignored, as the parallel CSV reader ignores them." + ) val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - if (hasHeader) readCall - else - s"""$readCall - |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin + // The schema's own names, which every downstream operator was configured + // against: sourceSchema below rewrites a blank header to `column-N`, where + // pandas writes `Unnamed: 1`. Taken by position, so a header the user really + // did spell `Unnamed: 1` survives too. See CSVScanSourceOpDesc. + val schemaNames: Seq[String] = + Try(sourceSchema()).toOption.toSeq + .flatMap(_.getAttributes.map(a => pyStringLiteral(a.getName))) + + val body = + if (schemaNames.nonEmpty) + s"""$readCall + |out1df.columns = [${schemaNames.mkString(", ")}]""".stripMargin + else if (hasHeader) readCall + else + // Unresolved file: fall back to Texera's headerless naming. + s"""$readCall + |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin + + (ignoredWindow :+ body).mkString("\n") } override def sourceSchema(): Schema = { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index a7b7607c6bb..079f487085d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -29,6 +29,7 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper @@ -79,8 +80,9 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat ) } + override def standaloneSourcePath(): Option[String] = fileName + override def generateStandaloneCode(): String = { - val basename = sourceBasename(fileName.getOrElse("")) // First character, empty means comma — the same resolution the reader below does — // and escaped, so every value the field accepts survives being spliced into Python. // See CSVScanSourceOpDesc for what handing pandas the raw value did. @@ -89,11 +91,20 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat val headerArg = if (hasHeader) "0" else "None" val args = scala.collection.mutable.ArrayBuffer[String]() - args += s"""filepath_or_buffer=${pyStringLiteral(basename)}""" + args += s"filepath_or_buffer=$SourceFilePlaceholder" args += s"sep=${pyStringLiteral(sep)}" args += s"""encoding=${pyStringLiteral(encoding)}""" args += s"header=$headerArg" + // This reader has NO missing value at all: scala-csv hands an omitted field + // back as the empty string, and every other text stands for itself, so a + // blank cell is "" and even widens its column to STRING. pandas instead + // reads a list of words as missing by default, "NA" and "null" among them, + // and reads a blank as NaN. Dropping the default list settles both, and no + // na_values is named — where the other CSV readers null a blank, this one + // keeps it. See CSVScanSourceOpDesc. + args += "keep_default_na=False" + // Name the columns this operator inferred as timestamps, so pandas parses // the same ones instead of leaving them as text. See CSVScanSourceOpDesc. val dateColumns: Seq[String] = @@ -104,6 +115,20 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat ) if (dateColumns.nonEmpty) args += s"parse_dates=[${dateColumns.mkString(", ")}]" + // Read a LONG column as the nullable integer, so a hole does not widen it + // through a float and round the values it carries. See CSVScanSourceOpDesc. + val longColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes.zipWithIndex + .filter(_._1.getType == AttributeType.LONG) + .map { + case (a, i) => + val key = if (hasHeader) pyStringLiteral(a.getName) else i.toString + s"""$key: "Int64"""" + } + ) + if (longColumns.nonEmpty) args += s"dtype={${longColumns.mkString(", ")}}" + offset.foreach { o => if (hasHeader) args += s"skiprows=range(1, ${o + 1})" else args += s"skiprows=$o" @@ -112,8 +137,20 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - if (hasHeader) readCall + // The schema's own names, which every downstream operator was configured + // against: sourceSchema below rewrites a blank header to `column-N`, where + // pandas writes `Unnamed: 1`. Taken by position, so a header the user really + // did spell `Unnamed: 1` survives too. See CSVScanSourceOpDesc. + val schemaNames: Seq[String] = + Try(sourceSchema()).toOption.toSeq + .flatMap(_.getAttributes.map(a => pyStringLiteral(a.getName))) + + if (schemaNames.nonEmpty) + s"""$readCall + |out1df.columns = [${schemaNames.mkString(", ")}]""".stripMargin + else if (hasHeader) readCall else + // Unresolved file: fall back to Texera's headerless naming. s"""$readCall |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala index 3948298d16e..0218944490a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala @@ -97,14 +97,9 @@ class FileScanOpDesc val enc = fileEncoding.toString.replace("_", "-").toLowerCase val buf = scala.collection.mutable.ArrayBuffer[String]() - if (extract) - buf += "# WARNING: extract=true is not supported in standalone mode; files are read as-is, not unpacked from archives." - val isBinary = attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY - val openArgs = - if (isBinary) """"rb"""" - else s""""r", encoding=${pyStringLiteral(enc)}""" + val encLit = pyStringLiteral(enc) // The executor takes the row's first String field, not its first column, so a row that // carries an id ahead of the path still finds the path. Reading column 0 opened the id. @@ -118,7 +113,33 @@ class FileScanOpDesc buf += "" buf += "_rows = []" buf += "for _fn in (_texera_file_name(r) for r in in1df.itertuples(index=False)):" - buf += s" with open(_fn, $openArgs) as _f:" + + // With extract on the engine reads the files INSIDE the archive, one tuple + // per entry, and skips the entries macOS adds. `zipfile` is the reader that + // matches: the platform casts what it opened to a ZipArchiveInputStream, so + // an archive that is not a zip fails on both sides. A directory entry is + // read, not skipped, there and here alike — it yields nothing. + val (indent, nameExpr, readWhole, lines) = + if (extract) { + buf += " with zipfile.ZipFile(_fn) as _z:" + buf += " for _name in _z.namelist():" + buf += """ if _name.startswith("__MACOSX"):""" + buf += " continue" + buf += " with _z.open(_name) as _f:" + // A zip entry only opens binary, so text is decoded here rather than by + // the reader. TextIOWrapper, not splitlines: it ends a line where + // `open(..., "r")` does, which is what the non-extract branch reads with. + ( + " " * 16, + "_name", + if (isBinary) "_f.read()" else s"_f.read().decode($encLit)", + s"io.TextIOWrapper(_f, encoding=$encLit)" + ) + } else { + val openArgs = if (isBinary) """"rb"""" else s""""r", encoding=$encLit""" + buf += s" with open(_fn, $openArgs) as _f:" + (" " * 8, "_fn", "_f.read()", "_f") + } // Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line // branch ignores outputFileName and emits only the value, so the filename @@ -126,29 +147,32 @@ class FileScanOpDesc val emitFilename = outputFileName && attributeType.isSingle if (attributeType.isSingle) { - if (emitFilename) buf += " _rows.append((_fn, _f.read()))" - else buf += " _rows.append(_f.read())" + if (emitFilename) buf += s"${indent}_rows.append(($nameExpr, $readWhole))" + else buf += s"${indent}_rows.append($readWhole)" } else { val castExpr = attributeType match { case FileAttributeType.INTEGER => "int(l.rstrip())" case FileAttributeType.LONG => "int(l.rstrip())" case FileAttributeType.DOUBLE => "float(l.rstrip())" - case FileAttributeType.BOOLEAN => """l.rstrip().lower() == "true"""" + case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" case _ => """l.rstrip("\n")""" } // The slice applies to the raw lines, as the engine drops and takes // before parsing: a line outside the window is never converted, so an // unparseable one there costs nothing. Taking after dropping also keeps - // a large limit from overflowing the end index. + // a large limit from overflowing the end index. The window is per entry, + // as the engine's is: it drops and takes inside the flatMap over entries. val linesExpr = - if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) "_f" + if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) lines else { val dropped = - fileScanOffset.filter(_ > 0).fold("_f.readlines()")(o => s"_f.readlines()[$o:]") + fileScanOffset + .filter(_ > 0) + .fold(s"$lines.readlines()")(o => s"$lines.readlines()[$o:]") fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") } - buf += s" _rows.extend($castExpr for l in $linesExpr)" + buf += s"${indent}_rows.extend($castExpr for l in $linesExpr)" } val colLit = pyStringLiteral(col) @@ -160,4 +184,13 @@ class FileScanOpDesc buf.mkString("\n") } + + override def standaloneHelpers(): Seq[String] = + if (attributeType == FileAttributeType.BOOLEAN) Seq(TextSourceOpDesc.BooleanParser) + else Seq.empty + + override def standaloneImports(): Seq[String] = + if (!extract) Seq.empty + else if (attributeType.isSingle) Seq("import zipfile") + else Seq("import io", "import zipfile") } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala index e5a82a1a418..e397d9988b9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala @@ -30,6 +30,7 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.metadata.annotations.HideAnnotation import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc import org.apache.texera.amber.operator.source.scan.{ @@ -73,61 +74,104 @@ class FileScanSourceOpDesc fileTypeName = Option("") + override def standaloneSourcePath(): Option[String] = fileName + override def generateStandaloneCode(): String = { - val basename = sourceBasename(fileName.getOrElse("")) val col = attributeName + // `encoding` is the charset the panel offers, which is the one to honour. + // The executor reads the inherited `fileEncoding` instead, and that one is in + // this class's @JsonIgnoreProperties, so it never survives the trip and the + // engine decodes UTF-8 whatever the user chose. Following the executor here + // would mean ignoring the field as well; the export states what was asked for. val enc = encoding.toString.replace("_", "-").toLowerCase - val basenameLit = pyStringLiteral(basename) val colLit = pyStringLiteral(col) val encLit = pyStringLiteral(enc) val buf = scala.collection.mutable.ArrayBuffer[String]() - if (extract) - buf += s"""# WARNING: extract=true is not supported in standalone mode; provide the unarchived $basenameLit directly.""" - val isBinary = attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY - if (attributeType.isSingle) { + val castExpr = attributeType match { + case FileAttributeType.INTEGER => "int(l.rstrip())" + case FileAttributeType.LONG => "int(l.rstrip())" + case FileAttributeType.DOUBLE => "float(l.rstrip())" + case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall + case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" + case _ => """l.rstrip("\n")""" + } + + // The slice applies to the raw lines, as the engine drops and takes before + // parsing: a line outside the window is never converted, so an unparseable + // one there costs nothing. Taking after dropping also keeps a large limit + // from overflowing the end index. With extract on, the window is per entry, + // as the engine's is: it drops and takes inside the flatMap over entries. + def windowed(lines: String): String = + if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) lines + else { + val dropped = + fileScanOffset.filter(_ > 0).fold(s"$lines.readlines()")(o => s"$lines.readlines()[$o:]") + fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") + } + + // Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line + // branch emits only the value, so the filename column is added ONLY in + // single-value mode, whatever the flag says. + val emitFilename = outputFileName && attributeType.isSingle + + if (extract) { + // The engine reads the files INSIDE the archive, one tuple per entry, and + // skips the entries macOS adds. `zipfile` is the reader that matches: the + // platform casts what it opened to a ZipArchiveInputStream, so an archive + // that is not a zip fails on both sides. A directory entry is read, not + // skipped, there and here alike — it yields nothing. The filename column + // carries the ENTRY's name, which is what the engine puts there. + buf += "_rows = []" + buf += s"with zipfile.ZipFile($SourceFilePlaceholder) as _z:" + buf += " for _name in _z.namelist():" + buf += """ if _name.startswith("__MACOSX"):""" + buf += " continue" + buf += " with _z.open(_name) as _f:" + if (attributeType.isSingle) { + // A zip entry only opens binary, so text is decoded here rather than by + // the reader. + val readWhole = if (isBinary) "_f.read()" else s"_f.read().decode($encLit)" + if (emitFilename) buf += s" _rows.append((_name, $readWhole))" + else buf += s" _rows.append($readWhole)" + } else { + // TextIOWrapper, not splitlines: it ends a line where `open(..., "r")` + // does, which is what the branch below reads with. + val linesExpr = windowed(s"io.TextIOWrapper(_f, encoding=$encLit)") + buf += s" _rows.extend($castExpr for l in $linesExpr)" + } + if (emitFilename) buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])""" + else buf += s"""out1df = pd.DataFrame({$colLit: _rows})""" + } else if (attributeType.isSingle) { val openArgs = - if (isBinary) s"""$basenameLit, "rb"""" - else s"""$basenameLit, "r", encoding=$encLit""" + if (isBinary) s"""$SourceFilePlaceholder, "rb"""" + else s"""$SourceFilePlaceholder, "r", encoding=$encLit""" val dfCols = - if (outputFileName) s"""{"filename": $basenameLit, $colLit: [_f.read()]}""" + if (emitFilename) s"""{"filename": $SourceFilePlaceholder, $colLit: [_f.read()]}""" else s"""{$colLit: [_f.read()]}""" buf += s"""with open($openArgs) as _f:""" buf += s""" out1df = pd.DataFrame($dfCols)""" } else { - val castExpr = attributeType match { - case FileAttributeType.INTEGER => "int(l.rstrip())" - case FileAttributeType.LONG => "int(l.rstrip())" - case FileAttributeType.DOUBLE => "float(l.rstrip())" - case FileAttributeType.BOOLEAN => """l.rstrip().lower() == "true"""" - case FileAttributeType.TIMESTAMP => "pd.Timestamp(l.rstrip())" - case _ => """l.rstrip("\n")""" - } - // The slice applies to the raw lines, as the engine drops and takes - // before parsing: a line outside the window is never converted, so an - // unparseable one there costs nothing. Taking after dropping also keeps - // a large limit from overflowing the end index. - val linesExpr = - if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) "_f" - else { - val dropped = - fileScanOffset.filter(_ > 0).fold("_f.readlines()")(o => s"_f.readlines()[$o:]") - fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") - } - val dfCols = - if (outputFileName) - s"""{"filename": $basenameLit, $colLit: [$castExpr for l in $linesExpr]}""" - else s"""{$colLit: [$castExpr for l in $linesExpr]}""" - buf += s"""with open($basenameLit, "r", encoding=$encLit) as _f:""" - buf += s""" out1df = pd.DataFrame($dfCols)""" + val linesExpr = windowed("_f") + buf += s"""with open($SourceFilePlaceholder, "r", encoding=$encLit) as _f:""" + buf += s""" out1df = pd.DataFrame({$colLit: [$castExpr for l in $linesExpr]})""" } buf.mkString("\n") } + override def standaloneHelpers(): Seq[String] = + if (attributeType == FileAttributeType.BOOLEAN) Seq(TextSourceOpDesc.BooleanParser) + else Seq.empty + + override def standaloneImports(): Seq[String] = + if (!extract) Seq.empty + else if (attributeType.isSingle) Seq("import zipfile") + else Seq("import io", "import zipfile") + override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index 944a8588b4e..48a076c75b8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -28,6 +28,7 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.{JSONToMap, objectMapper} @@ -46,8 +47,9 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato fileTypeName = Option("JSONL") + override def standaloneSourcePath(): Option[String] = fileName + override def generateStandaloneCode(): String = { - val basename = sourceBasename(fileName.getOrElse("")) val enc = fileEncoding.toString.replace("_", "-").toLowerCase // The executor drops and takes on the RAW lines, before any of them is @@ -55,18 +57,29 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato // unparseable one there costs nothing. Reading the whole file first and // slicing the frame would end the export on a line the workflow skipped. val windowed = offset.exists(_ > 0) || limit.isDefined - val source = - if (!windowed) pyStringLiteral(basename) - else { - val dropped = offset.filter(_ > 0).fold("_lines")(o => s"_lines[$o:]") - val taken = limit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") - s"""io.StringIO("".join($taken))""" - } + + // read_json parses a JSON number into a float before any dtype it is handed + // can apply, so a LONG past 2^53 is already rounded by the time the column + // exists: 9007199254740993 arrives as ...992, a value the file never held. + // Reading the lines a second time with Python's own parser, which keeps an + // integer exact, is the only way to put the column back. + val longColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes.filter(_.getType == AttributeType.LONG).map(_.getName) + ) + + // The lines are read into the script whenever something below needs them, + // and read_json is then fed from those rather than from the file, so the + // file is opened once either way. + val readsLines = windowed || longColumns.nonEmpty + val dropped = offset.filter(_ > 0).fold("_lines")(o => s"_lines[$o:]") + val taken = limit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") val readArgs = scala.collection.mutable.ArrayBuffer[String]() - readArgs += source + readArgs += (if (readsLines) s"""io.StringIO("".join($taken))""" else SourceFilePlaceholder) readArgs += "lines=True" - if (!windowed) readArgs += s"""encoding=${pyStringLiteral(enc)}""" + // Text already decoded by the open above carries no encoding of its own. + if (!readsLines) readArgs += s"""encoding=${pyStringLiteral(enc)}""" // JSON has no timestamp of its own, so both readers infer from the text and // do not infer alike: the schema below tries TIMESTAMP and parses what it @@ -90,18 +103,46 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato else readExpr val lines = scala.collection.mutable.ArrayBuffer[String]() - if (windowed) { - lines += "import io" - lines += s"""with open(${pyStringLiteral(basename)}, "r", encoding=${pyStringLiteral( + if (readsLines) { + lines += s"""with open($SourceFilePlaceholder, "r", encoding=${pyStringLiteral( enc )}) as _f:""" lines += " _lines = _f.readlines()" } lines += s"out1df = $baseExpr" + if (longColumns.nonEmpty) { + lines += s"_records = [json.loads(_l) for _l in $taken]" + longColumns.foreach { name => + val nameLit = pyStringLiteral(name) + // Flattening joins a nested key to its parent with a dot, so the schema's + // name is a path into the record rather than a key of it. Without + // flattening it is a key, and a key is free to hold a dot of its own. + val valueExpr = + if (flatten) s"_texera_json_value(_r, $nameLit)" else s"_r.get($nameLit)" + lines += s"""out1df[$nameLit] = pd.array([$valueExpr for _r in _records], dtype="Int64")""" + } + } + lines.mkString("\n") } + override def standaloneHelpers(): Seq[String] = + if (flatten && longColumnCount > 0) Seq(JSONLScanSourceOpDesc.JsonValueAtPath) else Seq.empty + + override def standaloneImports(): Seq[String] = { + val windowed = offset.exists(_ > 0) || limit.isDefined + val longs = longColumnCount + (if (windowed || longs > 0) Seq("import io") else Seq.empty) ++ + (if (longs > 0) Seq("import json") else Seq.empty) + } + + /** How many columns the schema types LONG, or none when it cannot be read. */ + private def longColumnCount: Int = + Try(sourceSchema()).toOption + .map(_.getAttributes.count(_.getType == AttributeType.LONG)) + .getOrElse(0) + @throws[IOException] override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -183,3 +224,22 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato } } + +object JSONLScanSourceOpDesc { + + /** + * A flattened column's name read back out of the record it came from. + * + * Only the columns an exact re-read has to rebuild need this, and only when + * flattening is on: the name is then a dotted path rather than a key. A path + * the record does not hold reads as nothing, which is what the flattened + * frame carries there. + */ + val JsonValueAtPath: String = + """def _texera_json_value(record, path): + | for part in path.split("."): + | if not isinstance(record, dict) or part not in record: + | return None + | record = record[part] + | return record""".stripMargin +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala index b9ecfa89d1c..28fa2ab8a1a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala @@ -94,7 +94,7 @@ class TextInputSourceOpDesc case FileAttributeType.INTEGER => "int(l)" case FileAttributeType.LONG => "int(l)" case FileAttributeType.DOUBLE => "float(l)" - case FileAttributeType.BOOLEAN => """l.lower() == "true"""" + case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall case FileAttributeType.TIMESTAMP => "pd.Timestamp(l)" case _ => "l" } @@ -110,4 +110,8 @@ class TextInputSourceOpDesc buf.mkString("\n") } + + override def standaloneHelpers(): Seq[String] = + if (attributeType == FileAttributeType.BOOLEAN) Seq(TextSourceOpDesc.BooleanParser) + else Seq.empty } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala index 654e11c02d2..aae94ed8494 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala @@ -89,3 +89,27 @@ trait TextSourceOpDesc { ) var fileScanOffset: Option[Int] = None } + +object TextSourceOpDesc { + + /** + * The line-to-boolean rule the engine uses, as Python. + * + * `parseField` reads "true" and "false" in any case, then falls back to an + * integer and calls only 1 true. Anything else ends the run. Comparing the + * lowercased line to "true" read 1 as false and let text the engine refuses + * through as a row of false. + */ + val BooleanParser: String = + """def _texera_parse_bool(line): + | text = line.strip() + | lowered = text.lower() + | if lowered == "true": + | return True + | if lowered == "false": + | return False + | return int(text) == 1""".stripMargin + + /** The call the per-line cast makes, over the loop variable the readers share. */ + val BooleanParserCall: String = "_texera_parse_bool(l)" +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 54dd677913e..bbdb8525d87 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -162,4 +162,35 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val ex = intercept[RuntimeException](d.inferSchema()) ex.getMessage shouldBe "Failed to read the .arrow file. Please ensure it is a valid Arrow file." } + + // pd.read_feather carries a null in a float, which rounds every value past 2^53: + // the file's 9007199254740993 reads back as ...992, where the executor hands the + // exact value on. The file states its own types, so only the hole causes this. + "ArrowSourceOpDesc.generateStandaloneCode" should "keep a nullable long exact" in { + val schema = Schema(List(new Attribute("big", AttributeType.LONG))) + val file = writeArrowFile( + schema, + Seq(Array[Any](9007199254740993L), Array[Any](null), Array[Any](9007199254740995L)) + ) + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + + val exec = new ArrowSourceOpExec(objectMapper.writeValueAsString(d)) + exec.open() + val fromEngine = + try exec.produceTuple().map(_.getFields.head).toList + finally exec.close() + fromEngine shouldBe List(9007199254740993L, null, 9007199254740995L) + + d.generateStandaloneCode() should include( + """out1df["big"] = pd.read_feather(sourceFile, columns=["big"], dtype_backend="numpy_nullable")["big"]""" + ) + } + + // The base's sourceSchema returns null here: this operator reads its types out of + // the file. Asking for them without a file must leave the export alone, not throw. + it should "emit a plain read when the file cannot be inspected" in { + val d = new ArrowSourceOpDesc + d.generateStandaloneCode() shouldBe """out1df = pd.read_feather(sourceFile)""" + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index 5fe9b31a919..108f5127a49 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -32,6 +32,7 @@ import org.apache.texera.amber.operator.{LogicalOp, TestOperators} import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.operator.source.scan.csvOld.CSVOldScanSourceOpDesc +import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec @@ -97,6 +98,38 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { opDesc.sourceSchema().getAttributes.map(_.getName).toList } + // Writes a CSV whose `big` column holds values past 2^53 and one omitted cell, + // and returns the absolute path. + private def writeNullableLongCsv(): String = { + val tmpFile = Files.createTempFile("nullable-long-", ".csv") + tmpFile.toFile.deleteOnExit() + Files.write( + tmpFile, + "id,big\n1,9007199254740993\n2,\n3,9007199254740995\n".getBytes(StandardCharsets.UTF_8) + ) + tmpFile.toString + } + + // Writes a CSV holding the country code NA and an omitted field, and returns + // the absolute path. + private def writeNaCsv(): String = { + val tmpFile = Files.createTempFile("na-code-", ".csv") + tmpFile.toFile.deleteOnExit() + Files.write(tmpFile, "code,note\nNA,x\n,y\n".getBytes(StandardCharsets.UTF_8)) + tmpFile.toString + } + + // Writes a headered CSV with six data rows and returns the absolute path. + private def writeSixRowCsv(): String = { + val tmpFile = Files.createTempFile("six-row-", ".csv") + tmpFile.toFile.deleteOnExit() + Files.write( + tmpFile, + "id,name\n1,a\n2,b\n3,c\n4,d\n5,e\n6,f\n".getBytes(StandardCharsets.UTF_8) + ) + tmpFile.toString + } + // Writes a numeric column with one blank cell and returns the absolute path. private def writeCsvWithBlankNumericCell(): String = { val tmpFile = Files.createTempFile("blank-cell-", ".csv") @@ -211,7 +244,10 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { ) } - it should "use the csv basename in standalone code" in { + // The name is left to the translator, which is the only thing that can see a + // second source wanting it. What the operator owes is the path and the name it + // would like. + it should "offer the csv basename and read the file by placeholder" in { csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) csvScanSourceOpDesc.customDelimiter = Some(",") csvScanSourceOpDesc.hasHeader = true @@ -219,19 +255,26 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { val code = csvScanSourceOpDesc.generateStandaloneCode() - assert(code.contains("""filepath_or_buffer="country_sales_small_multi_line.csv"""")) + assert(code.contains("filepath_or_buffer=sourceFile")) + assert(csvScanSourceOpDesc.standaloneSourcePath() == csvScanSourceOpDesc.fileName) + assert( + csvScanSourceOpDesc.standaloneSourceName().contains("country_sales_small_multi_line.csv") + ) assert(!code.contains("base64.b64decode")) assert(!code.contains("io.BytesIO")) } - it should "use the unresolved csv basename in standalone code" in { + it should "offer the unresolved csv basename" in { csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) csvScanSourceOpDesc.customDelimiter = Some(",") csvScanSourceOpDesc.hasHeader = true val code = csvScanSourceOpDesc.generateStandaloneCode() - assert(code.contains("""filepath_or_buffer="country_sales_small_multi_line.csv"""")) + assert(code.contains("filepath_or_buffer=sourceFile")) + assert( + csvScanSourceOpDesc.standaloneSourceName().contains("country_sales_small_multi_line.csv") + ) assert(!code.contains("base64.b64decode")) assert(!code.contains("io.BytesIO")) } @@ -265,6 +308,122 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(code.contains("""out1df.columns = ["id", "name", "column-3", "age"]""")) } + // A null is what forces the widening: pandas carries the hole in a float, and a + // long past 2^53 does not survive the trip. 9007199254740993 came back as ...992, + // a value the file never held and the executor never produced. Int64 is the + // nullable integer, so the column keeps both its values and its hole. + it should "read a nullable long as an exact integer, the way the parser does" in { + val path = writeNullableLongCsv() + csvScanSourceOpDesc.fileName = Some(path) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + // This reader's blank is a null, which inferField passes over, so the column + // keeps the type its values have. + assert(csvScanSourceOpDesc.sourceSchema().getAttribute("big").getType == AttributeType.LONG) + + val code = csvScanSourceOpDesc.generateStandaloneCode() + assert(code.contains("""dtype={"big": "Int64"}""")) + // `id` has no hole, so a float would carry it exactly. Only the column the + // schema calls LONG is asked for. + assert(!code.contains(""""id": "Int64"""")) + } + + // sourceSchema reads this operator's file with scala-csv, which hands a blank + // back as "", so one blank cell types the whole column STRING — while the + // executor's block reader nulls the blank and parses the rest as that STRING. + // pandas sees the blank as missing and infers a number instead, which read a + // column of ids back as floats and rounded 9007199254740993 to ...992. + it should "read a parallel CSV column the schema typed STRING as text" in { + val path = writeNullableLongCsv() + parallelCsvScanSourceOpDesc.fileName = Some(path) + parallelCsvScanSourceOpDesc.customDelimiter = Some(",") + parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + assert( + parallelCsvScanSourceOpDesc.sourceSchema().getAttribute("big").getType == + AttributeType.STRING + ) + + val exec = new ParallelCSVScanSourceOpExec( + objectMapper.writeValueAsString(parallelCsvScanSourceOpDesc) + ) + exec.open() + val rows = + try exec.produceTuple().map(_.getFields.toList).toList + finally exec.close() + assert(rows.map(_(1)) == List("9007199254740993", null, "9007199254740995")) + + assert( + parallelCsvScanSourceOpDesc + .generateStandaloneCode() + .contains("""dtype={"big": "string"}""") + ) + } + + // The block reader nulls an omitted field and leaves every other text alone, so + // "NA" is the country code it says it is. pandas reads it as missing by default, + // so the export read a column of codes as a column of nulls. + it should "read only an empty field as null for parallel CSV, the way its reader does" in { + val path = writeNaCsv() + parallelCsvScanSourceOpDesc.fileName = Some(path) + parallelCsvScanSourceOpDesc.customDelimiter = Some(",") + parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + val exec = new ParallelCSVScanSourceOpExec( + objectMapper.writeValueAsString(parallelCsvScanSourceOpDesc) + ) + exec.open() + val rows = + try exec.produceTuple().map(_.getFields.toList).toList + finally exec.close() + assert(rows == List(List("NA", "x"), List(null, "y"))) + + val code = parallelCsvScanSourceOpDesc.generateStandaloneCode() + assert(code.contains("keep_default_na=False")) + assert(code.contains("""na_values=[""]""")) + } + + it should "give the parallel CSV frame the names the schema gives it" in { + val path = writeCsvWithEmptyHeader() + parallelCsvScanSourceOpDesc.fileName = Some(path) + parallelCsvScanSourceOpDesc.customDelimiter = Some(",") + parallelCsvScanSourceOpDesc.hasHeader = true + parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + assert( + parallelCsvScanSourceOpDesc + .generateStandaloneCode() + .contains("""out1df.columns = ["id", "name", "column-3", "age"]""") + ) + } + + // ParallelCSVScanSourceOpExec.open carves the file into byte ranges and leaves + // limit and offset as TODOs, so the window the panel offers never reaches the + // rows. Slicing in the export handed back fewer rows than the workflow did. + it should "leave limit and offset out of the parallel CSV read, as its reader ignores them" in { + val path = writeSixRowCsv() + parallelCsvScanSourceOpDesc.fileName = Some(path) + parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + parallelCsvScanSourceOpDesc.customDelimiter = Some(",") + parallelCsvScanSourceOpDesc.offset = Some(2) + parallelCsvScanSourceOpDesc.limit = Some(2) + + val exec = new ParallelCSVScanSourceOpExec( + objectMapper.writeValueAsString(parallelCsvScanSourceOpDesc) + ) + exec.open() + val rowsRead = + try exec.produceTuple().size + finally exec.close() + assert(rowsRead == 6) + + val code = parallelCsvScanSourceOpDesc.generateStandaloneCode() + assert(!code.contains("skiprows")) + assert(!code.contains("nrows")) + assert(code.startsWith("# NOTE: this operator's limit and offset are ignored")) + } + it should "use comma as the default delimiter when customDelimiter is not set for parallel CSV" in { parallelCsvScanSourceOpDesc.customDelimiter = None diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala index b72476641b0..f261531198f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala @@ -20,6 +20,8 @@ package org.apache.texera.amber.operator.source.scan.csvOld import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.storage.FileResolver +import org.apache.texera.amber.core.tuple.AttributeType import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants @@ -28,6 +30,9 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files + class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) @@ -95,4 +100,62 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { r.limit shouldBe Some(10) r.offset shouldBe Some(5) } + + // scala-csv hands back the text of every field and nothing else: a blank cell is + // "", not a null, and "NA" is the country code it says it is. pandas reads both + // as missing by default, so the export read a column of codes as a column of + // nulls and turned the blank into NaN. + "CSVOldScanSourceOpDesc.generateStandaloneCode" should + "read a literal NA as text and a blank as an empty string, as its reader does" in { + val d = describing(writeCsv("code,note\nNA,x\n,y\n")) + + rowsFromEngine(d) shouldBe List(List("NA", "x"), List("", "y")) + + val code = d.generateStandaloneCode() + code should include("keep_default_na=False") + // No na_values: where the other CSV readers null a blank, this one keeps it. + code should not include "na_values" + } + + // The same reason keeps a large integer exact here: a blank types the column + // STRING, and with no missing value named, pandas reads every cell as the text + // it is. Nothing widens through a float, so 9007199254740993 stays itself. + it should "keep a nullable large integer exact, as text" in { + val d = describing(writeCsv("id,big\n1,9007199254740993\n2,\n3,9007199254740995\n")) + + d.sourceSchema().getAttribute("big").getType shouldBe AttributeType.STRING + rowsFromEngine(d).map(_(1)) shouldBe List("9007199254740993", "", "9007199254740995") + } + + // sourceSchema names a blank header column-N; pandas names it "Unnamed: N", and a + // downstream operator asks for the name the schema gave. + it should "give the frame the names the schema gives it" in { + val d = describing(writeCsv("id,name,,age\n1,Alice,x,30\n")) + d.generateStandaloneCode() should include( + """out1df.columns = ["id", "name", "column-3", "age"]""" + ) + } + + private def writeCsv(content: String): String = { + val file = Files.createTempFile("csv-old-", ".csv") + file.toFile.deleteOnExit() + Files.write(file, content.getBytes(StandardCharsets.UTF_8)) + file.toString + } + + private def describing(path: String): CSVOldScanSourceOpDesc = { + val d = new CSVOldScanSourceOpDesc + d.fileName = Some(path) + d.customDelimiter = Some(",") + d.hasHeader = true + d.setResolvedFileName(FileResolver.resolve(path)) + d + } + + private def rowsFromEngine(d: CSVOldScanSourceOpDesc): List[List[Any]] = { + val exec = new CSVOldScanSourceOpExec(objectMapper.writeValueAsString(d)) + exec.open() + try exec.produceTuple().map(_.getFields.toList).toList + finally exec.close() + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala index 26ac6372b4a..67ef0446e56 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala @@ -30,6 +30,7 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.source.scan.{FileAttributeType, FileDecodingMethod} +import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec @@ -212,7 +213,7 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { FileAttributeType.INTEGER -> "int(l.rstrip())", FileAttributeType.LONG -> "int(l.rstrip())", FileAttributeType.DOUBLE -> "float(l.rstrip())", - FileAttributeType.BOOLEAN -> """l.rstrip().lower() == "true"""", + FileAttributeType.BOOLEAN -> "_texera_parse_bool(l)", FileAttributeType.TIMESTAMP -> "pd.Timestamp(l.rstrip())", FileAttributeType.STRING -> """l.rstrip("\n")""" ) @@ -225,6 +226,12 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { .contains(s" _rows.extend($cast for l in _f)") ) } + // The boolean cast is the one that calls out of the body, so the script has + // to carry the definition. + fileScanOpDesc.attributeType = FileAttributeType.BOOLEAN + assert(fileScanOpDesc.standaloneHelpers() == Seq(TextSourceOpDesc.BooleanParser)) + fileScanOpDesc.attributeType = FileAttributeType.STRING + assert(fileScanOpDesc.standaloneHelpers().isEmpty) } it should "slice the raw lines before converting them when a limit or offset is set" in { @@ -255,12 +262,42 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { ) } - it should "warn that archive extraction is unsupported when extract is on" in { - // `extract` is a val, so it can only be set through deserialization. + // With extract on, FileScanUtils opens the file as a zip and emits a tuple per + // entry. Reading the archive itself as text was the export's answer before, and + // it produced whichever bytes the compression happened to leave readable. + it should "read the entries inside the archive when extract is on" in { + // `extract` and `outputFileName` are vals, so they are set by deserialization. val desc = objectMapper.readValue( """{"operatorType":"FileScanOp","extract":true}""", classOf[FileScanOpDesc] ) - assert(desc.generateStandaloneCode().startsWith("# WARNING: extract=true is not supported")) + val code = desc.generateStandaloneCode() + + assert(code.contains(" with zipfile.ZipFile(_fn) as _z:")) + assert(code.contains(" for _name in _z.namelist():")) + assert(code.contains(""" if _name.startswith("__MACOSX"):""")) + assert(code.contains(" with _z.open(_name) as _f:")) + // A zip entry opens binary whatever the column holds, so a text line is + // decoded rather than read through a text-mode handle. + assert( + code.contains( + """ _rows.extend(l.rstrip("\n") for l in io.TextIOWrapper(_f, encoding="utf-8"))""" + ) + ) + assert(desc.standaloneImports() == Seq("import io", "import zipfile")) + assert(!code.contains("WARNING")) + } + + it should "name the entry, not the archive, in the filename column" in { + val desc = objectMapper.readValue( + """{"operatorType":"FileScanOp","extract":true,"outputFileName":true, + |"attributeType":"single string"}""".stripMargin, + classOf[FileScanOpDesc] + ) + val code = desc.generateStandaloneCode() + + assert(code.contains(""" _rows.append((_name, _f.read().decode("utf-8")))""")) + assert(code.endsWith("""out1df = pd.DataFrame(_rows, columns=["filename", "line"])""")) + assert(desc.standaloneImports() == Seq("import zipfile")) } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala index 2f3d76af9de..3bcc1c65a73 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala @@ -20,16 +20,25 @@ package org.apache.texera.amber.operator.source.scan.file import com.fasterxml.jackson.databind.node.ObjectNode +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.storage.FileResolver import org.apache.texera.amber.core.tuple.{AttributeType, Schema, SchemaEnforceable, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.operator.TestOperators +import org.apache.texera.amber.operator.{StandaloneCodeGenerator, TestOperators} import org.apache.texera.amber.operator.source.scan.{FileAttributeType, FileDecodingMethod} +import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import java.util.concurrent.TimeUnit +import java.util.zip.{ZipEntry, ZipOutputStream} +import scala.io.Source +import scala.util.Try + class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { var fileScanSourceOpDesc: FileScanSourceOpDesc = _ @@ -238,6 +247,15 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { ) } + it should "parse a boolean line through the shared helper, and declare it" in { + fileScanSourceOpDesc.attributeType = FileAttributeType.BOOLEAN + assert(fileScanSourceOpDesc.generateStandaloneCode().contains("_texera_parse_bool(l)")) + assert(fileScanSourceOpDesc.standaloneHelpers() == Seq(TextSourceOpDesc.BooleanParser)) + + fileScanSourceOpDesc.attributeType = FileAttributeType.STRING + assert(fileScanSourceOpDesc.standaloneHelpers().isEmpty) + } + "FileScanSourceOpDesc.getPhysicalOp" should "wire the FileScanSourceOpExec class as a source op and propagate its schema" in { val physical = @@ -271,4 +289,156 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(schema.getAttribute("line").getType == AttributeType.STRING) } + // With extract on the engine reads the files INSIDE the archive. The export + // used to open the archive itself and hand back whatever bytes the compression + // left readable, behind a warning comment. + "FileScanSourceOpDesc.generateStandaloneCode" should + "read the same entries out of an archive as the engine does" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("file-scan-archive-") + dir.toFile.deleteOnExit() + val archive = writeArchive( + dir, + "a.txt" -> "line1\nline2\n", + "b.txt" -> "line3\n", + // macOS puts these beside the real entries; the engine skips them by name. + "__MACOSX/a.txt" -> "junk\n" + ) + + val desc = extractingDesc(archive, """"attributeType":"string"""") + assert(linesFromEngine(desc) == Seq("line1", "line2", "line3")) + assert( + runStandalone(python, dir, desc, """print(list(out1df["line"]))""")._2.trim + .endsWith("""['line1', 'line2', 'line3']""") + ) + } + + it should "take an archive entry's own name for the filename column" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("file-scan-archive-named-") + dir.toFile.deleteOnExit() + val archive = writeArchive(dir, "a.txt" -> "first", "b.txt" -> "second") + + val desc = extractingDesc( + archive, + """"attributeType":"single string"""", + """"outputFileName":true""" + ) + val engine = + tuplesFromEngine(desc).map(t => (t.getField[String]("filename"), t.getField[String]("line"))) + assert(engine == Seq(("a.txt", "first"), ("b.txt", "second"))) + + val out = runStandalone( + python, + dir, + desc, + """print(list(out1df.itertuples(index=False, name=None)))""" + )._2 + assert(out.trim.endsWith("""[('a.txt', 'first'), ('b.txt', 'second')]""")) + } + + /** A zip at `dir/archive.zip` holding the given entries. */ + private def writeArchive(dir: Path, entries: (String, String)*): Path = { + val archive = dir.resolve("archive.zip") + val out = new ZipOutputStream(Files.newOutputStream(archive)) + try entries.foreach { + case (name, content) => + out.putNextEntry(new ZipEntry(name)) + out.write(content.getBytes(StandardCharsets.UTF_8)) + out.closeEntry() + } finally out.close() + archive + } + + /** `extract` and `outputFileName` are vals, so the flags are deserialized in. */ + private def extractingDesc(archive: Path, fields: String*): FileScanSourceOpDesc = { + val desc = objectMapper.readValue( + (Seq(""""operatorType":"FileScan"""", """"extract":true""") ++ fields) + .mkString("{", ",", "}"), + classOf[FileScanSourceOpDesc] + ) + desc.setResolvedFileName(FileResolver.resolve(archive.toString)) + desc + } + + private def tuplesFromEngine(desc: FileScanSourceOpDesc): Seq[Tuple] = { + val exec = new FileScanSourceOpExec(objectMapper.writeValueAsString(desc)) + exec.open() + try exec + .produceTuple() + .map(_.asInstanceOf[SchemaEnforceable].enforceSchema(desc.sourceSchema())) + .toSeq + finally exec.close() + } + + private def linesFromEngine(desc: FileScanSourceOpDesc): Seq[String] = + tuplesFromEngine(desc).map(_.getField[String]("line")) + + /** + * The operator's exported body, run from `dir`, with the placeholder bound as + * the translator binds it and the imports it declared written out. + */ + private def runStandalone( + python: String, + dir: Path, + desc: FileScanSourceOpDesc, + tail: String + ): (Int, String) = { + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${desc.standaloneImports().mkString("\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${desc.standaloneSourceName().get}" + |${desc.generateStandaloneCode()} + |$tail + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n")(assert(process.exitValue() == 0)) + (process.exitValue(), out) + } + + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path + // (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption + .exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index 17cc308354f..1cce3efc2ca 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -21,8 +21,10 @@ package org.apache.texera.amber.operator.source.scan.json import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.storage.FileResolver +import org.apache.texera.amber.core.tuple.{AttributeType, SchemaEnforceable} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.source.scan.FileDecodingMethod import org.apache.texera.amber.util.JSONUtils.objectMapper @@ -116,10 +118,19 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { op.fileName = Some(data.toString) op.offset = Some(1) + // The body names its file by placeholder and the translator puts a name + // there. Standing in for the translator is all this test needs, and the + // process runs from the file's own directory, so the bare name resolves. + val bindSourceFile = + s"""${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}"""" + val script = dir.resolve("run.py") Files.write( script, s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |$bindSourceFile |${op.generateStandaloneCode()} |print(list(out1df["id"])) |""".stripMargin.getBytes(StandardCharsets.UTF_8) @@ -137,6 +148,67 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // read_json parses a JSON number into a float before any dtype it is handed can + // apply, so a long past 2^53 arrives already rounded: 9007199254740993 came back + // as ...992, a value the file never held and the executor never produced. The + // hole is what forces the widening, so the column needs one to show it. + it should "keep a nullable long exact, as the executor does" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-long-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + Files.write( + data, + ("""{"id":1,"big":9007199254740993}""" + "\n" + + """{"id":2}""" + "\n" + + """{"id":3,"big":9007199254740995}""" + "\n").getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + op.sourceSchema().getAttribute("big").getType shouldBe AttributeType.LONG + + val exec = new JSONLScanSourceOpExec(objectMapper.writeValueAsString(op)) + exec.open() + val fromEngine = + try exec + .produceTuple() + .map( + _.asInstanceOf[SchemaEnforceable].enforceSchema(op.sourceSchema()).getField[Any]("big") + ) + .toList + finally exec.close() + fromEngine shouldBe List(9007199254740993L, null, 9007199254740995L) + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}" + |${op.generateStandaloneCode()} + |print([None if pd.isna(v) else int(v) for v in out1df["big"]]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + out.trim should endWith("[9007199254740993, None, 9007199254740995]") + } + } + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path // (UDF_PYTHON_PATH), then python3 / python / py. private def resolvePython(): Option[String] = { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala index dcede030447..081d73108f6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala @@ -19,8 +19,15 @@ package org.apache.texera.amber.operator.source.scan.text +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{AttributeType, Schema, SchemaEnforceable, Tuple} +import org.apache.texera.amber.core.tuple.{ + AttributeType, + AttributeTypeUtils, + Schema, + SchemaEnforceable, + Tuple +} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.source.scan.FileAttributeType @@ -30,6 +37,9 @@ import org.scalatest.flatspec.AnyFlatSpec import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths} +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { var textInputSourceOpDesc: TextInputSourceOpDesc = _ @@ -306,6 +316,128 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { ) } + it should "parse a boolean line through the shared helper, and declare it" in { + textInputSourceOpDesc.attributeType = FileAttributeType.BOOLEAN + textInputSourceOpDesc.textInput = "true" + assert(textInputSourceOpDesc.generateStandaloneCode().contains("_texera_parse_bool(l)")) + assert(textInputSourceOpDesc.standaloneHelpers() == Seq(TextSourceOpDesc.BooleanParser)) + + textInputSourceOpDesc.attributeType = FileAttributeType.STRING + assert(textInputSourceOpDesc.standaloneHelpers().isEmpty) + } + + // `parseField` reads a BOOLEAN line as "true" or "false" in any case, then as an + // integer that is true only at 1, and refuses anything else. Comparing the + // lowercased line to "true" called 1 false and passed text the engine refuses + // off as a row of false. + it should "read a boolean line the way the engine does" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + textInputSourceOpDesc.attributeType = FileAttributeType.BOOLEAN + textInputSourceOpDesc.textInput = Seq("true", "TRUE", " true ", "False", "1", "0", "2", "-1") + .mkString("\n") + + val expected = booleansFromEngine().map(b => if (b) "True" else "False").mkString(", ") + val (exitCode, out) = runStandalone(python) + withClue(s"python said:\n$out\n") { + assert(exitCode == 0) + assert(out.trim.endsWith(s"[$expected]")) + } + } + + it should "refuse a boolean line the engine refuses" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + textInputSourceOpDesc.attributeType = FileAttributeType.BOOLEAN + textInputSourceOpDesc.textInput = "true\nyes" + + assertThrows[AttributeTypeUtils.AttributeTypeException](booleansFromEngine()) + val (exitCode, out) = runStandalone(python) + withClue(s"python said:\n$out\n") { + assert(exitCode != 0) + assert(out.contains("ValueError")) + } + } + + /** The configured text read as booleans by the executor the engine runs. */ + private def booleansFromEngine(): Seq[Boolean] = { + val exec = new TextInputSourceOpExec(objectMapper.writeValueAsString(textInputSourceOpDesc)) + exec.open() + try { + exec + .produceTuple() + .map( + _.asInstanceOf[SchemaEnforceable] + .enforceSchema(textInputSourceOpDesc.sourceSchema()) + .getField[Boolean]("line") + ) + .toSeq + } finally exec.close() + } + + /** + * The configured operator's exported script, run, printing its one column. + * + * `bool` on each value because a numpy scalar does not repr as Python's own + * True and False. + */ + private def runStandalone(python: String): (Int, String) = { + val dir = Files.createTempDirectory("text-input-standalone-") + dir.toFile.deleteOnExit() + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${textInputSourceOpDesc.standaloneHelpers().mkString("\n\n")} + |${textInputSourceOpDesc.generateStandaloneCode()} + |print([bool(v) for v in out1df["line"]]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + (process.exitValue(), out) + } + + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path + // (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePython(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption + .exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + "TextInputSourceOpDesc.getPhysicalOp" should "wire the TextInputSourceOpExec class as a source op with one output port" in { val physical = diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala index 8e873403787..7fa1be9c752 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -25,7 +25,9 @@ import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.common.compiler.model.LogicalPlan import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral +import java.util.regex.Matcher import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ @@ -67,6 +69,25 @@ class WorkflowToPythonTranslator extends LazyLogging { // getTopologicalOpIds() uses jgrapht internally — no need for a custom topo sort val topoOrder = logicalPlan.getTopologicalOpIds.asScala.toList + // What each source's file is called in the directory the script runs from. + // A source offers the last segment of its resolved path, which is the name a + // person would give the file, but two sources reading different files can + // offer the same one — and did, leaving the script to read one of them twice + // and say nothing. The second gets a name of its own, for the same reason a + // chart's output file is numbered. Keyed by the resolved path, so one file + // read by two operators keeps one name. + val sourceFileNames = mutable.Map[String, String]() + topoOrder.map(logicalPlan.getOperator).foreach { + case gen: StandaloneCodeGenerator => + gen.standaloneSourcePath().foreach { path => + sourceFileNames.getOrElseUpdate( + path, + distinctName(gen.standaloneSourceName().getOrElse(""), sourceFileNames.values.toSet) + ) + } + case _ => () + } + // pandas is the one module every generator uses: an operator body reads and // writes frames whatever else it does. Everything beyond that is asked of the // operators in the plan, so a script that draws nothing does not require a @@ -147,6 +168,7 @@ class WorkflowToPythonTranslator extends LazyLogging { inVars, outVars, fileBase(displayName, fileBaseCounts), + gen.standaloneSourcePath().flatMap(sourceFileNames.get).getOrElse(""), displayName ) @@ -214,6 +236,18 @@ class WorkflowToPythonTranslator extends LazyLogging { s"${stem}_$n" } + // The offered name if no other source has taken it, otherwise the same name + // numbered before its extension: data.csv, then data-2.csv. Numbering the stem + // rather than appending keeps the suffix, which is what a reader opens the + // file by. + private def distinctName(offered: String, taken: Set[String]): String = { + if (!taken.contains(offered)) return offered + val dot = offered.lastIndexOf('.') + val (stem, ext) = + if (dot <= 0) (offered, "") else (offered.substring(0, dot), offered.substring(dot)) + Iterator.from(2).map(n => s"$stem-$n$ext").find(!taken.contains(_)).get + } + // Replaces in{N}df / out{N}df placeholders with concrete variable names. // Substitutes in reverse index order to prevent partial matches (e.g. in1df // inside in10df). Only the code parts are rewritten: a generator writes a @@ -226,6 +260,7 @@ class WorkflowToPythonTranslator extends LazyLogging { inVars: List[String], outVars: List[String], fileBase: String, + sourceFile: String, displayName: String ): String = { def substitute(fragment: String): String = { @@ -238,6 +273,15 @@ class WorkflowToPythonTranslator extends LazyLogging { result = result.replaceAll("""\boutputHtml\b""", "\"" + fileBase + ".html\"") result = result.replaceAll("""\boutputJson\b""", "\"" + fileBase + ".json\"") + // A source names the file it reads sourceFile and gets back the name + // assigned where sourceFileNames is built. Quoted through the escaper the + // operators use, and then quoted again for the replacement: a file name is + // the user's text, so it can hold both a backslash and a `$`. + result = result.replaceAll( + s"""\\b${StandaloneCodeGenerator.SourceFilePlaceholder}\\b""", + Matcher.quoteReplacement(pyStringLiteral(sourceFile)) + ) + // A variadic port takes as many upstream links as the user draws, and an // operator reading one cannot name them: `in1df`/`in2df` state a count, and // whichever count it states is wrong for every other workflow. This one diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala index ba0c3f3ae70..b1321a3bc68 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -163,6 +163,47 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { script should include("""fig.write_html("stub_2.html")""") } + /** A source offers the last segment of its resolved path, which two sources + * reading different files can spell the same. Handing both that one name left + * the script reading one file twice and saying nothing about it. + */ + it should "give each source reading a different file a name of its own" in { + val script = translateSources("/alice/sales/v1/data.csv", "/bob/ops/v3/data.csv") + script should include("""pd.read_csv("data.csv")""") + script should include("""pd.read_csv("data-2.csv")""") + } + + /** The other half: one file read twice is still one file, so numbering it would + * send the reader looking for a second copy that was never there. + */ + it should "give two sources reading one file the same name" in { + val script = translateSources("/alice/sales/v1/data.csv", "/alice/sales/v1/data.csv") + script.linesIterator.count(_.contains("""pd.read_csv("data.csv")""")) shouldBe 2 + script should not include "data-2.csv" + } + + /** A file name is the user's text, so it reaches the script through the escaper + * the operators use and through the replacement quoting on top of that: a bare + * `$` in a name is a group reference to `replaceAll`. + */ + it should "escape a source file name that Python or the replacement would read" in { + val script = translateSources("""/alice/v1/we"ird$1\x.csv""") + script should include("""pd.read_csv("we\"ird$1\\x.csv")""") + } + + private def translateSources(paths: String*): String = { + val ops = paths.zipWithIndex.map { + case (path, i) => + val op = + new StubOp(s"out1df = pd.read_csv(${StandaloneCodeGenerator.SourceFilePlaceholder})") { + override def standaloneSourcePath(): Option[String] = Some(path) + } + op.setOperatorId(s"source$i") + op + } + new WorkflowToPythonTranslator().translate(LogicalPlan(ops.toList, List.empty)) + } + /** The translator's own contract when it meets an operator it cannot render: * a comment rather than a silently wrong line. */ From d938d35b469a97d3f34eae5768499b7e740cf1fc Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 15:35:56 -0700 Subject: [PATCH 19/33] feat(workflow-operator): clamp the exported scan window at zero pandas reads a negative bound from the end where the native readers read it as no skip or no rows: iloc[-1:] is the last row, and read_csv rejects a negative nrows outright. The Arrow export and the two CSV exports now take the window from the clamped value, so the script answers what the executor answers for a value only the property editor refuses. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/arrow/ArrowSourceOpDesc.scala | 6 +++++- .../source/scan/csv/CSVScanSourceOpDesc.scala | 7 +++++-- .../scan/csvOld/CSVOldScanSourceOpDesc.scala | 6 ++++-- .../scan/arrow/ArrowSourceOpDescSpec.scala | 12 ++++++++++++ .../scan/csv/CSVScanSourceOpDescSpec.scala | 16 ++++++++++++++++ .../scan/csvOld/CSVOldScanSourceOpDescSpec.scala | 13 +++++++++++++ 6 files changed, 55 insertions(+), 5 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index 262600291e3..cddbc8dac77 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -61,7 +61,11 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // // The executor drops `offset` rows and then takes `limit` of them. Feather has // no row-range read, so the same window is taken once the frame is in memory. - val window = (offset, limit) match { + // + // Clamped first: the property editor refuses a negative, but a plan posted to + // the API can still carry one, and `iloc` reads it from the end where `drop` + // skips nothing and `take` keeps nothing. + val window = (offset.map(_.max(0)), limit.map(_.max(0))) match { case (Some(o), Some(l)) => Some(s"$o:${o + l}") case (Some(o), None) => Some(s"$o:") case (None, Some(l)) => Some(s":$l") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index 7089b5ac922..d1aa51df4d5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -210,12 +210,15 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator ) if (longColumns.nonEmpty) args += s"dtype={${longColumns.mkString(", ")}}" - offset.foreach { o => + // Clamped: the property editor refuses a negative, but a plan posted to the API + // can still carry one, and pandas rejects a negative `nrows` outright where the + // executor's `take` simply keeps no rows. + offset.map(_.max(0)).foreach { o => // With a header, skip offset rows after row 0; without, skip offset rows from the start. if (hasHeader) args += s"skiprows=range(1, ${o + 1})" else args += s"skiprows=$o" } - limit.foreach(l => args += s"nrows=$l") + limit.map(_.max(0)).foreach(l => args += s"nrows=$l") val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index 079f487085d..5f253004768 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -129,11 +129,13 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat ) if (longColumns.nonEmpty) args += s"dtype={${longColumns.mkString(", ")}}" - offset.foreach { o => + // Clamped, as in the newer CSV scan: pandas rejects a negative `nrows` where + // the executor's `take` keeps no rows, and only the editor refuses one. + offset.map(_.max(0)).foreach { o => if (hasHeader) args += s"skiprows=range(1, ${o + 1})" else args += s"skiprows=$o" } - limit.foreach(l => args += s"nrows=$l") + limit.map(_.max(0)).foreach(l => args += s"nrows=$l") val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 4a8fb81659a..7f6515e16e3 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -214,6 +214,18 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // Only the property editor refuses a negative window; a plan posted to the API + // arrives with one intact. `iloc` counts a negative bound from the end, so -1 + // asked for the last row where the executor's drop skips none, and for all but + // the last where its take keeps none. + it should "take a negative window from the front, as the executor reads it" in { + val d = new ArrowSourceOpDesc + d.offset = Some(-1) + d.limit = Some(-1) + + d.generateStandaloneCode() should include("out1df.iloc[0:0]") + } + it should "throw a friendly error when the file is not a valid Arrow file" in { val bogus = File.createTempFile("not-arrow-", ".arrow") bogus.deleteOnExit() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index 108f5127a49..3718cd1256d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -279,6 +279,22 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(!code.contains("io.BytesIO")) } + // Only the property editor refuses a negative window; a plan posted to the API + // arrives with one intact. pandas rejects a negative nrows outright, where the + // executor's take just keeps no rows, so the export asks for the empty window. + it should "ask pandas for the empty window a negative limit means to the executor" in { + csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + csvScanSourceOpDesc.limit = Some(-1) + csvScanSourceOpDesc.offset = Some(-1) + + val code = csvScanSourceOpDesc.generateStandaloneCode() + + assert(code.contains("nrows=0")) + assert(code.contains("skiprows=range(1, 1)")) + } + // The parser sets no null value, so only an empty field is null. pandas reads a list of // words as missing by default, which turned the country code NA into a null. it should "read only an empty field as null, the way the parser does" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala index f261531198f..e18e310f597 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala @@ -136,6 +136,19 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { ) } + // Only the property editor refuses a negative window; a plan posted to the API + // arrives with one intact. pandas rejects a negative nrows outright, where this + // reader's take just keeps no rows, so the export asks for the empty window. + it should "ask pandas for the empty window a negative limit means to the reader" in { + val d = describing(writeCsv("id\n1\n2\n3\n")) + d.limit = Some(-1) + d.offset = Some(-1) + + val code = d.generateStandaloneCode() + code should include("nrows=0") + code should include("skiprows=range(1, 1)") + } + private def writeCsv(content: String): String = { val file = Files.createTempFile("csv-old-", ".csv") file.toFile.deleteOnExit() From a2290cdfbebcf3af22b6d04c49f2a6b195a17c63 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 16:20:52 -0700 Subject: [PATCH 20/33] fix(workflow-operator): give a flattened array the executor's column names The executor gives every element of a nested array a column of its own, named for its position counted from one, so {"items":[{"id":1},{"id":2}]} is items1.id and items2.id and the schema declares those. json_normalize opens a nested object and leaves an array whole, so the export handed the plan one items column holding a list and a step reading items1.id found no such column. The script now flattens each record the way JSONToMap flattens it and leaves json_normalize records that are flat by then. The exact re-read a long needs looks its column up in that flattened record, which also settles a long nested inside an array: the name items1.id is a key there, where splitting it on the dot looked for an items1 no record holds. Co-Authored-By: Claude Opus 5 (1M context) --- .../scan/json/JSONLScanSourceOpDesc.scala | 59 +++++---- .../scan/json/JSONLScanSourceOpDescSpec.scala | 114 ++++++++++++++++++ 2 files changed, 152 insertions(+), 21 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index 48a076c75b8..35b893da690 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -98,8 +98,14 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato readArgs += s"convert_dates=[${dateColumns.mkString(", ")}]" val readExpr = s"pd.read_json(${readArgs.mkString(", ")})" + // json_normalize opens a nested object and leaves a nested array whole, so + // an array arrived as one column holding a list where the executor had + // already given each element a column of its own. The flattening the + // executor does is done here instead, and json_normalize is left the frame + // to build out of records that are flat by then. val baseExpr = - if (flatten) s"pd.json_normalize($readExpr.to_dict('records'))" + if (flatten) + s"pd.json_normalize([_texera_json_flatten(_r) for _r in $readExpr.to_dict('records')])" else readExpr val lines = scala.collection.mutable.ArrayBuffer[String]() @@ -112,15 +118,13 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato lines += s"out1df = $baseExpr" if (longColumns.nonEmpty) { - lines += s"_records = [json.loads(_l) for _l in $taken]" + // Flattened or not, a column's name is a key of the record the lookup + // below reads, because a flattened record is flattened first. + val parsed = if (flatten) "_texera_json_flatten(json.loads(_l))" else "json.loads(_l)" + lines += s"_records = [$parsed for _l in $taken]" longColumns.foreach { name => val nameLit = pyStringLiteral(name) - // Flattening joins a nested key to its parent with a dot, so the schema's - // name is a path into the record rather than a key of it. Without - // flattening it is a key, and a key is free to hold a dot of its own. - val valueExpr = - if (flatten) s"_texera_json_value(_r, $nameLit)" else s"_r.get($nameLit)" - lines += s"""out1df[$nameLit] = pd.array([$valueExpr for _r in _records], dtype="Int64")""" + lines += s"""out1df[$nameLit] = pd.array([_r.get($nameLit) for _r in _records], dtype="Int64")""" } } @@ -128,7 +132,7 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato } override def standaloneHelpers(): Seq[String] = - if (flatten && longColumnCount > 0) Seq(JSONLScanSourceOpDesc.JsonValueAtPath) else Seq.empty + if (flatten) Seq(JSONLScanSourceOpDesc.JsonFlatten) else Seq.empty override def standaloneImports(): Seq[String] = { val windowed = offset.exists(_ > 0) || limit.isDefined @@ -228,18 +232,31 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato object JSONLScanSourceOpDesc { /** - * A flattened column's name read back out of the record it came from. + * One record flattened the way `JSONUtils.JSONToMap` flattens it, so the + * names the two paths give a nested value are the same names. * - * Only the columns an exact re-read has to rebuild need this, and only when - * flattening is on: the name is then a dotted path rather than a key. A path - * the record does not hold reads as nothing, which is what the flattened - * frame carries there. + * A nested object joins its key to its parent's with a dot, and an array + * element takes its parent's name followed by its position counted from one: + * `{"items": [{"id": 1}, {"id": 2}]}` is `items1.id` and `items2.id` on both + * sides. A value nested no deeper keeps its own key. */ - val JsonValueAtPath: String = - """def _texera_json_value(record, path): - | for part in path.split("."): - | if not isinstance(record, dict) or part not in record: - | return None - | record = record[part] - | return record""".stripMargin + val JsonFlatten: String = + """def _texera_json_flatten(record): + | flat = {} + | stack = [(record, "")] + | while stack: + | node, parent = stack.pop() + | if isinstance(node, dict): + | for key, child in node.items(): + | path = parent + "." + key if parent else key + | if isinstance(child, (dict, list)): + | stack.append((child, path)) + | else: + | flat[path] = child + | elif isinstance(node, list): + | for index, child in enumerate(node): + | stack.append((child, parent + str(index + 1))) + | elif parent: + | flat[parent] = node + | return flat""".stripMargin } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index 1cce3efc2ca..71fed300cb7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -209,6 +209,120 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // The executor gives every element of a nested array a column of its own, + // named for its position counted from one, so {"items":[{"id":1},{"id":2}]} + // is items1.id and items2.id. json_normalize opens an object and leaves an + // array whole, so the export handed the plan one items column holding a list + // and a step reading items1.id found no such column. + it should "name a flattened array's columns the way the executor does" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-flatten-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + Files.write( + data, + ("""{"items":[{"id":1},{"id":2}],"tags":["x","y"],"name":"a"}""" + "\n" + + """{"items":[{"id":3},{"id":4}],"tags":["z"],"name":"b"}""" + "\n") + .getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + op.flatten = true + val schema = op.sourceSchema() + schema.getAttributeNames should contain allOf ("items1.id", "items2.id", "tags1", "tags2") + + val exec = new JSONLScanSourceOpExec(objectMapper.writeValueAsString(op)) + exec.open() + val fromEngine = + try exec + .produceTuple() + .map(_.asInstanceOf[SchemaEnforceable].enforceSchema(schema).getField[Any]("items2.id")) + .toList + finally exec.close() + fromEngine shouldBe List(2, 4) + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}" + |${op.generateStandaloneCode()} + |print(sorted(out1df.columns)) + |print(list(out1df["items2.id"])) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + // The columns the schema declares, and no items column standing for them. + out should include(s"${schema.getAttributeNames.sorted.mkString("['", "', '", "']")}") + out.trim should endWith("[2, 4]") + } + } + + // The exact re-read a long needs looks its column up by name, and under + // flattening that name belongs to the flattened record rather than to the one + // the file holds. + it should "keep a nullable long inside a flattened array exact" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-flatten-long-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + Files.write( + data, + ("""{"items":[{"id":9007199254740993}]}""" + "\n" + + """{"items":[{}]}""" + "\n" + + """{"items":[{"id":9007199254740995}]}""" + "\n").getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + op.flatten = true + op.sourceSchema().getAttribute("items1.id").getType shouldBe AttributeType.LONG + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}" + |${op.generateStandaloneCode()} + |print([None if pd.isna(v) else int(v) for v in out1df["items1.id"]]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + out.trim should endWith("[9007199254740993, None, 9007199254740995]") + } + } + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path // (UDF_PYTHON_PATH), then python3 / python / py. private def resolvePython(): Option[String] = { From 2ef4c1591e485c0027b4174830d79d7d253ab695 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 17:05:20 -0700 Subject: [PATCH 21/33] fix(operator): order the exported JSONL read the way the schema does A JSONL file states no column order, so the operator sorts the names it found and the rows the workflow sees follow that. read_json keeps the order the first record happened to use, so the export handed the plan the same columns in another order, which is what a positional read downstream and a file export both go by. Co-Authored-By: Claude Opus 5 (1M context) --- .../scan/json/JSONLScanSourceOpDesc.scala | 7 +++ .../scan/json/JSONLScanSourceOpDescSpec.scala | 49 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index 35b893da690..7fde7ea1ed1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -128,6 +128,13 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato } } + // A JSONL file states no column order, so the schema this operator infers + // sorts the names it found and the rows the workflow sees follow that + // order. read_json keeps the order the first record happened to use, which + // is the same columns in a different order, and column order is what a + // positional read downstream and a file export both go by. + lines += "out1df = out1df[sorted(out1df.columns)]" + lines.mkString("\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index 71fed300cb7..6eb00c642d1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -274,6 +274,55 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // A JSONL file states no column order, so the operator sorts the names it + // found and the rows the workflow sees follow that. read_json keeps the order + // the first record used, which left the export writing the same columns in + // another order than the run did. + it should "order its columns the way the schema it infers does" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-order-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + Files.write( + data, + ("""{"zeta":1,"alpha":"a","mid":2.5}""" + "\n" + + """{"zeta":2,"alpha":"b","mid":3.5}""" + "\n").getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + val declared = op.sourceSchema().getAttributeNames + declared shouldBe List("alpha", "mid", "zeta") + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}" + |${op.generateStandaloneCode()} + |print(list(out1df.columns)) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + out.trim should endWith(declared.mkString("['", "', '", "']")) + } + } + // The exact re-read a long needs looks its column up by name, and under // flattening that name belongs to the flattened record rather than to the one // the file holds. From 46e62a840a0ddfd69b64d79d02e6b022eb5dec19 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 22:28:11 -0700 Subject: [PATCH 22/33] fix(operator): read an Arrow file's narrow numeric widths on both paths A single-precision or 16-bit column arrived null. `parseField` had no case for the Float and Short those vectors hand back, so the parse threw and the Arrow source's own catch wrote null for the whole column. Texera writes every double it owns as eight bytes and every integer as 32 or 64 bits, which is why nothing had reached the gap before; a file written elsewhere from a numpy float32 or int16 array does. The exported script kept the file's width where the executor reads the Texera column's, so 16777216 and 1 summed to 16777216 against the 16777217 doubles give. It now normalizes the two widths, as the Parquet source already does. Co-Authored-By: Claude Opus 5 (1M context) --- .../amber/core/tuple/AttributeTypeUtils.scala | 21 +++- .../source/scan/arrow/ArrowSourceOpDesc.scala | 15 ++- .../scan/arrow/ArrowSourceOpDescSpec.scala | 109 +++++++++++++++++- 3 files changed, 137 insertions(+), 8 deletions(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala index 17c3dfef33c..21f9e6faab9 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala @@ -144,8 +144,12 @@ object AttributeTypeUtils extends Serializable { } else { str.trim.toInt } - case int: Integer => int - case long: java.lang.Long => long.toInt + case int: Integer => int + case long: java.lang.Long => long.toInt + // An Arrow file states the width of its own integers, and a 16-bit + // column hands its values over as Shorts. Without this the parse threw, + // the Arrow source caught it, and every value in the column arrived null. + case short: java.lang.Short => short.toInt case double: java.lang.Double => double.toInt case boolean: java.lang.Boolean => if (boolean) 1 else 0 // Timestamp and Binary are considered to be illegal here. @@ -232,10 +236,15 @@ object AttributeTypeUtils extends Serializable { def parseDouble(fieldValue: Any): java.lang.Double = { val attempt: Try[Double] = Try { fieldValue match { - case str: String => str.trim.toDouble - case int: Integer => int.toDouble - case long: java.lang.Long => long.toDouble - case double: java.lang.Double => double + case str: String => str.trim.toDouble + case int: Integer => int.toDouble + case long: java.lang.Long => long.toDouble + case double: java.lang.Double => double + // A single-precision column hands its values over as Floats, which only + // an Arrow file produces: Texera writes every double it owns as eight + // bytes. Without this the parse threw, the Arrow source caught it, and + // the whole column arrived null. + case float: java.lang.Float => float.toDouble case boolean: java.lang.Boolean => if (boolean) 1 else 0 // Timestamp and Binary are considered to be illegal here. case _ => diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index cddbc8dac77..ed31ee0d723 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -53,6 +53,19 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // is not rounded on its way through a float. val read = s"""out1df = pd.read_feather($SourceFilePlaceholder, dtype_backend="numpy_nullable")""" + // The widths pandas keeps and Texera has no column for. A file states the + // width of each of its numbers, and pandas reads every one of them back, + // where a Texera column is a double or a 32-bit integer and nothing + // narrower. Left alone, a single-precision column summed to a different + // number on the two sides: 16777216 and 1 add to 16777217 as doubles and to + // 16777216 as floats. See ParquetScanSourceOpDesc, which normalizes the + // same widths for the same reason. + val widths = + """|for _column, _values in out1df.items(): + | if _values.dtype == "Float32": + | out1df[_column] = _values.astype("Float64") + | elif _values.dtype == "Int16": + | out1df[_column] = _values.astype("Int32")""".stripMargin // A timestamp column needs nothing here. The file names UTC and holds the // wall clock as UTC, so pd.read_feather and the executor read the same // reading off it — no zone of the reader's own enters either side. The other @@ -72,7 +85,7 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { case _ => None } - (read +: window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)").toSeq) + (Seq(read, widths) ++ window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)")) .mkString("\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 7f6515e16e3..11c0fdb143a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -21,8 +21,10 @@ package org.apache.texera.amber.operator.source.scan.arrow import com.typesafe.config.ConfigFactory import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.VectorSchemaRoot +import org.apache.arrow.vector.{Float4Vector, SmallIntVector, VectorSchemaRoot} import org.apache.arrow.vector.ipc.ArrowFileWriter +import org.apache.arrow.vector.types.FloatingPointPrecision +import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema => ArrowFileSchema} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -40,6 +42,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source +import scala.jdk.CollectionConverters._ import scala.util.Try class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { @@ -73,6 +76,51 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { file } + /** A file holding the narrow numeric widths [[ArrowUtils.fromTexeraSchema]] + * cannot name. + * + * It writes every float as a double and every integer as 32 or 64 bits, so a + * single-precision or 16-bit column can only be built from the Arrow schema + * itself. Real files carry them: anything pyarrow writes from a numpy + * `float32` or `int16` array does. + */ + private def writeNarrowArrowFile(): File = { + val fields = List( + new Field( + "f", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null + ), + new Field("s", FieldType.nullable(new ArrowType.Int(16, true)), null) + ) + val file = File.createTempFile("arrow-narrow-", ".arrow") + file.deleteOnExit() + val allocator = new RootAllocator() + val root = VectorSchemaRoot.create(new ArrowFileSchema(fields.asJava), allocator) + val out = new FileOutputStream(file) + val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) + try { + writer.start() + root.allocateNew() + val f = root.getVector("f").asInstanceOf[Float4Vector] + val s = root.getVector("s").asInstanceOf[SmallIntVector] + // 2^24 and 1: the pair a single-precision sum cannot tell from 2^24 alone. + f.setSafe(0, 16777216.0f) + f.setSafe(1, 1.0f) + s.setSafe(0, 7.toShort) + s.setSafe(1, 8.toShort) + root.setRowCount(2) + writer.writeBatch() + writer.end() + } finally { + writer.close() + root.close() + allocator.close() + out.close() + } + file + } + "ArrowSourceOpDesc.operatorInfo" should "advertise the Arrow file-scan name in the Data Input group with no input and one output" in { val info = (new ArrowSourceOpDesc).operatorInfo @@ -285,6 +333,65 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // A file states the width of each of its numbers, and Texera has no column + // narrower than a double or a 32-bit integer. Both sides have to land on the + // Texera width: the executor was reading a whole single-precision column as + // null, and the script was keeping a width whose arithmetic gives another + // answer. + it should "read the narrow numeric widths as the Texera column, on both paths" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val file = writeNarrowArrowFile() + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + + d.inferSchema().getAttributes.map(a => (a.getName, a.getType)) shouldBe List( + ("f", AttributeType.DOUBLE), + ("s", AttributeType.INTEGER) + ) + + val exec = new ArrowSourceOpExec(objectMapper.writeValueAsString(d)) + exec.open() + val fromEngine = + try exec.produceTuple().map(_.getFields.toList).toList + finally exec.close() + fromEngine shouldBe List(List(16777216.0d, 7), List(1.0d, 8)) + + val workDir = Files.createTempDirectory("arrow-widths-") + workDir.toFile.deleteOnExit() + Files.copy(file.toPath, workDir.resolve(file.getName)) + + val script = workDir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${bindSourceFile(d)} + |${d.generateStandaloneCode()} + |print(str(out1df["f"].dtype), str(out1df["s"].dtype)) + |print([float(v) for v in out1df["f"]], [int(v) for v in out1df["s"]]) + |# The pair the single-precision column could not add: kept as Float32 + |# this printed 16777216.0, where the executor's doubles give 16777217.0. + |print(float(out1df["f"].sum())) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(workDir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "Float64 Int32" + lines(1) shouldBe "[16777216.0, 1.0] [7, 8]" + lines(2) shouldBe "16777217.0" + } + } + /** The block names its file by placeholder; the translator puts a name there. */ private def bindSourceFile(d: ArrowSourceOpDesc): String = s"""${StandaloneCodeGenerator.SourceFilePlaceholder} = "${d.standaloneSourceName().get}"""" From b3108555851363d195dc5f75323cc73e8b485cc2 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 22:44:39 -0700 Subject: [PATCH 23/33] fix(operator): read an Arrow file's unsigned integers as the numbers they count to Arrow's unsigned vectors hand back the raw storage, signed: a column counting to 4294967295 arrived as -1, and an unsigned 16-bit one arrived null, its Character being no kind of integer to the parse. The mapping read only the bit width and never the sign, so there was no width above to read the value into. Each unsigned column now takes the Texera type its own values need, as the Parquet source's mapping already does, and an unsigned 64-bit one is refused rather than read as the -1 it is stored as. The exported script lands on the same types. The two upstream specs pinning the old mapping move with it, and the one refusing an 8-bit width now asserts the INTEGER it reads as. Co-Authored-By: Claude Opus 5 (1M context) --- .../amber/core/tuple/AttributeTypeUtils.scala | 8 +- .../apache/texera/amber/util/ArrowUtils.scala | 58 +++++++-- .../texera/amber/util/ArrowUtilsSpec.scala | 24 +++- .../source/scan/arrow/ArrowSourceOpDesc.scala | 20 +-- .../scan/arrow/ArrowSourceOpDescSpec.scala | 114 ++++++++++++++---- .../texera/amber/util/ArrowUtilsSpec.scala | 7 +- 6 files changed, 180 insertions(+), 51 deletions(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala index 21f9e6faab9..66a955f49ad 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/AttributeTypeUtils.scala @@ -146,10 +146,12 @@ object AttributeTypeUtils extends Serializable { } case int: Integer => int case long: java.lang.Long => long.toInt - // An Arrow file states the width of its own integers, and a 16-bit - // column hands its values over as Shorts. Without this the parse threw, - // the Arrow source caught it, and every value in the column arrived null. + // An Arrow file states the width of its own integers, and a narrow + // column hands its values over as Shorts or Bytes. Without this the parse + // threw, the Arrow source caught it, and every value in the column + // arrived null. case short: java.lang.Short => short.toInt + case byte: java.lang.Byte => byte.toInt case double: java.lang.Double => double.toInt case boolean: java.lang.Boolean => if (boolean) 1 else 0 // Timestamp and Binary are considered to be illegal here. diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala index 434519b3dea..7fc2fd64efe 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala @@ -84,11 +84,16 @@ object ArrowUtils extends LazyLogging { // Use the attribute type from the schema (which includes metadata) // instead of deriving it from the Arrow type val attributeType = schema.getAttributes(index).getType - // A timestamp is the one type whose field says more than the - // schema does, so it is the only one that reads the field. + // A timestamp and an unsigned integer are the types whose field + // says more than the schema does, so they are the ones that read + // the field. attributeType match { case AttributeType.TIMESTAMP => wallClockOf(value, fieldVector.getField.getType) - case _ => AttributeTypeUtils.parseField(value, attributeType) + case _ => + AttributeTypeUtils.parseField( + unsignedValueOf(value, fieldVector.getField.getType), + attributeType + ) } } catch { case e: Exception => @@ -101,6 +106,29 @@ object ArrowUtils extends LazyLogging { .build() } + /** The number an unsigned column counts to, out of the storage it counts in. + * + * Arrow's unsigned vectors hand back the raw storage, signed: `UInt1Vector` a + * Byte, `UInt4Vector` an Int, each reading the file's largest value as -1. + * `UInt2Vector` is the exception and hands back a Character, which is already + * the number but is no kind of integer to the parse below. Every other value + * passes through untouched. + * + * Nothing wider is handled because nothing wider arrives: [[toAttributeType]] + * refuses an unsigned 64-bit column, having no Texera type to hold it. + */ + private def unsignedValueOf(value: AnyRef, arrowType: ArrowType): AnyRef = + arrowType match { + case int: ArrowType.Int if !int.getIsSigned => + value match { + case byte: java.lang.Byte => Int.box(byte & 0xff) + case char: java.lang.Character => Int.box(char.toInt) + case integer: Integer => Long.box(integer.toLong & 0xffffffffL) + case other => other + } + case _ => value + } + /** The wall clock a timestamp column holds, read the way its own field states. * * A Texera TIMESTAMP carries no zone, so the wall clock is the whole of what @@ -182,15 +210,23 @@ object ArrowUtils extends LazyLogging { @throws[AttributeTypeException] def toAttributeType(srcType: ArrowType): AttributeType = { srcType match { + // An unsigned column counts up where its storage counts down: the largest + // unsigned 32-bit value is stored as -1, so read as its storage it would + // arrive as -1 where the file means 4294967295. The next Texera integer up + // holds it, and [[unsignedValueOf]] does the reading. Past 64 bits there is + // no next one. The same widening covers the narrow widths, Texera having no + // column shorter than a 32-bit integer. case int: ArrowType.Int => - int.getBitWidth match { - case 16 | 32 => - AttributeType.INTEGER - - case 64 => - AttributeType.LONG - - case other => + (int.getBitWidth, int.getIsSigned) match { + case (8 | 16 | 32, true) => AttributeType.INTEGER + case (8 | 16, false) => AttributeType.INTEGER + case (64, true) => AttributeType.LONG + case (32, false) => AttributeType.LONG + case (64, false) => + throw new AttributeTypeUtils.AttributeTypeException( + "Unsupported unsigned 64-bit Int, which is wider than any Texera column" + ) + case (other, _) => throw new AttributeTypeUtils.AttributeTypeException( s"Unsupported Int bit width: $other" ) diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala index e3f8014462d..ead7813744b 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala @@ -50,15 +50,27 @@ class ArrowUtilsSpec extends AnyFlatSpec with Matchers { ArrowUtils.toAttributeType(new ArrowType.Int(64, true)) shouldBe AttributeType.LONG } - it should "throw AttributeTypeException for non-standard Int bit-widths" in { - // Only 16/32 (INTEGER) and 64 (LONG) are supported. Other widths used to - // be silently coerced to LONG by a `case 64 | _` catch-all; they now - // raise rather than masquerade as Int64. + it should "map every integer to the narrowest Texera type that holds it" in { + // Texera has no column shorter than a 32-bit integer, so a narrower width + // reads as one. An unsigned column needs the width above its own, counting + // up where its storage counts down: the largest unsigned 32-bit value is + // past what an INTEGER holds. + ArrowUtils.toAttributeType(new ArrowType.Int(8, true)) shouldBe AttributeType.INTEGER + ArrowUtils.toAttributeType(new ArrowType.Int(8, false)) shouldBe AttributeType.INTEGER + ArrowUtils.toAttributeType(new ArrowType.Int(16, false)) shouldBe AttributeType.INTEGER + ArrowUtils.toAttributeType(new ArrowType.Int(32, false)) shouldBe AttributeType.LONG + } + + it should "throw AttributeTypeException for an integer no Texera type holds" in { + // A width above 64 used to be silently coerced to LONG by a `case 64 | _` + // catch-all; it raises rather than masquerade as Int64. An unsigned 64-bit + // column raises for the same reason, there being no width above it to read + // it as. assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.Int(8, true)) + ArrowUtils.toAttributeType(new ArrowType.Int(128, true)) } assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.Int(128, true)) + ArrowUtils.toAttributeType(new ArrowType.Int(64, false)) } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index ed31ee0d723..31e417a1f25 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -54,18 +54,22 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { val read = s"""out1df = pd.read_feather($SourceFilePlaceholder, dtype_backend="numpy_nullable")""" // The widths pandas keeps and Texera has no column for. A file states the - // width of each of its numbers, and pandas reads every one of them back, - // where a Texera column is a double or a 32-bit integer and nothing - // narrower. Left alone, a single-precision column summed to a different - // number on the two sides: 16777216 and 1 add to 16777217 as doubles and to - // 16777216 as floats. See ParquetScanSourceOpDesc, which normalizes the - // same widths for the same reason. + // width and the sign of each of its numbers, and pandas reads every one of + // them back, where a Texera column is a double or a 32-bit integer and + // nothing narrower. Left alone, a single-precision column summed to a + // different number on the two sides: 16777216 and 1 add to 16777217 as + // doubles and to 16777216 as floats. An unsigned column lands on the Texera + // type its own values need, which is the executor's rule too: one counting + // to 4294967295 has to be a long. See ParquetScanSourceOpDesc, which + // normalizes the same widths for the same reason. val widths = """|for _column, _values in out1df.items(): | if _values.dtype == "Float32": | out1df[_column] = _values.astype("Float64") - | elif _values.dtype == "Int16": - | out1df[_column] = _values.astype("Int32")""".stripMargin + | elif _values.dtype in ("Int8", "Int16", "UInt8", "UInt16"): + | out1df[_column] = _values.astype("Int32") + | elif _values.dtype == "UInt32": + | out1df[_column] = _values.astype("Int64")""".stripMargin // A timestamp column needs nothing here. The file names UTC and holds the // wall clock as UTC, so pd.read_feather and the executor read the same // reading off it — no zone of the reader's own enters either side. The other diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 11c0fdb143a..f1c531b8a48 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -21,7 +21,16 @@ package org.apache.texera.amber.operator.source.scan.arrow import com.typesafe.config.ConfigFactory import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.{Float4Vector, SmallIntVector, VectorSchemaRoot} +import org.apache.arrow.vector.{ + Float4Vector, + SmallIntVector, + TinyIntVector, + UInt1Vector, + UInt2Vector, + UInt4Vector, + UInt8Vector, + VectorSchemaRoot +} import org.apache.arrow.vector.ipc.ArrowFileWriter import org.apache.arrow.vector.types.FloatingPointPrecision import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema => ArrowFileSchema} @@ -76,13 +85,16 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { file } - /** A file holding the narrow numeric widths [[ArrowUtils.fromTexeraSchema]] - * cannot name. + /** A file holding the numeric widths [[ArrowUtils.fromTexeraSchema]] cannot + * name. + * + * It writes every float as a double and every integer as a signed 32 or 64 + * bits, so a single-precision, narrow or unsigned column can only be built + * from the Arrow schema itself. Real files carry them: anything pyarrow writes + * from a numpy `float32`, `int16` or `uint32` array does. * - * It writes every float as a double and every integer as 32 or 64 bits, so a - * single-precision or 16-bit column can only be built from the Arrow schema - * itself. Real files carry them: anything pyarrow writes from a numpy - * `float32` or `int16` array does. + * Each column holds the value its own width cannot be read out of by + * accident, and one the reading has to leave alone. */ private def writeNarrowArrowFile(): File = { val fields = List( @@ -91,7 +103,11 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), null ), - new Field("s", FieldType.nullable(new ArrowType.Int(16, true)), null) + new Field("s", FieldType.nullable(new ArrowType.Int(16, true)), null), + new Field("b", FieldType.nullable(new ArrowType.Int(8, true)), null), + new Field("u8", FieldType.nullable(new ArrowType.Int(8, false)), null), + new Field("u16", FieldType.nullable(new ArrowType.Int(16, false)), null), + new Field("u32", FieldType.nullable(new ArrowType.Int(32, false)), null) ) val file = File.createTempFile("arrow-narrow-", ".arrow") file.deleteOnExit() @@ -104,11 +120,26 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { root.allocateNew() val f = root.getVector("f").asInstanceOf[Float4Vector] val s = root.getVector("s").asInstanceOf[SmallIntVector] + val b = root.getVector("b").asInstanceOf[TinyIntVector] + val u8 = root.getVector("u8").asInstanceOf[UInt1Vector] + val u16 = root.getVector("u16").asInstanceOf[UInt2Vector] + val u32 = root.getVector("u32").asInstanceOf[UInt4Vector] // 2^24 and 1: the pair a single-precision sum cannot tell from 2^24 alone. f.setSafe(0, 16777216.0f) f.setSafe(1, 1.0f) s.setSafe(0, 7.toShort) s.setSafe(1, 8.toShort) + // A signed 8-bit column, whose negative must survive the unsigned reading. + b.setSafe(0, (-3).toByte) + b.setSafe(1, 4.toByte) + // The largest value each unsigned width counts to, every one of them + // stored as -1, beside a value the two readings agree on. + u8.setSafe(0, 0xff) + u8.setSafe(1, 1) + u16.setSafe(0, 0xffff) + u16.setSafe(1, 1) + u32.setSafe(0, -1) + u32.setSafe(1, 1) root.setRowCount(2) writer.writeBatch() writer.end() @@ -333,12 +364,12 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { } } - // A file states the width of each of its numbers, and Texera has no column - // narrower than a double or a 32-bit integer. Both sides have to land on the - // Texera width: the executor was reading a whole single-precision column as - // null, and the script was keeping a width whose arithmetic gives another - // answer. - it should "read the narrow numeric widths as the Texera column, on both paths" in { + // A file states the width and the sign of each of its numbers, and Texera has + // no column narrower than a double or a 32-bit integer. Both sides have to land + // on the Texera one: the executor was reading a whole single-precision, narrow + // or unsigned column as null, or as the -1 its storage counts down to, and the + // script was keeping a width whose arithmetic gives another answer. + it should "read the numeric widths and signs as the Texera column, on both paths" in { val python = resolvePython().getOrElse(cancel("No runnable python executable")) if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") @@ -346,9 +377,15 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val d = new ArrowSourceOpDesc d.fileName = Some(file.toURI.toString) + // An unsigned 32-bit column is a LONG: its largest value is past what an + // INTEGER holds. Every narrower width fits in one. d.inferSchema().getAttributes.map(a => (a.getName, a.getType)) shouldBe List( ("f", AttributeType.DOUBLE), - ("s", AttributeType.INTEGER) + ("s", AttributeType.INTEGER), + ("b", AttributeType.INTEGER), + ("u8", AttributeType.INTEGER), + ("u16", AttributeType.INTEGER), + ("u32", AttributeType.LONG) ) val exec = new ArrowSourceOpExec(objectMapper.writeValueAsString(d)) @@ -356,7 +393,10 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val fromEngine = try exec.produceTuple().map(_.getFields.toList).toList finally exec.close() - fromEngine shouldBe List(List(16777216.0d, 7), List(1.0d, 8)) + fromEngine shouldBe List( + List(16777216.0d, 7, -3, 255, 65535, 4294967295L), + List(1.0d, 8, 4, 1, 1, 1L) + ) val workDir = Files.createTempDirectory("arrow-widths-") workDir.toFile.deleteOnExit() @@ -368,8 +408,9 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { s"""import pandas as pd |${bindSourceFile(d)} |${d.generateStandaloneCode()} - |print(str(out1df["f"].dtype), str(out1df["s"].dtype)) - |print([float(v) for v in out1df["f"]], [int(v) for v in out1df["s"]]) + |print(" ".join(str(out1df[c].dtype) for c in out1df.columns)) + |print([float(v) for v in out1df["f"]]) + |print([[int(v) for v in out1df[c]] for c in ("s", "b", "u8", "u16", "u32")]) |# The pair the single-precision column could not add: kept as Float32 |# this printed 16777216.0, where the executor's doubles give 16777217.0. |print(float(out1df["f"].sum())) @@ -386,12 +427,43 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { withClue(s"python said:\n$out") { process.exitValue() shouldBe 0 val lines = out.trim.linesIterator.toSeq - lines.head shouldBe "Float64 Int32" - lines(1) shouldBe "[16777216.0, 1.0] [7, 8]" - lines(2) shouldBe "16777217.0" + lines.head shouldBe "Float64 Int32 Int32 Int32 Int32 Int64" + lines(1) shouldBe "[16777216.0, 1.0]" + lines(2) shouldBe "[[7, 8], [-3, 4], [255, 1], [65535, 1], [4294967295, 1]]" + lines(3) shouldBe "16777217.0" } } + // Nothing in Texera holds a value past 2^63 - 1, so the column is refused at + // the schema rather than read as the -1 its storage counts down to. + it should "refuse an unsigned 64-bit column, having no Texera type for it" in { + val fields = List(new Field("u", FieldType.nullable(new ArrowType.Int(64, false)), null)) + val file = File.createTempFile("arrow-u64-", ".arrow") + file.deleteOnExit() + val allocator = new RootAllocator() + val root = VectorSchemaRoot.create(new ArrowFileSchema(fields.asJava), allocator) + val out = new FileOutputStream(file) + val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) + try { + writer.start() + root.allocateNew() + root.getVector("u").asInstanceOf[UInt8Vector].setSafe(0, -1L) + root.setRowCount(1) + writer.writeBatch() + writer.end() + } finally { + writer.close() + root.close() + allocator.close() + out.close() + } + + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + val ex = intercept[RuntimeException](d.inferSchema()) + ex.getMessage shouldBe "Failed to read the .arrow file. Please ensure it is a valid Arrow file." + } + /** The block names its file by placeholder; the translator puts a name there. */ private def bindSourceFile(d: ArrowSourceOpDesc): String = s"""${StandaloneCodeGenerator.SourceFilePlaceholder} = "${d.standaloneSourceName().get}"""" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala index 94abd91b8ae..94721397ba8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala @@ -70,10 +70,13 @@ class ArrowUtilsSpec extends AnyFlatSpec { it should "convert to AttributeTypes correctly" in { assert(ArrowUtils.toAttributeType(unsignedShortInt) == AttributeType.INTEGER) assert(ArrowUtils.toAttributeType(signedShortInt) == AttributeType.INTEGER) - assert(ArrowUtils.toAttributeType(unsignedInt) == AttributeType.INTEGER) assert(ArrowUtils.toAttributeType(signedInt) == AttributeType.INTEGER) - assert(ArrowUtils.toAttributeType(unsignedLongInt) == AttributeType.LONG) assert(ArrowUtils.toAttributeType(signedLongInt) == AttributeType.LONG) + // An unsigned column needs the width above its own: read at its own width it + // would hand back the storage, and the largest unsigned 32-bit value is + // stored as -1. Past 64 bits there is no width above to read it as. + assert(ArrowUtils.toAttributeType(unsignedInt) == AttributeType.LONG) + assertThrows[AttributeTypeException](ArrowUtils.toAttributeType(unsignedLongInt)) assert(ArrowUtils.toAttributeType(boolean) == AttributeType.BOOLEAN) From 3700e19ca1dd0630ce405f5fdc54d5ba3e7895a9 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 22:54:23 -0700 Subject: [PATCH 24/33] fix(workflow-operator): count the end of an exported scan window in Long Both bounds are Ints the operator accepts, and their sum is not one. The Arrow window ended at a negative, which `iloc` reads from the end, so it asked for all but the last two rows where the executor takes every row from the offset on. The two CSV readers add 1 to the offset to step past the header, which overflows at the largest offset and leaves an empty range, skipping nothing where the executor's drop keeps no rows. Each addition is now carried in Long, as the Parquet source already does. JSONL and the file scans take two independent slices and add nothing, so they were already right. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/arrow/ArrowSourceOpDesc.scala | 5 ++++- .../source/scan/csv/CSVScanSourceOpDesc.scala | 5 ++++- .../source/scan/csvOld/CSVOldScanSourceOpDesc.scala | 3 ++- .../source/scan/arrow/ArrowSourceOpDescSpec.scala | 12 ++++++++++++ .../source/scan/csv/CSVScanSourceOpDescSpec.scala | 12 ++++++++++++ .../scan/csvOld/CSVOldScanSourceOpDescSpec.scala | 10 ++++++++++ 6 files changed, 44 insertions(+), 3 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index 31e417a1f25..a81405efb67 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -83,7 +83,10 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // the API can still carry one, and `iloc` reads it from the end where `drop` // skips nothing and `take` keeps nothing. val window = (offset.map(_.max(0)), limit.map(_.max(0))) match { - case (Some(o), Some(l)) => Some(s"$o:${o + l}") + // The end of the window is counted in Long: two Ints the operator accepts + // can add up past what an Int holds, and the slice would come out negative + // and take the wrong rows. As in ParquetScanSourceOpDesc. + case (Some(o), Some(l)) => Some(s"$o:${o.toLong + l}") case (Some(o), None) => Some(s"$o:") case (None, Some(l)) => Some(s":$l") case _ => None diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index d1aa51df4d5..cb7afdeba6b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -215,7 +215,10 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator // executor's `take` simply keeps no rows. offset.map(_.max(0)).foreach { o => // With a header, skip offset rows after row 0; without, skip offset rows from the start. - if (hasHeader) args += s"skiprows=range(1, ${o + 1})" + // The end of the range is counted in Long: the largest offset the operator + // accepts overflows an Int on the way past the header, and the range came + // out empty, skipping nothing where the executor's `drop` keeps no rows. + if (hasHeader) args += s"skiprows=range(1, ${o.toLong + 1})" else args += s"skiprows=$o" } limit.map(_.max(0)).foreach(l => args += s"nrows=$l") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index 5f253004768..17598449e5d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -132,7 +132,8 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat // Clamped, as in the newer CSV scan: pandas rejects a negative `nrows` where // the executor's `take` keeps no rows, and only the editor refuses one. offset.map(_.max(0)).foreach { o => - if (hasHeader) args += s"skiprows=range(1, ${o + 1})" + // Counted in Long past the header, as in the newer CSV scan. + if (hasHeader) args += s"skiprows=range(1, ${o.toLong + 1})" else args += s"skiprows=$o" } limit.map(_.max(0)).foreach(l => args += s"nrows=$l") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index f1c531b8a48..42b66e8296d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -305,6 +305,18 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { d.generateStandaloneCode() should include("out1df.iloc[0:0]") } + // Both bounds are Ints the operator accepts, and their sum is not one. Added + // as Ints the window ended at -2, which `iloc` reads from the end: it asked + // for everything but the last two rows where the executor takes every row + // from the offset on. + it should "count the end of the window past what an Int holds" in { + val d = new ArrowSourceOpDesc + d.offset = Some(Int.MaxValue) + d.limit = Some(Int.MaxValue) + + d.generateStandaloneCode() should include("out1df.iloc[2147483647:4294967294]") + } + it should "throw a friendly error when the file is not a valid Arrow file" in { val bogus = File.createTempFile("not-arrow-", ".arrow") bogus.deleteOnExit() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index 3718cd1256d..6bd304cf6b5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -295,6 +295,18 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(code.contains("skiprows=range(1, 1)")) } + // The largest offset the operator accepts is an Int, and the row past the + // header is not. Added as Ints the range ran to a negative and came out empty, + // so pandas skipped nothing where the executor's drop keeps no rows. + it should "count the skipped range past what an Int holds" in { + csvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallMultiLineCsvPath) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + csvScanSourceOpDesc.offset = Some(Int.MaxValue) + + assert(csvScanSourceOpDesc.generateStandaloneCode().contains("skiprows=range(1, 2147483648)")) + } + // The parser sets no null value, so only an empty field is null. pandas reads a list of // words as missing by default, which turned the country code NA into a null. it should "read only an empty field as null, the way the parser does" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala index e18e310f597..4dcbf96ae1e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala @@ -149,6 +149,16 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { code should include("skiprows=range(1, 1)") } + // The largest offset the operator accepts is an Int, and the row past the + // header is not. Added as Ints the range ran to a negative and came out empty, + // so pandas skipped nothing where this reader's take keeps no rows. + it should "count the skipped range past what an Int holds" in { + val d = describing(writeCsv("id\n1\n2\n3\n")) + d.offset = Some(Int.MaxValue) + + d.generateStandaloneCode() should include("skiprows=range(1, 2147483648)") + } + private def writeCsv(content: String): String = { val file = Files.createTempFile("csv-old-", ".csv") file.toFile.deleteOnExit() From d8a9d1384b40fb7cc469bd91042446a599cc3238 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 23:02:01 -0700 Subject: [PATCH 25/33] fix(operator): decode a file scan with the charset its Encoding field names The panel writes `encoding`, but the executor decoded with the inherited `fileEncoding`, which this descriptor names in its own @JsonIgnoreProperties. It never survived the trip, so it was always its default and choosing any other charset changed nothing: a UTF-16 file came back as its bytes read as UTF-8. The executor reads the field the panel writes. The name stays as the panel spells it, a saved workflow carrying `encoding` and not the other. The export already followed the panel, so its note about parting from the engine goes with the defect. The spec that claimed to read US_ASCII was setting the field the executor never saw, and now sets the one it does. Closes #8596 Co-Authored-By: Claude Opus 5 (1M context) --- .../scan/file/FileScanSourceOpDesc.scala | 14 +++--- .../scan/file/FileScanSourceOpExec.scala | 4 +- .../scan/file/FileScanSourceOpDescSpec.scala | 46 +++++++++++++++---- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala index e397d9988b9..62cfbb809ba 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala @@ -55,7 +55,12 @@ class FileScanSourceOpDesc new JsonSchemaString(path = HideAnnotation.hideExpectedValue, value = "binary") ) ) - private val encoding: FileDecodingMethod = FileDecodingMethod.UTF_8 + // The charset the panel offers, and the one the executor decodes with. The + // inherited `fileEncoding` is named in this class's @JsonIgnoreProperties, so + // it never survives the trip into the executor: reading that one there left + // every file decoded as UTF-8 whatever was chosen. The name is kept as the + // panel spells it, a saved workflow carrying `encoding` and not the other. + val encoding: FileDecodingMethod = FileDecodingMethod.UTF_8 @JsonProperty(defaultValue = "false") @JsonSchemaTitle("Extract") @@ -78,11 +83,8 @@ class FileScanSourceOpDesc override def generateStandaloneCode(): String = { val col = attributeName - // `encoding` is the charset the panel offers, which is the one to honour. - // The executor reads the inherited `fileEncoding` instead, and that one is in - // this class's @JsonIgnoreProperties, so it never survives the trip and the - // engine decodes UTF-8 whatever the user chose. Following the executor here - // would mean ignoring the field as well; the export states what was asked for. + // `encoding` is the charset the panel offers, and the one the executor now + // decodes with. val enc = encoding.toString.replace("_", "-").toLowerCase val colLit = pyStringLiteral(col) val encLit = pyStringLiteral(enc) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpExec.scala index d47cf3681c2..5a0677787de 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpExec.scala @@ -36,7 +36,9 @@ class FileScanSourceOpExec private[scan] ( FileScanUtils.createTuplesFromFile( fileName = desc.fileName.get, attributeType = desc.attributeType, - fileEncoding = desc.fileEncoding, + // `encoding`, the field the panel writes, and not the inherited + // `fileEncoding` this descriptor ignores on the way over. + fileEncoding = desc.encoding, extract = desc.extract, outputFileName = desc.outputFileName, fileScanOffset = desc.fileScanOffset, diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala index 3bcc1c65a73..d7f7846d59d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala @@ -33,7 +33,7 @@ import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} +import java.nio.file.{Files, Path, Paths} import java.util.concurrent.TimeUnit import java.util.zip.{ZipEntry, ZipOutputStream} import scala.io.Source @@ -193,11 +193,13 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { FileScanSourceOpExec.close() } + // `encoding` and not the inherited `fileEncoding`: the descriptor drops that + // one on the way over, so setting it never reached the executor at all. it should "read first 5 lines of the input text file with US_ASCII encoding" in { - fileScanSourceOpDesc.setResolvedFileName( - FileResolver.resolve(TestOperators.TestCRLFTextFilePath) + fileScanSourceOpDesc = describing( + Paths.get(TestOperators.TestCRLFTextFilePath), + """"encoding":"US_ASCII"""" ) - fileScanSourceOpDesc.fileEncoding = FileDecodingMethod.ASCII fileScanSourceOpDesc.attributeType = FileAttributeType.STRING fileScanSourceOpDesc.fileScanLimit = Option(5) val FileScanSourceOpExec = @@ -317,6 +319,28 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { ) } + // The Encoding field the panel offers is `encoding`, and the executor was + // decoding with the inherited `fileEncoding` this descriptor drops on the way + // over, so every file came back read as UTF-8 whatever was chosen. A UTF-16 + // file is the one that shows it: read as UTF-8 its text is not its text. + it should "decode with the charset the Encoding field names" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("file-scan-encoding-") + dir.toFile.deleteOnExit() + val file = dir.resolve("utf16.txt") + Files.write(file, "première\ndeuxième\n".getBytes(StandardCharsets.UTF_16)) + + val desc = describing(file, """"encoding":"UTF_16"""") + assert(linesFromEngine(desc) == Seq("première", "deuxième")) + + val out = runStandalone(python, dir, desc, """print(list(out1df["line"]))""")._2 + assert(out.trim.endsWith("""['première', 'deuxième']""")) + } + it should "take an archive entry's own name for the filename column" in { val python = resolvePython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") @@ -358,17 +382,21 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { archive } - /** `extract` and `outputFileName` are vals, so the flags are deserialized in. */ - private def extractingDesc(archive: Path, fields: String*): FileScanSourceOpDesc = { + /** `extract`, `outputFileName` and `encoding` are vals, so the fields are + * deserialized in. + */ + private def describing(file: Path, fields: String*): FileScanSourceOpDesc = { val desc = objectMapper.readValue( - (Seq(""""operatorType":"FileScan"""", """"extract":true""") ++ fields) - .mkString("{", ",", "}"), + (""""operatorType":"FileScan"""" +: fields).mkString("{", ",", "}"), classOf[FileScanSourceOpDesc] ) - desc.setResolvedFileName(FileResolver.resolve(archive.toString)) + desc.setResolvedFileName(FileResolver.resolve(file.toString)) desc } + private def extractingDesc(archive: Path, fields: String*): FileScanSourceOpDesc = + describing(archive, """"extract":true""" +: fields: _*) + private def tuplesFromEngine(desc: FileScanSourceOpDesc): Seq[Tuple] = { val exec = new FileScanSourceOpExec(objectMapper.writeValueAsString(desc)) exec.open() From 95ab40ce2b8b8b9083255cba81832fc71025e0f8 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 23:07:21 -0700 Subject: [PATCH 26/33] fix(operator): name the file each line came from when a file scan was asked to A file scan with Extract and Include Filename on could not run at all in any attribute type that reads line by line, which is the default. The two halves disagreed about how wide a row is: the schema prepends a `filename` column whenever the flag is set, while the line-by-line branch emitted the value alone, so the first tuple was one field against two columns and could not be built. Include Filename is only offered once Extract is on, so every configuration reaching it is one the panel invites. The branch now pairs each entry with its name, as the single-value branch already did. Both exports follow: their `&& isSingle` was there to mirror the defect, and the spec pinning that a line-mode export carries no filename column now pins the column it does carry. Closes #8598 Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/file/FileScanOpDesc.scala | 11 +++--- .../scan/file/FileScanSourceOpDesc.scala | 17 +++++---- .../source/scan/file/FileScanUtils.scala | 35 +++++++++++-------- .../source/scan/file/FileScanOpDescSpec.scala | 15 ++++++-- .../scan/file/FileScanSourceOpDescSpec.scala | 30 ++++++++++++++++ 5 files changed, 80 insertions(+), 28 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala index 0218944490a..e577d79329d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala @@ -141,10 +141,10 @@ class FileScanOpDesc (" " * 8, "_fn", "_f.read()", "_f") } - // Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line - // branch ignores outputFileName and emits only the value, so the filename - // column is added ONLY in single-value mode. - val emitFilename = outputFileName && attributeType.isSingle + // Whatever the flag says, as the platform now reads it: every row carries the + // name of the file its value came from, a line's as much as a whole file's. + // See FileScanUtils.createTuplesFromFile. + val emitFilename = outputFileName if (attributeType.isSingle) { if (emitFilename) buf += s"${indent}_rows.append(($nameExpr, $readWhole))" @@ -172,7 +172,8 @@ class FileScanOpDesc .fold(s"$lines.readlines()")(o => s"$lines.readlines()[$o:]") fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") } - buf += s"${indent}_rows.extend($castExpr for l in $linesExpr)" + val row = if (emitFilename) s"($nameExpr, $castExpr)" else castExpr + buf += s"${indent}_rows.extend($row for l in $linesExpr)" } val colLit = pyStringLiteral(col) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala index 62cfbb809ba..bfa640b2a95 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala @@ -115,10 +115,10 @@ class FileScanSourceOpDesc fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") } - // Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line - // branch emits only the value, so the filename column is added ONLY in - // single-value mode, whatever the flag says. - val emitFilename = outputFileName && attributeType.isSingle + // Whatever the flag says, as the platform now reads it: every row carries + // the name of the file its value came from, a line's as much as a whole + // file's. See FileScanUtils.createTuplesFromFile. + val emitFilename = outputFileName if (extract) { // The engine reads the files INSIDE the archive, one tuple per entry, and @@ -143,7 +143,8 @@ class FileScanSourceOpDesc // TextIOWrapper, not splitlines: it ends a line where `open(..., "r")` // does, which is what the branch below reads with. val linesExpr = windowed(s"io.TextIOWrapper(_f, encoding=$encLit)") - buf += s" _rows.extend($castExpr for l in $linesExpr)" + val row = if (emitFilename) s"(_name, $castExpr)" else castExpr + buf += s" _rows.extend($row for l in $linesExpr)" } if (emitFilename) buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])""" else buf += s"""out1df = pd.DataFrame({$colLit: _rows})""" @@ -158,8 +159,12 @@ class FileScanSourceOpDesc buf += s""" out1df = pd.DataFrame($dfCols)""" } else { val linesExpr = windowed("_f") + val dfCols = + if (emitFilename) + s"""{"filename": $SourceFilePlaceholder, $colLit: [$castExpr for l in $linesExpr]}""" + else s"""{$colLit: [$castExpr for l in $linesExpr]}""" buf += s"""with open($SourceFilePlaceholder, "r", encoding=$encLit) as _f:""" - buf += s""" out1df = pd.DataFrame({$colLit: [$castExpr for l in $linesExpr]})""" + buf += s""" out1df = pd.DataFrame($dfCols)""" } buf.mkString("\n") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanUtils.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanUtils.scala index b6cbdc1b936..b6d44d758c8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanUtils.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanUtils.scala @@ -138,20 +138,27 @@ private[file] object FileScanUtils { TupleLike(fields.toSeq: _*) } } else { - fileEntries.flatMap { entry => - val lines = new BufferedReader(new InputStreamReader(entry, fileEncoding.getCharset)) - .lines() - .iterator() - .asScala - .drop(fileScanOffset.getOrElse(0)) - fileScanLimit - .fold(lines)(lines.take) - .map(line => - TupleLike(attributeType match { - case FileAttributeType.SINGLE_STRING => line - case _ => parseField(line, attributeType.getType) - }) - ) + // Paired with the entry names the same way the single-value branch is: + // a row carries the name of the file its line came from, which is what + // the schema declares when the filename was asked for. Emitting the value + // alone left a one-field row against a two-column schema, and the tuple + // could not be built at all. + fileEntries.zipAll(filenameIt, null, null).flatMap { + case (entry, entryFileName) => + val lines = new BufferedReader(new InputStreamReader(entry, fileEncoding.getCharset)) + .lines() + .iterator() + .asScala + .drop(fileScanOffset.getOrElse(0)) + fileScanLimit + .fold(lines)(lines.take) + .map { line => + val value = attributeType match { + case FileAttributeType.SINGLE_STRING => line + case _ => parseField(line, attributeType.getType) + } + if (outputFileName) TupleLike(entryFileName, value) else TupleLike(value) + } } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala index 67ef0446e56..f1fe01c06bc 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala @@ -191,11 +191,20 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(withoutName.contains(" _rows.append(_f.read())")) assert(withoutName.endsWith("""out1df = pd.DataFrame({"line": _rows})""")) - // The platform's line-by-line branch (FileScanUtils.createTuplesFromFile) - // emits only the value, so line mode must drop the filename column too. + // A line carries the name of the file it came from, as the whole file does, + // which is what the platform's line-by-line branch now emits and what the + // schema declares either way. It used to emit the value alone, leaving a + // one-field row against a two-column schema. fileScanOpDesc.attributeType = FileAttributeType.STRING fileScanOpDesc.outputFileName = true - assert(!fileScanOpDesc.generateStandaloneCode().contains("filename")) + val lines = fileScanOpDesc.generateStandaloneCode() + assert(lines.contains(""" _rows.extend((_fn, l.rstrip("\n")) for l in _f)""")) + assert(lines.endsWith("""out1df = pd.DataFrame(_rows, columns=["filename", "line"])""")) + + fileScanOpDesc.outputFileName = false + val bareLines = fileScanOpDesc.generateStandaloneCode() + assert(bareLines.contains(""" _rows.extend(l.rstrip("\n") for l in _f)""")) + assert(bareLines.endsWith("""out1df = pd.DataFrame({"line": _rows})""")) } it should "open binary attribute types in binary mode" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala index d7f7846d59d..db9f6e377cb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala @@ -369,6 +369,36 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(out.trim.endsWith("""[('a.txt', 'first'), ('b.txt', 'second')]""")) } + // Include Filename is only offered once Extract is on, and reading the entries + // line by line is the default there, so this is the configuration the panel + // invites first. The line-by-line branch carried no name, leaving a one-field + // row against the two-column schema, and the first tuple could not be built. + it should "name the entry each line came from, reading an archive line by line" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("file-scan-archive-lines-") + dir.toFile.deleteOnExit() + val archive = writeArchive(dir, "a.txt" -> "one\ntwo\n", "b.txt" -> "three\n") + + val desc = extractingDesc(archive, """"outputFileName":true""") + val engine = + tuplesFromEngine(desc).map(t => (t.getField[String]("filename"), t.getField[String]("line"))) + assert(engine == Seq(("a.txt", "one"), ("a.txt", "two"), ("b.txt", "three"))) + + val out = runStandalone( + python, + dir, + desc, + """print(list(out1df.itertuples(index=False, name=None)))""" + )._2 + assert( + out.trim.endsWith("""[('a.txt', 'one'), ('a.txt', 'two'), ('b.txt', 'three')]""") + ) + } + /** A zip at `dir/archive.zip` holding the given entries. */ private def writeArchive(dir: Path, entries: (String, String)*): Path = { val archive = dir.resolve("archive.zip") From f3d0db2b95dfd5d0ce5477beb8fa0ebbf636fda3 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 19 Sep 2026 01:07:17 -0700 Subject: [PATCH 27/33] fix(operator): type a flattened JSONL timestamp as the schema declares it `read_json` was handed the names the schema gives nested values, but under flattening those columns do not exist yet: the file still holds the object around them, so it found nothing to convert and `json_normalize` built the column out of text. A plan sorting on it read January 2025 as coming before March 2024, where the executor, which parses the value on its way into the tuple, had them the other way around. `convert_dates` now goes to the reader only when the names it is given are the file's own. Under flattening the declared timestamp columns are converted once the frame that holds them exists. A format is inferred per value, as this operator's own parser reads each value on its own, so a column that wrote the same instant two ways still converts whole; `read_json` was already lenient that way. Co-Authored-By: Claude Opus 5 (1M context) --- .../scan/json/JSONLScanSourceOpDesc.scala | 17 ++++- .../scan/json/JSONLScanSourceOpDescSpec.scala | 68 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index 7fde7ea1ed1..16e9196367c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -95,7 +95,13 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato .filter(_.getType == AttributeType.TIMESTAMP) .map(a => pyStringLiteral(a.getName)) ) - readArgs += s"convert_dates=[${dateColumns.mkString(", ")}]" + // Under flattening the schema names a nested value for the column the + // flattening is about to build, and read_json is asked about that name + // while the file still holds the object around it. It finds no such column, + // converts nothing, and the value reaches the plan as text, where a sort + // puts a 2025 date before a 2024 one. Those columns are converted once the + // frame that holds them exists, below. + if (!flatten) readArgs += s"convert_dates=[${dateColumns.mkString(", ")}]" val readExpr = s"pd.read_json(${readArgs.mkString(", ")})" // json_normalize opens a nested object and leaves a nested array whole, so @@ -128,6 +134,15 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato } } + if (flatten) { + // A format is inferred for each value on its own, the way this operator's + // own parser reads each value on its own, so a column whose lines wrote + // the same instant two ways still converts whole. + dateColumns.foreach { nameLit => + lines += s"""out1df[$nameLit] = pd.to_datetime(out1df[$nameLit], format="mixed")""" + } + } + // A JSONL file states no column order, so the schema this operator infers // sorts the names it found and the rows the workflow sees follow that // order. read_json keeps the order the first record happened to use, which diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index 6eb00c642d1..6b39ac4fcba 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -372,6 +372,74 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // read_json is handed the name the flattening will give a nested value, so it + // looks for a column the file does not yet have and converts nothing. The + // value stayed text, and a plan sorting on it read January 2025 as coming + // before March 2024. + it should "give a flattened timestamp the type the schema declares" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-flatten-date-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + // Written so that the text and the instant disagree on the order: as text + // "01/15/2025" comes first, as a moment "03/01/2024" does. + Files.write( + data, + ("""{"id":1,"meta":{"when":"03/01/2024 10:00:00"}}""" + "\n" + + """{"id":2,"meta":{"when":"01/15/2025 08:30:00"}}""" + "\n") + .getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + op.flatten = true + val schema = op.sourceSchema() + schema.getAttribute("meta.when").getType shouldBe AttributeType.TIMESTAMP + + val exec = new JSONLScanSourceOpExec(objectMapper.writeValueAsString(op)) + exec.open() + val fromEngine = + try exec + .produceTuple() + .map(_.asInstanceOf[SchemaEnforceable].enforceSchema(schema)) + .toList + .sortBy(_.getField[java.sql.Timestamp]("meta.when").getTime) + .map(_.getField[Any]("id")) + finally exec.close() + fromEngine shouldBe List(1, 2) + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}" + |${op.generateStandaloneCode()} + |print(out1df["meta.when"].dtype.kind) + |print(list(out1df.sort_values("meta.when")["id"])) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + // "M" is a datetime column; text would print "O". + out.linesIterator.map(_.trim).toList should contain("M") + out.trim should endWith("[1, 2]") + } + } + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path // (UDF_PYTHON_PATH), then python3 / python / py. private def resolvePython(): Option[String] = { From 3d1a4be422398d3a3d8116ece57c8777492986d0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 19 Sep 2026 01:29:09 -0700 Subject: [PATCH 28/33] fix(operator): keep a file's columns when its scan window asks for no rows The Limit bounds the rows a file scan outputs, and it also bounded the sample `sourceSchema` reads to infer their types, so a Limit of 0 left the inference nothing to look at. The three readers then failed differently on the same file: the CSV and JSONL scans declared a schema with no attributes at all, and the old CSV scan threw, taking its names from the header row and its types from the sample and then asking an empty array for the first one. The parallel CSV scan threw for the same reason. A file's columns do not depend on how many of its rows were asked for, so a window of no rows reads the sample it would have read without one. A Limit of 1 or more is unchanged, still typing the columns from the rows the operator will actually emit. Closes #8602 Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/csv/CSVScanSourceOpDesc.scala | 6 ++++- .../csv/ParallelCSVScanSourceOpDesc.scala | 6 ++++- .../scan/csvOld/CSVOldScanSourceOpDesc.scala | 6 ++++- .../scan/json/JSONLScanSourceOpDesc.scala | 6 ++++- .../scan/csv/CSVScanSourceOpDescSpec.scala | 23 ++++++++++++++++++ .../csvOld/CSVOldScanSourceOpDescSpec.scala | 15 ++++++++++++ .../scan/json/JSONLScanSourceOpDescSpec.scala | 24 +++++++++++++++++++ 7 files changed, 82 insertions(+), 4 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index cb7afdeba6b..4e40a01e99c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -122,7 +122,11 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator parser.beginParsing(inputReader) var data: Array[Array[String]] = Array() - val readLimit = limit.getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT) + // A window of no rows is still a window on this file, and the file's columns + // do not depend on how many of its rows were asked for. Reading the sample + // through the limit left a Limit of 0 nothing to infer from, and the operator + // declared a schema of no columns at all. + val readLimit = limit.filter(_ > 0).getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT) for (_ <- 0 until readLimit) { val row = CSVScanSourceOpExec.parseNextRow(parser, maxColumns) if (row != null) { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala index 2d2740f06ff..0537cd317ac 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala @@ -193,9 +193,13 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe if (hasHeader) reader.readNext() + // A window of no rows is still a window on this file, and the file's columns + // do not depend on how many of its rows were asked for. Reading the sample + // through the limit left a Limit of 0 nothing to infer from, and the types + // came back empty while the header below still asked each column for one. val attributeTypeList: Array[AttributeType] = inferSchemaFromRows( reader.iterator - .take(limit.getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT)) + .take(limit.filter(_ > 0).getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT)) .map(seq => seq.toArray) ) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index 17598449e5d..75de0a1e79c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -185,8 +185,12 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat reader = CSVReader.open(file, fileEncoding.getCharset.name())(CustomFormat) val startOffset = offset.getOrElse(0) + (if (hasHeader) 1 else 0) + // A window of no rows is still a window on this file, and the file's columns + // do not depend on how many of its rows were asked for. Reading the sample + // through the limit left a Limit of 0 nothing to infer from, and the types + // came back empty while the header below still asked each column for one. val endOffset = - startOffset + limit.getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT) + startOffset + limit.filter(_ > 0).getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT) val attributeTypeList: Array[AttributeType] = inferSchemaFromRows( reader.iterator .slice(startOffset, endOffset) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index 16e9196367c..8d53c4d2e9f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -213,8 +213,12 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato val allFields: ArrayBuffer[Map[String, String]] = ArrayBuffer() val startOffset = offset.getOrElse(0) + // A window of no rows is still a window on this file, and the file's columns + // do not depend on how many of its rows were asked for. Reading the sample + // through the limit left a Limit of 0 nothing to infer from, and the operator + // declared a schema of no columns at all. val endOffset = - startOffset + limit.getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT) + startOffset + limit.filter(_ > 0).getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT) reader .lines() .iterator() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index 6bd304cf6b5..fa9ba41dc89 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -562,4 +562,27 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(columnNames(oldCsv, path) == List("id", "name", "age")) } + // The limit bounded the sample the inference reads as well as the rows the + // operator emits, so a Limit of 0 had nothing to infer from. The three readers + // then failed differently on the same file: these two declared a schema of no + // columns at all, and the old one threw, its header still asking each column + // for a type the empty sample could not give. A file's columns do not depend + // on how many of its rows were asked for. + it should "keep the file's columns when the window asks for no rows" in { + val path = writeSemicolonCsv() + val csv = new CSVScanSourceOpDesc() + csv.customDelimiter = Some(";") + csv.limit = Some(0) + val parallelCsv = new ParallelCSVScanSourceOpDesc() + parallelCsv.customDelimiter = Some(";") + parallelCsv.limit = Some(0) + val oldCsv = new CSVOldScanSourceOpDesc() + oldCsv.customDelimiter = Some(";") + oldCsv.limit = Some(0) + + assert(columnNames(csv, path) == List("id", "name", "age")) + assert(columnNames(parallelCsv, path) == List("id", "name", "age")) + assert(columnNames(oldCsv, path) == List("id", "name", "age")) + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala index 4dcbf96ae1e..8d031db4426 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala @@ -159,6 +159,21 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { d.generateStandaloneCode() should include("skiprows=range(1, 2147483648)") } + // The limit bounded the sample the inference reads as well as the rows the + // operator emits, so a Limit of 0 had nothing to infer from. The types came + // back empty while the header still asked each column for one, and the + // operator threw before a row was read. A file's columns do not depend on how + // many of its rows were asked for. + it should "keep the file's columns when the window asks for no rows" in { + val d = describing(writeCsv("id,name\n1,alice\n2,bob\n")) + d.limit = Some(0) + + val schema = d.sourceSchema() + schema.getAttributeNames shouldBe List("id", "name") + schema.getAttribute("id").getType shouldBe AttributeType.INTEGER + schema.getAttribute("name").getType shouldBe AttributeType.STRING + } + private def writeCsv(content: String): String = { val file = Files.createTempFile("csv-old-", ".csv") file.toFile.deleteOnExit() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index 6b39ac4fcba..dba9da1cd2c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -440,6 +440,30 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // The limit bounded the sample the inference reads as well as the rows the + // operator emits, so a Limit of 0 had nothing to infer from and the operator + // declared a schema of no columns at all. A file's columns do not depend on + // how many of its rows were asked for. + it should "keep the file's columns when the window asks for no rows" in { + val data = Files.createTempFile("jsonl-zero-window-", ".jsonl") + data.toFile.deleteOnExit() + Files.write( + data, + "{\"id\":1,\"name\":\"alice\"}\n{\"id\":2,\"name\":\"bob\"}\n".getBytes( + StandardCharsets.UTF_8 + ) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + op.limit = Some(0) + + val schema = op.sourceSchema() + schema.getAttributeNames shouldBe List("id", "name") + schema.getAttribute("id").getType shouldBe AttributeType.INTEGER + } + // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path // (UDF_PYTHON_PATH), then python3 / python / py. private def resolvePython(): Option[String] = { From ce52ee210f8d3735cb39a731221ac7089b2bea66 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 21 Sep 2026 01:05:59 -0700 Subject: [PATCH 29/33] fix(operator): read every column an Arrow file states, index or not A file pandas wrote from a frame keyed by one of its columns says so in its schema, and pandas reads those columns back as the frame's index rather than as columns. The executor reads the columns the file states and has no notion of an index, so a file written from a frame keyed by `customer_id` kept that column on the one side and dropped it on the other, where an operator naming it raised. The columns the file states are asked of it and the index put back under those names, which is `__index_level_0__` for an index that had none, and in the file's own order, which is where pandas wrote them: last. Before the widths are looked at, a column restored from an index being as narrow as any other. The Parquet source parted the same way over the same note. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/arrow/ArrowSourceOpDesc.scala | 27 +++++- .../scan/arrow/ArrowSourceOpDescSpec.scala | 94 +++++++++++++++++-- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index a81405efb67..8ae83bf39c6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -44,6 +44,10 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { fileTypeName = Option("Arrow") + // pyarrow is what pandas reads an Arrow file with; the columns the file states + // are asked of it directly below. + override def standaloneImports(): Seq[String] = Seq("import pyarrow as pa") + override def standaloneSourcePath(): Option[String] = fileName override def generateStandaloneCode(): String = { @@ -53,6 +57,22 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // is not rounded on its way through a float. val read = s"""out1df = pd.read_feather($SourceFilePlaceholder, dtype_backend="numpy_nullable")""" + // A file pandas wrote from a frame keyed by one of its columns says so in its + // schema, and pandas reads those columns back as the frame's index rather + // than as columns. The executor reads the columns the file states and has no + // notion of an index, so a file written from a frame keyed by `customer_id` + // kept that column on the one side and dropped it on the other, where an + // operator naming it raised. Put back under the name the file gives it, which + // is `__index_level_0__` for an index that had none, and in the file's own + // order, which is where pandas wrote them: last. The Parquet source strips + // the same note, which its reader lets it do before the columns are read. + val index = + s"""|with pa.ipc.open_file($SourceFilePlaceholder) as _file: + | _names = _file.schema.names + |if list(out1df.columns) != _names: + | _index = out1df.index.to_frame(index=False) + | _index.columns = [_name for _name in _names if _name not in out1df.columns] + | out1df = pd.concat([out1df.reset_index(drop=True), _index], axis=1)[_names]""".stripMargin // The widths pandas keeps and Texera has no column for. A file states the // width and the sign of each of its numbers, and pandas reads every one of // them back, where a Texera column is a double or a 32-bit integer and @@ -92,8 +112,11 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { case _ => None } - (Seq(read, widths) ++ window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)")) - .mkString("\n") + // The index is put back before the widths are looked at, a column restored + // from one being as narrow as any other. + (Seq(read, index, widths) ++ window.map(w => + s"out1df = out1df.iloc[$w].reset_index(drop=True)" + )).mkString("\n") } @throws[IOException] diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 42b66e8296d..be62473d52e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -59,11 +59,18 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) - private def writeArrowFile(schema: Schema, rows: Seq[Array[Any]]): File = { + private def writeArrowFile( + schema: Schema, + rows: Seq[Array[Any]], + schemaMetadata: Map[String, String] = Map.empty + ): File = { val file = File.createTempFile("arrow-src-", ".arrow") file.deleteOnExit() val allocator = new RootAllocator() - val root = VectorSchemaRoot.create(ArrowUtils.fromTexeraSchema(schema), allocator) + val root = VectorSchemaRoot.create( + new ArrowFileSchema(ArrowUtils.fromTexeraSchema(schema).getFields, schemaMetadata.asJava), + allocator + ) val out = new FileOutputStream(file) val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) try { @@ -267,8 +274,7 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val script = workDir.resolve("run.py") Files.write( script, - s"""import pandas as pd - |${bindSourceFile(d)} + s"""${scriptHeader(d)} |${d.generateStandaloneCode()} |print(str(out1df["d"].dtype), str(out1df["i"].dtype)) |print(repr(out1df["d"].iloc[0]), repr(out1df["d"].iloc[1])) @@ -357,8 +363,7 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val script = workDir.resolve("run.py") Files.write( script, - s"""import pandas as pd - |${bindSourceFile(d)} + s"""${scriptHeader(d)} |${d.generateStandaloneCode()} |print([None if pd.isna(v) else int(v) for v in out1df["big"]]) |""".stripMargin.getBytes(StandardCharsets.UTF_8) @@ -417,8 +422,7 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { val script = workDir.resolve("run.py") Files.write( script, - s"""import pandas as pd - |${bindSourceFile(d)} + s"""${scriptHeader(d)} |${d.generateStandaloneCode()} |print(" ".join(str(out1df[c].dtype) for c in out1df.columns)) |print([float(v) for v in out1df["f"]]) @@ -446,6 +450,74 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // pandas writes into an Arrow file's schema which of a frame's columns was its + // index, and reads those columns back as the frame's index rather than as + // columns. The executor reads the columns the file states and has no notion of + // an index, so a file written from a frame keyed by `customer_id` kept that + // column on the one side and dropped it on the other. + it should "keep the column a pandas index was written from" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val schema = Schema( + List( + new Attribute("name", AttributeType.STRING), + new Attribute("amt", AttributeType.DOUBLE), + new Attribute("customer_id", AttributeType.INTEGER) + ) + ) + // The note pandas leaves, which is the whole of what parts the two readings. + // Written by hand: the Java writer here has nothing to say about an index. + val pandasMetadata = + """|{"index_columns": ["customer_id"], + | "column_indexes": [], + | "columns": [{"name": "name", "field_name": "name", + | "pandas_type": "unicode", "numpy_type": "object", "metadata": null}, + | {"name": "amt", "field_name": "amt", + | "pandas_type": "float64", "numpy_type": "float64", "metadata": null}, + | {"name": "customer_id", "field_name": "customer_id", + | "pandas_type": "int32", "numpy_type": "int32", "metadata": null}], + | "pandas_version": "2.2.3"}""".stripMargin + val file = writeArrowFile( + schema, + Seq(Array[Any]("alice", 1.5d, 101), Array[Any]("bob", 2.5d, 102)), + Map("pandas" -> pandasMetadata) + ) + + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + d.inferSchema().getAttributeNames.toList shouldBe List("name", "amt", "customer_id") + + val workDir = Files.createTempDirectory("arrow-index-") + workDir.toFile.deleteOnExit() + Files.copy(file.toPath, workDir.resolve(file.getName)) + + val script = workDir.resolve("run.py") + Files.write( + script, + s"""${scriptHeader(d)} + |${d.generateStandaloneCode()} + |print(list(out1df.columns)) + |print([int(v) for v in out1df["customer_id"]]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(workDir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + // The same columns, under the names the file gives them and in its order. + lines.head shouldBe "['name', 'amt', 'customer_id']" + lines(1) shouldBe "[101, 102]" + } + } + // Nothing in Texera holds a value past 2^63 - 1, so the column is refused at // the schema rather than read as the -1 its storage counts down to. it should "refuse an unsigned 64-bit column, having no Texera type for it" in { @@ -480,6 +552,10 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { private def bindSourceFile(d: ArrowSourceOpDesc): String = s"""${StandaloneCodeGenerator.SourceFilePlaceholder} = "${d.standaloneSourceName().get}"""" + /** What a script needs before the block: its imports, and the file it reads. */ + private def scriptHeader(d: ArrowSourceOpDesc): String = + (d.standaloneImports() :+ "import pandas as pd" :+ bindSourceFile(d)).mkString("\n") + private def resolvePython(): Option[String] = { def fromConfig: Option[String] = Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption @@ -500,7 +576,7 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { private def canImportPandas(python: String): Boolean = Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + new ProcessBuilder(python, "-c", "import pandas, pyarrow").redirectErrorStream(true).start() ).toOption.exists { p => if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } else p.exitValue() == 0 From ce2e4840eef25077d584aa40692c1d2eff00c13d Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 21 Sep 2026 14:24:00 -0700 Subject: [PATCH 30/33] fix(operator): read an Arrow file's columns under the names it states A file pandas wrote notes in its schema what the frame it came from looked like, and reading that note back parts the two readings twice over: a column the frame was keyed by returns as the index, and labels that were numbers return as numbers where the file says "1" and "2". The second broke the first, the restoration matching a physical name against a numeric label and assigning two names to a one-column frame. Refuse the note instead of undoing it, which is what the Parquet source does. pd.read_feather takes no say in it, so the table is read through pyarrow and converted with ignore_metadata, the nullable dtypes being named in the script now that read_feather's dtype_backend is out of the picture. Co-Authored-By: Claude Opus 5 (1M context) --- .../source/scan/arrow/ArrowSourceOpDesc.scala | 58 +++++++++-------- .../scan/arrow/ArrowSourceOpDescSpec.scala | 63 +++++++++++++++++++ 2 files changed, 95 insertions(+), 26 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index 8ae83bf39c6..277ae42a39d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -44,8 +44,8 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { fileTypeName = Option("Arrow") - // pyarrow is what pandas reads an Arrow file with; the columns the file states - // are asked of it directly below. + // pyarrow is what reads an Arrow file, pandas going through it; the block + // below asks it for the file's own columns. override def standaloneImports(): Seq[String] = Seq("import pyarrow as pa") override def standaloneSourcePath(): Option[String] = fileName @@ -54,25 +54,33 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { // Arrow says of every value whether it is there, and a numpy column has // nowhere to put that: a missing double and a stored NaN both land on NaN. // The nullable dtypes keep a holed integer integral too, so a long past 2^53 - // is not rounded on its way through a float. + // is not rounded on its way through a float. Named here because the read + // below cannot ask pd.read_feather for them, and named for the types + // ArrowUtils.toAttributeType reads; a timestamp needs none, datetime64 + // having a slot of its own for a missing value. + val dtypes = + """|_nullable = { + | pa.bool_(): pd.BooleanDtype(), + | pa.string(): pd.StringDtype(), + | pa.float32(): pd.Float32Dtype(), + | pa.float64(): pd.Float64Dtype(), + | pa.int8(): pd.Int8Dtype(), + | pa.int16(): pd.Int16Dtype(), + | pa.int32(): pd.Int32Dtype(), + | pa.int64(): pd.Int64Dtype(), + | pa.uint8(): pd.UInt8Dtype(), + | pa.uint16(): pd.UInt16Dtype(), + | pa.uint32(): pd.UInt32Dtype(), + |}""".stripMargin + // A file pandas wrote notes in its schema what the frame it came from looked + // like, and pandas reads that note back: a column the frame was keyed by + // returns as the index, and numbered labels return as numbers where the file + // says "1" and "2". The executor reads the columns the file states, so the + // note is refused rather than undone, as the Parquet source refuses it. val read = - s"""out1df = pd.read_feather($SourceFilePlaceholder, dtype_backend="numpy_nullable")""" - // A file pandas wrote from a frame keyed by one of its columns says so in its - // schema, and pandas reads those columns back as the frame's index rather - // than as columns. The executor reads the columns the file states and has no - // notion of an index, so a file written from a frame keyed by `customer_id` - // kept that column on the one side and dropped it on the other, where an - // operator naming it raised. Put back under the name the file gives it, which - // is `__index_level_0__` for an index that had none, and in the file's own - // order, which is where pandas wrote them: last. The Parquet source strips - // the same note, which its reader lets it do before the columns are read. - val index = s"""|with pa.ipc.open_file($SourceFilePlaceholder) as _file: - | _names = _file.schema.names - |if list(out1df.columns) != _names: - | _index = out1df.index.to_frame(index=False) - | _index.columns = [_name for _name in _names if _name not in out1df.columns] - | out1df = pd.concat([out1df.reset_index(drop=True), _index], axis=1)[_names]""".stripMargin + | _table = _file.read_all() + |out1df = _table.to_pandas(ignore_metadata=True, types_mapper=_nullable.get)""".stripMargin // The widths pandas keeps and Texera has no column for. A file states the // width and the sign of each of its numbers, and pandas reads every one of // them back, where a Texera column is a double or a 32-bit integer and @@ -91,10 +99,10 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { | elif _values.dtype == "UInt32": | out1df[_column] = _values.astype("Int64")""".stripMargin // A timestamp column needs nothing here. The file names UTC and holds the - // wall clock as UTC, so pd.read_feather and the executor read the same - // reading off it — no zone of the reader's own enters either side. The other - // scan sources have to name their date columns, CSV and JSONL carrying no - // types to go on, but Arrow states its own. + // wall clock as UTC, so pandas and the executor read the same reading off + // it: no zone of the reader's own enters either side. The other scan sources + // have to name their date columns, CSV and JSONL carrying no types to go on, + // but Arrow states its own. // // The executor drops `offset` rows and then takes `limit` of them. Feather has // no row-range read, so the same window is taken once the frame is in memory. @@ -112,9 +120,7 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { case _ => None } - // The index is put back before the widths are looked at, a column restored - // from one being as narrow as any other. - (Seq(read, index, widths) ++ window.map(w => + (Seq(dtypes, read, widths) ++ window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)" )).mkString("\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index be62473d52e..a29f8949617 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -518,6 +518,69 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // The same note also says what type the frame's column labels had, and pandas + // casts the names back to it. A frame whose columns were numbered is written + // under the names "1" and "2", which is what the executor reads, and comes + // back out of pandas labelled 1 and 2. + it should "keep the names a pandas frame of numbered columns was written under" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val schema = + Schema( + List(new Attribute("1", AttributeType.INTEGER), new Attribute("2", AttributeType.INTEGER)) + ) + // What `pd.DataFrame({1: [7, 8], 2: [10, 20]}).to_feather(...)` leaves: the + // names are strings, and `column_indexes` is the whole of what turns them + // back into numbers. + val pandasMetadata = + """|{"index_columns": [{"kind": "range", "name": null, "start": 0, "stop": 2, "step": 1}], + | "column_indexes": [{"name": null, "field_name": null, + | "pandas_type": "int64", "numpy_type": "int64", "metadata": null}], + | "columns": [{"name": "1", "field_name": "1", + | "pandas_type": "int32", "numpy_type": "int32", "metadata": null}, + | {"name": "2", "field_name": "2", + | "pandas_type": "int32", "numpy_type": "int32", "metadata": null}], + | "pandas_version": "2.2.3"}""".stripMargin + val file = writeArrowFile( + schema, + Seq(Array[Any](7, 10), Array[Any](8, 20)), + Map("pandas" -> pandasMetadata) + ) + + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + d.inferSchema().getAttributeNames.toList shouldBe List("1", "2") + + val workDir = Files.createTempDirectory("arrow-labels-") + workDir.toFile.deleteOnExit() + Files.copy(file.toPath, workDir.resolve(file.getName)) + + val script = workDir.resolve("run.py") + Files.write( + script, + s"""${scriptHeader(d)} + |${d.generateStandaloneCode()} + |print(list(out1df.columns)) + |print([int(v) for v in out1df["2"]]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(workDir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + + withClue(s"python said:\n$out") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "['1', '2']" + lines(1) shouldBe "[10, 20]" + } + } + // Nothing in Texera holds a value past 2^63 - 1, so the column is refused at // the schema rather than read as the -1 its storage counts down to. it should "refuse an unsigned 64-bit column, having no Texera type for it" in { From 719cb935db58cfea73fb5a8f78258d475d485238 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 23 Sep 2026 15:59:51 -0700 Subject: [PATCH 31/33] fix(operator): read a zoned Arrow timestamp as the wall clock it holds A timestamp field may name a zone, and pandas kept it on the column where the executor keeps only the wall clock in that zone, a Texera TIMESTAMP having none. A downstream cast to a plain datetime refuses to drop a zone, so Arrow Source followed by Extract DateTime ended the script on a UTC file. The zone is taken off and the clock left as it is: pandas already holds the wall clock in the file's own zone, which is what ArrowUtils reads. The test uses a zone other than UTC so the clock taken is the file's. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../source/scan/arrow/ArrowSourceOpDesc.scala | 21 +++-- .../scan/arrow/ArrowSourceOpDescSpec.scala | 81 +++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala index 277ae42a39d..2d7820ca4ee 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDesc.scala @@ -98,12 +98,19 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { | out1df[_column] = _values.astype("Int32") | elif _values.dtype == "UInt32": | out1df[_column] = _values.astype("Int64")""".stripMargin - // A timestamp column needs nothing here. The file names UTC and holds the - // wall clock as UTC, so pandas and the executor read the same reading off - // it: no zone of the reader's own enters either side. The other scan sources - // have to name their date columns, CSV and JSONL carrying no types to go on, - // but Arrow states its own. - // + // A timestamp column may name a zone, and pandas keeps it on the column + // where the executor keeps only the wall clock in that zone: a Texera + // TIMESTAMP has none. Left zoned, the column reached a downstream `astype` + // that refuses to drop a zone and ended the script. pandas already holds the + // wall clock in the file's own zone, so the zone is taken off and the clock + // left as it is, which is what ArrowUtils reads. The other scan sources have + // to name their date columns, CSV and JSONL carrying no types to go on, but + // Arrow states its own. + val zones = + """|for _column, _values in out1df.items(): + | if isinstance(_values.dtype, pd.DatetimeTZDtype): + | out1df[_column] = _values.dt.tz_localize(None)""".stripMargin + // The executor drops `offset` rows and then takes `limit` of them. Feather has // no row-range read, so the same window is taken once the frame is in memory. // @@ -120,7 +127,7 @@ class ArrowSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { case _ => None } - (Seq(dtypes, read, widths) ++ window.map(w => + (Seq(dtypes, read, widths, zones) ++ window.map(w => s"out1df = out1df.iloc[$w].reset_index(drop=True)" )).mkString("\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index a29f8949617..0db99b4cbf9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -24,6 +24,7 @@ import org.apache.arrow.memory.RootAllocator import org.apache.arrow.vector.{ Float4Vector, SmallIntVector, + TimeStampMilliTZVector, TinyIntVector, UInt1Vector, UInt2Vector, @@ -611,6 +612,86 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { ex.getMessage shouldBe "Failed to read the .arrow file. Please ensure it is a valid Arrow file." } + // A zoned timestamp reaches the executor as the wall clock in the zone the + // field names, and nothing of the zone is left. pandas kept the zone on the + // column, and a downstream cast to a plain datetime refused it. A zone other + // than UTC, so that the wall clock taken is the file's and not UTC's. + it should "read a zoned timestamp as the wall clock the executor reads" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val fields = List( + new Field( + "ts", + FieldType.nullable( + new ArrowType.Timestamp( + org.apache.arrow.vector.types.TimeUnit.MILLISECOND, + "Asia/Tokyo" + ) + ), + null + ) + ) + val file = File.createTempFile("arrow-zoned-", ".arrow") + file.deleteOnExit() + val allocator = new RootAllocator() + val root = VectorSchemaRoot.create(new ArrowFileSchema(fields.asJava), allocator) + val out = new FileOutputStream(file) + val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) + try { + writer.start() + root.allocateNew() + val ts = root.getVector("ts").asInstanceOf[TimeStampMilliTZVector] + ts.setSafe(0, 0L) + ts.setNull(1) + root.setRowCount(2) + writer.writeBatch() + writer.end() + } finally { + writer.close() + root.close() + allocator.close() + out.close() + } + + val d = new ArrowSourceOpDesc + d.fileName = Some(file.toURI.toString) + + val exec = new ArrowSourceOpExec(objectMapper.writeValueAsString(d)) + exec.open() + val fromEngine = + try exec.produceTuple().map(_.getFields.head).toList + finally exec.close() + fromEngine shouldBe List(java.sql.Timestamp.valueOf("1970-01-01 09:00:00"), null) + + val workDir = Files.createTempDirectory("arrow-zoned-") + workDir.toFile.deleteOnExit() + Files.copy(file.toPath, workDir.resolve(file.getName)) + + val script = workDir.resolve("run.py") + Files.write( + script, + s"""${scriptHeader(d)} + |${d.generateStandaloneCode()} + |print(out1df["ts"].dtype) + |print([None if pd.isna(v) else str(v) for v in out1df["ts"].astype("datetime64[ns]")]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(workDir.toFile) + .redirectErrorStream(true) + .start() + val stdout = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$stdout") { + process.exitValue() shouldBe 0 + val lines = stdout.trim.linesIterator.toSeq + lines.head shouldBe "datetime64[ms]" + lines(1) shouldBe "['1970-01-01 09:00:00', None]" + } + } + /** The block names its file by placeholder; the translator puts a name there. */ private def bindSourceFile(d: ArrowSourceOpDesc): String = s"""${StandaloneCodeGenerator.SourceFilePlaceholder} = "${d.standaloneSourceName().get}"""" From 5bc1a0df2ef8d5acc0a728776fd2a46b6fe23300 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 23 Sep 2026 15:59:52 -0700 Subject: [PATCH 32/33] fix(operator): ask pandas for a CSV column's type by its position The schema calls a blank header `column-2` and pandas calls it `Unnamed: 1` until the rename that follows the read, so a column asked for by the schema's name is one pandas does not have. parse_dates raised on it and ended the read; dtype passed over it without a word, and a long past 2^53 was inferred as a float and rounded. All three CSV scans now name those columns by position, header or not. pandas reads an integer there as a position even where a header spells one. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../source/scan/csv/CSVScanSourceOpDesc.scala | 29 ++++---- .../csv/ParallelCSVScanSourceOpDesc.scala | 7 +- .../scan/csvOld/CSVOldScanSourceOpDesc.scala | 15 ++-- .../scan/csv/CSVScanSourceOpDescSpec.scala | 70 ++++++++++++++++++- 4 files changed, 92 insertions(+), 29 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index 4e40a01e99c..9be784baea9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -185,32 +185,31 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator // A CSV carries no types, so both readers infer, and they do not infer // alike: the schema above tries TIMESTAMP and parses what it can, while // pd.read_csv leaves a date column as text. Name the columns this operator - // decided were timestamps so pandas parses the same ones — by position when - // there is no header, the frame's columns having no names until the rename - // below. A schema that cannot be read (an unresolved file) leaves the - // argument off rather than failing the export. + // decided were timestamps so pandas parses the same ones. They are named by + // position, header or not, because the schema's names are not pandas' until + // the rename below: a blank header is `column-2` here and `Unnamed: 1` + // there, and asking for `column-2` ended the read. pandas takes an integer + // here as a position even where a header spells one. A schema that cannot be + // read (an unresolved file) leaves the argument off rather than failing the + // export. val dateColumns: Seq[String] = Try(sourceSchema()).toOption.toSeq.flatMap( _.getAttributes.zipWithIndex .filter(_._1.getType == AttributeType.TIMESTAMP) - .map { case (a, i) => if (hasHeader) pyStringLiteral(a.getName) else i.toString } + .map(_._2.toString) ) if (dateColumns.nonEmpty) args += s"parse_dates=[${dateColumns.mkString(", ")}]" - // A LONG column holding a null has to be asked for by name, or pandas widens - // it through a float to carry the hole: 9007199254740993 comes back as - // ...992, a value the file never held and the executor never produced. - // Int64 is the nullable integer, so the hole costs the column nothing. - // INTEGER needs none of this — every int32 is exact in a float64. + // A LONG column holding a null has to be asked for, or pandas widens it + // through a float to carry the hole: 9007199254740993 comes back as ...992, + // a value the file never held and the executor never produced. Int64 is the + // nullable integer, so the hole costs the column nothing. INTEGER needs none + // of this — every int32 is exact in a float64. By position, as the dates are. val longColumns: Seq[String] = Try(sourceSchema()).toOption.toSeq.flatMap( _.getAttributes.zipWithIndex .filter(_._1.getType == AttributeType.LONG) - .map { - case (a, i) => - val key = if (hasHeader) pyStringLiteral(a.getName) else i.toString - s"""$key: "Int64"""" - } + .map { case (_, i) => s"""$i: "Int64"""" } ) if (longColumns.nonEmpty) args += s"dtype={${longColumns.mkString(", ")}}" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala index 0537cd317ac..e598bc720d7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala @@ -114,7 +114,9 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe // executor nulls it and parses the rest as that STRING. pandas infers a number // instead, so a column of ids came back as floats, one past 2^53 rounded: // 9007199254740993 as ...992. A LONG needs the nullable integer for the same - // reason. See CSVScanSourceOpDesc. + // reason. By position, as in CSVScanSourceOpDesc: under a blank header the + // schema's `column-2` is pandas' `Unnamed: 1`, and pandas passes over a + // name it has no column for, so the type was never applied. val dtypes: Seq[String] = Try(sourceSchema()).toOption.toSeq.flatMap( _.getAttributes.zipWithIndex @@ -125,8 +127,7 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe case AttributeType.STRING => Some("string") case _ => None } - val key = if (hasHeader) pyStringLiteral(a.getName) else i.toString - pandasType.map(t => s"""$key: "$t"""") + pandasType.map(t => s"""$i: "$t"""") } ) if (dtypes.nonEmpty) args += s"dtype={${dtypes.mkString(", ")}}" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index 75de0a1e79c..b2aa83a1da4 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -106,26 +106,25 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat args += "keep_default_na=False" // Name the columns this operator inferred as timestamps, so pandas parses - // the same ones instead of leaving them as text. See CSVScanSourceOpDesc. + // the same ones instead of leaving them as text. By position, header or + // not, since a blank header is not yet the schema's name. See + // CSVScanSourceOpDesc. val dateColumns: Seq[String] = Try(sourceSchema()).toOption.toSeq.flatMap( _.getAttributes.zipWithIndex .filter(_._1.getType == AttributeType.TIMESTAMP) - .map { case (a, i) => if (hasHeader) pyStringLiteral(a.getName) else i.toString } + .map(_._2.toString) ) if (dateColumns.nonEmpty) args += s"parse_dates=[${dateColumns.mkString(", ")}]" // Read a LONG column as the nullable integer, so a hole does not widen it - // through a float and round the values it carries. See CSVScanSourceOpDesc. + // through a float and round the values it carries. By position, as the + // dates are. See CSVScanSourceOpDesc. val longColumns: Seq[String] = Try(sourceSchema()).toOption.toSeq.flatMap( _.getAttributes.zipWithIndex .filter(_._1.getType == AttributeType.LONG) - .map { - case (a, i) => - val key = if (hasHeader) pyStringLiteral(a.getName) else i.toString - s"""$key: "Int64"""" - } + .map { case (_, i) => s"""$i: "Int64"""" } ) if (longColumns.nonEmpty) args += s"dtype={${longColumns.mkString(", ")}}" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index fa9ba41dc89..7ea1405ac26 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -351,10 +351,74 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(csvScanSourceOpDesc.sourceSchema().getAttribute("big").getType == AttributeType.LONG) val code = csvScanSourceOpDesc.generateStandaloneCode() - assert(code.contains("""dtype={"big": "Int64"}""")) + assert(code.contains("""dtype={1: "Int64"}""")) // `id` has no hole, so a float would carry it exactly. Only the column the // schema calls LONG is asked for. - assert(!code.contains(""""id": "Int64"""")) + assert(!code.contains("""0: "Int64"""")) + } + + // The schema calls a blank header `column-2` and pandas calls it `Unnamed: 1` + // until the rename, so a date column asked for by the schema's name ended the + // read on a missing column. By position, the two agree. + it should "ask pandas for a date under a blank header by its position" in { + val tmpFile = Files.createTempFile("blank-date-header-", ".csv") + tmpFile.toFile.deleteOnExit() + Files.write(tmpFile, "id,\n1,2024-01-01 00:00:00\n".getBytes(StandardCharsets.UTF_8)) + val path = tmpFile.toString + csvScanSourceOpDesc.fileName = Some(path) + csvScanSourceOpDesc.customDelimiter = Some(",") + csvScanSourceOpDesc.hasHeader = true + csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + assert( + csvScanSourceOpDesc.sourceSchema().getAttribute("column-2").getType == + AttributeType.TIMESTAMP + ) + + val code = csvScanSourceOpDesc.generateStandaloneCode() + assert(code.contains("parse_dates=[1]")) + assert(code.contains("""out1df.columns = ["id", "column-2"]""")) + } + + it should "ask pandas for a date under a blank header by its position for old CSV" in { + val tmpFile = Files.createTempFile("blank-date-header-", ".csv") + tmpFile.toFile.deleteOnExit() + Files.write(tmpFile, "id,\n1,2024-01-01 00:00:00\n".getBytes(StandardCharsets.UTF_8)) + val path = tmpFile.toString + val oldCsvScanSourceOpDesc = new CSVOldScanSourceOpDesc() + oldCsvScanSourceOpDesc.fileName = Some(path) + oldCsvScanSourceOpDesc.customDelimiter = Some(",") + oldCsvScanSourceOpDesc.hasHeader = true + oldCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + assert( + oldCsvScanSourceOpDesc.sourceSchema().getAttribute("column-2").getType == + AttributeType.TIMESTAMP + ) + assert(oldCsvScanSourceOpDesc.generateStandaloneCode().contains("parse_dates=[1]")) + } + + // pandas raises nothing for a type asked of a name it has no column for, so + // under a blank header the type was dropped without a word and the column + // was inferred as floats, rounding 9007199254740993 to ...992. + it should "ask pandas for a type under a blank header by its position for parallel CSV" in { + val tmpFile = Files.createTempFile("blank-long-header-", ".csv") + tmpFile.toFile.deleteOnExit() + Files.write( + tmpFile, + "id,\n1,9007199254740993\n2,\n3,9007199254740995\n".getBytes(StandardCharsets.UTF_8) + ) + val path = tmpFile.toString + parallelCsvScanSourceOpDesc.fileName = Some(path) + parallelCsvScanSourceOpDesc.customDelimiter = Some(",") + parallelCsvScanSourceOpDesc.hasHeader = true + parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path)) + + assert( + parallelCsvScanSourceOpDesc.sourceSchema().getAttribute("column-2").getType == + AttributeType.STRING + ) + assert(parallelCsvScanSourceOpDesc.generateStandaloneCode().contains("""dtype={1: "string"}""")) } // sourceSchema reads this operator's file with scala-csv, which hands a blank @@ -385,7 +449,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert( parallelCsvScanSourceOpDesc .generateStandaloneCode() - .contains("""dtype={"big": "string"}""") + .contains("""dtype={1: "string"}""") ) } From 053a6fdf08880851618e6ce987d986085674c983 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 23 Sep 2026 15:59:52 -0700 Subject: [PATCH 33/33] fix(operator): keep a JSONL boolean with a missing key boolean pandas has no plain boolean that carries a hole, so a record missing the key widened the column to floats, and a downstream cast to text gave 1.0 and 0.0 where the executor has true and false. Such a column is cast to the nullable boolean, which carries both values and the hole. A column with no hole arrives as bool already and is left alone. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scan/json/JSONLScanSourceOpDesc.scala | 16 +++++ .../scan/json/JSONLScanSourceOpDescSpec.scala | 63 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala index 8d53c4d2e9f..79880d7faba 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala @@ -143,6 +143,22 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato } } + // pandas has no plain boolean that carries a hole, so a record missing the + // key widens the column to floats, and a later cast to text read 1.0 and 0.0 + // where the executor has true and false. The nullable boolean carries both + // values and the hole. A column with no hole arrives as bool already and is + // left alone. + val booleanColumns: Seq[String] = + Try(sourceSchema()).toOption.toSeq.flatMap( + _.getAttributes + .filter(_.getType == AttributeType.BOOLEAN) + .map(a => pyStringLiteral(a.getName)) + ) + booleanColumns.foreach { nameLit => + lines += s"""if out1df[$nameLit].dtype == "float64":""" + lines += s""" out1df[$nameLit] = out1df[$nameLit].astype("boolean")""" + } + // A JSONL file states no column order, so the schema this operator infers // sorts the names it found and the rows the workflow sees follow that // order. read_json keeps the order the first record happened to use, which diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index dba9da1cd2c..61e890ca6d0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -209,6 +209,69 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } + // pandas has no plain boolean with a hole, so a record missing the key widened + // the column to floats, and a cast to text downstream read 1.0 and 0.0 where + // the executor has true and false. `c` has no hole and stays the bool it was. + it should "keep a boolean with a missing key boolean, as the executor does" in { + val python = resolvePython().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val dir = Files.createTempDirectory("jsonl-bool-") + dir.toFile.deleteOnExit() + val data = dir.resolve("input.jsonl") + Files.write( + data, + ("""{"b":true,"c":true}""" + "\n" + + """{"c":false}""" + "\n" + + """{"b":false,"c":true}""" + "\n").getBytes(StandardCharsets.UTF_8) + ) + + val op = new JSONLScanSourceOpDesc + op.fileName = Some(data.toString) + op.setResolvedFileName(FileResolver.resolve(data.toString)) + op.sourceSchema().getAttribute("b").getType shouldBe AttributeType.BOOLEAN + + val exec = new JSONLScanSourceOpExec(objectMapper.writeValueAsString(op)) + exec.open() + val fromEngine = + try exec + .produceTuple() + .map( + _.asInstanceOf[SchemaEnforceable].enforceSchema(op.sourceSchema()).getField[Any]("b") + ) + .toList + finally exec.close() + fromEngine shouldBe List(true, null, false) + + val script = dir.resolve("run.py") + Files.write( + script, + s"""import pandas as pd + |${op.standaloneImports().mkString("\n")} + |${op.standaloneHelpers().mkString("\n\n")} + |${StandaloneCodeGenerator.SourceFilePlaceholder} = "${op.standaloneSourceName().get}" + |${op.generateStandaloneCode()} + |print(out1df["b"].dtype, out1df["c"].dtype) + |print([None if pd.isna(v) else str(v) for v in out1df["b"]]) + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) + + val process = new ProcessBuilder(python, script.toString) + .directory(dir.toFile) + .redirectErrorStream(true) + .start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\n") { + process.exitValue() shouldBe 0 + val lines = out.trim.linesIterator.toSeq + lines.head shouldBe "boolean bool" + lines(1) shouldBe "['True', None, 'False']" + } + } + // The executor gives every element of a nested array a column of its own, // named for its position counted from one, so {"items":[{"id":1},{"id":2}]} // is items1.id and items2.id. json_normalize opens an object and leaves an