From 539c96947ac55380c9b40c7fb02a669b0e7a6ed0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 10 Sep 2026 20:41:42 -0700 Subject: [PATCH 1/3] feat(visualization): export the charts drawn over a field or a sequence Ten charts implement the trait: Filled Area, Contour, Dumbbell, Time Series, Quiver, Choropleth Map, 3D Scatter, Volcano, Gantt and Carpet. What they share is the question they answer, which is how a value varies across a domain rather than where a row sits on an axis. Split out of the coordinate-system change on review, which had grown past what one reading can hold. Co-Authored-By: Claude Opus 5 (1M context) --- .../carpetPlot/CarpetPlotOpDesc.scala | 38 +++++++- .../choroplethMap/ChoroplethMapOpDesc.scala | 47 +++++++++- .../contourPlot/ContourPlotOpDesc.scala | 88 +++++++++++++++++- .../dumbbellPlot/DumbbellPlotOpDesc.scala | 78 +++++++++++++++- .../filledAreaPlot/FilledAreaPlotOpDesc.scala | 90 ++++++++++++++++++- .../ganttChart/GanttChartOpDesc.scala | 44 ++++++++- .../quiverPlot/QuiverPlotOpDesc.scala | 54 ++++++++++- .../scatter3DChart/Scatter3dChartOpDesc.scala | 46 +++++++++- .../timeSeriesplot/TimeSeriesOpDesc.scala | 60 ++++++++++++- .../volcanoPlot/VolcanoPlotOpDesc.scala | 45 +++++++++- .../contourPlot/ContourPlotOpDescSpec.scala | 43 +++++++++ .../FilledAreaPlotOpDescSpec.scala | 26 ++++++ 12 files changed, 636 insertions(+), 23 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala index 6c7981390d0..8f97d47d518 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.carpetPlot import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import javax.validation.constraints.NotNull @@ -41,7 +45,7 @@ import javax.validation.constraints.NotNull } } """) -class CarpetPlotOpDesc extends PythonOperatorDescriptor { +class CarpetPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "a", required = true) @NotNull(message = "A-axis Attribute cannot be empty") @@ -132,4 +136,34 @@ class CarpetPlotOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val aLit = pyStringLiteral(a) + val bLit = pyStringLiteral(b) + val yLit = pyStringLiteral(y) + // Both empty cases write the page the operator yields rather than printing: + // a reason the reader can see is the whole output of a chart that cannot be + // drawn, and a run that writes nothing at all looks like a crash. + s"""table = in1df.dropna(subset=[$aLit, $bLit, $yLit]).copy() + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write("

Input table is empty

") + |elif table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write("

No valid rows after removing nulls

") + |else: + | table[$aLit] = table[$aLit].astype(float) + | table[$bLit] = table[$bLit].astype(float) + | table[$yLit] = table[$yLit].astype(float) + | fig = go.Figure(go.Carpet( + | a=table[$aLit], + | b=table[$bLit], + | y=table[$yLit] + | )) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Carpet plot saved to " + outputJson + " and " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala index 84e4ce999a1..1dee0093f82 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala @@ -22,11 +22,15 @@ package org.apache.texera.amber.operator.visualization.choroplethMap import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -44,7 +48,7 @@ import javax.validation.constraints.NotNull } } """) -class ChoroplethMapOpDesc extends PythonOperatorDescriptor { +class ChoroplethMapOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "locations", required = true) @JsonSchemaTitle("Locations Column") @@ -52,6 +56,7 @@ class ChoroplethMapOpDesc extends PythonOperatorDescriptor { "Column used to describe location. Currently only supports countries and needs to be three-letter ISO country code" ) @AutofillAttributeName + @SampleColumn("iso_country") @NotNull(message = "Locations Column cannot be empty") var locations: EncodableString = "" @@ -128,4 +133,40 @@ class ChoroplethMapOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val locationsLit = pyStringLiteral(locations) + val colorLit = pyStringLiteral(color) + // The error page is written to the same file a plotted chart lands in, so a + // reason for "no chart" is where the reader looks for the chart — printing it + // to the terminal alone left that file absent. render_error's continuation + // line keeps the runtime path's indentation, since the HTML is triple-quoted + // and those spaces reach the browser. + s"""def render_error(error_msg): + | return '''

Choropleth map is not available.

+ |

Reason is: {}

+ | '''.format(error_msg) + | + |def fail(error_msg): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(error_msg)) + | print(f"Choropleth map error: {error_msg}") + | + |if in1df.empty: + | fail("Input table is empty.") + |else: + | # Bound to a name of its own: the same frame can feed another branch + | # of the plan, which must still see every row. + | chart_df = in1df.dropna(subset=[$locationsLit, $colorLit]) + | if chart_df.empty: + | fail("No valid rows left (every row has at least 1 missing value).") + | else: + | fig = px.choropleth(chart_df, locations=$locationsLit, color=$colorLit, color_continuous_scale=px.colors.sequential.Plasma) + | fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0}) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Choropleth map saved to " + outputJson + " and " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala index e548ea389e4..88adfc2f93e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala @@ -23,10 +23,14 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.{EncodableString, PythonLiteral} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} @@ -43,7 +47,7 @@ import javax.validation.constraints.NotNull } } """) -class ContourPlotOpDesc extends PythonOperatorDescriptor { +class ContourPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "x", required = true) @JsonSchemaTitle("x") @@ -112,12 +116,32 @@ class ContourPlotOpDesc extends PythonOperatorDescriptor { |import plotly.io as pio | |class ProcessTableOperator(UDFTableOperator): + | def render_error(self, error_msg) -> str: + | return '''

Contour plot is not available.

+ |

Reason is: {}

+ | '''.format(error_msg) | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + | # A row missing any of the three has no point to contribute, and + | # griddata refuses a NaN coordinate outright. + | table = table.dropna(subset=[$x, $y, $z]) + | if table.empty: + | yield {'html-content': self.render_error("Table should not have any empty/null values or fields.")} + | return + | | x = table[$x].values | y = table[$y].values | z = table[$z].values + | + | # Cubic interpolation triangulates the points before it interpolates, + | # which needs them to span a plane. Points that all fall on one line + | # leave Qhull without a simplex to start from and it raises instead. + | points = np.unique(np.column_stack((x, y)), axis=0) + | if np.linalg.matrix_rank(points - points.mean(axis=0)) < 2: + | yield {'html-content': self.render_error("The x and y values all fall on one line, so there is no area to contour.")} + | return + | | grid_size = $gridSizeLiteral | connGaps = True if '$connectGaps' == 'true' else False | @@ -137,6 +161,66 @@ class ContourPlotOpDesc extends PythonOperatorDescriptor { | yield {'html-content': html} |""".encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = + // render_error's continuation line keeps the runtime path's indentation — the + // HTML is triple-quoted, so those spaces reach the browser. + s"""import numpy as np + |from scipy.interpolate import griddata + | + |def render_error(error_msg): + | return '''

Contour plot is not available.

+ |

Reason is: {}

+ | '''.format(error_msg) + | + |def _write_error(message): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(message)) + | + |# A row missing any of the three has no point to contribute, and griddata + |# refuses a NaN coordinate outright. Bound to a name of its own: the same + |# frame can feed another branch of the plan, which must still see every row. + |chart_df = in1df.dropna(subset=[${pyStringLiteral(x)}, ${pyStringLiteral( + y + )}, ${pyStringLiteral( + z + )}]) + |if chart_df.empty: + | _write_error("Table should not have any empty/null values or fields.") + |else: + | x = chart_df[${pyStringLiteral(x)}].values + | y = chart_df[${pyStringLiteral(y)}].values + | z = chart_df[${pyStringLiteral(z)}].values + | + | # Cubic interpolation triangulates the points before it interpolates, which + | # needs them to span a plane. Points that all fall on one line leave Qhull + | # without a simplex to start from and it raises instead. + | points = np.unique(np.column_stack((x, y)), axis=0) + | # An else rather than an exit: this block shares a script with the rest of + | # the plan, which has to go on running after a plot that cannot be drawn. + | if np.linalg.matrix_rank(points - points.mean(axis=0)) < 2: + | _write_error("The x and y values all fall on one line, so there is no area to contour.") + | else: + | grid_size = ${gridSize.getOrElse(ContourPlotOpDesc.DefaultGridSize)} + | connGaps = True if ${pyStringLiteral(connectGaps.toString)} == "true" else False + | + | grid_x, grid_y = np.meshgrid(np.linspace(min(x), max(x), grid_size), np.linspace(min(y), max(y), grid_size)) + | grid_z = griddata((x, y), z, (grid_x, grid_y), method='cubic') + | + | fig = go.Figure(data=go.Contour( + | x=np.linspace(min(x), max(x), grid_size), + | y=np.linspace(min(y), max(y), grid_size), + | z=grid_z, + | connectgaps=connGaps, + | contours_coloring='${coloringMethod.getColoringMethod}', + | colorbar_title=${pyStringLiteral(z)} + | )) + | fig.update_layout(title='Contour Plot') + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Contour plot saved to " + outputJson + " and " + outputHtml)""".stripMargin } object ContourPlotOpDesc { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala index e855507e564..f27a4d74f91 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.dumbbellPlot import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -43,7 +47,7 @@ import scala.jdk.CollectionConverters.CollectionHasAsScala } } """) -class DumbbellPlotOpDesc extends PythonOperatorDescriptor { +class DumbbellPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "categoryColumnName", required = true) @JsonSchemaTitle("Category Column Name") @@ -196,4 +200,74 @@ class DumbbellPlotOpDesc extends PythonOperatorDescriptor { | |""".encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + // Typed-in values and column names become escaped Python literals; the runtime + // path splices them as decode expressions, which a standalone script cannot use. + val comparedLit = pyStringLiteral(comparedColumnName) + val measurementLit = pyStringLiteral(measurementColumnName) + val showLegendsOption = if (showLegends) "showlegend=True" else "showlegend=False" + // Python list literal of dot column names, matching addPlotlyDots(). + val dotColumnNames = + if (dots != null && dots.size() != 0) + dots.asScala.map(dot => pyStringLiteral(dot.dotValue)).mkString(", ") + else "" + s"""import plotly.graph_objects as go + | + |def render_error(error_msg): + | return '''

DumbbellPlot is not available.

+ |

Reason is: {}

+ | '''.format(error_msg) + | + |table = in1df + |_error = None + |if table.empty: + | _error = "input table is empty." + |else: + | table = table.dropna(subset=[$comparedLit, ${pyStringLiteral( + categoryColumnName + )}, $measurementLit]) + | if table.empty: + | _error = "input table has no rows with all of the configured columns filled in." + |if _error is not None: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(_error)) + |else: + | entityNames = list(table[$comparedLit].unique()) + | entityNames = sorted(entityNames, reverse=True) + | categoryValues = [${pyStringLiteral(dumbbellStartValue)}, ${pyStringLiteral( + dumbbellEndValue + )}] + | filtered_table = table[(table[$comparedLit].isin(entityNames)) & + | (table[${pyStringLiteral(categoryColumnName)}].isin(categoryValues))] + | fig = go.Figure() + | color = 'black' + | for entity in entityNames: + | entity_data = filtered_table[filtered_table[$comparedLit] == entity] + | fig.add_trace(go.Scatter(x=entity_data[$measurementLit], + | y=[entity] * len(entity_data), + | mode='lines', + | name=entity, + | line=dict(color=color))) + | fig.update_layout(xaxis_title=$measurementLit, + | yaxis_title=$comparedLit, + | yaxis=dict(categoryorder='array', categoryarray=entityNames), + | $showLegendsOption, + | margin=dict(l=0, r=0, b=60, t=0)) + | dotColumnNames = [$dotColumnNames] + | for dotColumn in dotColumnNames: + | for entity in entityNames: + | entity_dot_data = filtered_table[filtered_table[$comparedLit] == entity] + | x_values = entity_dot_data[dotColumn].values + | y_values = [entity] * len(x_values) + | fig.add_trace(go.Scatter(x=x_values, y=y_values, + | mode='markers', + | name=entity + ' ' + dotColumn, + | marker=dict(color='black', size=5))) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Dumbbell plot saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala index 84ca6082d24..adbb877e41f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala @@ -20,25 +20,57 @@ package org.apache.texera.amber.operator.visualization.filledAreaPlot import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} -import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class FilledAreaPlotOpDesc extends PythonOperatorDescriptor { +// `lineGroup` names the column the plot is split on, so it is required exactly when +// that switch is on: with the switch off nothing reads it, and with the switch on +// code generation asserts it and the run ends. Conditional rather than a plain +// required, so a freshly dropped operator is not flagged for a field it has no use +// for. +@JsonSchemaInject(json = """ +{ + "allOf": [ + { + "if": { + "properties": { + "facetColumn": { "const": true } + } + }, + "then": { + "required": ["lineGroup"] + } + } + ] +} +""") +class FilledAreaPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("X-axis Attribute") @JsonPropertyDescription("The attribute for your x-axis") @AutofillAttributeName @NotNull(message = "X-axis Attribute cannot be empty") + // Test-only steering, for x and the two fields below. The operator refuses line + // groups whose x sets are disjoint, so which three columns are named together + // decides whether a chart is drawn at all: every `node_src` group shares a + // `comp_a` value with the first. Left to the first column of each type, the + // verification plots a string against an index and renders an error page, which + // compares equal on both paths and asks nothing. + @SampleColumn("comp_a") var x: EncodableString = "" @JsonProperty(required = true) @@ -46,12 +78,14 @@ class FilledAreaPlotOpDesc extends PythonOperatorDescriptor { @JsonPropertyDescription("The attribute for your y-axis") @AutofillAttributeName @NotNull(message = "Y-axis Attribute cannot be empty") + @SampleColumn("score") var y: EncodableString = "" @JsonProperty(required = false) @JsonSchemaTitle("Line Group") @JsonPropertyDescription("The attribute for group of each line") @AutofillAttributeName + @SampleColumn("node_src") var lineGroup: EncodableString = "" @JsonProperty(required = false) @@ -172,4 +206,52 @@ class FilledAreaPlotOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val xLit = pyStringLiteral(x) + val yLit = pyStringLiteral(y) + val lineGroupLit = pyStringLiteral(lineGroup) + val colorArg = if (color.nonEmpty) s""", color=${pyStringLiteral(color)}""" else "" + val facetColumnArg = if (facetColumn) s""", facet_col=$lineGroupLit""" else "" + val lineGroupArg = if (lineGroup.nonEmpty) s""", line_group=$lineGroupLit""" else "" + val patternParam = + if (pattern.nonEmpty) s""", pattern_shape=${pyStringLiteral(pattern)}""" else "" + s"""columns = list(in1df.columns) + |error = "" + |if $xLit not in columns or $yLit not in columns: + | error = "missing attributes" + |elif $lineGroupLit != "": + | grouped = in1df.groupby($lineGroupLit) + | x_values = None + | tolerance = (len(grouped) // 100) * 5 + | count = 0 + | for _, group in grouped: + | if x_values == None: + | x_values = set(group[$xLit].unique()) + | elif set(group[$xLit].unique()).intersection(x_values): + | x_values = x_values.union(set(group[$xLit].unique())) + | elif not set(group[$xLit].unique()).intersection(x_values): + | count += 1 + | if count > tolerance: + | error = "X attributes not shared across groups" + | + |if error == "": + | fig = px.area(in1df, x=$xLit, y=$yLit$colorArg$facetColumnArg$lineGroupArg$patternParam) + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Filled area plot saved to " + outputHtml) + |elif error == "X attributes not shared across groups": + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write('''

Plot is not available, because:

+ |
  • X attribute is not shared across all line groups
    • + | ''') + |elif error == "missing attributes": + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write('''

    Plot is not available, because:

    + |
  • X or Y attribute does not exist
    • + | ''')""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala index ed5a85d2ad3..680e14509d8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.ganttChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -44,7 +48,7 @@ import javax.validation.constraints.NotNull } } """) -class GanttChartOpDesc extends PythonOperatorDescriptor { +class GanttChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "start", required = true) @JsonSchemaTitle("Start Datetime Column") @@ -145,4 +149,40 @@ class GanttChartOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val colorLit = pyStringLiteral(color) + val startLit = pyStringLiteral(start) + val finishLit = pyStringLiteral(finish) + val taskLit = pyStringLiteral(task) + val optionalFilter = + if (color.nonEmpty) s""" & (in1df[$colorLit].notnull())""" else "" + val colorSetting = if (color.nonEmpty) s""", color=$colorLit""" else "" + val patternSetting = + if (pattern.nonEmpty) s""", pattern_shape=${pyStringLiteral(pattern)}""" else "" + + s"""def render_error(error_msg): + | return '''

    Gantt Chart is not available.

    + |

    Reason: {}

    + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df[(in1df[$startLit].notnull()) & (in1df[$finishLit].notnull())$optionalFilter].copy() + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("One or more of your input columns have all missing values")) + | else: + | fig = px.timeline(table, x_start=$startLit, x_end=$finishLit, y=$taskLit$colorSetting$patternSetting) + | fig.update_yaxes(autorange='reversed') + | fig.update_layout(margin=dict(t=0, b=0, l=0, r=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Gantt chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala index 991ba362a10..99201de207f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala @@ -25,10 +25,11 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -50,7 +51,7 @@ import javax.validation.constraints.NotNull } } """) -class QuiverPlotOpDesc extends PythonOperatorDescriptor { +class QuiverPlotOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { //property panel variable: 4 requires: {x,y,u,v}, all columns should only contain numerical data @@ -72,6 +73,7 @@ class QuiverPlotOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("u") @JsonPropertyDescription("Column for the vector component in the x-direction") @AutofillAttributeName + @SampleColumn("uvec") @NotNull(message = "u cannot be empty") var u: EncodableString = "" @@ -79,6 +81,7 @@ class QuiverPlotOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("v") @JsonPropertyDescription("Column for the vector component in the y-direction") @AutofillAttributeName + @SampleColumn("log2fc") @NotNull(message = "v cannot be empty") var v: EncodableString = "" @@ -159,4 +162,49 @@ class QuiverPlotOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val xLit = pyStringLiteral(x) + val yLit = pyStringLiteral(y) + val uLit = pyStringLiteral(u) + val vLit = pyStringLiteral(v) + s"""import plotly.figure_factory as ff + | + |def render_error(error_msg): + | return '''

    Quiver Plot is not available.

    + |

    Reasons are: {}

    + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df + | required_columns = {$xLit, $yLit, $uLit, $vLit} + | if not required_columns.issubset(table.columns): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(f"Input table must contain columns: {', '.join(required_columns)}")) + | else: + | table = table.dropna() + | def type_check(value): + | return isinstance(value, (int, float)) + | if any(not table[col].apply(type_check).all() for col in required_columns): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write("Type error: All columns should only contain numerical data") + | else: + | try: + | fig = ff.create_quiver( + | table[$xLit], table[$yLit], + | table[$uLit], table[$vLit], + | scale=0.1 + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Quiver plot saved to " + outputHtml) + | except Exception as e: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(f"Plotly error: {str(e)}"))""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala index c86d9e3cc61..ea410612e53 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala @@ -26,12 +26,14 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull -class Scatter3dChartOpDesc extends PythonOperatorDescriptor { +class Scatter3dChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "x", required = true) @JsonSchemaTitle("X Column") @JsonPropertyDescription("Data column for the x-axis") @@ -126,4 +128,46 @@ class Scatter3dChartOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val xLit = pyStringLiteral(x) + val yLit = pyStringLiteral(y) + val zLit = pyStringLiteral(z) + s"""def render_error(error_msg): + | return '''

    Chart is not available.

    + |

    Reason is: {}

    + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df + | fig = go.Figure(data=[go.Scatter3d( + | x=table[$xLit], + | y=table[$yLit], + | z=table[$zLit], + | mode='markers', + | marker=dict( + | size=12, + | colorscale='Viridis', + | opacity=0.8 + | ) + | )]) + | fig.update_traces(marker=dict(size=5, opacity=0.8)) + | fig.update_layout( + | scene=dict( + | xaxis_title='X:' + $xLit, + | yaxis_title='Y:' + $yLit, + | zaxis_title='Z:' + $zLit + | ), + | margin=dict(t=0, b=0, l=0, r=0) + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Scatter3D chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala index 6e5e15f6329..f31ab2a901d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala @@ -25,8 +25,10 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.{NotBlank, NotNull} @@ -44,12 +46,13 @@ import javax.validation.constraints.{NotBlank, NotNull} } } """) -class TimeSeriesOpDesc extends PythonOperatorDescriptor { +class TimeSeriesOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "timeColumn", required = true) @JsonSchemaTitle("Time Column") @JsonPropertyDescription("The column containing time/date values (e.g., Date, Timestamp).") @AutofillAttributeName + @SampleColumn("start_ts") @NotNull(message = "Time Column cannot be empty") var timeColumn: EncodableString = "" @@ -65,12 +68,14 @@ class TimeSeriesOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Category Column") @JsonPropertyDescription("Optional - A categorical column to create separate lines.") @AutofillAttributeName + @SampleColumn("node_src") var CategoryColumn: EncodableString = "No Selection" @JsonProperty(value = "facetColumn", required = false, defaultValue = "No Selection") @JsonSchemaTitle("Facet Column") @JsonPropertyDescription("Optional - A column to create separate subplots.") @AutofillAttributeName + @SampleColumn("node_dst") var facetColumn: EncodableString = "No Selection" // Declared as a schema enum rather than named only in the description: the code @@ -159,4 +164,55 @@ class TimeSeriesOpDesc extends PythonOperatorDescriptor { | yield {'html-content': self.render_error(str(e))} |""".encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val dropnaCols = List(timeColumn, valueColumn) ++ + (if (CategoryColumn != "No Selection") Some(CategoryColumn) else None) ++ + (if (facetColumn != "No Selection") Some(facetColumn) else None) + val dropnaStr = dropnaCols.map(pyStringLiteral).mkString("[", ", ", "]") + val colorArg = + if (CategoryColumn != "No Selection") s""", color=${pyStringLiteral(CategoryColumn)}""" + else "" + val facetArg = + if (facetColumn != "No Selection") s""", facet_col=${pyStringLiteral(facetColumn)}""" else "" + val timeLit = pyStringLiteral(timeColumn) + val valueLit = pyStringLiteral(valueColumn) + val plotFunc = if (plotType == "area") "px.area" else "px.line" + val showSlider = if (showRangeSlider) "True" else "False" + + s"""def render_error(msg): + | return f"

    Time Series Plot is not available.

    Reason: {msg}

    " + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | try: + | table = in1df.copy() + | table[$timeLit] = pd.to_datetime(table[$timeLit], errors='coerce') + | table = table.dropna(subset=$dropnaStr).sort_values(by=$timeLit) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Table became empty after filtering.")) + | else: + | fig = $plotFunc(table, x=$timeLit, y=$valueLit$colorArg$facetArg) + | if $showSlider: + | fig.update_xaxes(rangeslider_visible=True) + | fig.update_layout( + | margin=dict(l=0, r=0, t=30, b=0), + | title=dict(text="Time Series Plot", x=0.5), + | xaxis_title=$timeLit, + | yaxis_title=$valueLit, + | template="plotly_white" + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Time series plot saved to " + outputHtml) + | except Exception as e: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(str(e)))""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/volcanoPlot/VolcanoPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/volcanoPlot/VolcanoPlotOpDesc.scala index 6fcc7085be1..9ceac44472a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/volcanoPlot/VolcanoPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/volcanoPlot/VolcanoPlotOpDesc.scala @@ -26,8 +26,10 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -41,7 +43,7 @@ import javax.validation.constraints.NotNull } } """) -class VolcanoPlotOpDesc extends PythonOperatorDescriptor { +class VolcanoPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("Effect Size (log2 Fold Change)") @@ -51,6 +53,7 @@ class VolcanoPlotOpDesc extends PythonOperatorDescriptor { "and is used for the x-axis of the volcano plot." ) @AutofillAttributeName + @SampleColumn("log2fc") @NotNull(message = "Effect Size (log2 Fold Change) cannot be empty") var effectColumn: EncodableString = "" @@ -62,6 +65,7 @@ class VolcanoPlotOpDesc extends PythonOperatorDescriptor { "plotted on the y-axis to indicate statistical significance." ) @AutofillAttributeName + @SampleColumn("pvalue") @NotNull(message = "P-Value Column cannot be empty") var pvalueColumn: EncodableString = "" @@ -125,4 +129,41 @@ class VolcanoPlotOpDesc extends PythonOperatorDescriptor { |""".encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val pvalueLit = pyStringLiteral(pvalueColumn) + val effectLit = pyStringLiteral(effectColumn) + s"""import numpy as np + | + |def render_error(msg): + | return f"

    Volcano Plot failed

    {msg}

    " + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |elif $pvalueLit not in in1df.columns or $effectLit not in in1df.columns: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Missing required columns in table.")) + |else: + | table = in1df[in1df[$pvalueLit] > 0].copy() + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No rows with valid p-values.")) + | else: + | table["-log10(pvalue)"] = -np.log10(table[$pvalueLit]) + | fig = px.scatter( + | table, + | x=$effectLit, + | y="-log10(pvalue)", + | hover_name=table.columns[0], + | color=$effectLit, + | color_continuous_scale="RdBu", + | title="Volcano Plot" + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Volcano plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDescSpec.scala index 3a1fa4ba9b7..263aff03dfb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDescSpec.scala @@ -114,4 +114,47 @@ class ContourPlotOpDescSpec extends AnyFlatSpec with Matchers { d.gridSize = Some(25) d.generatePythonCode() should include("grid_size = 25") } + + it should "drop the rows a blank leaves unusable, and answer an emptied table" in { + // The drop names all three columns: griddata refuses a NaN coordinate, and a + // NaN in z quietly interpolates to an all-NaN grid, which draws an empty chart. + val code = plottingCode + code should include("table = table.dropna(subset=[") + code should include("if table.empty:") + code should include("Table should not have any empty/null values or fields.") + } + + it should "answer a point set that cannot be triangulated" in { + // Guarded by a rank test rather than by catching the QhullError, so a genuine + // failure inside griddata still surfaces as one. + val code = plottingCode + code should include("np.linalg.matrix_rank(points - points.mean(axis=0)) < 2") + code should include("no area to contour") + } + + it should "define the render_error it now calls" in { + plottingCode should include("def render_error(self, error_msg)") + } + + "ContourPlotOpDesc.generateStandaloneCode" should + "leave the rest of the script running when it cannot contour" in { + // The exported block is one operator among many in a single script, so the + // guard branches around the plot rather than ending the process. + val code = configured.generateStandaloneCode() + code should include("no area to contour") + code should not include "SystemExit" + code should not include "sys.exit" + } + + /** The emitted code for a fully configured operator, which the guards above read. */ + private def plottingCode: String = configured.generatePythonCode() + + private def configured: ContourPlotOpDesc = { + val d = new ContourPlotOpDesc + d.x = "lon" + d.y = "lat" + d.z = "elev" + d.coloringMethod = ContourPlotColoringFunction.HEATMAP + d + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala index 8f93c3fd0f0..f92d56d2bc5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala @@ -22,6 +22,7 @@ package org.apache.texera.amber.operator.visualization.filledAreaPlot import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec @@ -30,6 +31,7 @@ import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit +import scala.jdk.CollectionConverters._ import scala.util.Try class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { @@ -370,4 +372,28 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc fp.facetColumn shouldBe true fp.pattern shouldBe "p" } + + // The assertion tested above is the last line of defence, reached only once the + // user has hit run. The schema is what refuses the configuration while it is + // still being written, and only under the switch: with it off nothing reads the + // field, so a freshly dropped operator is not flagged for it. + "FilledAreaPlotOpDesc JSON schema" should + "require the line group only when the plot is split by it" in { + val schema = + OperatorMetadataGenerator.generateOperatorJsonSchema(classOf[FilledAreaPlotOpDesc]) + + val baseRequired = schema.get("required").elements().asScala.map(_.asText()).toSet + baseRequired should contain("x") + baseRequired should not contain "lineGroup" + + val rule = schema + .get("allOf") + .elements() + .asScala + .find(node => node.has("if") && node.has("then")) + .getOrElse(fail("expected a conditional if/then rule in the FilledAreaPlot schema")) + rule.get("if").get("properties").get("facetColumn").get("const").asBoolean() shouldBe true + val thenRequired = rule.get("then").get("required").elements().asScala.map(_.asText()).toList + thenRequired should contain("lineGroup") + } } From 50f987e1d8c4438a430bc399ddba6a6d9e4bc8bb Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 18 Sep 2026 16:20:38 -0700 Subject: [PATCH 2/3] feat(visualization): answer the same input the operator answers The exported block for the filled area plot rounded its tolerance the other way round: five percent of the groups is (n * 5) // 100, not (n // 100) * 5, and at 150 groups that is 7 rather than 5. The script refused three tables the operator draws. The loop around it is now the operator's own, down to the break that ends it. The carpet plot dropped nulls before it looked for the columns to drop them from, so a column that is not in the table reached pandas as a KeyError and a value that is not a number reached astype as a ValueError. Both ended the whole exported run, where the operator answers each of them with a page saying what is wrong. The block now checks in the operator's order and writes the same pages. Both are checked by running the exported code rather than by reading it: the filled area plot at the boundary the tolerance sets, 7 disjoint groups of 150 drawn and 8 refused, and the carpet plot over a missing column, a column of words, an empty table, a table of nulls, and a table it can draw. Co-Authored-By: Claude Opus 5 (1M context) --- .../carpetPlot/CarpetPlotOpDesc.scala | 57 ++++--- .../filledAreaPlot/FilledAreaPlotOpDesc.scala | 14 +- .../carpetPlot/CarpetPlotOpDescSpec.scala | 155 ++++++++++++++++++ .../FilledAreaPlotOpDescSpec.scala | 78 +++++++++ 4 files changed, 278 insertions(+), 26 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala index 8f97d47d518..4c4b43447bd 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDesc.scala @@ -142,28 +142,45 @@ class CarpetPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCod val aLit = pyStringLiteral(a) val bLit = pyStringLiteral(b) val yLit = pyStringLiteral(y) - // Both empty cases write the page the operator yields rather than printing: - // a reason the reader can see is the whole output of a chart that cannot be - // drawn, and a run that writes nothing at all looks like a crash. - s"""table = in1df.dropna(subset=[$aLit, $bLit, $yLit]).copy() - |if in1df.empty: - | with open(outputHtml, "w", encoding="utf-8") as output: - | output.write("

    Input table is empty

    ") - |elif table.empty: + // Every case the operator answers with a page writes that page here rather + // than printing or raising: a reason the reader can see is the whole output + // of a chart that cannot be drawn, and a run that writes nothing at all + // looks like a crash. A column that is not there and a value that is not a + // number are checked in the operator's own order, before the drop that + // would otherwise raise on the missing name. + s"""def _write_page(html): | with open(outputHtml, "w", encoding="utf-8") as output: - | output.write("

    No valid rows after removing nulls

    ") + | output.write(html) + | + |missing = [column for column in [$aLit, $bLit, $yLit] if column not in in1df.columns] + |if in1df.empty: + | _write_page("

    Input table is empty

    ") + |elif missing: + | _write_page(f"

    Column '{missing[0]}' not found

    ") |else: - | table[$aLit] = table[$aLit].astype(float) - | table[$bLit] = table[$bLit].astype(float) - | table[$yLit] = table[$yLit].astype(float) - | fig = go.Figure(go.Carpet( - | a=table[$aLit], - | b=table[$bLit], - | y=table[$yLit] - | )) - | fig.write_json(outputJson) - | fig.write_html(outputHtml) - | print("Carpet plot saved to " + outputJson + " and " + outputHtml)""".stripMargin + | table = in1df.dropna(subset=[$aLit, $bLit, $yLit]).copy() + | if table.empty: + | _write_page("

    No valid rows after removing nulls

    ") + | else: + | try: + | table[$aLit] = table[$aLit].astype(float) + | table[$bLit] = table[$bLit].astype(float) + | table[$yLit] = table[$yLit].astype(float) + | except Exception as e: + | _write_page(f"

    Error converting input columns to numeric values: {str(e)}

    ") + | else: + | try: + | fig = go.Figure(go.Carpet( + | a=table[$aLit], + | b=table[$bLit], + | y=table[$yLit] + | )) + | except Exception as e: + | _write_page(f"

    Error generating carpet plot: {str(e)}

    ") + | else: + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Carpet plot saved to " + outputJson + " and " + outputHtml)""".stripMargin } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala index adbb877e41f..e35862aa12a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDesc.scala @@ -224,17 +224,19 @@ class FilledAreaPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandalon |elif $lineGroupLit != "": | grouped = in1df.groupby($lineGroupLit) | x_values = None - | tolerance = (len(grouped) // 100) * 5 + | tolerance = (len(grouped) * 5) // 100 | count = 0 | for _, group in grouped: - | if x_values == None: - | x_values = set(group[$xLit].unique()) - | elif set(group[$xLit].unique()).intersection(x_values): - | x_values = x_values.union(set(group[$xLit].unique())) - | elif not set(group[$xLit].unique()).intersection(x_values): + | group_x_values = set(group[$xLit].unique()) + | if x_values is None: + | x_values = group_x_values + | elif group_x_values.intersection(x_values): + | x_values = x_values.union(group_x_values) + | else: | count += 1 | if count > tolerance: | error = "X attributes not shared across groups" + | break | |if error == "": | fig = px.area(in1df, x=$xLit, y=$yLit$colorArg$facetColumnArg$lineGroupArg$patternParam) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala index 05976ef9c21..8003f73eace 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.visualization.carpetPlot +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants @@ -26,6 +27,11 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.util.Try + class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { "CarpetPlotOpDesc.operatorInfo" should @@ -63,6 +69,70 @@ class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { code should include("go.Carpet(") } + // The operator answers bad input with a page saying what is wrong. The script + // has to answer the same, rather than raising: a column that is not there used + // to reach the drop as a KeyError, and a value that is not a number reached + // astype as a ValueError, both of them ending the whole exported run. + "CarpetPlotOpDesc.generateStandaloneCode" should + "write the page the operator yields for input it cannot plot" in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandasAndPlotly(python)) { + cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") + } + + val d = new CarpetPlotOpDesc + d.a = "ax" + d.b = "bx" + d.y = "yx" + val moduleFile = Files.createTempFile("carpet_standalone_", ".py") + val driverFile = Files.createTempFile("carpet_driver_", ".py") + try { + Files.write(moduleFile, d.generateStandaloneCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, driverScript.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Carpet driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + val verdicts = "CASE (\\S+) (\\S+)".r + .findAllMatchIn(output) + .map(m => m.group(1) -> m.group(2)) + .toMap + verdicts shouldBe Map( + "missing" -> "COLUMN_NOT_FOUND", + "words" -> "NOT_NUMERIC", + "empty" -> "EMPTY", + "nulls" -> "NO_VALID_ROWS", + "good" -> "CHART" + ) + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // The generated code the operator emits for the native path, which the pages + // above are written to match. + "CarpetPlotOpDesc.generatePythonCode" should "answer bad input with a page of its own" in { + val d = new CarpetPlotOpDesc + d.a = "ax" + d.b = "bx" + d.y = "yx" + val code = d.generatePythonCode() + code should include("Column '{col}' not found") + code should include("Error converting input columns to numeric values") + } + "CarpetPlotOpDesc" should "round-trip a / b / y through the polymorphic base" in { val d = new CarpetPlotOpDesc d.a = "ax" @@ -75,4 +145,89 @@ class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { c.b shouldBe "bx" c.y shouldBe "yx" } + + // Runs the exported block over one frame per case and reports the page it + // wrote, so a branch that raises instead of answering shows up as a failure + // rather than as a missing file. + private val driverScript: String = + """import pathlib + |import sys + |import tempfile + | + |import pandas as pd + |import plotly.graph_objects as go + | + |source = pathlib.Path(sys.argv[1]).read_text() + | + |cases = { + | "good": pd.DataFrame({"ax": [1.0, 2.0], "bx": [1.0, 2.0], "yx": [3.0, 4.0]}), + | "missing": pd.DataFrame({"ax": [1.0], "yx": [3.0]}), + | "words": pd.DataFrame({"ax": ["one", "two"], "bx": [1.0, 2.0], "yx": [3.0, 4.0]}), + | "empty": pd.DataFrame({"ax": [], "bx": [], "yx": []}), + | "nulls": pd.DataFrame({"ax": [None, None], "bx": [1.0, 2.0], "yx": [3.0, 4.0]}), + |} + | + |for name, frame in cases.items(): + | directory = tempfile.mkdtemp() + | scope = { + | "in1df": frame, + | "outputHtml": directory + "/chart.html", + | "outputJson": directory + "/chart.json", + | "pd": pd, + | "go": go, + | } + | exec(compile(source, "standalone", "exec"), scope) + | # An answer page is the whole file and opens with the heading, where a + | # chart is a plotly document that happens to contain any wording. + | page = pathlib.Path(scope["outputHtml"]).read_text().strip() + | if not page.startswith("

    "): + | verdict = "CHART" + | elif page.startswith("

    Column '"): + | verdict = "COLUMN_NOT_FOUND" + | elif page.startswith("

    Error converting input columns"): + | verdict = "NOT_NUMERIC" + | elif page.startswith("

    Input table is empty"): + | verdict = "EMPTY" + | elif page.startswith("

    No valid rows"): + | verdict = "NO_VALID_ROWS" + | else: + | verdict = "OTHER_PAGE" + | print("CASE %s %s" % (name, verdict)) + |""".stripMargin + + // Python executable resolution, following FilledAreaPlotOpDescSpec: + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = { + val configOpt = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + configOpt + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + } + + def isRunnable(exe: String): Boolean = { + val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) + pTry.toOption.exists { p => + val finished = p.waitFor(5, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) + } + + private def canImportPandasAndPlotly(python: String): Boolean = { + val pTry = Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ) + pTry.toOption.exists { p => + val finished = p.waitFor(60, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala index f92d56d2bc5..fe80dade087 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala @@ -396,4 +396,82 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc val thenRequired = rule.get("then").get("required").elements().asScala.map(_.asText()).toList thenRequired should contain("lineGroup") } + + // The tolerance is five percent of the groups, and the operator multiplies + // before it divides. At 150 groups that is 7, where dividing first gives 5, so + // the script would have refused three tables the operator draws. Run at the + // boundary rather than asserted: 7 disjoint groups are within the tolerance + // and 8 are past it. + "FilledAreaPlotOpDesc.generateStandaloneCode" should + "refuse the same tables the operator refuses" in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandasAndPlotly(python)) { + cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") + } + + opDesc.x = "x" + opDesc.y = "y" + opDesc.lineGroup = "grp" + val moduleFile = Files.createTempFile("filled_area_standalone_", ".py") + val driverFile = Files.createTempFile("filled_area_boundary_", ".py") + try { + Files.write(moduleFile, opDesc.generateStandaloneCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, boundaryDriverScript.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Boundary driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + val verdicts = "CASE (\\S+) (\\S+)".r + .findAllMatchIn(output) + .map(m => m.group(1) -> m.group(2)) + .toMap + verdicts shouldBe Map("7" -> "CHART", "8" -> "FALLBACK") + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // Runs the exported block over 150 line groups, as many of them disjoint from + // the first as the case names, and reports the page it wrote. + private val boundaryDriverScript: String = + """import pathlib + |import sys + |import tempfile + | + |import pandas as pd + |import plotly.express as px + | + |source = pathlib.Path(sys.argv[1]).read_text() + | + |for disjoint in (7, 8): + | rows = [] + | for index in range(150): + | value = "z%03d" % index if 1 <= index <= disjoint else "a" + | rows.append({"x": value, "y": 1.0, "grp": "g%03d" % index}) + | rows.append({"x": value, "y": 2.0, "grp": "g%03d" % index}) + | directory = tempfile.mkdtemp() + | scope = { + | "in1df": pd.DataFrame(rows), + | "outputHtml": directory + "/chart.html", + | "outputJson": directory + "/chart.json", + | "pd": pd, + | "px": px, + | } + | exec(compile(source, "standalone", "exec"), scope) + | page = pathlib.Path(scope["outputHtml"]).read_text() + | verdict = "FALLBACK" if "not shared across all line groups" in page else "CHART" + | print("CASE %d %s" % (disjoint, verdict)) + |""".stripMargin } From 7e662c0a82d57bab43b494286d99fb8ecded2da1 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 26 Sep 2026 15:27:07 -0700 Subject: [PATCH 3/3] test(operator): run the cases that execute the exported code in amber-integration Co-Authored-By: Claude Opus 5.5 (1M context) --- .../visualization/carpetPlot/CarpetPlotOpDescSpec.scala | 6 +++++- .../filledAreaPlot/FilledAreaPlotOpDescSpec.scala | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala index 8003f73eace..4386f601ba4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala @@ -23,7 +23,9 @@ import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,6 +36,8 @@ import scala.util.Try class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + "CarpetPlotOpDesc.operatorInfo" should "advertise the name and Scientific visualization group" in { val info = (new CarpetPlotOpDesc).operatorInfo @@ -74,7 +78,7 @@ class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { // to reach the drop as a KeyError, and a value that is not a number reached // astype as a ValueError, both of them ending the whole exported run. "CarpetPlotOpDesc.generateStandaloneCode" should - "write the page the operator yields for input it cannot plot" in { + "write the page the operator yields for input it cannot plot" taggedAs NeedsPythonPackages in { val python = resolvePythonExecutable().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala index fe80dade087..f80e3d0a131 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala @@ -23,8 +23,10 @@ import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,6 +38,8 @@ import scala.util.Try class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + var opDesc: FilledAreaPlotOpDesc = _ before { @@ -306,7 +310,7 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc | print("CASE %s %s" % (cid, verdict)) |""".stripMargin - it should "enforce the five-percent tolerance at runtime boundaries" in { + it should "enforce the five-percent tolerance at runtime boundaries" taggedAs NeedsPythonPackages in { val python = resolvePythonExecutable().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) @@ -403,7 +407,7 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc // boundary rather than asserted: 7 disjoint groups are within the tolerance // and 8 are past it. "FilledAreaPlotOpDesc.generateStandaloneCode" should - "refuse the same tables the operator refuses" in { + "refuse the same tables the operator refuses" taggedAs NeedsPythonPackages in { val python = resolvePythonExecutable().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") )