diff --git a/docs/source/contributor-guide/expression-audits/predicate_funcs.md b/docs/source/contributor-guide/expression-audits/predicate_funcs.md index 0731abf7f7..8aa9fe219c 100644 --- a/docs/source/contributor-guide/expression-audits/predicate_funcs.md +++ b/docs/source/contributor-guide/expression-audits/predicate_funcs.md @@ -94,7 +94,7 @@ ## ilike - Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8. -- Spark 3.5.8 (audited 2026-05-27): baseline. `ILike(left, right, escapeChar) extends RuntimeReplaceable`; the analyzer rewrites to `Like(Lower(left), Lower(right), escapeChar)`. Comet handles via `CometLike` and `CometLower` (case-conversion path, gated by `spark.comet.caseConversion.enabled=false` by default). +- Spark 3.5.8 (audited 2026-07-07): baseline. `ILike(left, right, escapeChar) extends RuntimeReplaceable`; the analyzer rewrites to `Like(Lower(left), Lower(right), escapeChar)`. Comet handles via `CometLike` and `CometLower` (the case-conversion path is Spark-compatible by default via codegen dispatch, with a faster native path available behind `spark.comet.expression.Lower.allowIncompatible=true`). - Spark 4.0.1 (audited 2026-05-27): identical to 3.5.8. - Spark 4.1.1 (audited 2026-05-27): identical to 4.0.1. diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 926fc6bc95..d0468c0d8d 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -779,14 +779,6 @@ object CometConf extends ShimCometConf { .toSequence .createWithDefault(Seq("Range,InMemoryTableScan,RDDScan,OneRowRelation")) - val COMET_CASE_CONVERSION_ENABLED: ConfigEntry[Boolean] = - conf("spark.comet.caseConversion.enabled") - .category(CATEGORY_EXEC) - .doc("Java uses locale-specific rules when converting strings to upper or lower case and " + - "Rust does not, so we disable upper and lower by default.") - .booleanConf - .createWithDefault(false) - val COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK: ConfigEntry[Boolean] = conf("spark.comet.scan.unsignedSmallIntSafetyCheck") .category(CATEGORY_SCAN) diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala index 00b339a195..efe8d9278e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/strings.scala +++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala @@ -56,18 +56,16 @@ class CometCaseConversionBase[T <: Expression](function: String) override def getIncompatibleReasons(): Seq[String] = Seq("Results can vary depending on locale and character set") - override def nativeOptInConfigKeyOverride: Option[String] = - Some(CometConf.COMET_CASE_CONVERSION_ENABLED.key) - override def getSupportLevel(expr: T): SupportLevel = - if (!CometConf.COMET_CASE_CONVERSION_ENABLED.get()) { - Compatible(nativeOptIn = Some(NativeOptIn(CometConf.COMET_CASE_CONVERSION_ENABLED.key))) + if (!CometConf.isExprAllowIncompat(getExprConfigName(expr))) { + Compatible(nativeOptIn = + Some(NativeOptIn(CometConf.getExprAllowIncompatConfigKey(getExprConfigName(expr))))) } else { Compatible() } override def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = { - if (CometConf.COMET_CASE_CONVERSION_ENABLED.get()) { + if (CometConf.isExprAllowIncompat(getExprConfigName(expr))) { // Native scalar function: faster but does not match Spark for locale-specific characters // (e.g. Turkish dotted/dotless I). Opt-in. super.convert(expr, inputs, binding) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt index d285d92d0f..befec50933 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt @@ -9,7 +9,7 @@ CometNativeColumnarToRow : +- CometExchange : +- CometHashAggregate : +- CometProject - : +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] + : +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] : :- CometProject : : +- CometBroadcastHashJoin : : :- CometProject @@ -51,7 +51,7 @@ CometNativeColumnarToRow +- CometExchange +- CometHashAggregate +- CometProject - +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] + +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] :- CometProject : +- CometBroadcastHashJoin : :- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt index d285d92d0f..befec50933 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt @@ -9,7 +9,7 @@ CometNativeColumnarToRow : +- CometExchange : +- CometHashAggregate : +- CometProject - : +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] + : +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] : :- CometProject : : +- CometBroadcastHashJoin : : :- CometProject @@ -51,7 +51,7 @@ CometNativeColumnarToRow +- CometExchange +- CometHashAggregate +- CometProject - +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] + +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] :- CometProject : +- CometBroadcastHashJoin : :- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt index 0a03e76aa8..2a36bd81c2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt @@ -11,7 +11,7 @@ CometNativeColumnarToRow : +- CometExchange : +- CometHashAggregate : +- CometProject - : +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] + : +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] : :- CometProject : : +- CometBroadcastHashJoin : : :- CometProject @@ -53,7 +53,7 @@ CometNativeColumnarToRow +- CometExchange +- CometHashAggregate +- CometProject - +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] + +- CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)] :- CometProject : +- CometBroadcastHashJoin : :- CometProject diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 33bd58c55e..5d2bc2c08a 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -3378,14 +3378,14 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("upper shows native opt-in via caseConversion config key") { + test("upper shows native opt-in via per-expression allowIncompatible config key") { withTable("t_upper") { spark.sql("create table t_upper(s string) using parquet") spark.sql("insert into t_upper values ('abc'), ('xyz')") val df = spark.sql("select upper(s) as u from t_upper") val explain = new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan) assert(explain.contains("native implementation of Upper")) - assert(explain.contains("spark.comet.caseConversion.enabled")) + assert(explain.contains("spark.comet.expression.Upper.allowIncompatible")) } }