diff --git a/datafusion/expr/src/window_frame.rs b/datafusion/expr/src/window_frame.rs index a61d9d689ae7a..d9db35875ab1e 100644 --- a/datafusion/expr/src/window_frame.rs +++ b/datafusion/expr/src/window_frame.rs @@ -282,7 +282,13 @@ impl WindowFrame { /// Returns whether the window frame is "free range"; i.e. its start/end /// bounds are UNBOUNDED or CURRENT ROW. - fn free_range(&self) -> bool { + /// + /// This inspects only the bounds, not the frame units, so it returns `true` + /// for ROWS and GROUPS frames with such bounds as well. For RANGE frames, + /// such bounds are located by comparing order key values, whereas a finite + /// offset bound (e.g. `5 PRECEDING`) has to be computed arithmetically from + /// the current row's order key value. + pub fn free_range(&self) -> bool { (self.start_bound.is_unbounded() || self.start_bound == WindowFrameBound::CurrentRow) && (self.end_bound.is_unbounded() diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index d11c3e7435fde..a89d5d71122d2 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -1068,6 +1068,7 @@ fn coerce_frame_bound( fn extract_window_frame_target_type(col_type: &DataType) -> Result { if col_type.is_numeric() || col_type.is_string() + || col_type.is_binary() || col_type.is_null() || matches!( col_type, @@ -1102,7 +1103,20 @@ fn coerce_window_frame( .map(|s| s.expr.get_type(schema)) .transpose()?; if let Some(col_type) = current_types { - extract_window_frame_target_type(&col_type)? + let target_type = extract_window_frame_target_type(&col_type)?; + // A finite offset bound (e.g. `5 PRECEDING`) is computed as + // `current_value ± offset`, so it is only meaningful for target + // types that support arithmetic. Strings, binaries, booleans + // and lists are orderable -- which is all a free range frame + // needs -- but have no such arithmetic. + let supports_offset_arithmetic = + target_type.is_numeric() || is_interval(&target_type); + if !supports_offset_arithmetic && !window_frame.free_range() { + return plan_err!( + "RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type {target_type}" + ); + } + target_type } else { return internal_err!("ORDER BY column cannot be empty"); } diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index 6374cbf4f4b80..8a2f3b11f30c7 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -6874,3 +6874,138 @@ ORDER BY id 2 NULL 3 3 3 3 3 NULL NULL NULL NULL NULL 4 7 7 7 7 7 + +# RANGE window frame over a binary ORDER BY key. The default frame for an +# ORDER BY without an explicit frame is RANGE BETWEEN UNBOUNDED PRECEDING AND +# CURRENT ROW, which used to fail with +# "Internal error: Cannot run range queries on datatype: Binary". +# Binary is orderable, so peer/range comparison is well defined just like Utf8. +query ?I +SELECT x, COUNT(*) OVER (ORDER BY x) +FROM (VALUES (arrow_cast('a', 'Binary')), + (arrow_cast('b', 'Binary')), + (arrow_cast('b', 'Binary')), + (arrow_cast('c', 'Binary'))) t(x) +ORDER BY x +---- +61 1 +62 3 +62 3 +63 4 + +query ?I +SELECT x, COUNT(*) OVER (ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) +FROM (VALUES (arrow_cast('a', 'LargeBinary')), + (arrow_cast('b', 'LargeBinary')), + (arrow_cast('b', 'LargeBinary'))) t(x) +ORDER BY x +---- +61 3 +62 2 +62 2 + +query ?I +SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) +FROM (VALUES (arrow_cast('a', 'BinaryView')), + (arrow_cast('b', 'BinaryView')), + (arrow_cast('b', 'BinaryView'))) t(x) +ORDER BY x +---- +61 3 +62 2 +62 2 + +query ?I +SELECT x, COUNT(*) OVER (ORDER BY x) +FROM (VALUES (arrow_cast(arrow_cast('a', 'Binary'), 'FixedSizeBinary(1)')), + (arrow_cast(arrow_cast('b', 'Binary'), 'FixedSizeBinary(1)'))) t(x) +ORDER BY x +---- +61 1 +62 2 + +# A binary ORDER BY key nested in a dictionary resolves through the same arm. +query ?I +SELECT x, COUNT(*) OVER (ORDER BY x) +FROM (VALUES (arrow_cast(arrow_cast('a', 'Binary'), 'Dictionary(Int32, Binary)')), + (arrow_cast(arrow_cast('b', 'Binary'), 'Dictionary(Int32, Binary)')), + (arrow_cast(arrow_cast('b', 'Binary'), 'Dictionary(Int32, Binary)'))) t(x) +ORDER BY x +---- +61 1 +62 3 +62 3 + +# A non-aggregate window function over a binary ORDER BY key. +query ?I +SELECT x, RANK() OVER (ORDER BY x) +FROM (VALUES (arrow_cast('a', 'Binary')), + (arrow_cast('b', 'Binary')), + (arrow_cast('b', 'Binary'))) t(x) +ORDER BY x +---- +61 1 +62 2 +62 2 + +# Unsupported RANGE ORDER BY types still propagate the type-coercion error. +query error DataFusion error: type_coercion\ncaused by\nInternal error: Cannot run range queries on datatype: Struct\("c0": Int64\)\. +SELECT COUNT(*) OVER (ORDER BY x) +FROM (VALUES (struct(1))) t(x) + +# An order key only needs to be comparable for a free range frame, whose bounds +# are all UNBOUNDED or CURRENT ROW. A finite offset such as `1 PRECEDING` instead +# has to be computed as `current_value - 1`, so it additionally requires +# arithmetic on the order key type. Reject such frames during planning for every +# order key type that has no arithmetic, rather than silently widening the frame +# to the whole partition. +query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Binary +SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM (VALUES (arrow_cast('a', 'Binary'))) t(x) + +query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Utf8 +SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM (VALUES ('a')) t(x) + +# The end bound is rejected on the same grounds as the start bound. +query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Boolean +SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) +FROM (VALUES (true)) t(x) + +query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type List\(Int64\) +SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM (VALUES ([1,2])) t(x) + +query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Null +SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM (VALUES (NULL)) t(x) + +# Numeric order keys keep their finite offsets, decimals included. +query RI +SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM (VALUES (arrow_cast(1.5, 'Decimal128(10, 2)')), + (arrow_cast(2.0, 'Decimal128(10, 2)'))) t(x) +ORDER BY x +---- +1.5 1 +2 2 + +# Free range frames over those same types keep working, so the rejection above is +# narrowed to the offset form rather than to the order key type as a whole. +query TI +SELECT x, COUNT(*) OVER (ORDER BY x) +FROM (VALUES ('a'), ('b'), ('b')) t(x) +ORDER BY x +---- +a 1 +b 3 +b 3 + +query BI +SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) +FROM (VALUES (false), (true), (true)) t(x) +ORDER BY x +---- +false 3 +true 2 +true 2 diff --git a/dev/update_function_docs.sh b/dev/update_function_docs.sh index 04266b8cbc0d3..2ead1253ac9c0 100755 --- a/dev/update_function_docs.sh +++ b/dev/update_function_docs.sh @@ -317,6 +317,8 @@ where **offset** is an non-negative integer. RANGE and GROUPS modes require an ORDER BY clause (with RANGE the ORDER BY must specify exactly one column). +In RANGE mode an **offset** is measured in ORDER BY values rather than in rows, so the bound is computed by adding it to or subtracting it from the current row's ORDER BY value. That restricts `offset PRECEDING` and `offset FOLLOWING` to ORDER BY types supporting such arithmetic, namely the numeric, date, and timestamp types. Other orderable types, such as strings and binaries, can still be used with `UNBOUNDED PRECEDING`, `CURRENT ROW` and `UNBOUNDED FOLLOWING`, which are located by comparing ORDER BY values. + ## Filter clause for aggregate window functions Aggregate window functions support the SQL `FILTER (WHERE ...)` clause to include only rows that satisfy the predicate from the window frame in the aggregation. diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index d64f287ea0b52..508250775718b 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -1291,3 +1291,47 @@ let table_opts = TableParquetOptions::try_from(&proto_table_opts)?; ``` See [issue #24019](https://github.com/apache/datafusion/issues/24019) for details. + +### RANGE window frames with offsets now reject ORDER BY types without arithmetic + +A `RANGE` frame offset such as `1 PRECEDING` is measured in `ORDER BY` values +rather than in rows, so the frame bound is computed as `current_value - 1`. That +requires arithmetic on the `ORDER BY` type. For types that have no such +arithmetic -- `Utf8`, `Binary`, `Boolean`, `List` and `Null` -- such frames are +now rejected when the query is planned: + +```sql +SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM (VALUES ('a'), ('b'), ('c')) t(x) +ORDER BY x; +-- Error during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Utf8 +``` + +What this replaces depends on the `ORDER BY` type. Over `Utf8`, `Boolean` and +`List` keys such queries planned successfully, and in DataFusion 54 the failing +frame-bound arithmetic was silently treated as an overflow +([#22140](https://github.com/apache/datafusion/pull/22140)): the bound collapsed +to the partition edge, and the query returned results computed over that wrongly +widened frame instead of an offset window. DataFusion 53 and earlier failed +during execution with an arithmetic error. Rows whose order key value was `NULL` +never reached the arithmetic, so queries where every order key value was `NULL` +returned correct results throughout; those also now fail at planning. Over a +`Null`-typed key the offset itself failed to coerce, erroring at planning time +with `Cast error: Casting from Utf8 to Null not supported`, and binary keys were +rejected for every `RANGE` frame with the internal error quoted below -- for +those two, only the error message changes. + +Such an `ORDER BY` type is still usable with the `UNBOUNDED PRECEDING`, +`CURRENT ROW` and `UNBOUNDED FOLLOWING` bounds, which are located by comparing +`ORDER BY` values rather than by computing them. Binary order keys additionally +gained support for those bounds in this release, having previously failed with +`Internal error: Cannot run range queries on datatype: Binary`. + +**Who is affected:** + +- Queries using `offset PRECEDING` or `offset FOLLOWING` in a `RANGE` frame over + a non-numeric, non-temporal `ORDER BY` key. To count rows rather than compare + values, use a `ROWS` frame; to include everything up to the partition edge, + use `UNBOUNDED PRECEDING` or `UNBOUNDED FOLLOWING`. + +See [issue #24327](https://github.com/apache/datafusion/issues/24327) for details. diff --git a/docs/source/user-guide/sql/window_functions.md b/docs/source/user-guide/sql/window_functions.md index 2c8050ce1f9ca..6496da11029c4 100644 --- a/docs/source/user-guide/sql/window_functions.md +++ b/docs/source/user-guide/sql/window_functions.md @@ -145,6 +145,8 @@ where **offset** is an non-negative integer. RANGE and GROUPS modes require an ORDER BY clause (with RANGE the ORDER BY must specify exactly one column). +In RANGE mode an **offset** is measured in ORDER BY values rather than in rows, so the bound is computed by adding it to or subtracting it from the current row's ORDER BY value. That restricts `offset PRECEDING` and `offset FOLLOWING` to ORDER BY types supporting such arithmetic, namely the numeric, date, and timestamp types. Other orderable types, such as strings and binaries, can still be used with `UNBOUNDED PRECEDING`, `CURRENT ROW` and `UNBOUNDED FOLLOWING`, which are located by comparing ORDER BY values. + ## Filter clause for aggregate window functions Aggregate window functions support the SQL `FILTER (WHERE ...)` clause to include only rows that satisfy the predicate from the window frame in the aggregation.