diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_integer_object_nulls.csv b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_integer_object_nulls.csv new file mode 100644 index 000000000000..e28e175f4107 --- /dev/null +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_integer_object_nulls.csv @@ -0,0 +1,140 @@ +test case pyarrow array integer_object_nulls=False integer_object_nulls=True spark pandas_options +int8:standard [0, 1, -1, 127, -128]@int8 [0, 1, -1, 127, -128]@Series[int8] [0, 1, -1, 127, -128]@Series[int8] [0, 1, -1, 127, -128]@Series[int8] +int8:nullable [0, 1, None]@int8 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +int8:empty []@int8 []@Series[int8] []@Series[int8] []@Series[int8] +int16:standard [0, 1, -1, 32767, -32768]@int16 [0, 1, -1, 32767, -32768]@Series[int16] [0, 1, -1, 32767, -32768]@Series[int16] [0, 1, -1, 32767, -32768]@Series[int16] +int16:nullable [0, 1, None]@int16 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +int16:empty []@int16 []@Series[int16] []@Series[int16] []@Series[int16] +int32:standard [0, 1, -1, 2147483647, -2147483648]@int32 [0, 1, -1, 2147483647, -2147483648]@Series[int32] [0, 1, -1, 2147483647, -2147483648]@Series[int32] [0, 1, -1, 2147483647, -2147483648]@Series[int32] +int32:nullable [0, 1, None]@int32 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +int32:empty []@int32 []@Series[int32] []@Series[int32] []@Series[int32] +int64:standard [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64] [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64] [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64] +int64:nullable [0, 1, None]@int64 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +int64:empty []@int64 []@Series[int64] []@Series[int64] []@Series[int64] +uint8:standard [0, 1, 255]@uint8 [0, 1, 255]@Series[uint8] [0, 1, 255]@Series[uint8] [0, 1, 255]@Series[uint8] +uint8:nullable [0, 1, None]@uint8 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +uint8:empty []@uint8 []@Series[uint8] []@Series[uint8] []@Series[uint8] +uint16:standard [0, 1, 65535]@uint16 [0, 1, 65535]@Series[uint16] [0, 1, 65535]@Series[uint16] [0, 1, 65535]@Series[uint16] +uint16:nullable [0, 1, None]@uint16 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +uint16:empty []@uint16 []@Series[uint16] []@Series[uint16] []@Series[uint16] +uint32:standard [0, 1, 4294967295]@uint32 [0, 1, 4294967295]@Series[uint32] [0, 1, 4294967295]@Series[uint32] [0, 1, 4294967295]@Series[uint32] +uint32:nullable [0, 1, None]@uint32 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +uint32:empty []@uint32 []@Series[uint32] []@Series[uint32] []@Series[uint32] +uint64:standard [0, 1, 18446744073709551615]@uint64 [0, 1, 18446744073709551615]@Series[uint64] [0, 1, 18446744073709551615]@Series[uint64] [0, 1, 18446744073709551615]@Series[uint64] +uint64:nullable [0, 1, None]@uint64 [0.0, 1.0, nan]@Series[float64] [0, 1, None]@Series[object] [0, 1, None]@Series[object] +uint64:empty []@uint64 []@Series[uint64] []@Series[uint64] []@Series[uint64] +float32:standard [0.0, 1.5, -1.5]@float32 [0.0, 1.5, -1.5]@Series[float32] [0.0, 1.5, -1.5]@Series[float32] [0.0, 1.5, -1.5]@Series[float32] +float32:nullable [0.0, 1.5, None]@float32 [0.0, 1.5, nan]@Series[float32] [0.0, 1.5, nan]@Series[float32] [0.0, 1.5, nan]@Series[float32] +float32:empty []@float32 []@Series[float32] []@Series[float32] []@Series[float32] +float64:standard [0.0, 1.5, -1.5]@float64 [0.0, 1.5, -1.5]@Series[float64] [0.0, 1.5, -1.5]@Series[float64] [0.0, 1.5, -1.5]@Series[float64] +float64:nullable [0.0, 1.5, None]@float64 [0.0, 1.5, nan]@Series[float64] [0.0, 1.5, nan]@Series[float64] [0.0, 1.5, nan]@Series[float64] +float64:special [nan, inf, -inf]@float64 [nan, inf, -inf]@Series[float64] [nan, inf, -inf]@Series[float64] [nan, inf, -inf]@Series[float64] +float64:empty []@float64 []@Series[float64] []@Series[float64] []@Series[float64] +bool:standard [True, False, True]@bool [True, False, True]@Series[bool] [True, False, True]@Series[bool] [True, False, True]@Series[bool] +bool:nullable [True, False, None]@bool [True, False, None]@Series[object] [True, False, None]@Series[object] [True, False, None]@Series[object] +bool:empty []@bool []@Series[bool] []@Series[bool] []@Series[bool] +string:standard [hello, world, ]@string ['hello', 'world', '']@Series[object] ['hello', 'world', '']@Series[object] ['hello', 'world', '']@Series[object] +string:nullable [hello, None, world]@string ['hello', None, 'world']@Series[object] ['hello', None, 'world']@Series[object] ['hello', None, 'world']@Series[object] +string:empty []@string []@Series[object] []@Series[object] []@Series[object] +large_string:standard [hello, world]@large_string ['hello', 'world']@Series[object] ['hello', 'world']@Series[object] ['hello', 'world']@Series[object] +large_string:nullable [hello, None]@large_string ['hello', None]@Series[object] ['hello', None]@Series[object] ['hello', None]@Series[object] +large_string:empty []@large_string []@Series[object] []@Series[object] []@Series[object] +binary:standard [b'hello', b'world']@binary [b'hello', b'world']@Series[object] [b'hello', b'world']@Series[object] [b'hello', b'world']@Series[object] +binary:nullable [b'hello', None]@binary [b'hello', None]@Series[object] [b'hello', None]@Series[object] [b'hello', None]@Series[object] +binary:empty []@binary []@Series[object] []@Series[object] []@Series[object] +large_binary:standard [b'hello', b'world']@large_binary [b'hello', b'world']@Series[object] [b'hello', b'world']@Series[object] [b'hello', b'world']@Series[object] +large_binary:nullable [b'hello', None]@large_binary [b'hello', None]@Series[object] [b'hello', None]@Series[object] [b'hello', None]@Series[object] +large_binary:empty []@large_binary []@Series[object] []@Series[object] []@Series[object] +decimal128:standard [1.23, 4.56, -7.89]@decimal128(5, 2) [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[object] [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[object] [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[object] +decimal128:nullable [1.23, None, 4.56]@decimal128(5, 2) [Decimal('1.23'), None, Decimal('4.56')]@Series[object] [Decimal('1.23'), None, Decimal('4.56')]@Series[object] [Decimal('1.23'), None, Decimal('4.56')]@Series[object] +decimal128:empty []@decimal128(5, 2) []@Series[object] []@Series[object] []@Series[object] +date32:standard [2024-01-01, 2024-06-15]@date32[day] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] +date32:nullable [2024-01-01, None]@date32[day] [datetime.date(2024, 1, 1), None]@Series[object] [datetime.date(2024, 1, 1), None]@Series[object] [datetime.date(2024, 1, 1), None]@Series[object] +date32:empty []@date32[day] []@Series[object] []@Series[object] []@Series[object] +date64:standard [2024-01-01, 2024-06-15]@date64[ms] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] +date64:nullable [2024-01-01, None]@date64[ms] [datetime.date(2024, 1, 1), None]@Series[object] [datetime.date(2024, 1, 1), None]@Series[object] [datetime.date(2024, 1, 1), None]@Series[object] +date64:empty []@date64[ms] []@Series[object] []@Series[object] []@Series[object] +timestamp[s]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[s] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[s]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[s]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] +timestamp[s]:nullable [2024-01-01 12:00:00, None]@timestamp[s] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[s]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[s]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] +timestamp[s]:empty []@timestamp[s] []@Series[datetime64[s]] []@Series[datetime64[s]] []@Series[datetime64[ns]] +timestamp[ms]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ms] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ms]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ms]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] +timestamp[ms]:nullable [2024-01-01 12:00:00, None]@timestamp[ms] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ms]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ms]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] +timestamp[ms]:empty []@timestamp[ms] []@Series[datetime64[ms]] []@Series[datetime64[ms]] []@Series[datetime64[ns]] +timestamp[us]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[us] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[us]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[us]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] +timestamp[us]:nullable [2024-01-01 12:00:00, None]@timestamp[us] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[us]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[us]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] +timestamp[us]:empty []@timestamp[us] []@Series[datetime64[us]] []@Series[datetime64[us]] []@Series[datetime64[ns]] +timestamp[ns]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ns] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] +timestamp[ns]:nullable [2024-01-01 12:00:00, None]@timestamp[ns] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] +timestamp[ns]:empty []@timestamp[ns] []@Series[datetime64[ns]] []@Series[datetime64[ns]] []@Series[datetime64[ns]] +timestamp[us,tz=UTC]:standard [2024-01-01 12:00:00+00:00, 2024-06-15 18:30:00+00:00]@timestamp[us, tz=UTC] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[datetime64[us, UTC]] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[datetime64[us, UTC]] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[datetime64[ns, UTC]] +timestamp[us,tz=UTC]:nullable [2024-01-01 12:00:00+00:00, None]@timestamp[us, tz=UTC] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), NaT]@Series[datetime64[us, UTC]] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), NaT]@Series[datetime64[us, UTC]] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), NaT]@Series[datetime64[ns, UTC]] +timestamp[us,tz=UTC]:empty []@timestamp[us, tz=UTC] []@Series[datetime64[us, UTC]] []@Series[datetime64[us, UTC]] []@Series[datetime64[ns, UTC]] +duration[s]:standard [1 day, 0:00:00, 2:30:00]@duration[s] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[s]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[s]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] +duration[s]:nullable [1 day, 0:00:00, None]@duration[s] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[s]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[s]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] +duration[s]:empty []@duration[s] []@Series[timedelta64[s]] []@Series[timedelta64[s]] []@Series[timedelta64[ns]] +duration[ms]:standard [1 day, 0:00:00, 2:30:00]@duration[ms] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ms]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ms]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] +duration[ms]:nullable [1 day, 0:00:00, None]@duration[ms] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ms]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ms]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] +duration[ms]:empty []@duration[ms] []@Series[timedelta64[ms]] []@Series[timedelta64[ms]] []@Series[timedelta64[ns]] +duration[us]:standard [1 day, 0:00:00, 2:30:00]@duration[us] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[us]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[us]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] +duration[us]:nullable [1 day, 0:00:00, None]@duration[us] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[us]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[us]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] +duration[us]:empty []@duration[us] []@Series[timedelta64[us]] []@Series[timedelta64[us]] []@Series[timedelta64[ns]] +duration[ns]:standard [1 days 00:00:00, 0 days 02:30:00]@duration[ns] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] +duration[ns]:nullable [1 days 00:00:00, None]@duration[ns] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] +duration[ns]:empty []@duration[ns] []@Series[timedelta64[ns]] []@Series[timedelta64[ns]] []@Series[timedelta64[ns]] +time32[s]:standard [12:30:00, 18:45:30]@time32[s] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] +time32[s]:nullable [12:30:00, None]@time32[s] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] +time32[s]:empty []@time32[s] []@Series[object] []@Series[object] []@Series[object] +time32[ms]:standard [12:30:00, 18:45:30]@time32[ms] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] +time32[ms]:nullable [12:30:00, None]@time32[ms] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] +time32[ms]:empty []@time32[ms] []@Series[object] []@Series[object] []@Series[object] +time64[us]:standard [12:30:00, 18:45:30]@time64[us] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] +time64[us]:nullable [12:30:00, None]@time64[us] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] +time64[us]:empty []@time64[us] []@Series[object] []@Series[object] []@Series[object] +time64[ns]:standard [12:30:00, 18:45:30]@time64[ns] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] +time64[ns]:nullable [12:30:00, None]@time64[ns] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] [datetime.time(12, 30), None]@Series[object] +time64[ns]:empty []@time64[ns] []@Series[object] []@Series[object] []@Series[object] +null:standard [None, None, None]@null [None, None, None]@Series[object] [None, None, None]@Series[object] [None, None, None]@Series[object] +null:empty []@null []@Series[object] []@Series[object] []@Series[object] +list:standard [[1, 2], [3, 4, 5]]@list [array([1, 2]), array([3, 4, 5])]@Series[object] [array([1, 2]), array([3, 4, 5])]@Series[object] [array([1, 2]), array([3, 4, 5])]@Series[object] +list:nullable [[1, 2], None, [3]]@list [array([1, 2]), None, array([3])]@Series[object] [array([1, 2]), None, array([3])]@Series[object] [array([1, 2]), None, array([3])]@Series[object] +list:empty []@list []@Series[object] []@Series[object] []@Series[object] +list:standard [['a', 'b'], ['c']]@list [array(['a', 'b'], dtype=object), array(['c'], dtype=object)]@Series[object] [array(['a', 'b'], dtype=object), array(['c'], dtype=object)]@Series[object] [array(['a', 'b'], dtype=object), array(['c'], dtype=object)]@Series[object] +large_list:standard [[1, 2], [3, 4]]@large_list [array([1, 2]), array([3, 4])]@Series[object] [array([1, 2]), array([3, 4])]@Series[object] [array([1, 2]), array([3, 4])]@Series[object] +large_list:empty []@large_list []@Series[object] []@Series[object] []@Series[object] +fixed_size_list[3]:standard [[1, 2, 3], [4, 5, 6]]@fixed_size_list[3] [array([1, 2, 3]), array([4, 5, 6])]@Series[object] [array([1, 2, 3]), array([4, 5, 6])]@Series[object] [array([1, 2, 3]), array([4, 5, 6])]@Series[object] +fixed_size_list[3]:empty []@fixed_size_list[3] []@Series[object] []@Series[object] []@Series[object] +struct:standard [[('x', 1), ('y', 'a')], [('x', 2), ('y', 'b')]]@struct [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[object] [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[object] [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[object] +struct:nullable [[('x', 1), ('y', 'a')], None]@struct [{'x': 1, 'y': 'a'}, None]@Series[object] [{'x': 1, 'y': 'a'}, None]@Series[object] [{'x': 1, 'y': 'a'}, None]@Series[object] +struct:empty []@struct []@Series[object] []@Series[object] []@Series[object] +map:standard [[('a', 1), ('b', 2)], [('c', 3)]]@map [[('a', 1), ('b', 2)], [('c', 3)]]@Series[object] [[('a', 1), ('b', 2)], [('c', 3)]]@Series[object] [[('a', 1), ('b', 2)], [('c', 3)]]@Series[object] +map:empty []@map []@Series[object] []@Series[object] []@Series[object] +list>:standard [[[1, 2], [3]], [[4, 5, 6]]]@list> [array([array([1, 2]), array([3])], dtype=object), array([array([4, 5, 6])], dtype=object)]@Series[object] [array([array([1, 2]), array([3])], dtype=object), array([array([4, 5, 6])], dtype=object)]@Series[object] [array([array([1, 2]), array([3])], dtype=object), array([array([4, 5, 6])], dtype=object)]@Series[object] +list:standard [[{'x': 1}, {'x': 2}], [{'x': 3}]]@list> [array([{'x': 1}, {'x': 2}], dtype=object), array([{'x': 3}], dtype=object)]@Series[object] [array([{'x': 1}, {'x': 2}], dtype=object), array([{'x': 3}], dtype=object)]@Series[object] [array([{'x': 1}, {'x': 2}], dtype=object), array([{'x': 3}], dtype=object)]@Series[object] +list>:standard [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@list> [array([list([('a', 1)]), list([('b', 2)])], dtype=object), array([list([('c', 3)])], dtype=object)]@Series[object] [array([list([('a', 1)]), list([('b', 2)])], dtype=object), array([list([('c', 3)])], dtype=object)]@Series[object] [array([list([('a', 1)]), list([('b', 2)])], dtype=object), array([list([('c', 3)])], dtype=object)]@Series[object] +struct:standard [[('outer', {'inner': 1})], [('outer', {'inner': 2})]]@struct> [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[object] [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[object] [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[object] +struct>:standard [[('items', [1, 2, 3])], [('items', [4, 5])]]@struct> [{'items': array([1, 2, 3])}, {'items': array([4, 5])}]@Series[object] [{'items': array([1, 2, 3])}, {'items': array([4, 5])}]@Series[object] [{'items': array([1, 2, 3])}, {'items': array([4, 5])}]@Series[object] +struct>:standard [[('mapping', [('a', 1)])], [('mapping', [('b', 2)])]]@struct> [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[object] [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[object] [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[object] +map>:standard [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@map> [[('a', array([1, 2])), ('b', array([3]))], [('c', array([4, 5, 6]))]]@Series[object] [[('a', array([1, 2])), ('b', array([3]))], [('c', array([4, 5, 6]))]]@Series[object] [[('a', array([1, 2])), ('b', array([3]))], [('c', array([4, 5, 6]))]]@Series[object] +map:standard [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@map> [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[object] [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[object] [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[object] +map>:standard [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@map> [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[object] [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[object] [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[object] +dictionary:standard [a, b, a, b]@dictionary ['a', 'b', 'a', 'b']@Series[category] ['a', 'b', 'a', 'b']@Series[category] ['a', 'b', 'a', 'b']@Series[category] +dictionary:nullable [a, b, None, a]@dictionary ['a', 'b', nan, 'a']@Series[category] ['a', 'b', nan, 'a']@Series[category] ['a', 'b', nan, 'a']@Series[category] +dictionary:empty []@dictionary []@Series[category] []@Series[category] []@Series[category] +int8:extremes-nullable [127, -128, None]@int8 [127.0, -128.0, nan]@Series[float64] [127, -128, None]@Series[object] [127, -128, None]@Series[object] +int16:extremes-nullable [32767, -32768, None]@int16 [32767.0, -32768.0, nan]@Series[float64] [32767, -32768, None]@Series[object] [32767, -32768, None]@Series[object] +int32:extremes-nullable [2147483647, -2147483648, None]@int32 [2147483647.0, -2147483648.0, nan]@Series[float64] [2147483647, -2147483648, None]@Series[object] [2147483647, -2147483648, None]@Series[object] +int64:extremes-nullable [9223372036854775807, -9223372036854775808, None]@int64 [9.223372036854776e+18, -9.223372036854776e+18, nan]@Series[float64] [9223372036854775807, -9223372036854775808, None]@Series[object] [9223372036854775807, -9223372036854775808, None]@Series[object] +uint8:extremes-nullable [255, 0, None]@uint8 [255.0, 0.0, nan]@Series[float64] [255, 0, None]@Series[object] [255, 0, None]@Series[object] +uint16:extremes-nullable [65535, 0, None]@uint16 [65535.0, 0.0, nan]@Series[float64] [65535, 0, None]@Series[object] [65535, 0, None]@Series[object] +uint32:extremes-nullable [4294967295, 0, None]@uint32 [4294967295.0, 0.0, nan]@Series[float64] [4294967295, 0, None]@Series[object] [4294967295, 0, None]@Series[object] +uint64:extremes-nullable [18446744073709551615, 0, None]@uint64 [1.8446744073709552e+19, 0.0, nan]@Series[float64] [18446744073709551615, 0, None]@Series[object] [18446744073709551615, 0, None]@Series[object] +int64:all-null [None, None]@int64 [nan, nan]@Series[float64] [None, None]@Series[object] [None, None]@Series[object] +int64:multi-chunk-nullable [1, None, 2]@int64 [1.0, nan, 2.0]@Series[float64] [1, None, 2]@Series[object] [1, None, 2]@Series[object] +list:null-element [[1, None], [2, 3]]@list [array([ 1., nan]), array([2., 3.])]@Series[object] [array([1, None], dtype=object), array([2, 3], dtype=object)]@Series[object] [array([1, None], dtype=object), array([2, 3], dtype=object)]@Series[object] +list:null-element-extreme [[9223372036854775807, None]]@list [array([9.22337204e+18, nan])]@Series[object] [array([9223372036854775807, None], dtype=object)]@Series[object] [array([9223372036854775807, None], dtype=object)]@Series[object] +large_list:null-element [[1, None]]@large_list [array([ 1., nan])]@Series[object] [array([1, None], dtype=object)]@Series[object] [array([1, None], dtype=object)]@Series[object] +fixed_size_list[3]:null-element [[1, None, 3]]@fixed_size_list[3] [array([ 1., nan, 3.])]@Series[object] [array([1, None, 3], dtype=object)]@Series[object] [array([1, None, 3], dtype=object)]@Series[object] +list>:null-element [[[1, None], [2]]]@list> [array([array([ 1., nan]), array([2.])], dtype=object)]@Series[object] [array([array([1, None], dtype=object), array([2], dtype=object)], dtype=object)]@Series[object] [array([array([1, None], dtype=object), array([2], dtype=object)], dtype=object)]@Series[object] +struct:null-int-field [[('x', 1), ('y', 'a')], [('x', None), ('y', 'b')]]@struct [{'x': 1.0, 'y': 'a'}, {'x': None, 'y': 'b'}]@Series[object] [{'x': 1, 'y': 'a'}, {'x': None, 'y': 'b'}]@Series[object] [{'x': 1, 'y': 'a'}, {'x': None, 'y': 'b'}]@Series[object] +map:null-value [[('a', 1), ('b', None)]]@map [[('a', 1.0), ('b', None)]]@Series[object] [[('a', 1), ('b', None)]]@Series[object] [[('a', 1), ('b', None)]]@Series[object] +dictionary:nullable [1, None, 1]@dictionary [1, nan, 1]@Series[category] [1, nan, 1]@Series[category] [1, nan, 1]@Series[category] diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_integer_object_nulls.md b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_integer_object_nulls.md new file mode 100644 index 000000000000..9545c071aa08 --- /dev/null +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_integer_object_nulls.md @@ -0,0 +1,141 @@ +| test case | pyarrow array | integer_object_nulls=False | integer_object_nulls=True | spark pandas_options | +|----------------------------------------|-----------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| +| int8:standard | [0, 1, -1, 127, -128]@int8 | [0, 1, -1, 127, -128]@Series[int8] | [0, 1, -1, 127, -128]@Series[int8] | [0, 1, -1, 127, -128]@Series[int8] | +| int8:nullable | [0, 1, None]@int8 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| int8:empty | []@int8 | []@Series[int8] | []@Series[int8] | []@Series[int8] | +| int16:standard | [0, 1, -1, 32767, -32768]@int16 | [0, 1, -1, 32767, -32768]@Series[int16] | [0, 1, -1, 32767, -32768]@Series[int16] | [0, 1, -1, 32767, -32768]@Series[int16] | +| int16:nullable | [0, 1, None]@int16 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| int16:empty | []@int16 | []@Series[int16] | []@Series[int16] | []@Series[int16] | +| int32:standard | [0, 1, -1, 2147483647, -2147483648]@int32 | [0, 1, -1, 2147483647, -2147483648]@Series[int32] | [0, 1, -1, 2147483647, -2147483648]@Series[int32] | [0, 1, -1, 2147483647, -2147483648]@Series[int32] | +| int32:nullable | [0, 1, None]@int32 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| int32:empty | []@int32 | []@Series[int32] | []@Series[int32] | []@Series[int32] | +| int64:standard | [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 | [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64] | [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64] | [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64] | +| int64:nullable | [0, 1, None]@int64 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| int64:empty | []@int64 | []@Series[int64] | []@Series[int64] | []@Series[int64] | +| uint8:standard | [0, 1, 255]@uint8 | [0, 1, 255]@Series[uint8] | [0, 1, 255]@Series[uint8] | [0, 1, 255]@Series[uint8] | +| uint8:nullable | [0, 1, None]@uint8 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| uint8:empty | []@uint8 | []@Series[uint8] | []@Series[uint8] | []@Series[uint8] | +| uint16:standard | [0, 1, 65535]@uint16 | [0, 1, 65535]@Series[uint16] | [0, 1, 65535]@Series[uint16] | [0, 1, 65535]@Series[uint16] | +| uint16:nullable | [0, 1, None]@uint16 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| uint16:empty | []@uint16 | []@Series[uint16] | []@Series[uint16] | []@Series[uint16] | +| uint32:standard | [0, 1, 4294967295]@uint32 | [0, 1, 4294967295]@Series[uint32] | [0, 1, 4294967295]@Series[uint32] | [0, 1, 4294967295]@Series[uint32] | +| uint32:nullable | [0, 1, None]@uint32 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| uint32:empty | []@uint32 | []@Series[uint32] | []@Series[uint32] | []@Series[uint32] | +| uint64:standard | [0, 1, 18446744073709551615]@uint64 | [0, 1, 18446744073709551615]@Series[uint64] | [0, 1, 18446744073709551615]@Series[uint64] | [0, 1, 18446744073709551615]@Series[uint64] | +| uint64:nullable | [0, 1, None]@uint64 | [0.0, 1.0, nan]@Series[float64] | [0, 1, None]@Series[object] | [0, 1, None]@Series[object] | +| uint64:empty | []@uint64 | []@Series[uint64] | []@Series[uint64] | []@Series[uint64] | +| float32:standard | [0.0, 1.5, -1.5]@float32 | [0.0, 1.5, -1.5]@Series[float32] | [0.0, 1.5, -1.5]@Series[float32] | [0.0, 1.5, -1.5]@Series[float32] | +| float32:nullable | [0.0, 1.5, None]@float32 | [0.0, 1.5, nan]@Series[float32] | [0.0, 1.5, nan]@Series[float32] | [0.0, 1.5, nan]@Series[float32] | +| float32:empty | []@float32 | []@Series[float32] | []@Series[float32] | []@Series[float32] | +| float64:standard | [0.0, 1.5, -1.5]@float64 | [0.0, 1.5, -1.5]@Series[float64] | [0.0, 1.5, -1.5]@Series[float64] | [0.0, 1.5, -1.5]@Series[float64] | +| float64:nullable | [0.0, 1.5, None]@float64 | [0.0, 1.5, nan]@Series[float64] | [0.0, 1.5, nan]@Series[float64] | [0.0, 1.5, nan]@Series[float64] | +| float64:special | [nan, inf, -inf]@float64 | [nan, inf, -inf]@Series[float64] | [nan, inf, -inf]@Series[float64] | [nan, inf, -inf]@Series[float64] | +| float64:empty | []@float64 | []@Series[float64] | []@Series[float64] | []@Series[float64] | +| bool:standard | [True, False, True]@bool | [True, False, True]@Series[bool] | [True, False, True]@Series[bool] | [True, False, True]@Series[bool] | +| bool:nullable | [True, False, None]@bool | [True, False, None]@Series[object] | [True, False, None]@Series[object] | [True, False, None]@Series[object] | +| bool:empty | []@bool | []@Series[bool] | []@Series[bool] | []@Series[bool] | +| string:standard | [hello, world, ]@string | ['hello', 'world', '']@Series[object] | ['hello', 'world', '']@Series[object] | ['hello', 'world', '']@Series[object] | +| string:nullable | [hello, None, world]@string | ['hello', None, 'world']@Series[object] | ['hello', None, 'world']@Series[object] | ['hello', None, 'world']@Series[object] | +| string:empty | []@string | []@Series[object] | []@Series[object] | []@Series[object] | +| large_string:standard | [hello, world]@large_string | ['hello', 'world']@Series[object] | ['hello', 'world']@Series[object] | ['hello', 'world']@Series[object] | +| large_string:nullable | [hello, None]@large_string | ['hello', None]@Series[object] | ['hello', None]@Series[object] | ['hello', None]@Series[object] | +| large_string:empty | []@large_string | []@Series[object] | []@Series[object] | []@Series[object] | +| binary:standard | [b'hello', b'world']@binary | [b'hello', b'world']@Series[object] | [b'hello', b'world']@Series[object] | [b'hello', b'world']@Series[object] | +| binary:nullable | [b'hello', None]@binary | [b'hello', None]@Series[object] | [b'hello', None]@Series[object] | [b'hello', None]@Series[object] | +| binary:empty | []@binary | []@Series[object] | []@Series[object] | []@Series[object] | +| large_binary:standard | [b'hello', b'world']@large_binary | [b'hello', b'world']@Series[object] | [b'hello', b'world']@Series[object] | [b'hello', b'world']@Series[object] | +| large_binary:nullable | [b'hello', None]@large_binary | [b'hello', None]@Series[object] | [b'hello', None]@Series[object] | [b'hello', None]@Series[object] | +| large_binary:empty | []@large_binary | []@Series[object] | []@Series[object] | []@Series[object] | +| decimal128:standard | [1.23, 4.56, -7.89]@decimal128(5, 2) | [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[object] | [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[object] | [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[object] | +| decimal128:nullable | [1.23, None, 4.56]@decimal128(5, 2) | [Decimal('1.23'), None, Decimal('4.56')]@Series[object] | [Decimal('1.23'), None, Decimal('4.56')]@Series[object] | [Decimal('1.23'), None, Decimal('4.56')]@Series[object] | +| decimal128:empty | []@decimal128(5, 2) | []@Series[object] | []@Series[object] | []@Series[object] | +| date32:standard | [2024-01-01, 2024-06-15]@date32[day] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] | +| date32:nullable | [2024-01-01, None]@date32[day] | [datetime.date(2024, 1, 1), None]@Series[object] | [datetime.date(2024, 1, 1), None]@Series[object] | [datetime.date(2024, 1, 1), None]@Series[object] | +| date32:empty | []@date32[day] | []@Series[object] | []@Series[object] | []@Series[object] | +| date64:standard | [2024-01-01, 2024-06-15]@date64[ms] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[object] | +| date64:nullable | [2024-01-01, None]@date64[ms] | [datetime.date(2024, 1, 1), None]@Series[object] | [datetime.date(2024, 1, 1), None]@Series[object] | [datetime.date(2024, 1, 1), None]@Series[object] | +| date64:empty | []@date64[ms] | []@Series[object] | []@Series[object] | []@Series[object] | +| timestamp[s]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[s] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[s]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[s]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] | +| timestamp[s]:nullable | [2024-01-01 12:00:00, None]@timestamp[s] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[s]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[s]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] | +| timestamp[s]:empty | []@timestamp[s] | []@Series[datetime64[s]] | []@Series[datetime64[s]] | []@Series[datetime64[ns]] | +| timestamp[ms]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ms] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ms]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ms]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] | +| timestamp[ms]:nullable | [2024-01-01 12:00:00, None]@timestamp[ms] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ms]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ms]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] | +| timestamp[ms]:empty | []@timestamp[ms] | []@Series[datetime64[ms]] | []@Series[datetime64[ms]] | []@Series[datetime64[ns]] | +| timestamp[us]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[us] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[us]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[us]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] | +| timestamp[us]:nullable | [2024-01-01 12:00:00, None]@timestamp[us] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[us]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[us]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] | +| timestamp[us]:empty | []@timestamp[us] | []@Series[datetime64[us]] | []@Series[datetime64[us]] | []@Series[datetime64[ns]] | +| timestamp[ns]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ns] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[datetime64[ns]] | +| timestamp[ns]:nullable | [2024-01-01 12:00:00, None]@timestamp[ns] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] | [Timestamp('2024-01-01 12:00:00'), NaT]@Series[datetime64[ns]] | +| timestamp[ns]:empty | []@timestamp[ns] | []@Series[datetime64[ns]] | []@Series[datetime64[ns]] | []@Series[datetime64[ns]] | +| timestamp[us,tz=UTC]:standard | [2024-01-01 12:00:00+00:00, 2024-06-15 18:30:00+00:00]@timestamp[us, tz=UTC] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[datetime64[us, UTC]] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[datetime64[us, UTC]] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[datetime64[ns, UTC]] | +| timestamp[us,tz=UTC]:nullable | [2024-01-01 12:00:00+00:00, None]@timestamp[us, tz=UTC] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), NaT]@Series[datetime64[us, UTC]] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), NaT]@Series[datetime64[us, UTC]] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), NaT]@Series[datetime64[ns, UTC]] | +| timestamp[us,tz=UTC]:empty | []@timestamp[us, tz=UTC] | []@Series[datetime64[us, UTC]] | []@Series[datetime64[us, UTC]] | []@Series[datetime64[ns, UTC]] | +| duration[s]:standard | [1 day, 0:00:00, 2:30:00]@duration[s] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[s]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[s]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] | +| duration[s]:nullable | [1 day, 0:00:00, None]@duration[s] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[s]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[s]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] | +| duration[s]:empty | []@duration[s] | []@Series[timedelta64[s]] | []@Series[timedelta64[s]] | []@Series[timedelta64[ns]] | +| duration[ms]:standard | [1 day, 0:00:00, 2:30:00]@duration[ms] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ms]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ms]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] | +| duration[ms]:nullable | [1 day, 0:00:00, None]@duration[ms] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ms]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ms]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] | +| duration[ms]:empty | []@duration[ms] | []@Series[timedelta64[ms]] | []@Series[timedelta64[ms]] | []@Series[timedelta64[ns]] | +| duration[us]:standard | [1 day, 0:00:00, 2:30:00]@duration[us] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[us]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[us]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] | +| duration[us]:nullable | [1 day, 0:00:00, None]@duration[us] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[us]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[us]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] | +| duration[us]:empty | []@duration[us] | []@Series[timedelta64[us]] | []@Series[timedelta64[us]] | []@Series[timedelta64[ns]] | +| duration[ns]:standard | [1 days 00:00:00, 0 days 02:30:00]@duration[ns] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[timedelta64[ns]] | +| duration[ns]:nullable | [1 days 00:00:00, None]@duration[ns] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] | [Timedelta('1 days 00:00:00'), NaT]@Series[timedelta64[ns]] | +| duration[ns]:empty | []@duration[ns] | []@Series[timedelta64[ns]] | []@Series[timedelta64[ns]] | []@Series[timedelta64[ns]] | +| time32[s]:standard | [12:30:00, 18:45:30]@time32[s] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | +| time32[s]:nullable | [12:30:00, None]@time32[s] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | +| time32[s]:empty | []@time32[s] | []@Series[object] | []@Series[object] | []@Series[object] | +| time32[ms]:standard | [12:30:00, 18:45:30]@time32[ms] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | +| time32[ms]:nullable | [12:30:00, None]@time32[ms] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | +| time32[ms]:empty | []@time32[ms] | []@Series[object] | []@Series[object] | []@Series[object] | +| time64[us]:standard | [12:30:00, 18:45:30]@time64[us] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | +| time64[us]:nullable | [12:30:00, None]@time64[us] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | +| time64[us]:empty | []@time64[us] | []@Series[object] | []@Series[object] | []@Series[object] | +| time64[ns]:standard | [12:30:00, 18:45:30]@time64[ns] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[object] | +| time64[ns]:nullable | [12:30:00, None]@time64[ns] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | [datetime.time(12, 30), None]@Series[object] | +| time64[ns]:empty | []@time64[ns] | []@Series[object] | []@Series[object] | []@Series[object] | +| null:standard | [None, None, None]@null | [None, None, None]@Series[object] | [None, None, None]@Series[object] | [None, None, None]@Series[object] | +| null:empty | []@null | []@Series[object] | []@Series[object] | []@Series[object] | +| list:standard | [[1, 2], [3, 4, 5]]@list | [array([1, 2]), array([3, 4, 5])]@Series[object] | [array([1, 2]), array([3, 4, 5])]@Series[object] | [array([1, 2]), array([3, 4, 5])]@Series[object] | +| list:nullable | [[1, 2], None, [3]]@list | [array([1, 2]), None, array([3])]@Series[object] | [array([1, 2]), None, array([3])]@Series[object] | [array([1, 2]), None, array([3])]@Series[object] | +| list:empty | []@list | []@Series[object] | []@Series[object] | []@Series[object] | +| list:standard | [['a', 'b'], ['c']]@list | [array(['a', 'b'], dtype=object), array(['c'], dtype=object)]@Series[object] | [array(['a', 'b'], dtype=object), array(['c'], dtype=object)]@Series[object] | [array(['a', 'b'], dtype=object), array(['c'], dtype=object)]@Series[object] | +| large_list:standard | [[1, 2], [3, 4]]@large_list | [array([1, 2]), array([3, 4])]@Series[object] | [array([1, 2]), array([3, 4])]@Series[object] | [array([1, 2]), array([3, 4])]@Series[object] | +| large_list:empty | []@large_list | []@Series[object] | []@Series[object] | []@Series[object] | +| fixed_size_list[3]:standard | [[1, 2, 3], [4, 5, 6]]@fixed_size_list[3] | [array([1, 2, 3]), array([4, 5, 6])]@Series[object] | [array([1, 2, 3]), array([4, 5, 6])]@Series[object] | [array([1, 2, 3]), array([4, 5, 6])]@Series[object] | +| fixed_size_list[3]:empty | []@fixed_size_list[3] | []@Series[object] | []@Series[object] | []@Series[object] | +| struct:standard | [[('x', 1), ('y', 'a')], [('x', 2), ('y', 'b')]]@struct | [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[object] | [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[object] | [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[object] | +| struct:nullable | [[('x', 1), ('y', 'a')], None]@struct | [{'x': 1, 'y': 'a'}, None]@Series[object] | [{'x': 1, 'y': 'a'}, None]@Series[object] | [{'x': 1, 'y': 'a'}, None]@Series[object] | +| struct:empty | []@struct | []@Series[object] | []@Series[object] | []@Series[object] | +| map:standard | [[('a', 1), ('b', 2)], [('c', 3)]]@map | [[('a', 1), ('b', 2)], [('c', 3)]]@Series[object] | [[('a', 1), ('b', 2)], [('c', 3)]]@Series[object] | [[('a', 1), ('b', 2)], [('c', 3)]]@Series[object] | +| map:empty | []@map | []@Series[object] | []@Series[object] | []@Series[object] | +| list>:standard | [[[1, 2], [3]], [[4, 5, 6]]]@list> | [array([array([1, 2]), array([3])], dtype=object), array([array([4, 5, 6])], dtype=object)]@Series[object] | [array([array([1, 2]), array([3])], dtype=object), array([array([4, 5, 6])], dtype=object)]@Series[object] | [array([array([1, 2]), array([3])], dtype=object), array([array([4, 5, 6])], dtype=object)]@Series[object] | +| list:standard | [[{'x': 1}, {'x': 2}], [{'x': 3}]]@list> | [array([{'x': 1}, {'x': 2}], dtype=object), array([{'x': 3}], dtype=object)]@Series[object] | [array([{'x': 1}, {'x': 2}], dtype=object), array([{'x': 3}], dtype=object)]@Series[object] | [array([{'x': 1}, {'x': 2}], dtype=object), array([{'x': 3}], dtype=object)]@Series[object] | +| list>:standard | [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@list> | [array([list([('a', 1)]), list([('b', 2)])], dtype=object), array([list([('c', 3)])], dtype=object)]@Series[object] | [array([list([('a', 1)]), list([('b', 2)])], dtype=object), array([list([('c', 3)])], dtype=object)]@Series[object] | [array([list([('a', 1)]), list([('b', 2)])], dtype=object), array([list([('c', 3)])], dtype=object)]@Series[object] | +| struct:standard | [[('outer', {'inner': 1})], [('outer', {'inner': 2})]]@struct> | [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[object] | [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[object] | [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[object] | +| struct>:standard | [[('items', [1, 2, 3])], [('items', [4, 5])]]@struct> | [{'items': array([1, 2, 3])}, {'items': array([4, 5])}]@Series[object] | [{'items': array([1, 2, 3])}, {'items': array([4, 5])}]@Series[object] | [{'items': array([1, 2, 3])}, {'items': array([4, 5])}]@Series[object] | +| struct>:standard | [[('mapping', [('a', 1)])], [('mapping', [('b', 2)])]]@struct> | [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[object] | [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[object] | [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[object] | +| map>:standard | [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@map> | [[('a', array([1, 2])), ('b', array([3]))], [('c', array([4, 5, 6]))]]@Series[object] | [[('a', array([1, 2])), ('b', array([3]))], [('c', array([4, 5, 6]))]]@Series[object] | [[('a', array([1, 2])), ('b', array([3]))], [('c', array([4, 5, 6]))]]@Series[object] | +| map:standard | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@map> | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[object] | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[object] | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[object] | +| map>:standard | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@map> | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[object] | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[object] | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[object] | +| dictionary:standard | [a, b, a, b]@dictionary | ['a', 'b', 'a', 'b']@Series[category] | ['a', 'b', 'a', 'b']@Series[category] | ['a', 'b', 'a', 'b']@Series[category] | +| dictionary:nullable | [a, b, None, a]@dictionary | ['a', 'b', nan, 'a']@Series[category] | ['a', 'b', nan, 'a']@Series[category] | ['a', 'b', nan, 'a']@Series[category] | +| dictionary:empty | []@dictionary | []@Series[category] | []@Series[category] | []@Series[category] | +| int8:extremes-nullable | [127, -128, None]@int8 | [127.0, -128.0, nan]@Series[float64] | [127, -128, None]@Series[object] | [127, -128, None]@Series[object] | +| int16:extremes-nullable | [32767, -32768, None]@int16 | [32767.0, -32768.0, nan]@Series[float64] | [32767, -32768, None]@Series[object] | [32767, -32768, None]@Series[object] | +| int32:extremes-nullable | [2147483647, -2147483648, None]@int32 | [2147483647.0, -2147483648.0, nan]@Series[float64] | [2147483647, -2147483648, None]@Series[object] | [2147483647, -2147483648, None]@Series[object] | +| int64:extremes-nullable | [9223372036854775807, -9223372036854775808, None]@int64 | [9.223372036854776e+18, -9.223372036854776e+18, nan]@Series[float64] | [9223372036854775807, -9223372036854775808, None]@Series[object] | [9223372036854775807, -9223372036854775808, None]@Series[object] | +| uint8:extremes-nullable | [255, 0, None]@uint8 | [255.0, 0.0, nan]@Series[float64] | [255, 0, None]@Series[object] | [255, 0, None]@Series[object] | +| uint16:extremes-nullable | [65535, 0, None]@uint16 | [65535.0, 0.0, nan]@Series[float64] | [65535, 0, None]@Series[object] | [65535, 0, None]@Series[object] | +| uint32:extremes-nullable | [4294967295, 0, None]@uint32 | [4294967295.0, 0.0, nan]@Series[float64] | [4294967295, 0, None]@Series[object] | [4294967295, 0, None]@Series[object] | +| uint64:extremes-nullable | [18446744073709551615, 0, None]@uint64 | [1.8446744073709552e+19, 0.0, nan]@Series[float64] | [18446744073709551615, 0, None]@Series[object] | [18446744073709551615, 0, None]@Series[object] | +| int64:all-null | [None, None]@int64 | [nan, nan]@Series[float64] | [None, None]@Series[object] | [None, None]@Series[object] | +| int64:multi-chunk-nullable | [1, None, 2]@int64 | [1.0, nan, 2.0]@Series[float64] | [1, None, 2]@Series[object] | [1, None, 2]@Series[object] | +| list:null-element | [[1, None], [2, 3]]@list | [array([ 1., nan]), array([2., 3.])]@Series[object] | [array([1, None], dtype=object), array([2, 3], dtype=object)]@Series[object] | [array([1, None], dtype=object), array([2, 3], dtype=object)]@Series[object] | +| list:null-element-extreme | [[9223372036854775807, None]]@list | [array([9.22337204e+18, nan])]@Series[object] | [array([9223372036854775807, None], dtype=object)]@Series[object] | [array([9223372036854775807, None], dtype=object)]@Series[object] | +| large_list:null-element | [[1, None]]@large_list | [array([ 1., nan])]@Series[object] | [array([1, None], dtype=object)]@Series[object] | [array([1, None], dtype=object)]@Series[object] | +| fixed_size_list[3]:null-element | [[1, None, 3]]@fixed_size_list[3] | [array([ 1., nan, 3.])]@Series[object] | [array([1, None, 3], dtype=object)]@Series[object] | [array([1, None, 3], dtype=object)]@Series[object] | +| list>:null-element | [[[1, None], [2]]]@list> | [array([array([ 1., nan]), array([2.])], dtype=object)]@Series[object] | [array([array([1, None], dtype=object), array([2], dtype=object)], dtype=object)]@Series[object] | [array([array([1, None], dtype=object), array([2], dtype=object)], dtype=object)]@Series[object] | +| struct:null-int-field | [[('x', 1), ('y', 'a')], [('x', None), ('y', 'b')]]@struct | [{'x': 1.0, 'y': 'a'}, {'x': None, 'y': 'b'}]@Series[object] | [{'x': 1, 'y': 'a'}, {'x': None, 'y': 'b'}]@Series[object] | [{'x': 1, 'y': 'a'}, {'x': None, 'y': 'b'}]@Series[object] | +| map:null-value | [[('a', 1), ('b', None)]]@map | [[('a', 1.0), ('b', None)]]@Series[object] | [[('a', 1), ('b', None)]]@Series[object] | [[('a', 1), ('b', None)]]@Series[object] | +| dictionary:nullable | [1, None, 1]@dictionary | [1, nan, 1]@Series[category] | [1, nan, 1]@Series[category] | [1, nan, 1]@Series[category] | \ No newline at end of file diff --git a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py index ed660f163aab..742b7db7389f 100644 --- a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py +++ b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py @@ -32,8 +32,8 @@ the per-cell conversion helper. Each concrete test class supplies its own source arrays (the rows relevant to the argument under test) and one test method per argument combination (each producing its own golden file). New ``to_pandas`` -arguments (e.g. ``zero_copy_only``) can be added as additional classes without -touching the existing ones. +arguments (e.g. ``zero_copy_only``, ``integer_object_nulls``) can be added as +additional classes without touching the existing ones. ## Golden File Cell Format @@ -493,6 +493,178 @@ def compute_cell(row_name, col_name): ) +@unittest.skipIf( + not have_pyarrow or not have_pandas or not have_numpy, + pyarrow_requirement_message or pandas_requirement_message or numpy_requirement_message, +) +class PyArrowArrayToPandasIntegerObjectNullsTests(_PyArrowToPandasTestBase): + """ + Tests pa.Array.to_pandas(integer_object_nulls=True) via golden file comparison. + + numpy integers have no null, so by default PyArrow widens a null-bearing integer + array to ``float64`` with ``NaN`` -- which cannot hold every int64 exactly, so a + large value silently changes. ``integer_object_nulls=True`` keeps ``object`` + dtype (Python ``int`` and ``None``) instead, preserving the values. + + PySpark passes it in ``ArrowArrayToPandasConversion.convert_legacy`` + (``python/pyspark/sql/conversion.py``), bundled with ``date_as_object`` and + ``coerce_temporal_nanoseconds``, then narrows the object Series to a nullable + extension dtype (``Int8Dtype`` .. ``Int64Dtype``) -- the only bridge from Arrow to + those dtypes that avoids ``float64``. + + Three output columns are recorded per source array: the argument off, on, and the + full ``pandas_options`` dict ``convert_legacy`` passes. The last is not a + duplicate of the second -- ``coerce_temporal_nanoseconds`` shifts the temporal + rows to ``ns`` -- and pinning the call as Spark makes it also catches PyArrow + changing the ``date_as_object=True`` default it relies on. + + The row set mirrors ``test_pyarrow_arrow_to_pandas_default.py``'s rows so both + golden files pin the same types, and appends the integer variants those rows do + not reach (see ``_build_source_arrays``). + """ + + def _build_source_arrays(self): + """ + Reuse every row of the sibling default test, then add the integer variants + that only matter for this argument: the shared rows' nullable values are + small enough to survive ``float64``, and their nested rows are missing whole + sub-lists rather than a single integer inside one. + """ + default_tests = test_pyarrow_arrow_to_pandas_default.PyArrowArrayToPandasDefaultTests + sources = default_tests._build_source_arrays(self) + + # Each width's min and max alongside a null. The shared rows use small values, + # which float64 represents exactly; only at 64 bits does the range exceed its + # 53-bit mantissa and the value itself change. + for pa_type in [ + pa.int8(), + pa.int16(), + pa.int32(), + pa.int64(), + pa.uint8(), + pa.uint16(), + pa.uint32(), + pa.uint64(), + ]: + if pa.types.is_signed_integer(pa_type): + bounds = [2 ** (pa_type.bit_width - 1) - 1, -(2 ** (pa_type.bit_width - 1))] + else: + bounds = [2**pa_type.bit_width - 1, 0] + sources[f"{pa_type}:extremes-nullable"] = pa.array(bounds + [None], pa_type) + + # Every value is null, so there is no integer left to convert. + sources["int64:all-null"] = pa.array([None, None], pa.int64()) + # Values split across buffers, with the null in one chunk and not the other. + # Reachable in production: cogrouped applyInPandas is the one UDF path that does + # not call combine_chunks() first. + sources["int64:multi-chunk-nullable"] = pa.chunked_array( + [pa.array([1, None], pa.int64()), pa.array([2], pa.int64())] + ) + + # Nested types whose null is an integer ELEMENT, not a missing sub-list: the + # shared rows only cover the latter, which this argument does not affect. + # These are also the types convert_legacy still serves. + sources["list:null-element"] = pa.array([[1, None], [2, 3]], pa.list_(pa.int64())) + sources["list:null-element-extreme"] = pa.array( + [[2**63 - 1, None]], pa.list_(pa.int64()) + ) + sources["large_list:null-element"] = pa.array([[1, None]], pa.large_list(pa.int64())) + sources["fixed_size_list[3]:null-element"] = pa.array( + [[1, None, 3]], pa.list_(pa.int64(), 3) + ) + sources["list>:null-element"] = pa.array( + [[[1, None], [2]]], pa.list_(pa.list_(pa.int64())) + ) + sources["struct:null-int-field"] = pa.array( + [{"x": 1, "y": "a"}, {"x": None, "y": "b"}], + pa.struct([("x", pa.int64()), ("y", pa.string())]), + ) + sources["map:null-value"] = pa.array( + [[("a", 1), ("b", None)]], pa.map_(pa.string(), pa.int64()) + ) + # Control: dictionary encoding stores the distinct values once plus an index per + # row, so the null lives in the indices and the int64 values hold none. With no + # null to represent there, the argument has nothing to decide -- both columns + # stay category, whose own values also remain int64. + sources["dictionary:nullable"] = pa.array( + [1, None, 1], pa.int64() + ).dictionary_encode() + + return sources + + # Output column for the default: a null-bearing integer array widens to float64. + COL_INTEGER_OBJECT_NULLS_OFF = "integer_object_nulls=False" + # Output column for the argument on: the result stays object dtype. + COL_INTEGER_OBJECT_NULLS_ON = "integer_object_nulls=True" + # Output column for all three arguments as convert_legacy passes them together. + COL_SPARK_PANDAS_OPTIONS = "spark pandas_options" + + # Kept as one dict so the column cannot drift from the call site it mirrors. + SPARK_PANDAS_OPTIONS = { + "date_as_object": True, + "coerce_temporal_nanoseconds": True, + "integer_object_nulls": True, + } + + def test_to_pandas_integer_object_nulls(self): + """Test pa.Array.to_pandas(integer_object_nulls=True/False) against golden file.""" + sources = self._build_source_arrays() + row_names = list(sources.keys()) + col_names = [ + "pyarrow array", + self.COL_INTEGER_OBJECT_NULLS_OFF, + self.COL_INTEGER_OBJECT_NULLS_ON, + self.COL_SPARK_PANDAS_OPTIONS, + ] + + # Version-specific expected values go here, keyed by (row, col), when a newer + # pandas/PyArrow/NumPy legitimately changes a cell's output. This argument does + # not touch strings, so a string row shifts in every output column at once. + overrides: dict[tuple[str, str], str] = {} + + def override_outputs(row: str, expected: str) -> None: + for col in col_names[1:]: # every column but the "pyarrow array" input + overrides[(row, col)] = expected + + pandas_3_or_later = LooseVersion(pd.__version__) >= LooseVersion("3.0.0") + pyarrow_24_or_later = LooseVersion(pa.__version__) >= LooseVersion("24.0.0") + + # Pandas 3 renders Arrow string arrays with its dedicated string dtype. + if pandas_3_or_later: + override_outputs("string:standard", "['hello', 'world', '']@Series[str]") + override_outputs("string:nullable", "['hello', nan, 'world']@Series[str]") + override_outputs("large_string:standard", "['hello', 'world']@Series[str]") + override_outputs("large_string:nullable", "['hello', nan]@Series[str]") + + # Empty ones stay object until PyArrow 24, so the baseline holds before that. + # Spark supports PyArrow 18+, so both branches are reachable. + if pandas_3_or_later and pyarrow_24_or_later: + override_outputs("string:empty", "[]@Series[str]") + override_outputs("large_string:empty", "[]@Series[str]") + + def compute_cell(row_name, col_name): + arr = sources[row_name] + if col_name == "pyarrow array": + return self.repr_value(arr, max_len=0) + elif col_name == self.COL_INTEGER_OBJECT_NULLS_OFF: + return self._to_pandas_cell(arr, integer_object_nulls=False) + elif col_name == self.COL_INTEGER_OBJECT_NULLS_ON: + return self._to_pandas_cell(arr, integer_object_nulls=True) + elif col_name == self.COL_SPARK_PANDAS_OPTIONS: + return self._to_pandas_cell(arr, **self.SPARK_PANDAS_OPTIONS) + else: + raise ValueError(f"unknown column: {col_name}") + + self.compare_or_generate_golden_matrix( + row_names=row_names, + col_names=col_names, + compute_cell=compute_cell, + golden_file_prefix="golden_pyarrow_arrow_to_pandas_integer_object_nulls", + index_name="test case", + overrides=overrides, + ) + + if __name__ == "__main__": from pyspark.testing import main