diff --git a/paimon-python/pypaimon/common/where_parser.py b/paimon-python/pypaimon/common/where_parser.py index 3286c81a5c2d..1ba78986a566 100644 --- a/paimon-python/pypaimon/common/where_parser.py +++ b/paimon-python/pypaimon/common/where_parser.py @@ -40,6 +40,8 @@ "age > 18 OR (name = 'Bob' AND status = 'active')" """ +import datetime +import decimal import re from typing import Any, Dict, List, Optional @@ -114,10 +116,79 @@ def _build_field_type_map(fields: List[DataField]) -> Dict[str, Optional[str]]: return result +def _cast_decimal(value_str: str, type_name: str) -> decimal.Decimal: + """Cast a DECIMAL literal, rescaled to the column's scale. + + A DECIMAL column reads back as decimal.Decimal, and the pushed-down Arrow + filter binds the literal's own scale rather than the column's, so a literal + written at a different scale (e.g. ``50`` against a ``DECIMAL(10, 2)`` + ``50.00``) would silently match nothing. Rescale to the column scale when the + value is exact there; otherwise keep it as written (it then correctly matches + no row). A malformed literal is re-raised as ValueError, which + parse_where_clause documents and the CLI relies on. + """ + try: + value = decimal.Decimal(value_str) + except decimal.InvalidOperation: + raise ValueError(f"Invalid DECIMAL literal: {value_str!r}") from None + + match = re.search(r'\(\s*(\d+)\s*,\s*(\d+)\s*\)', type_name) + precision = int(match.group(1)) if match else 38 + scale = int(match.group(2)) if match else 0 + # Quantize in a context wide enough for the column precision. The default + # context caps precision at 28, so a DECIMAL(38, 2) literal in integer form + # would raise InvalidOperation here, fall through unscaled, and then match + # nothing once the Arrow filter binds it at scale 0. + context = decimal.Context(prec=max(precision, 1)) + try: + rescaled = value.quantize(decimal.Decimal(1).scaleb(-scale), context=context) + except decimal.InvalidOperation: + return value + return rescaled if rescaled == value else value + + +def _cast_date(value_str: str) -> datetime.date: + """Parse a DATE literal as ``YYYY-MM-DD``. + + ``datetime.date.fromisoformat`` only exists on Python 3.7+, while pypaimon + still declares ``python_requires >= 3.6`` and runs a 3.6 test lane, so parse + through ``strptime`` instead. + """ + try: + return datetime.datetime.strptime(value_str.strip(), '%Y-%m-%d').date() + except ValueError: + raise ValueError(f"Invalid DATE literal: {value_str!r}") from None + + +_TIME_PATTERN = re.compile(r'(\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?)?') + + +def _cast_time(value_str: str) -> datetime.time: + """Parse a TIME literal as a wall-clock ``HH:MM[:SS[.ffffff]]``. + + ``datetime.time.fromisoformat`` only exists on Python 3.7+, and it also + accepts a UTC offset (``12:30:00+01:00``) that a Paimon TIME -- which has no + time zone -- cannot represent; Arrow would then compare only the wall-clock + part and match a value the user did not ask for. Parse a plain time here and + reject any offset. + """ + match = _TIME_PATTERN.fullmatch(value_str.strip()) + if match is None: + raise ValueError(f"Invalid TIME literal: {value_str!r}") + hour, minute, second, fraction = match.groups() + microsecond = int(fraction.ljust(6, '0')) if fraction else 0 + try: + return datetime.time(int(hour), int(minute), + int(second) if second else 0, microsecond) + except ValueError: + raise ValueError(f"Invalid TIME literal: {value_str!r}") from None + + def _cast_literal(value_str: str, type_name: str) -> Any: """Cast a literal string to the appropriate Python type based on the field type.""" integer_types = {'TINYINT', 'SMALLINT', 'INT', 'INTEGER', 'BIGINT'} float_types = {'FLOAT', 'DOUBLE'} + decimal_types = {'DECIMAL', 'NUMERIC', 'DEC'} base_type = type_name.split('(')[0].strip() @@ -125,10 +196,18 @@ def _cast_literal(value_str: str, type_name: str) -> Any: return int(value_str) if base_type in float_types: return float(value_str) - if base_type.startswith('DECIMAL') or base_type in ('DECIMAL', 'NUMERIC', 'DEC'): - return float(value_str) + if base_type in decimal_types: + return _cast_decimal(value_str, type_name) if base_type == 'BOOLEAN': return value_str.lower() in ('true', '1', 'yes') + if base_type == 'DATE': + # DATE/TIME columns read back as datetime.date / datetime.time; leaving the + # literal a string makes the arrow comparison kernel raise instead of + # filtering. TIMESTAMP is intentionally left out: its LOCAL TIME ZONE form + # reads back tz-aware and needs dedicated normalization. + return _cast_date(value_str) + if base_type == 'TIME': + return _cast_time(value_str) return value_str diff --git a/paimon-python/pypaimon/tests/py36/where_literal_cast_test.py b/paimon-python/pypaimon/tests/py36/where_literal_cast_test.py new file mode 100644 index 000000000000..98daef9bb3cd --- /dev/null +++ b/paimon-python/pypaimon/tests/py36/where_literal_cast_test.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""WHERE literal casting on the Python 3.6/3.7 lane. + +The full ``where_parser_test`` is excluded from the 3.6/3.7 subset, yet DATE and +TIME casting must stay off ``date.fromisoformat`` / ``time.fromisoformat`` (both +3.7+). These pure-casting checks run on that lane to guard the floor. +""" + +import datetime +import decimal +import unittest + +from pypaimon.common.where_parser import _cast_literal + + +class WhereLiteralCastPy36Test(unittest.TestCase): + + def test_cast_date(self): + self.assertEqual(_cast_literal('2024-01-01', 'DATE'), + datetime.date(2024, 1, 1)) + + def test_cast_time(self): + self.assertEqual(_cast_literal('12:30:00', 'TIME(0)'), + datetime.time(12, 30, 0)) + + def test_cast_time_with_fraction(self): + self.assertEqual(_cast_literal('12:30:00.5', 'TIME(3)'), + datetime.time(12, 30, 0, 500000)) + + def test_cast_time_rejects_offset(self): + with self.assertRaises(ValueError): + _cast_literal('12:30:00+01:00', 'TIME(0)') + + def test_cast_date_rejects_malformed(self): + with self.assertRaises(ValueError): + _cast_literal('not-a-date', 'DATE') + + def test_cast_high_precision_decimal_rescales(self): + value = _cast_literal('123456789012345678901234567890123456', + 'DECIMAL(38, 2)') + self.assertEqual(value.as_tuple().exponent, -2) + self.assertEqual( + value, decimal.Decimal('123456789012345678901234567890123456.00')) + + +if __name__ == '__main__': + unittest.main() diff --git a/paimon-python/pypaimon/tests/where_parser_test.py b/paimon-python/pypaimon/tests/where_parser_test.py index c5ba552ac789..4f38d71a910b 100644 --- a/paimon-python/pypaimon/tests/where_parser_test.py +++ b/paimon-python/pypaimon/tests/where_parser_test.py @@ -15,8 +15,16 @@ # specific language governing permissions and limitations # under the License. +import datetime +import decimal +import os +import shutil +import tempfile import unittest +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema from pypaimon.common.where_parser import parse_where_clause, _tokenize, _cast_literal from pypaimon.schema.data_types import ArrayType, AtomicType, DataField @@ -83,7 +91,43 @@ def test_cast_float(self): self.assertAlmostEqual(_cast_literal('3.14', 'DOUBLE'), 3.14) def test_cast_decimal(self): - self.assertAlmostEqual(_cast_literal('99.99', 'DECIMAL(10,2)'), 99.99) + # A DECIMAL literal must stay exact (decimal.Decimal); a float would never + # compare equal to the Decimal a DECIMAL column reads back as. + value = _cast_literal('99.99', 'DECIMAL(10,2)') + self.assertIsInstance(value, decimal.Decimal) + self.assertEqual(value, decimal.Decimal('99.99')) + + def test_cast_decimal_high_precision_rescales(self): + # A DECIMAL(38, 2) literal in integer form exceeds the default decimal + # context (precision 28); without a column-wide context the rescale would + # be dropped and the literal would bind at scale 0. + value = _cast_literal('123456789012345678901234567890123456', + 'DECIMAL(38, 2)') + self.assertEqual(value.as_tuple().exponent, -2) + self.assertEqual( + value, decimal.Decimal('123456789012345678901234567890123456.00')) + + def test_cast_date(self): + value = _cast_literal('2024-01-01', 'DATE') + self.assertEqual(value, datetime.date(2024, 1, 1)) + + def test_cast_time(self): + value = _cast_literal('12:30:00', 'TIME(0)') + self.assertEqual(value, datetime.time(12, 30, 0)) + + def test_cast_time_with_fraction(self): + value = _cast_literal('12:30:00.5', 'TIME(3)') + self.assertEqual(value, datetime.time(12, 30, 0, 500000)) + + def test_cast_time_rejects_offset(self): + # A Paimon TIME has no time zone; an offset-bearing literal must not + # silently drop the offset and match the wall-clock value. + with self.assertRaises(ValueError): + _cast_literal('12:30:00+01:00', 'TIME(0)') + + def test_cast_time_rejects_malformed(self): + with self.assertRaises(ValueError): + _cast_literal('25:00:00', 'TIME(0)') def test_cast_boolean(self): self.assertTrue(_cast_literal('true', 'BOOLEAN')) @@ -400,5 +444,89 @@ def test_error_non_atomic_type_field(self): self.assertIn("tags", str(context.exception)) +class WhereParserScanTest(unittest.TestCase): + """End-to-end: a WHERE clause on DECIMAL/DATE/TIME columns returns the row.""" + + @classmethod + def setUpClass(cls): + cls.tempdir = tempfile.mkdtemp() + cls.catalog = CatalogFactory.create( + {'warehouse': os.path.join(cls.tempdir, 'warehouse')}) + cls.catalog.create_database('default', False) + pa_schema = pa.schema([ + ('id', pa.int32()), + ('price', pa.decimal128(10, 2)), + ('big', pa.decimal128(38, 2)), + ('d', pa.date32()), + ('t', pa.time32('ms')), + ]) + cls.catalog.create_table( + 'default.where_literal_types', Schema.from_pyarrow_schema(pa_schema), False) + cls.table = cls.catalog.get_table('default.where_literal_types') + data = pa.table({ + 'id': pa.array([1, 2], pa.int32()), + 'price': pa.array( + [decimal.Decimal('99.99'), decimal.Decimal('50.00')], pa.decimal128(10, 2)), + 'big': pa.array( + [decimal.Decimal('123456789012345678901234567890123456.00'), + decimal.Decimal('1.00')], pa.decimal128(38, 2)), + 'd': pa.array( + [datetime.date(2024, 1, 1), datetime.date(2020, 1, 1)], pa.date32()), + 't': pa.array( + [datetime.time(12, 30, 0), datetime.time(1, 0, 0)], pa.time32('ms')), + }) + wb = cls.table.new_batch_write_builder() + writer = wb.new_write() + commit = wb.new_commit() + writer.write_arrow(data) + commit.commit(writer.prepare_commit()) + writer.close() + commit.close() + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tempdir, ignore_errors=True) + + def _scan_ids(self, where): + predicate = parse_where_clause(where, self.table.table_schema.fields) + read_builder = self.table.new_read_builder().with_filter(predicate) + splits = read_builder.new_scan().plan().splits() + return read_builder.new_read().to_arrow(splits).column('id').to_pylist() + + def test_decimal_equality_returns_matching_row(self): + # DECIMAL(10,2) reads back as Decimal; the old float literal dropped the row. + self.assertEqual(self._scan_ids("price = 99.99"), [1]) + + def test_decimal_literal_rescaled_to_column_scale(self): + # The pushed-down filter binds the literal's own scale, so an integer or + # trailing-zero literal must be rescaled to the column scale to match. + self.assertEqual(self._scan_ids("price = 50"), [2]) + self.assertEqual(self._scan_ids("price = 99.990"), [1]) + + def test_decimal_literal_finer_than_column_matches_nothing(self): + # A literal too precise for the column (99.999 on DECIMAL(10,2)) cannot equal + # any stored value; it must return no rows rather than round into a match. + self.assertEqual(self._scan_ids("price = 99.999"), []) + + def test_high_precision_decimal_integer_literal_matches(self): + # A DECIMAL(38, 2) integer-form literal exceeds the default decimal + # context; it must still rescale and match the stored scale-2 value. + self.assertEqual( + self._scan_ids("big = 123456789012345678901234567890123456"), [1]) + + def test_malformed_decimal_raises_value_error(self): + # parse_where_clause documents ValueError; a bad decimal must not leak + # decimal.InvalidOperation past the CLI's `except ValueError`. + with self.assertRaises(ValueError): + parse_where_clause("price = abc", self.table.table_schema.fields) + + def test_date_equality_returns_matching_row(self): + # DATE literal left as str raised in the arrow comparison kernel. + self.assertEqual(self._scan_ids("d = '2024-01-01'"), [1]) + + def test_time_equality_returns_matching_row(self): + self.assertEqual(self._scan_ids("t = '12:30:00'"), [1]) + + if __name__ == '__main__': unittest.main()