[python] Cast WHERE literals for DECIMAL, DATE and TIME columns

paimon-python/pypaimon/common/where_parser.py

@@ -40,6 +40,8 @@
   "age > 18 OR (name = 'Bob' AND status = 'active')"
 """
 
+import datetime
+import decimal
 import re
 from typing import Any, Dict, List, Optional
 
@@ -114,21 +116,98 @@ def _build_field_type_map(fields: List[DataField]) -> Dict[str, Optional[str]]:
     return result
 
 
+def _cast_decimal(value_str: str, type_name: str) -> decimal.Decimal:
+    """Cast a DECIMAL literal, rescaled to the column's scale.
+
+    A DECIMAL column reads back as decimal.Decimal, and the pushed-down Arrow
+    filter binds the literal's own scale rather than the column's, so a literal
+    written at a different scale (e.g. ``50`` against a ``DECIMAL(10, 2)``
+    ``50.00``) would silently match nothing. Rescale to the column scale when the
+    value is exact there; otherwise keep it as written (it then correctly matches
+    no row). A malformed literal is re-raised as ValueError, which
+    parse_where_clause documents and the CLI relies on.
+    """
+    try:
+        value = decimal.Decimal(value_str)
+    except decimal.InvalidOperation:
+        raise ValueError(f"Invalid DECIMAL literal: {value_str!r}") from None
+
+    match = re.search(r'\(\s*(\d+)\s*,\s*(\d+)\s*\)', type_name)
+    precision = int(match.group(1)) if match else 38
+    scale = int(match.group(2)) if match else 0
+    # Quantize in a context wide enough for the column precision. The default
+    # context caps precision at 28, so a DECIMAL(38, 2) literal in integer form
+    # would raise InvalidOperation here, fall through unscaled, and then match
+    # nothing once the Arrow filter binds it at scale 0.
+    context = decimal.Context(prec=max(precision, 1))
+    try:
+        rescaled = value.quantize(decimal.Decimal(1).scaleb(-scale), context=context)
+    except decimal.InvalidOperation:
+        return value
+    return rescaled if rescaled == value else value
+
+
+def _cast_date(value_str: str) -> datetime.date:
+    """Parse a DATE literal as ``YYYY-MM-DD``.
+
+    ``datetime.date.fromisoformat`` only exists on Python 3.7+, while pypaimon
+    still declares ``python_requires >= 3.6`` and runs a 3.6 test lane, so parse
+    through ``strptime`` instead.
+    """
+    try:
+        return datetime.datetime.strptime(value_str.strip(), '%Y-%m-%d').date()
+    except ValueError:
+        raise ValueError(f"Invalid DATE literal: {value_str!r}") from None
+
+
+_TIME_PATTERN = re.compile(r'(\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?)?')
+
+
+def _cast_time(value_str: str) -> datetime.time:
+    """Parse a TIME literal as a wall-clock ``HH:MM[:SS[.ffffff]]``.
+
+    ``datetime.time.fromisoformat`` only exists on Python 3.7+, and it also
+    accepts a UTC offset (``12:30:00+01:00``) that a Paimon TIME -- which has no
+    time zone -- cannot represent; Arrow would then compare only the wall-clock
+    part and match a value the user did not ask for. Parse a plain time here and
+    reject any offset.
+    """
+    match = _TIME_PATTERN.fullmatch(value_str.strip())
+    if match is None:
+        raise ValueError(f"Invalid TIME literal: {value_str!r}")
+    hour, minute, second, fraction = match.groups()
+    microsecond = int(fraction.ljust(6, '0')) if fraction else 0
+    try:
+        return datetime.time(int(hour), int(minute),
+                             int(second) if second else 0, microsecond)
+    except ValueError:
+        raise ValueError(f"Invalid TIME literal: {value_str!r}") from None
+
+
 def _cast_literal(value_str: str, type_name: str) -> Any:
     """Cast a literal string to the appropriate Python type based on the field type."""
     integer_types = {'TINYINT', 'SMALLINT', 'INT', 'INTEGER', 'BIGINT'}
     float_types = {'FLOAT', 'DOUBLE'}
+    decimal_types = {'DECIMAL', 'NUMERIC', 'DEC'}
 
     base_type = type_name.split('(')[0].strip()
 
     if base_type in integer_types:
         return int(value_str)
     if base_type in float_types:
         return float(value_str)
-    if base_type.startswith('DECIMAL') or base_type in ('DECIMAL', 'NUMERIC', 'DEC'):
-        return float(value_str)
+    if base_type in decimal_types:
+        return _cast_decimal(value_str, type_name)
     if base_type == 'BOOLEAN':
         return value_str.lower() in ('true', '1', 'yes')
+    if base_type == 'DATE':
+        # DATE/TIME columns read back as datetime.date / datetime.time; leaving the
+        # literal a string makes the arrow comparison kernel raise instead of
+        # filtering. TIMESTAMP is intentionally left out: its LOCAL TIME ZONE form
+        # reads back tz-aware and needs dedicated normalization.
+        return _cast_date(value_str)
+    if base_type == 'TIME':
+        return _cast_time(value_str)
     return value_str
 
 

paimon-python/pypaimon/tests/py36/where_literal_cast_test.py

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""WHERE literal casting on the Python 3.6/3.7 lane.
+
+The full ``where_parser_test`` is excluded from the 3.6/3.7 subset, yet DATE and
+TIME casting must stay off ``date.fromisoformat`` / ``time.fromisoformat`` (both
+3.7+). These pure-casting checks run on that lane to guard the floor.
+"""
+
+import datetime
+import decimal
+import unittest
+
+from pypaimon.common.where_parser import _cast_literal
+
+
+class WhereLiteralCastPy36Test(unittest.TestCase):
+
+    def test_cast_date(self):
+        self.assertEqual(_cast_literal('2024-01-01', 'DATE'),
+                         datetime.date(2024, 1, 1))
+
+    def test_cast_time(self):
+        self.assertEqual(_cast_literal('12:30:00', 'TIME(0)'),
+                         datetime.time(12, 30, 0))
+
+    def test_cast_time_with_fraction(self):
+        self.assertEqual(_cast_literal('12:30:00.5', 'TIME(3)'),
+                         datetime.time(12, 30, 0, 500000))
+
+    def test_cast_time_rejects_offset(self):
+        with self.assertRaises(ValueError):
+            _cast_literal('12:30:00+01:00', 'TIME(0)')
+
+    def test_cast_date_rejects_malformed(self):
+        with self.assertRaises(ValueError):
+            _cast_literal('not-a-date', 'DATE')
+
+    def test_cast_high_precision_decimal_rescales(self):
+        value = _cast_literal('123456789012345678901234567890123456',
+                              'DECIMAL(38, 2)')
+        self.assertEqual(value.as_tuple().exponent, -2)
+        self.assertEqual(
+            value, decimal.Decimal('123456789012345678901234567890123456.00'))
+
+
+if __name__ == '__main__':
+    unittest.main()

paimon-python/pypaimon/tests/where_parser_test.py

@@ -15,8 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import datetime
+import decimal
+import os
+import shutil
+import tempfile
 import unittest
 
+import pyarrow as pa
+
+from pypaimon import CatalogFactory, Schema
 from pypaimon.common.where_parser import parse_where_clause, _tokenize, _cast_literal
 from pypaimon.schema.data_types import ArrayType, AtomicType, DataField
 
@@ -83,7 +91,43 @@ def test_cast_float(self):
         self.assertAlmostEqual(_cast_literal('3.14', 'DOUBLE'), 3.14)
 
     def test_cast_decimal(self):
-        self.assertAlmostEqual(_cast_literal('99.99', 'DECIMAL(10,2)'), 99.99)
+        # A DECIMAL literal must stay exact (decimal.Decimal); a float would never
+        # compare equal to the Decimal a DECIMAL column reads back as.
+        value = _cast_literal('99.99', 'DECIMAL(10,2)')
+        self.assertIsInstance(value, decimal.Decimal)
+        self.assertEqual(value, decimal.Decimal('99.99'))
+
+    def test_cast_decimal_high_precision_rescales(self):
+        # A DECIMAL(38, 2) literal in integer form exceeds the default decimal
+        # context (precision 28); without a column-wide context the rescale would
+        # be dropped and the literal would bind at scale 0.
+        value = _cast_literal('123456789012345678901234567890123456',
+                              'DECIMAL(38, 2)')
+        self.assertEqual(value.as_tuple().exponent, -2)
+        self.assertEqual(
+            value, decimal.Decimal('123456789012345678901234567890123456.00'))
+
+    def test_cast_date(self):
+        value = _cast_literal('2024-01-01', 'DATE')
+        self.assertEqual(value, datetime.date(2024, 1, 1))
+
+    def test_cast_time(self):
+        value = _cast_literal('12:30:00', 'TIME(0)')
+        self.assertEqual(value, datetime.time(12, 30, 0))
+
+    def test_cast_time_with_fraction(self):
+        value = _cast_literal('12:30:00.5', 'TIME(3)')
+        self.assertEqual(value, datetime.time(12, 30, 0, 500000))
+
+    def test_cast_time_rejects_offset(self):
+        # A Paimon TIME has no time zone; an offset-bearing literal must not
+        # silently drop the offset and match the wall-clock value.
+        with self.assertRaises(ValueError):
+            _cast_literal('12:30:00+01:00', 'TIME(0)')
+
+    def test_cast_time_rejects_malformed(self):
+        with self.assertRaises(ValueError):
+            _cast_literal('25:00:00', 'TIME(0)')
 
     def test_cast_boolean(self):
         self.assertTrue(_cast_literal('true', 'BOOLEAN'))
@@ -400,5 +444,89 @@ def test_error_non_atomic_type_field(self):
         self.assertIn("tags", str(context.exception))
 
 
+class WhereParserScanTest(unittest.TestCase):
+    """End-to-end: a WHERE clause on DECIMAL/DATE/TIME columns returns the row."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tempdir = tempfile.mkdtemp()
+        cls.catalog = CatalogFactory.create(
+            {'warehouse': os.path.join(cls.tempdir, 'warehouse')})
+        cls.catalog.create_database('default', False)
+        pa_schema = pa.schema([
+            ('id', pa.int32()),
+            ('price', pa.decimal128(10, 2)),
+            ('big', pa.decimal128(38, 2)),
+            ('d', pa.date32()),
+            ('t', pa.time32('ms')),
+        ])
+        cls.catalog.create_table(
+            'default.where_literal_types', Schema.from_pyarrow_schema(pa_schema), False)
+        cls.table = cls.catalog.get_table('default.where_literal_types')
+        data = pa.table({
+            'id': pa.array([1, 2], pa.int32()),
+            'price': pa.array(
+                [decimal.Decimal('99.99'), decimal.Decimal('50.00')], pa.decimal128(10, 2)),
+            'big': pa.array(
+                [decimal.Decimal('123456789012345678901234567890123456.00'),
+                 decimal.Decimal('1.00')], pa.decimal128(38, 2)),
+            'd': pa.array(
+                [datetime.date(2024, 1, 1), datetime.date(2020, 1, 1)], pa.date32()),
+            't': pa.array(
+                [datetime.time(12, 30, 0), datetime.time(1, 0, 0)], pa.time32('ms')),
+        })
+        wb = cls.table.new_batch_write_builder()
+        writer = wb.new_write()
+        commit = wb.new_commit()
+        writer.write_arrow(data)
+        commit.commit(writer.prepare_commit())
+        writer.close()
+        commit.close()
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tempdir, ignore_errors=True)
+
+    def _scan_ids(self, where):
+        predicate = parse_where_clause(where, self.table.table_schema.fields)
+        read_builder = self.table.new_read_builder().with_filter(predicate)
+        splits = read_builder.new_scan().plan().splits()
+        return read_builder.new_read().to_arrow(splits).column('id').to_pylist()
+
+    def test_decimal_equality_returns_matching_row(self):
+        # DECIMAL(10,2) reads back as Decimal; the old float literal dropped the row.
+        self.assertEqual(self._scan_ids("price = 99.99"), [1])
+
+    def test_decimal_literal_rescaled_to_column_scale(self):
+        # The pushed-down filter binds the literal's own scale, so an integer or
+        # trailing-zero literal must be rescaled to the column scale to match.
+        self.assertEqual(self._scan_ids("price = 50"), [2])
+        self.assertEqual(self._scan_ids("price = 99.990"), [1])
+
+    def test_decimal_literal_finer_than_column_matches_nothing(self):
+        # A literal too precise for the column (99.999 on DECIMAL(10,2)) cannot equal
+        # any stored value; it must return no rows rather than round into a match.
+        self.assertEqual(self._scan_ids("price = 99.999"), [])
+
+    def test_high_precision_decimal_integer_literal_matches(self):
+        # A DECIMAL(38, 2) integer-form literal exceeds the default decimal
+        # context; it must still rescale and match the stored scale-2 value.
+        self.assertEqual(
+            self._scan_ids("big = 123456789012345678901234567890123456"), [1])
+
+    def test_malformed_decimal_raises_value_error(self):
+        # parse_where_clause documents ValueError; a bad decimal must not leak
+        # decimal.InvalidOperation past the CLI's `except ValueError`.
+        with self.assertRaises(ValueError):
+            parse_where_clause("price = abc", self.table.table_schema.fields)
+
+    def test_date_equality_returns_matching_row(self):
+        # DATE literal left as str raised in the arrow comparison kernel.
+        self.assertEqual(self._scan_ids("d = '2024-01-01'"), [1])
+
+    def test_time_equality_returns_matching_row(self):
+        self.assertEqual(self._scan_ids("t = '12:30:00'"), [1])
+
+
 if __name__ == '__main__':
     unittest.main()