From 001fc827f1e425de376ef9ebaffe57986c64610a Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Tue, 7 Jul 2026 13:55:07 +0200 Subject: [PATCH 1/2] feat: Allow to skip validating members in collection filter & validate --- dataframely/collection/collection.py | 50 ++++++++- dataframely/filter_result.py | 11 ++ .../collection/test_skip_member_validation.py | 105 ++++++++++++++++++ 3 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 tests/collection/test_skip_member_validation.py diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 447909b0..900f3fb8 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -386,6 +386,7 @@ def validate( *, cast: bool = False, eager: bool = True, + skip_member_validation: bool = False, **kwargs: Any, ) -> Self: """Validate that a set of data frames satisfy the collection's invariants. @@ -406,6 +407,11 @@ def validate( :meth:`~polars.LazyFrame.collect` on the individual member or :meth:`collect_all` on the collection. Note that, in the latter case, information from error messages is limited. + skip_member_validation: Whether to skip validating individual members and only + apply the collection filters. **Use this option with caution** as it + requires the caller to ensure that the individual members have been + validated. This option is particularly useful in performance-critical + scenarios where the members are known to be valid. kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` and :meth:`polars.LazyFrame.collect` when `eager=True`. @@ -429,7 +435,13 @@ def validate( if eager: # If we perform the validation eagerly, we call filter and check the failure # information to properly construct a useful error message. - filtered, failures = cls.filter(data, cast=cast, eager=True, **kwargs) + filtered, failures = cls.filter( + data, + cast=cast, + eager=True, + skip_member_validation=skip_member_validation, + **kwargs, + ) if any(len(failure) > 0 for failure in failures.values()): errors: dict[str, str] = {} for member, failure in failures.items(): @@ -460,7 +472,17 @@ def validate( # efficiently as we cannot easily propagate error messages from different # members anyways. members: dict[str, pl.LazyFrame] = { - name: member.schema.validate(data[name].lazy(), cast=cast, eager=False) + name: ( + ( + member.schema.cast(data[name].lazy()) + if cast + else data[name].lazy() + ) + if skip_member_validation + else member.schema.validate( + data[name].lazy(), cast=cast, eager=False + ) + ) for name, member in cls.members().items() if name in data } @@ -556,6 +578,7 @@ def filter( *, cast: bool = False, eager: bool = True, + skip_member_validation: bool = False, **kwargs: Any, ) -> CollectionFilterResult[Self]: """Filter the members data frame by their schemas and the collection's filters. @@ -572,6 +595,11 @@ def filter( eager: Whether the filter operation should be performed eagerly. Note that until https://github.com/pola-rs/polars/pull/24129 is released, eagerly filtering can provide significant speedups. + skip_member_validation: Whether to skip filtering individual members and only + apply the collection filters. **Use this option with caution** as it + requires the caller to ensure that the individual members have been + validated. This option is particularly useful in performance-critical + scenarios where the members are known to already be valid. kwargs: Keyword arguments passed directly to :meth:`polars.collect_all` and :meth:`polars.LazyFrame.collect` when `eager=True`. @@ -615,10 +643,20 @@ class HospitalInvoiceData(dy.Collection): if member.is_optional and member_name not in data: continue - member_result, failures[member_name] = member.schema.filter( - data[member_name].lazy(), cast=cast, eager=eager, **kwargs - ) - results[member_name] = member_result.lazy() + if skip_member_validation: + results[member_name] = ( + member.schema.cast(data[member_name].lazy()) + if cast + else data[member_name].lazy() + ) + failures[member_name] = FailureInfo._create_empty( + member.schema, with_casting_rules=cast + ) + else: + member_result, failures[member_name] = member.schema.filter( + data[member_name].lazy(), cast=cast, eager=eager, **kwargs + ) + results[member_name] = member_result.lazy() # Once we've done that, we can apply the filters on this collection. To this end, # we iterate over all filters and store the filter results. diff --git a/dataframely/filter_result.py b/dataframely/filter_result.py index f2f88b75..8e53e117 100644 --- a/dataframely/filter_result.py +++ b/dataframely/filter_result.py @@ -106,6 +106,17 @@ def __init__( self._rule_columns = rule_columns self.schema = schema + @classmethod + def _create_empty(cls, schema: type[S], with_casting_rules: bool) -> FailureInfo[S]: + rules = schema._validation_rules(with_cast=with_casting_rules) + lf = pl.LazyFrame( + schema={ + **schema.to_polars_schema(), # type: ignore + **{rule: pl.Boolean for rule in rules}, + } + ) + return cls(lf=lf, rule_columns=list(rules.keys()), schema=schema) + @cached_property def _df(self) -> pl.DataFrame: return self._lf.collect() diff --git a/tests/collection/test_skip_member_validation.py b/tests/collection/test_skip_member_validation.py new file mode 100644 index 00000000..804d065b --- /dev/null +++ b/tests/collection/test_skip_member_validation.py @@ -0,0 +1,105 @@ +import polars as pl +import polars.exceptions as plexc +import pytest + +import dataframely as dy + + +class FirstSchema(dy.Schema): + a = dy.Float64(min=5) + + +class SecondSchema(dy.Schema): + a = dy.String() + + +class Collection(dy.Collection): + first: dy.LazyFrame[FirstSchema] + second: dy.LazyFrame[SecondSchema] + + +@pytest.mark.parametrize("df_type", [pl.DataFrame, pl.LazyFrame]) +def test_validate_skip_member_validation_eager( + df_type: type[pl.DataFrame] | type[pl.LazyFrame], +) -> None: + first = df_type({"a": [3, 4, 5]}) # NOTE: first two rows are violations + second = df_type({"a": ["1", "2", "3"]}) + + with pytest.raises(dy.exc.ValidationError): + Collection.validate({"first": first, "second": second}, cast=True) # type: ignore + + Collection.validate( + {"first": first, "second": second}, # type: ignore + cast=True, + skip_member_validation=True, + ) + + +@pytest.mark.parametrize("df_type", [pl.DataFrame, pl.LazyFrame]) +def test_validate_skip_member_validation_lazy( + df_type: type[pl.DataFrame] | type[pl.LazyFrame], +) -> None: + first = df_type({"a": [3, 4, 5]}) # NOTE: first two rows are violations + second = df_type({"a": ["1", "2", "3"]}) + + with pytest.raises(plexc.ComputeError): + Collection.validate( + {"first": first, "second": second}, # type: ignore + cast=True, + eager=False, + ).collect_all() + + Collection.validate( + {"first": first, "second": second}, # type: ignore + cast=True, + skip_member_validation=True, + eager=False, + ).collect_all() + + +@pytest.mark.parametrize("df_type", [pl.DataFrame, pl.LazyFrame]) +def test_filter_skip_member_validation_eager( + df_type: type[pl.DataFrame] | type[pl.LazyFrame], +) -> None: + first = df_type({"a": [3, 4, 5]}) # NOTE: first two rows are violations + second = df_type({"a": ["1", "2", "3"]}) + + _, failure_info = Collection.filter( + {"first": first, "second": second}, # type: ignore + cast=True, + ) + assert failure_info["first"].counts() == {"a|min": 2} + assert failure_info["second"].counts() == {} + + _, failure_info = Collection.filter( + {"first": first, "second": second}, # type: ignore + cast=True, + skip_member_validation=True, + ) + assert failure_info["first"].counts() == {} + assert failure_info["second"].counts() == {} + + +@pytest.mark.parametrize("df_type", [pl.DataFrame, pl.LazyFrame]) +def test_filter_skip_member_validation_lazy( + df_type: type[pl.DataFrame] | type[pl.LazyFrame], +) -> None: + first = df_type({"a": [3, 4, 5]}) # NOTE: first two rows are violations + second = df_type({"a": ["1", "2", "3"]}) + + _, failure_info = Collection.filter( + {"first": first, "second": second}, # type: ignore + cast=True, + eager=False, + ) + assert failure_info["first"].counts() == {"a|min": 2} + assert failure_info["second"].counts() == {} + + _, failure_info = Collection.filter( + {"first": first, "second": second}, # type: ignore + cast=True, + skip_member_validation=True, + eager=False, + ) + assert failure_info["first"].counts() == {} + assert failure_info["second"].counts() == {} From 5820f8aacf2d03b4e2b54a34c4b29275659c9dad Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Tue, 7 Jul 2026 14:23:27 +0200 Subject: [PATCH 2/2] Fix --- tests/collection/test_skip_member_validation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/collection/test_skip_member_validation.py b/tests/collection/test_skip_member_validation.py index 804d065b..1777b231 100644 --- a/tests/collection/test_skip_member_validation.py +++ b/tests/collection/test_skip_member_validation.py @@ -1,3 +1,6 @@ +# Copyright (c) QuantCo 2025-2026 +# SPDX-License-Identifier: BSD-3-Clause + import polars as pl import polars.exceptions as plexc import pytest