feat: Generalize metrics and add nullability delta metric

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Set up pixi
         uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
         with:
-          environments: default lint
+          environments: default lint py311
       - name: Install repository
         run: pixi run -e default postinstall
       - name: pre-commit

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       # workaround for https://github.com/prefix-dev/pixi/issues/1482
       - id: pixi-install
         name: pixi-install
-        entry: pixi install -e default -e lint
+        entry: pixi install -e default -e lint -e py311
         language: system
         always_run: true
         require_serial: true
@@ -52,7 +52,7 @@ repos:
       # mypy
       - id: mypy
         name: mypy
-        entry: pixi run -e default mypy
+        entry: pixi run -e py311 mypy
         language: system
         types_or: [python, pyi]
         require_serial: true

diffly/cli.py

@@ -139,7 +139,7 @@ def main(
         list[str],
         typer.Option(
             help=(
-                "Metric presets to display per numerical column. Repeatable. "
+                "Metric presets to display per column. Repeatable. "
                 f"Available: {', '.join(DEFAULT_METRICS)}."
             )
         ),

diffly/comparison.py

@@ -25,7 +25,7 @@
     lazy_len,
     make_and_validate_mapping,
 )
-from .metrics import MetricFn, _make_numeric_metric
+from .metrics import Metric, MetricFn, _make_numeric_metric
 
 if TYPE_CHECKING:  # pragma: no cover
     # NOTE: We cannot import at runtime as we're otherwise running into circular
@@ -920,7 +920,7 @@ def summary(
         right_name: str = Side.RIGHT,
         slim: bool = False,
         hidden_columns: list[str] | None = None,
-        metrics: Mapping[str, MetricFn] | None = None,
+        metrics: Mapping[str, MetricFn | Metric] | None = None,
     ) -> Summary:
         """Generate a summary of all aspects of the comparison.
 
@@ -950,16 +950,18 @@ def summary(
                 advanced users who are familiar with the summary format.
             hidden_columns: Columns for which no values are printed, e.g. because they
                 contain sensitive information.
-            metrics: Optional mapping from display label to a metric callable
-                ``(left_expr, right_expr) -> pl.Expr``. Each callable receives two
+            metrics: Optional mapping from display label to a metric. A value may be a
+                callable ``(left_expr, right_expr) -> pl.Expr`` or a
+                :class:`~diffly.metrics.Metric`. Each callable receives two
                 :class:`polars.Expr` referring to the left and right values of a single
-                numerical column across all joined rows, and must return a scalar
-                aggregation expression. See :doc:`/api/metrics` for the full list of
-                presets and the :data:`~diffly.metrics.MetricFn` type. When ``None``
-                (default), no metrics are computed; presets are not applied
-                automatically. Metrics are only computed for numerical columns. Prefer
-                short labels — the summary has a fixed width and many or long labels
-                degrade rendering.
+                column across all joined rows, and must return a scalar aggregation
+                expression. Bare callables are only computed for numerical columns; wrap
+                one in a :class:`~diffly.metrics.Metric` with a column selector to target
+                other column types (e.g. ``Metric(fn, selector=cs.all())``).
+                See :doc:`/api/metrics` for the full list of presets and the
+                :data:`~diffly.metrics.MetricFn` type. When ``None`` (default), no metrics
+                are computed; presets are not applied automatically. Prefer short labels —
+                the summary has a fixed width and many or long labels degrade rendering.
 
         Returns:
             A summary which can be printed or written to a file.
@@ -976,7 +978,10 @@ def summary(
         from .summary import Summary
 
         resolved_metrics = (
-            {label: _make_numeric_metric(fn) for label, fn in metrics.items()}
+            {
+                label: v if isinstance(v, Metric) else _make_numeric_metric(v)
+                for label, v in metrics.items()
+            }
             if metrics is not None
             else None
         )

diffly/metrics.py

@@ -12,10 +12,7 @@
 
 @dataclass(frozen=True)
 class Metric:
-    """A metric function paired with a column-applicability selector.
-
-    Internal only.
-    """
+    """A metric function paired with a column-applicability selector."""
 
     fn: MetricFn
     selector: cs.Selector
@@ -30,10 +27,6 @@ class Metric:
 """
 
 
-def _make_numeric_metric(fn: MetricFn) -> Metric:
-    return Metric(fn=fn, selector=cs.numeric())
-
-
 def mean(left: pl.Expr, right: pl.Expr) -> pl.Expr:
     """Mean of ``right - left``."""
     return (right - left).mean()
@@ -70,6 +63,23 @@ def mean_relative_deviation(left: pl.Expr, right: pl.Expr) -> pl.Expr:
     return ((right - left) / left).abs().mean()
 
 
+def null_fraction_change(left: pl.Expr, right: pl.Expr) -> pl.Expr:
+    """Change in the fraction of null entries, rendered as ``<old> -> <new> (<delta>)``.
+
+    ``old`` and ``new`` are the null percentages of the left and right side, and
+    ``delta`` is their signed difference (``+`` when the right side has proportionally
+    more nulls, ``-`` when it has fewer). This metric function can be applied to columns
+    of any type.
+    """
+    return _render_change(
+        left.is_null().mean(),
+        right.is_null().mean(),
+        lambda value, signed: _percentage_string(
+            value, signed=signed, percent_sign=not signed
+        ),
+    )
+
+
 def quantile(q: float) -> MetricFn:
     """Factory returning a metric that computes the ``q``-quantile of
     ``right - left``."""
@@ -82,12 +92,52 @@ def _quantile(left: pl.Expr, right: pl.Expr) -> pl.Expr:
     return _quantile
 
 
-DEFAULT_METRICS: dict[str, MetricFn] = {
+DEFAULT_METRICS: dict[str, MetricFn | Metric] = {
     "Mean": mean,
     "Median": median,
     "Min": min,
     "Max": max,
     "Std": std,
     "Mean absolute deviation": mean_absolute_deviation,
     "Mean relative deviation": mean_relative_deviation,
+    "Null%": Metric(fn=null_fraction_change, selector=cs.all()),
 }
+
+
+# ------------------------------------------------------------------------------------ #
+#                                    UTILITY METHODS                                   #
+# ------------------------------------------------------------------------------------ #
+
+
+def _make_numeric_metric(fn: MetricFn) -> Metric:
+    return Metric(fn=fn, selector=cs.numeric())
+
+
+def _percentage_string(
+    fraction: pl.Expr, *, signed: bool = False, percent_sign: bool = True
+) -> pl.Expr:
+    """Format a fraction as a percentage string, optionally with an explicit sign."""
+    pct = (fraction * 100).round(2)
+    body = pl.format("{}%", pct) if percent_sign else pl.format("{}", pct)
+    if signed:
+        return pl.when(pct >= 0).then(pl.format("+{}", body)).otherwise(body)
+    return body
+
+
+def _render_change(
+    old: pl.Expr,
+    new: pl.Expr,
+    format_value: Callable[[pl.Expr, bool], pl.Expr],
+) -> pl.Expr:
+    """Render a change as ``<old> -> <new> (<delta>)``.
+
+    ``format_value(value, signed)`` formats a value for display; ``old`` and ``new`` are
+    rendered unsigned and the delta ``new - old`` is rendered signed (with an explicit
+    ``+`` prefix for positive values).
+    """
+    return pl.format(
+        "{} -> {} ({})",
+        format_value(old, False),
+        format_value(new, False),
+        format_value(new - old, True),
+    )

diffly/summary.py

@@ -1131,10 +1131,13 @@ def _format_value(value: Any, *, float_format: str | None = None) -> str:
 def _format_metric_value(value: Any) -> str:
     """Format a metric value for the column summary.
 
-    Blanks out ``None`` and renders floats with ``.4g`` precision.
+    Blanks out ``None``, renders string values verbatim, and renders floats with ``.4g``
+    precision.
     """
     if value is None:
         return ""
+    if isinstance(value, str):
+        return _yellow(value)
     return _format_value(value, float_format=".4g")
 
 

diffly/testing.py

@@ -19,7 +19,7 @@
 
 from ._compat import dy
 from .comparison import DataFrameComparison, compare_frames
-from .metrics import MetricFn
+from .metrics import Metric, MetricFn
 
 
 def assert_collection_equal(
@@ -40,7 +40,7 @@ def assert_collection_equal(
     right_name: str = Side.RIGHT,
     slim: bool = False,
     hidden_columns: list[str] | None = None,
-    metrics: Mapping[str, MetricFn] | None = None,
+    metrics: Mapping[str, MetricFn | Metric] | None = None,
 ) -> None:
     """Assert that two :mod:`dataframely` collections are equal.
 
@@ -85,9 +85,11 @@ def assert_collection_equal(
         hidden_columns: Columns for which no values are printed, e.g. because they
             contain sensitive information.
         metrics: Optional mapping from display label to a metric callable
-            ``(left_expr, right_expr) -> pl.Expr``. See :mod:`diffly.metrics` for
-            presets. When ``None`` (default), no metrics are computed; presets are
-            not applied automatically.
+            ``(left_expr, right_expr) -> pl.Expr`` or a :class:`~diffly.metrics.Metric`.
+            Bare callables are only computed for numerical columns; wrap one in a
+            :class:`~diffly.metrics.Metric` with a column selector to target other column
+            types. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
+            metrics are computed; presets are not applied automatically.
 
     Raises:
         AssertionError: If the collections are not equal.
@@ -174,7 +176,7 @@ def assert_frame_equal(
     right_name: str = Side.RIGHT,
     slim: bool = False,
     hidden_columns: list[str] | None = None,
-    metrics: Mapping[str, MetricFn] | None = None,
+    metrics: Mapping[str, MetricFn | Metric] | None = None,
 ) -> None:
     """Assert that two :mod:`polars` data frames are equal.
 
@@ -226,9 +228,11 @@ def assert_frame_equal(
         hidden_columns: Columns for which no values are printed, e.g. because they
             contain sensitive information.
         metrics: Optional mapping from display label to a metric callable
-            ``(left_expr, right_expr) -> pl.Expr``. See :mod:`diffly.metrics` for
-            presets. When ``None`` (default), no metrics are computed; presets are
-            not applied automatically.
+            ``(left_expr, right_expr) -> pl.Expr`` or a :class:`~diffly.metrics.Metric`.
+            Bare callables are only computed for numerical columns; wrap one in a
+            :class:`~diffly.metrics.Metric` with a column selector to target other column
+            types. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
+            metrics are computed; presets are not applied automatically.
 
     Raises:
         AssertionError: If the data frames are not equal.

docs/api/metrics.rst

@@ -4,15 +4,22 @@ Metrics
 
 .. currentmodule:: diffly.metrics
 
-Metrics are scalar aggregations computed per numerical column when generating a
+Metrics are scalar aggregations computed per column when generating a
 :meth:`~diffly.comparison.DataFrameComparison.summary`. Pass them via the
 ``metrics`` argument as a mapping from display label to a :data:`MetricFn`
 callable. :mod:`diffly.metrics` ships a set of presets; you can also supply
 your own callable ``(left_expr, right_expr) -> pl.Expr``.
 
+A bare callable is only computed for numerical columns. To target a different
+set of columns, wrap it in a :class:`Metric` with a column selector, e.g.
+``Metric(fn, selector=cs.all())``, ``Metric(fn, selector=cs.boolean())``, or
+``Metric(fn, selector=cs.by_name("my_column_name"))``.
+
 .. autodata:: MetricFn
    :no-value:
 
+.. autoclass:: Metric
+
 Presets
 =======
 
@@ -26,4 +33,5 @@ Presets
    std
    mean_absolute_deviation
    mean_relative_deviation
+   null_fraction_change
    quantile