fix(packaging): keep release artifacts below PyPI limit

.github/workflows/ci.yml

@@ -993,6 +993,7 @@ jobs:
           # The default build path creates the sdist first, then builds the
           # wheel from that sdist. This covers both VCS and sdist consumers.
           python -m build --outdir dist/
+          python scripts/ci/check_distribution_size.py dist/
           pip install dist/*.whl
 
       - name: Verify VCS-built wheel schemas
@@ -1019,9 +1020,9 @@ jobs:
               for child in schema_root.iterdir()
               if child.is_dir()
           }
-          # Preserve historical beta.6, rc.3 and rc.6 schemas offline.
+          # Ship released 3.0/3.1 and only the current 3.2 candidate offline.
           assert packaged_versions == {
-              "2.5", "3.0", "3.1", "3.2.0-beta.6", "3.2.0-rc.3", "3.2.0-rc.6", current_bundle
+              "3.0", "3.1", current_bundle
           }
           error_schema = json.loads(
               (schema_root / current_bundle / "enums/error-code.json").read_text(encoding="utf-8")

MANIFEST.in

@@ -4,28 +4,29 @@ include LICENSE
 include MIGRATION*.md
 recursive-include src/adcp py.typed
 recursive-include src/adcp/reporting/materializer/assets *.json
-recursive-include src/adcp/_compliance/3.2.0-rc.6 *.json *.jsonl *.yaml *.md
 recursive-include src/adcp/_compliance/3.2.0-rc.7 *.json *.jsonl *.yaml *.md
 # Bundled AdCP JSON schemas. ``scripts/bundle_schemas.py`` mirrors
 # ``schemas/cache/`` into ``src/adcp/_schemas/`` before ``python -m
 # build`` so the validator ships with the wheel. Keep distributions on
-# supported bundles, including historical v32's beta.6 and rc.3 continuations.
-recursive-include src/adcp/_schemas/2.5 *.json
+# supported released lines and the current 3.2 release candidate.
 recursive-include src/adcp/_schemas/3.0 *.json
 recursive-include src/adcp/_schemas/3.1 *.json
-recursive-include src/adcp/_schemas/3.2.0-beta.6 *.json
-recursive-include src/adcp/_schemas/3.2.0-rc.3 *.json
-recursive-include src/adcp/_schemas/3.2.0-rc.6 *.json
 recursive-include src/adcp/_schemas/3.2.0-rc.7 *.json
+prune src/adcp/_schemas/2.5
+prune src/adcp/_schemas/3.2.0-beta.6
+prune src/adcp/_schemas/3.2.0-rc.3
+prune src/adcp/_schemas/3.2.0-rc.6
+prune src/adcp/_compliance/3.2.0-rc.6
 prune src/adcp/_schemas/3.1.0-*
 # A clean VCS source tree has no ignored ``src/adcp/_schemas`` directory.
 # Keep the tracked inputs needed by the build_py hook in source distributions
 # so wheels built from an sdist have the same schema set as direct VCS wheels.
-recursive-include schemas/cache/2.5 *.json
 recursive-include schemas/cache/3.0 *.json
 recursive-include schemas/cache/3.1 *.json
-recursive-include schemas/cache/3.2.0-beta.6 *.json
-recursive-include schemas/cache/3.2.0-rc.3 *.json
-recursive-include schemas/cache/3.2.0-rc.6 *.json
 recursive-include schemas/cache/3.2.0-rc.7 *.json
-recursive-include schemas/releases *.json
+prune schemas/cache/2.5
+prune schemas/cache/3.2.0-beta.6
+prune schemas/cache/3.2.0-rc.3
+prune schemas/cache/3.2.0-rc.6
+prune schemas/releases
+include schemas/releases/3.2.0-rc.7.json

pyproject.toml

@@ -211,18 +211,10 @@ adcp = [
     # AdCP JSON schemas, mirrored from ``schemas/cache/`` by
     # ``scripts/bundle_schemas.py`` so the wheel ships them for
     # ``adcp.validation.schema_loader``.
-    "_schemas/2.5/**/*.json",
     "_schemas/3.0/**/*.json",
     "_schemas/3.1/**/*.json",
-    "_schemas/3.2.0-beta.6/**/*.json",
-    "_schemas/3.2.0-rc.3/**/*.json",
-    "_schemas/3.2.0-rc.6/**/*.json",
     "_schemas/3.2.0-rc.7/**/*.json",
     # Exact signed compliance inputs plus their immutable release provenance.
-    "_compliance/3.2.0-rc.6/**/*.json",
-    "_compliance/3.2.0-rc.6/**/*.jsonl",
-    "_compliance/3.2.0-rc.6/**/*.yaml",
-    "_compliance/3.2.0-rc.6/**/*.md",
     "_compliance/3.2.0-rc.7/**/*.json",
     "_compliance/3.2.0-rc.7/**/*.jsonl",
     "_compliance/3.2.0-rc.7/**/*.yaml",

scripts/ci/check_distribution_size.py

@@ -0,0 +1,53 @@
+"""Fail a release build before upload if either distribution approaches PyPI's limit."""
+
+from __future__ import annotations
+
+import sys
+import tarfile
+import zipfile
+from pathlib import Path
+
+MAX_BYTES = 90_000_000
+RETIRED_BUNDLES = {"2.5", "3.2.0-beta.6", "3.2.0-rc.3", "3.2.0-rc.6"}
+
+
+def check_distribution_size(directory: Path) -> tuple[Path, Path]:
+    """Require one wheel and one sdist, each below the release budget."""
+    artifacts = []
+    for pattern in ("*.whl", "*.tar.gz"):
+        found = sorted(directory.glob(pattern))
+        if len(found) != 1:
+            raise ValueError(f"expected one {pattern} in {directory}, found {len(found)}")
+        artifacts.append(found[0])
+    for artifact in artifacts:
+        size = artifact.stat().st_size
+        if size > MAX_BYTES:
+            raise ValueError(f"{artifact.name} is {size} bytes; limit is {MAX_BYTES} bytes")
+        print(f"{artifact.name}: {size} bytes (limit {MAX_BYTES})")
+    return artifacts[0], artifacts[1]
+
+
+def check_retired_schema_bundles(wheel: Path, source: Path) -> None:
+    """Reject superseded schema directories in either release artifact."""
+    with zipfile.ZipFile(wheel) as archive:
+        wheel_members = archive.namelist()
+    with tarfile.open(source) as archive:
+        source_members = archive.getnames()
+    for artifact, members in ((wheel, wheel_members), (source, source_members)):
+        for name in members:
+            path = Path(name)
+            retired = RETIRED_BUNDLES.intersection((*path.parts, path.stem))
+            if retired:
+                raise ValueError(
+                    f"{artifact.name} contains retired schema bundle {sorted(retired)[0]}"
+                )
+
+
+if __name__ == "__main__":
+    try:
+        wheel, source = check_distribution_size(
+            Path(sys.argv[1]) if len(sys.argv) == 2 else Path("dist")
+        )
+        check_retired_schema_bundles(wheel, source)
+    except ValueError as exc:
+        raise SystemExit(str(exc)) from exc

setup.py

@@ -15,11 +15,7 @@
     if "-" in _PINNED_ADCP_VERSION
     else ".".join(_PINNED_ADCP_VERSION.split(".")[:2])
 )
-_BUNDLED_SCHEMA_VERSIONS = tuple(
-    dict.fromkeys(
-        ("2.5", "3.0", "3.1", "3.2.0-beta.6", "3.2.0-rc.3", "3.2.0-rc.6", _CURRENT_SCHEMA_BUNDLE)
-    )
-)
+_BUNDLED_SCHEMA_VERSIONS = tuple(dict.fromkeys(("3.0", "3.1", _CURRENT_SCHEMA_BUNDLE)))
 
 
 class BuildPy(_build_py):

src/adcp/compat/purchase_continuation.py

@@ -34,6 +34,7 @@
 from adcp.types import AccountReference, CompatibilityPurchaseCoordinatorInput
 from adcp.types.core import TaskResult, TaskStatus
 from adcp.validation import (
+    ValidationOutcome,
     get_bundle_adcp_version,
     validate_request,
     validate_response,
@@ -57,6 +58,9 @@
 _SOURCE_VERSION_RE = re.compile(
     r"^(?:2\.5|3\.[01])\.\d+(?:-[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?)?$"
 )
+# This exact historical release was accepted while its bundle was shipped.
+# Keep continuation support after removing that bundle from distributions.
+_UNBUNDLED_SOURCE_VERSIONS = frozenset({"2.5.3"})
 _REQUIRED_LOSSES = frozenset({"feed_version_not_atomic", "pricing_version_not_atomic"})
 _MUTATION_LOSS = "mutation_idempotency_not_guaranteed"
 _ALLOWED_LOSSES = _REQUIRED_LOSSES | {_MUTATION_LOSS}
@@ -1128,7 +1132,7 @@ def _validate_bindings(
 
         request = payload["legacy_create_request"]
         outcome = validate_request("create_media_buy", request, version=record.source_adcp_version)
-        if not outcome.valid or outcome.variant == "skipped":
+        if not _source_schema_valid(outcome, record.source_adcp_version):
             raise _error(
                 CompatibilityContinuationErrorCode.INVALID_LEGACY_REQUEST,
                 "legacy_create_request does not validate against its exact source version",
@@ -1285,13 +1289,25 @@ def _validate_source_version(version: str) -> None:
             "source_adcp_version must be an exact 2.5.x, 3.0.x, or 3.1.x stable/prerelease bundle"
         )
     bundled_version = get_bundle_adcp_version(version=version)
+    if bundled_version is None and version in _UNBUNDLED_SOURCE_VERSIONS:
+        return
     if bundled_version != version:
         raise _invalid(
             "source_adcp_version must exactly match the bundled source schema release "
             f"(requested {version!r}, bundled {bundled_version!r})"
         )
 
 
+def _source_schema_valid(outcome: ValidationOutcome, version: str) -> bool:
+    """Allow a missing schema only for an exact supported cross-major release.
+
+    The continuation's account, product, pricing, target, loss and idempotency
+    bindings still run when the retired 2.5 bundle is unavailable.
+    """
+
+    return outcome.valid and (outcome.variant != "skipped" or version in _UNBUNDLED_SOURCE_VERSIONS)
+
+
 def _validate_loss_set(
     values: Any,
     *,
@@ -1488,7 +1504,7 @@ def _validate_source_discovery(
     request_outcome = validate_request("get_products", request, version=source_adcp_version)
     response_outcome = validate_response("get_products", response, version=source_adcp_version)
     for side, outcome in (("request", request_outcome), ("response", response_outcome)):
-        if not outcome.valid or outcome.variant == "skipped":
+        if not _source_schema_valid(outcome, source_adcp_version):
             raise _error(
                 CompatibilityContinuationErrorCode.INVALID_INPUT,
                 f"observed get_products {side} does not validate against its source version",
@@ -1729,7 +1745,7 @@ def _validated_result(
     except (TypeError, ValueError, CompatibilityContinuationError) as exc:
         raise _invalid_legacy_response(source_adcp_version, []) from exc
     outcome = validate_response("create_media_buy", payload, version=source_adcp_version)
-    if not outcome.valid or outcome.variant == "skipped":
+    if not _source_schema_valid(outcome, source_adcp_version):
         raise _invalid_legacy_response(
             source_adcp_version,
             [
@@ -1741,6 +1757,8 @@ def _validated_result(
                 for issue in outcome.issues
             ],
         )
+    if outcome.variant == "skipped":
+        _validate_unbundled_v25_result(payload, source_adcp_version)
     if outcome.variant in {"submitted", "working", "input-required"}:
         state = CompatibilityOperationState.PENDING
     elif "errors" in payload:
@@ -1764,6 +1782,43 @@ def _validated_result(
     return payload, state
 
 
+def _validate_unbundled_v25_result(payload: JsonObject, version: str) -> None:
+    """Retain the 2.5 success/error boundary without shipping its full schema."""
+
+    if "errors" in payload:
+        errors = payload["errors"]
+        valid_errors = (
+            isinstance(errors, list)
+            and bool(errors)
+            and all(
+                isinstance(error, Mapping)
+                and isinstance(error.get("code"), str)
+                and isinstance(error.get("message"), str)
+                for error in errors
+            )
+        )
+        if (
+            not valid_errors
+            or any(key in payload for key in ("media_buy_id", "buyer_ref", "packages"))
+            or payload.get("status") not in (None, "failed")
+        ):
+            raise _invalid_legacy_response(version, [])
+        return
+
+    packages = payload.get("packages")
+    if (
+        not isinstance(payload.get("media_buy_id"), str)
+        or not isinstance(payload.get("buyer_ref"), str)
+        or not isinstance(packages, list)
+        or not all(
+            isinstance(package, Mapping) and isinstance(package.get("package_id"), str)
+            for package in packages
+        )
+        or payload.get("status") not in (None, "completed")
+    ):
+        raise _invalid_legacy_response(version, [])
+
+
 def _aware_utc(value: datetime, *, field: str) -> datetime:
     if not isinstance(value, datetime) or value.tzinfo is None:
         raise _invalid(f"{field} must be a timezone-aware datetime")

tests/conformance/reporting/_production_installed.py

@@ -17,7 +17,7 @@
 
 
 def assert_installed_schema_keys(schemas, pinned):
-    assert set(schemas) == {"3.2.0-rc.6", pinned}
+    assert set(schemas) == {pinned}
 
 
 def main(settings):
@@ -70,33 +70,33 @@ def enter_phase(name):
         for name, expected in schemas.items():
             assert hashlib.sha256((schema_root / name).read_bytes()).hexdigest() == expected
     enter_phase("historical_reference_schemas")
-    reference = settings["historical_reference_schema"]
-    reference_root = Path(reference["root"])
-    assert not reference_root.is_relative_to(Path(sys.prefix))
-    assert not reference_root.is_relative_to(workspace)
-    assert reference["version"] not in settings["schemas"]
-    historical_resolved = schema_loader._resolve_schema_root(reference["version"])
-    assert historical_resolved is not None
-    historical_root = historical_resolved.root
-    assert historical_root.is_relative_to(Path(sys.prefix))
-    assert not historical_root.is_relative_to(workspace)
-    assert historical_root != reference_root
+    references = settings["historical_reference_schemas"]
+    installed_historical = []
 
-    def historical_manifest():
+    def schema_manifest(path):
         return {
-            str(p.relative_to(historical_root)): hashlib.sha256(p.read_bytes()).hexdigest()
-            for p in sorted(historical_root.rglob("*.json"))
+            str(p.relative_to(path)): hashlib.sha256(p.read_bytes()).hexdigest()
+            for p in sorted(path.rglob("*.json"))
         }
 
-    assert historical_manifest() == reference["files"]
-
-    def reference_manifest():
-        return {
-            str(p.relative_to(reference_root)): hashlib.sha256(p.read_bytes()).hexdigest()
-            for p in sorted(reference_root.rglob("*.json"))
-        }
-
-    assert reference_manifest() == reference["files"]
+    for reference in references:
+        version = reference["version"]
+        reference_root = Path(reference["root"])
+        assert not reference_root.is_relative_to(Path(sys.prefix))
+        assert not reference_root.is_relative_to(workspace)
+        assert version not in settings["schemas"]
+        # The wheel excludes superseded bundles. The immutable test input
+        # remains available through the existing source-layout fallback.
+        assert not (files("adcp") / "_schemas" / version / "bundled").is_dir()
+        historical_resolved = schema_loader._resolve_schema_root(version)
+        assert historical_resolved is not None
+        historical_root = historical_resolved.root
+        assert historical_root == reference_root
+        actual = schema_manifest(historical_root)
+        assert actual == reference["files"]
+        installed_historical.append(
+            {"version": version, "root": str(historical_root), "files": len(actual)}
+        )
     enter_phase("optional_driver_boundary")
     if settings["driver_absent"]:
         assert importlib.util.find_spec("psycopg") is None
@@ -105,7 +105,7 @@ def reference_manifest():
     else:
         assert os.environ.get("ADCP_PG_TEST_URL")
     reference_inputs = evidence / (settings["label"] + "-historical-schema-inputs.json")
-    reference_bytes = (json.dumps(reference, indent=2, sort_keys=True) + "\n").encode()
+    reference_bytes = (json.dumps(references, indent=2, sort_keys=True) + "\n").encode()
     reference_inputs.write_bytes(reference_bytes)
     identity = {
         "python": sys.version,
@@ -118,22 +118,11 @@ def reference_manifest():
         "wheel_sha256": settings["wheel_sha256"],
         "assets": settings["assets"],
         "schemas": settings["schemas"],
-        "historical_reference_schema": {
-            "version": reference["version"],
-            "root": str(reference_root),
-            "origin": (
-                "copied immutable test reference; independently compared with "
-                "the packaged historical bundle"
-            ),
-            "files": len(reference["files"]),
+        "historical_reference_schemas": {
+            "origin": "copied immutable test fixtures outside the wheel",
             "inputs": str(reference_inputs),
             "inputs_sha256": hashlib.sha256(reference_bytes).hexdigest(),
-        },
-        "installed_historical_schema": {
-            "version": reference["version"],
-            "root": str(historical_root),
-            "origin": "installed distribution",
-            "files": len(reference["files"]),
+            "versions": installed_historical,
         },
         "driver_absent": settings["driver_absent"],
     }
@@ -210,8 +199,11 @@ def reference_manifest():
     for name, module in tuple(sys.modules.items()):
         if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None):
             assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix))
-    assert reference_manifest() == reference["files"]
-    assert historical_manifest() == reference["files"]
+    for reference in references:
+        assert schema_manifest(Path(reference["root"])) == reference["files"]
+        resolved = schema_loader._resolve_schema_root(reference["version"])
+        assert resolved is not None
+        assert schema_manifest(resolved.root) == reference["files"]
     progress.start("complete")
     progress.close()
     journal = progress.path.with_suffix(".jsonl")