diff --git a/changelog.d/certify-us-populace-us-2024-f0af251-703bd81a565c-20260620T201958Z.changed.md b/changelog.d/certify-us-populace-us-2024-f0af251-703bd81a565c-20260620T201958Z.changed.md new file mode 100644 index 00000000..4cb9cea5 --- /dev/null +++ b/changelog.d/certify-us-populace-us-2024-f0af251-703bd81a565c-20260620T201958Z.changed.md @@ -0,0 +1 @@ +Certify the US data release `populace-us-2024-f0af251-703bd81a565c-20260620T201958Z` (populace_us_2024, policyengine-us 1.729.0) directly from its data release manifest. diff --git a/src/policyengine/data/release_manifests/us.json b/src/policyengine/data/release_manifests/us.json index ebdd1343..787a6d71 100644 --- a/src/policyengine/data/release_manifests/us.json +++ b/src/policyengine/data/release_manifests/us.json @@ -5,63 +5,63 @@ "certified_by": "policyengine.py certification", "certified_for_model_version": "1.729.0", "compatibility_basis": "built_with_model_package", - "data_build_id": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z" + "data_build_id": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z" }, "certified_data_artifact": { - "build_id": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", + "build_id": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", "data_package": { "name": "populace-data", "version": "0.1.0" }, "dataset": "populace_us_2024", - "sha256": "f0af25192d6c8a7efc2638da2bd8ec4278b066a1092cc89ef2275811efaff11d", - "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z" + "sha256": "16be6338f9d0b3c339883dae59949e995663b64cf145de6728b3dd0f916c5d5f", + "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-f0af251-703bd81a565c-20260620T201958Z" }, "country_id": "us", "data_package": { "name": "populace-data", - "release_manifest_path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/release_manifest.json", - "release_manifest_revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", + "release_manifest_path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/release_manifest.json", + "release_manifest_revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", "repo_id": "policyengine/populace-us", "repo_type": "dataset", "version": "0.1.0" }, "datasets": { "calibration_diagnostics": { - "path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/calibration_diagnostics.json", + "path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/calibration_diagnostics.json", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "sha256": "5dd5aef3cd06932d0af48beec7243177adf43dbfd426b2402e43f5cda91c9b0b" + "revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "sha256": "448a3c7ff0bda996332764d3c0a831f0e1df05a0eaaff8c2b3d65a40d9383c13" }, "demographics": { - "path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/demographics.json", + "path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/demographics.json", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "sha256": "ecda31ab2595650edb5b68ce50afb00d206b1875397e24c319ad5773919c0f2a" + "revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "sha256": "eea9b24a198ff654390cf8a14c2dc12eb5b9894d203d0fd9674747bbe3ab7815" }, "populace_us_2024": { "path": "populace_us_2024.h5", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "sha256": "f0af25192d6c8a7efc2638da2bd8ec4278b066a1092cc89ef2275811efaff11d" + "revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "sha256": "16be6338f9d0b3c339883dae59949e995663b64cf145de6728b3dd0f916c5d5f" }, "populace_us_2024_calibration": { "path": "populace_us_2024_calibration.npz", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "sha256": "aff274bf4b9cf4224ddcb76b48134766ce926518684ab007ce99b642731fdd78" + "revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "sha256": "bf79d64a5e18a028ecd1236f393b72fea26b32411b2e41c0a0e68e48e556adbb" }, "reform_validation": { - "path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/reform_validation.json", + "path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/reform_validation.json", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "sha256": "af05011f533fa4ade8746a7b4e00590ad981f0689f98111edf7ead661df811d4" + "revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "sha256": "c577e0b1419f9c12de0cad7a4b3bf32507e718f5ea9d79bc3a32d0c48160cc89" }, "us_source_coverage": { - "path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/us_source_coverage.json", + "path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/us_source_coverage.json", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "sha256": "48a1e5b178a2d54271ba0308b33341543c2bfe0e6931484d3e9e71415c6420ca" + "revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "sha256": "c3e0c388be2f53ca03b59dddc110db28bc4b0acaa21e2eb86003c735e4f47d8e" } }, "default_dataset": "populace_us_2024", diff --git a/src/policyengine/data/release_manifests/us.trace.tro.jsonld b/src/policyengine/data/release_manifests/us.trace.tro.jsonld index 5fabb2bb..e7a53686 100644 --- a/src/policyengine/data/release_manifests/us.trace.tro.jsonld +++ b/src/policyengine/data/release_manifests/us.trace.tro.jsonld @@ -17,7 +17,7 @@ "schema:name": "PolicyEngine", "schema:url": "https://policyengine.org" }, - "schema:dateCreated": "2026-06-19T01:03:19.741844+00:00", + "schema:dateCreated": "2026-06-20T20:33:09.794848+00:00", "schema:description": "TRACE TRO for certified runtime bundle us-4.17.10 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.", "schema:name": "policyengine us certified bundle TRO", "trov:createdWith": { @@ -45,7 +45,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/data_release_manifest" }, - "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/release_manifest.json" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/release_manifest.json" }, { "@id": "arrangement/1/location/dataset", @@ -53,7 +53,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/dataset" }, - "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/populace_us_2024.h5" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/populace_us_2024.h5" }, { "@id": "arrangement/1/location/model_wheel", @@ -75,21 +75,21 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "b8108ca3ada942977b1b82473647c60821ad689104288991f180400898abce98" + "trov:sha256": "a3f110ed0541a0512a20f0207e92da5a47d96b2011341c16b9da74b551c340e0" }, { "@id": "composition/1/artifact/data_release_manifest", "@type": "trov:ResearchArtifact", "schema:name": "populace-data release manifest 0.1.0", "trov:mimeType": "application/json", - "trov:sha256": "10ce09590dc16b8fb2978de0d4f0811b51a628684ac022168ae0bdcf936d6096" + "trov:sha256": "917655e713397ad18e52e0893ffeb1975c609616c15c2b08f9e989e7bad2a2d9" }, { "@id": "composition/1/artifact/dataset", "@type": "trov:ResearchArtifact", "schema:name": "populace_us_2024", "trov:mimeType": "application/x-hdf5", - "trov:sha256": "f0af25192d6c8a7efc2638da2bd8ec4278b066a1092cc89ef2275811efaff11d" + "trov:sha256": "16be6338f9d0b3c339883dae59949e995663b64cf145de6728b3dd0f916c5d5f" }, { "@id": "composition/1/artifact/model_wheel", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "0d2943af922484e6bbef219d982ab1dd3e7f01e1746e65a820629348edc7a76e" + "trov:sha256": "4a90cad4ccb0776acd8d35aca85e87a750a50af5f9742f9c9489e3aa65672fd7" } }, "trov:hasPerformance": { @@ -111,17 +111,14 @@ "pe:builtWithModelVersion": "1.729.0", "pe:certifiedBy": "policyengine.py certification", "pe:certifiedForModelVersion": "1.729.0", - "pe:ciGitRef": "refs/heads/main", - "pe:ciGitSha": "3ffa6825920f22e05a5bcd5c58416200ff7134c0", - "pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/27829003973", "pe:compatibilityBasis": "built_with_model_package", - "pe:dataBuildId": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z", - "pe:emittedIn": "github-actions", - "rdfs:comment": "Certification of build populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z for policyengine-us 1.729.0.", + "pe:dataBuildId": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", + "pe:emittedIn": "local", + "rdfs:comment": "Certification of build populace-us-2024-f0af251-703bd81a565c-20260620T201958Z for policyengine-us 1.729.0.", "trov:accessedArrangement": { "@id": "arrangement/1" }, - "trov:startedAtTime": "2026-06-19T01:03:19.741844+00:00", + "trov:startedAtTime": "2026-06-20T20:33:09.794848+00:00", "trov:wasConductedBy": { "@id": "trs" } diff --git a/src/policyengine/provenance/bundle.py b/src/policyengine/provenance/bundle.py index 14feb0ee..aeee68c4 100644 --- a/src/policyengine/provenance/bundle.py +++ b/src/policyengine/provenance/bundle.py @@ -951,12 +951,15 @@ def regenerate_trace_tro(country: str, manifest_dir: Path = MANIFEST_DIR) -> Pat from policyengine.provenance.manifest import ( DataReleaseManifestUnavailableError, get_data_release_manifest, + get_release_manifest, ) from policyengine.provenance.trace import ( build_trace_tro_from_release_bundle, serialize_trace_tro, ) + get_release_manifest.cache_clear() + get_data_release_manifest.cache_clear() release = get_release_manifest(country) try: data_release = get_data_release_manifest(country) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 0b378d06..69d01602 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -381,3 +381,55 @@ def test__given_vendored_us_manifest__then_tro_sidecar_binds_it(self): ) assert bundle_manifest["trov:sha256"] == expected + + +class TestTraceTroRegeneration: + def test__given_cached_manifest_readers__then_clears_before_regeneration( + self, monkeypatch, tmp_path + ): + """Certification writes the manifest then regenerates the sidecar in + the same process, so stale manifest caches must be invalidated first.""" + from policyengine.provenance import bundle, trace + from policyengine.provenance import manifest as manifest_module + + calls = [] + + class FakeCachedReader: + def __init__(self, name, result): + self.name = name + self.result = result + + def cache_clear(self): + calls.append(f"{self.name}.cache_clear") + + def __call__(self, country): + calls.append(f"{self.name}({country})") + return self.result + + monkeypatch.setattr( + manifest_module, + "get_release_manifest", + FakeCachedReader("release", object()), + ) + monkeypatch.setattr( + manifest_module, + "get_data_release_manifest", + FakeCachedReader("data_release", object()), + ) + monkeypatch.setattr( + trace, + "build_trace_tro_from_release_bundle", + lambda release, data_release: {"ok": True}, + ) + monkeypatch.setattr(trace, "serialize_trace_tro", lambda tro: b"{}\n") + + out_path = bundle.regenerate_trace_tro("us", tmp_path) + + assert out_path == tmp_path / "us.trace.tro.jsonld" + assert out_path.read_bytes() == b"{}\n" + assert calls == [ + "release.cache_clear", + "data_release.cache_clear", + "release(us)", + "data_release(us)", + ] diff --git a/tests/test_models.py b/tests/test_models.py index f6a3c10d..244c9841 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -120,7 +120,7 @@ def test_has_release_manifest_metadata(self): assert ( us_latest.default_dataset_uri == "hf://policyengine/populace-us/populace_us_2024.h5" - "@populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z" + "@populace-us-2024-f0af251-703bd81a565c-20260620T201958Z" ) def test_has_hundreds_of_parameters(self): diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index af9d9571..a66f3767 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -44,7 +44,7 @@ US_MODEL_VERSION = "1.729.0" US_BUILT_WITH_MODEL_VERSION = "1.729.0" US_DATA_RELEASE_VERSION = "0.1.0" -US_DATA_RELEASE_ID = "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z" +US_DATA_RELEASE_ID = "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z" US_DATA_RELEASE_REVISION = US_DATA_RELEASE_ID US_DATA_RELEASE_PATH = f"releases/{US_DATA_RELEASE_ID}/release_manifest.json" US_DATA_ARTIFACT_REVISION = US_DATA_RELEASE_ID