Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Certify the US data release `populace-us-2024-f0af251-703bd81a565c-20260620T201958Z` (populace_us_2024, policyengine-us 1.729.0) directly from its data release manifest.
44 changes: 22 additions & 22 deletions src/policyengine/data/release_manifests/us.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,63 +5,63 @@
"certified_by": "policyengine.py certification",
"certified_for_model_version": "1.729.0",
"compatibility_basis": "built_with_model_package",
"data_build_id": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z"
"data_build_id": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z"
},
"certified_data_artifact": {
"build_id": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"build_id": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"data_package": {
"name": "populace-data",
"version": "0.1.0"
},
"dataset": "populace_us_2024",
"sha256": "f0af25192d6c8a7efc2638da2bd8ec4278b066a1092cc89ef2275811efaff11d",
"uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z"
"sha256": "16be6338f9d0b3c339883dae59949e995663b64cf145de6728b3dd0f916c5d5f",
"uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-f0af251-703bd81a565c-20260620T201958Z"
},
"country_id": "us",
"data_package": {
"name": "populace-data",
"release_manifest_path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/release_manifest.json",
"release_manifest_revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"release_manifest_path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/release_manifest.json",
"release_manifest_revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"repo_id": "policyengine/populace-us",
"repo_type": "dataset",
"version": "0.1.0"
},
"datasets": {
"calibration_diagnostics": {
"path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/calibration_diagnostics.json",
"path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/calibration_diagnostics.json",
"repo_id": "policyengine/populace-us",
"revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"sha256": "5dd5aef3cd06932d0af48beec7243177adf43dbfd426b2402e43f5cda91c9b0b"
"revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"sha256": "448a3c7ff0bda996332764d3c0a831f0e1df05a0eaaff8c2b3d65a40d9383c13"
},
"demographics": {
"path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/demographics.json",
"path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/demographics.json",
"repo_id": "policyengine/populace-us",
"revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"sha256": "ecda31ab2595650edb5b68ce50afb00d206b1875397e24c319ad5773919c0f2a"
"revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"sha256": "eea9b24a198ff654390cf8a14c2dc12eb5b9894d203d0fd9674747bbe3ab7815"
},
"populace_us_2024": {
"path": "populace_us_2024.h5",
"repo_id": "policyengine/populace-us",
"revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"sha256": "f0af25192d6c8a7efc2638da2bd8ec4278b066a1092cc89ef2275811efaff11d"
"revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"sha256": "16be6338f9d0b3c339883dae59949e995663b64cf145de6728b3dd0f916c5d5f"
},
"populace_us_2024_calibration": {
"path": "populace_us_2024_calibration.npz",
"repo_id": "policyengine/populace-us",
"revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"sha256": "aff274bf4b9cf4224ddcb76b48134766ce926518684ab007ce99b642731fdd78"
"revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"sha256": "bf79d64a5e18a028ecd1236f393b72fea26b32411b2e41c0a0e68e48e556adbb"
},
"reform_validation": {
"path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/reform_validation.json",
"path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/reform_validation.json",
"repo_id": "policyengine/populace-us",
"revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"sha256": "af05011f533fa4ade8746a7b4e00590ad981f0689f98111edf7ead661df811d4"
"revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"sha256": "c577e0b1419f9c12de0cad7a4b3bf32507e718f5ea9d79bc3a32d0c48160cc89"
},
"us_source_coverage": {
"path": "releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/us_source_coverage.json",
"path": "releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/us_source_coverage.json",
"repo_id": "policyengine/populace-us",
"revision": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"sha256": "48a1e5b178a2d54271ba0308b33341543c2bfe0e6931484d3e9e71415c6420ca"
"revision": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"sha256": "c3e0c388be2f53ca03b59dddc110db28bc4b0acaa21e2eb86003c735e4f47d8e"
}
},
"default_dataset": "populace_us_2024",
Expand Down
25 changes: 11 additions & 14 deletions src/policyengine/data/release_manifests/us.trace.tro.jsonld
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"schema:name": "PolicyEngine",
"schema:url": "https://policyengine.org"
},
"schema:dateCreated": "2026-06-19T01:03:19.741844+00:00",
"schema:dateCreated": "2026-06-20T20:33:09.794848+00:00",
"schema:description": "TRACE TRO for certified runtime bundle us-4.17.10 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.",
"schema:name": "policyengine us certified bundle TRO",
"trov:createdWith": {
Expand Down Expand Up @@ -45,15 +45,15 @@
"trov:hasArtifact": {
"@id": "composition/1/artifact/data_release_manifest"
},
"trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/releases/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/release_manifest.json"
"trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/releases/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/release_manifest.json"
},
{
"@id": "arrangement/1/location/dataset",
"@type": "trov:ArtifactLocation",
"trov:hasArtifact": {
"@id": "composition/1/artifact/dataset"
},
"trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z/populace_us_2024.h5"
"trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-f0af251-703bd81a565c-20260620T201958Z/populace_us_2024.h5"
},
{
"@id": "arrangement/1/location/model_wheel",
Expand All @@ -75,21 +75,21 @@
"@type": "trov:ResearchArtifact",
"schema:name": "policyengine.py bundle manifest for us",
"trov:mimeType": "application/json",
"trov:sha256": "b8108ca3ada942977b1b82473647c60821ad689104288991f180400898abce98"
"trov:sha256": "a3f110ed0541a0512a20f0207e92da5a47d96b2011341c16b9da74b551c340e0"
},
{
"@id": "composition/1/artifact/data_release_manifest",
"@type": "trov:ResearchArtifact",
"schema:name": "populace-data release manifest 0.1.0",
"trov:mimeType": "application/json",
"trov:sha256": "10ce09590dc16b8fb2978de0d4f0811b51a628684ac022168ae0bdcf936d6096"
"trov:sha256": "917655e713397ad18e52e0893ffeb1975c609616c15c2b08f9e989e7bad2a2d9"
},
{
"@id": "composition/1/artifact/dataset",
"@type": "trov:ResearchArtifact",
"schema:name": "populace_us_2024",
"trov:mimeType": "application/x-hdf5",
"trov:sha256": "f0af25192d6c8a7efc2638da2bd8ec4278b066a1092cc89ef2275811efaff11d"
"trov:sha256": "16be6338f9d0b3c339883dae59949e995663b64cf145de6728b3dd0f916c5d5f"
},
{
"@id": "composition/1/artifact/model_wheel",
Expand All @@ -102,7 +102,7 @@
"trov:hasFingerprint": {
"@id": "composition/1/fingerprint",
"@type": "trov:CompositionFingerprint",
"trov:sha256": "0d2943af922484e6bbef219d982ab1dd3e7f01e1746e65a820629348edc7a76e"
"trov:sha256": "4a90cad4ccb0776acd8d35aca85e87a750a50af5f9742f9c9489e3aa65672fd7"
}
},
"trov:hasPerformance": {
Expand All @@ -111,17 +111,14 @@
"pe:builtWithModelVersion": "1.729.0",
"pe:certifiedBy": "policyengine.py certification",
"pe:certifiedForModelVersion": "1.729.0",
"pe:ciGitRef": "refs/heads/main",
"pe:ciGitSha": "3ffa6825920f22e05a5bcd5c58416200ff7134c0",
"pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/27829003973",
"pe:compatibilityBasis": "built_with_model_package",
"pe:dataBuildId": "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z",
"pe:emittedIn": "github-actions",
"rdfs:comment": "Certification of build populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z for policyengine-us 1.729.0.",
"pe:dataBuildId": "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z",
"pe:emittedIn": "local",
"rdfs:comment": "Certification of build populace-us-2024-f0af251-703bd81a565c-20260620T201958Z for policyengine-us 1.729.0.",
"trov:accessedArrangement": {
"@id": "arrangement/1"
},
"trov:startedAtTime": "2026-06-19T01:03:19.741844+00:00",
"trov:startedAtTime": "2026-06-20T20:33:09.794848+00:00",
"trov:wasConductedBy": {
"@id": "trs"
}
Expand Down
3 changes: 3 additions & 0 deletions src/policyengine/provenance/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -951,12 +951,15 @@ def regenerate_trace_tro(country: str, manifest_dir: Path = MANIFEST_DIR) -> Pat
from policyengine.provenance.manifest import (
DataReleaseManifestUnavailableError,
get_data_release_manifest,
get_release_manifest,
)
from policyengine.provenance.trace import (
build_trace_tro_from_release_bundle,
serialize_trace_tro,
)

get_release_manifest.cache_clear()
get_data_release_manifest.cache_clear()
release = get_release_manifest(country)
try:
data_release = get_data_release_manifest(country)
Expand Down
52 changes: 52 additions & 0 deletions tests/test_certify_data_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -381,3 +381,55 @@ def test__given_vendored_us_manifest__then_tro_sidecar_binds_it(self):
)

assert bundle_manifest["trov:sha256"] == expected


class TestTraceTroRegeneration:
def test__given_cached_manifest_readers__then_clears_before_regeneration(
self, monkeypatch, tmp_path
):
"""Certification writes the manifest then regenerates the sidecar in
the same process, so stale manifest caches must be invalidated first."""
from policyengine.provenance import bundle, trace
from policyengine.provenance import manifest as manifest_module

calls = []

class FakeCachedReader:
def __init__(self, name, result):
self.name = name
self.result = result

def cache_clear(self):
calls.append(f"{self.name}.cache_clear")

def __call__(self, country):
calls.append(f"{self.name}({country})")
return self.result

monkeypatch.setattr(
manifest_module,
"get_release_manifest",
FakeCachedReader("release", object()),
)
monkeypatch.setattr(
manifest_module,
"get_data_release_manifest",
FakeCachedReader("data_release", object()),
)
monkeypatch.setattr(
trace,
"build_trace_tro_from_release_bundle",
lambda release, data_release: {"ok": True},
)
monkeypatch.setattr(trace, "serialize_trace_tro", lambda tro: b"{}\n")

out_path = bundle.regenerate_trace_tro("us", tmp_path)

assert out_path == tmp_path / "us.trace.tro.jsonld"
assert out_path.read_bytes() == b"{}\n"
assert calls == [
"release.cache_clear",
"data_release.cache_clear",
"release(us)",
"data_release(us)",
]
2 changes: 1 addition & 1 deletion tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ def test_has_release_manifest_metadata(self):
assert (
us_latest.default_dataset_uri
== "hf://policyengine/populace-us/populace_us_2024.h5"
"@populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z"
"@populace-us-2024-f0af251-703bd81a565c-20260620T201958Z"
)

def test_has_hundreds_of_parameters(self):
Expand Down
2 changes: 1 addition & 1 deletion tests/test_release_manifests.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
US_MODEL_VERSION = "1.729.0"
US_BUILT_WITH_MODEL_VERSION = "1.729.0"
US_DATA_RELEASE_VERSION = "0.1.0"
US_DATA_RELEASE_ID = "populace-us-2024-c86a631-6e1bcd0271a5-20260619T002242Z"
US_DATA_RELEASE_ID = "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z"
US_DATA_RELEASE_REVISION = US_DATA_RELEASE_ID
US_DATA_RELEASE_PATH = f"releases/{US_DATA_RELEASE_ID}/release_manifest.json"
US_DATA_ARTIFACT_REVISION = US_DATA_RELEASE_ID
Expand Down