From 68cb8f8e9118e9a4451ac3be85ce99dd0515bb6f Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Thu, 17 Sep 2026 12:32:45 -0700 Subject: [PATCH 01/10] fix(zipapp): reduce bundled interpreter size Bazel reports source symlinks as regular files, causing Python executable aliases to be copied into zipapps as full binaries. Hermetic runtimes also include shared libpython artifacts even when the interpreter links Python statically. Preserve source symlinks through sandbox indirection and exclude libpython shared objects from runtime files while retaining them for explicit native dependencies. Add regression coverage for archive structure and native extension loading. --- news/py-zipapp-size.fixed.md | 3 + .../private/hermetic_runtime_repo_setup.bzl | 7 ++- tests/py_zipapp/main.py | 4 ++ tests/py_zipapp/venv_zipapp_test.py | 4 ++ tests/tools/zipapp/zipper_test.py | 61 +++++++++++++++++++ tools/zipapp/zipper.py | 24 ++++++-- 6 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 news/py-zipapp-size.fixed.md diff --git a/news/py-zipapp-size.fixed.md b/news/py-zipapp-size.fixed.md new file mode 100644 index 0000000000..a3294ca190 --- /dev/null +++ b/news/py-zipapp-size.fixed.md @@ -0,0 +1,3 @@ +(zipapp) Reduced self-contained archive sizes by preserving Python executable +symlinks and omitting shared `libpython` files when the hermetic interpreter +includes Python statically. \ No newline at end of file diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324894..7677dc8228 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -67,9 +67,10 @@ def define_hermetic_runtime_toolchain_impl( ] files_include += extra_files_glob_include files_exclude = [ - # Unused shared libraries. `python` executable and the `:libpython` target - # depend on `libpython{python_version}.so.1.0`. - "lib/libpython{major}.{minor}*.so".format(**version_dict), + # The hermetic Linux interpreter includes libpython statically. Keep + # shared libraries available through :libpython for explicit users, + # but don't include them in every Python runtime's runfiles. + "lib/libpython*.so*", # static libraries "lib/**/*.a", # tests for the standard libraries. diff --git a/tests/py_zipapp/main.py b/tests/py_zipapp/main.py index 5770170d2c..8bd959926c 100644 --- a/tests/py_zipapp/main.py +++ b/tests/py_zipapp/main.py @@ -4,6 +4,10 @@ def main(): print("Hello from zipapp") try: + import _ssl + + print(f"dep: {_ssl}") + import some_dep print(f"dep: {some_dep}") diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index bd26d533a3..2923c9b3ef 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -84,6 +84,10 @@ def test_zipapp_structure(self): # On Windows, pyvenv.cfg and bin/python3 are generated at runtime. if os.name != "nt": + self.assertFalse( + any("/lib/libpython" in name for name in namelist), + "Statically linked Python should not bundle libpython", + ) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") # The venv directory name depends on the target name, so find it diff --git a/tests/tools/zipapp/zipper_test.py b/tests/tools/zipapp/zipper_test.py index e4f25c5e21..412ee20c82 100644 --- a/tests/tools/zipapp/zipper_test.py +++ b/tests/tools/zipapp/zipper_test.py @@ -115,6 +115,67 @@ def test_create_zip_with_direct_symlink(tmp_path): ) +def test_create_zip_with_source_symlink_marked_as_regular(tmp_path): + manifest_path = tmp_path / "manifest.txt" + output_zip = tmp_path / "output.zip" + + target_path = tmp_path / "python3.14" + target_path.write_text("python") + symlink_path = tmp_path / "python" + symlink_path.symlink_to(target_path.name) + manifest_path.write_text(f"rf-file|0|bin/python|{symlink_path}") + + create_zip(manifest_path, output_zip) + + with zipfile.ZipFile(output_zip, "r") as zf: + assert_zip_file_content( + zf, + "runfiles/my_ws/bin/python", + is_symlink_file=True, + target=target_path.name, + ) + + +def test_create_zip_with_sandboxed_source_symlink(tmp_path): + manifest_path = tmp_path / "manifest.txt" + output_zip = tmp_path / "output.zip" + + source_dir = tmp_path / "source" + source_dir.mkdir() + target_path = source_dir / "python3.14" + target_path.write_text("python") + source_symlink = source_dir / "python" + source_symlink.symlink_to(target_path.name) + + sandbox_dir = tmp_path / "sandbox" + sandbox_dir.mkdir() + sandbox_symlink = sandbox_dir / "python" + sandbox_symlink.symlink_to(source_symlink) + sandbox_regular = sandbox_dir / "python3.14" + sandbox_regular.symlink_to(target_path) + manifest_path.write_text( + "\n".join( + [ + f"rf-file|0|bin/python|{sandbox_symlink}", + f"rf-file|0|bin/python3.14|{sandbox_regular}", + ] + ) + ) + + create_zip(manifest_path, output_zip) + + with zipfile.ZipFile(output_zip, "r") as zf: + assert_zip_file_content( + zf, + "runfiles/my_ws/bin/python", + is_symlink_file=True, + target=target_path.name, + ) + assert_zip_file_content( + zf, "runfiles/my_ws/bin/python3.14", content="python" + ) + + def test_pathsep_normalization(tmp_path): manifest_path = tmp_path / "manifest.txt" output_zip = tmp_path / "output.zip" diff --git a/tools/zipapp/zipper.py b/tools/zipapp/zipper.py index 5a8eb8c6fd..c0e547284a 100644 --- a/tools/zipapp/zipper.py +++ b/tools/zipapp/zipper.py @@ -118,6 +118,23 @@ def normalize_zip_path(path): return path.replace("\\", "/") +def _source_symlink_target(content_path, is_symlink_str): + if is_symlink_str == "1": + return os.readlink(content_path) + if is_symlink_str != "0" or not os.path.islink(content_path): + return None + + target = os.readlink(content_path) + if not os.path.isabs(target): + return target + + # Bazel sandboxes expose regular inputs as absolute symlinks. Look through + # that indirection to detect whether the original source is also a symlink. + if os.path.islink(target): + return os.readlink(target) + return None + + def _write_entry(zf, entry, compress_type, seen, platform_pathsep): type_, is_symlink_str, zip_path, content_path = entry # Normalize slashes, otherwise the `seen` logic doesn't @@ -155,14 +172,13 @@ def _write_entry(zf, entry, compress_type, seen, platform_pathsep): else: is_symlink_str = "0" - is_symlink = is_symlink_str == "1" - - if is_symlink: + symlink_target = _source_symlink_target(content_path, is_symlink_str) + if symlink_target is not None: zi = zipfile.ZipInfo(zip_path) zi.date_time = (1980, 1, 1, 0, 0, 0) zi.create_system = 3 # Unix zi.compress_type = compress_type - target = convert_symlink_target(os.readlink(content_path), platform_pathsep) + target = convert_symlink_target(symlink_target, platform_pathsep) # Set permissions to 777 for symlink (standard) zi.external_attr = (S_IFLNK | 0o777) << 16 zf.writestr(zi, target) From 08e599c5385419eb3e66014eaa9fd5182db98751 Mon Sep 17 00:00:00 2001 From: gfrankliu <17630355+gfrankliu@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:36:10 -0700 Subject: [PATCH 02/10] test(zipapp): allow macOS libpython dylibs Address PR review finding: the runtime-size assertion should mirror the Linux-only .so exclusion. macOS hermetic runtimes intentionally retain libpython dylibs, so only reject libpython shared-object entries. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/py_zipapp/venv_zipapp_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index 2923c9b3ef..cfadac74fb 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -85,7 +85,10 @@ def test_zipapp_structure(self): # On Windows, pyvenv.cfg and bin/python3 are generated at runtime. if os.name != "nt": self.assertFalse( - any("/lib/libpython" in name for name in namelist), + any( + "/lib/libpython" in name and ".so" in name + for name in namelist + ), "Statically linked Python should not bundle libpython", ) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") From d586969b07837cb31be2dbe1ad6af07188de87be Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Thu, 17 Sep 2026 13:51:51 -0700 Subject: [PATCH 03/10] style(zipapp): apply Ruff formatting --- tests/py_zipapp/venv_zipapp_test.py | 3 +-- tests/tools/zipapp/zipper_test.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index cfadac74fb..b906073fce 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -86,8 +86,7 @@ def test_zipapp_structure(self): if os.name != "nt": self.assertFalse( any( - "/lib/libpython" in name and ".so" in name - for name in namelist + "/lib/libpython" in name and ".so" in name for name in namelist ), "Statically linked Python should not bundle libpython", ) diff --git a/tests/tools/zipapp/zipper_test.py b/tests/tools/zipapp/zipper_test.py index 412ee20c82..e0ff1557d5 100644 --- a/tests/tools/zipapp/zipper_test.py +++ b/tests/tools/zipapp/zipper_test.py @@ -171,9 +171,7 @@ def test_create_zip_with_sandboxed_source_symlink(tmp_path): is_symlink_file=True, target=target_path.name, ) - assert_zip_file_content( - zf, "runfiles/my_ws/bin/python3.14", content="python" - ) + assert_zip_file_content(zf, "runfiles/my_ws/bin/python3.14", content="python") def test_pathsep_normalization(tmp_path): From 1e6601284bcce387571e411792d05621b2da666c Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Thu, 17 Sep 2026 15:22:51 -0700 Subject: [PATCH 04/10] fix(toolchains): retain versioned libpython libraries Custom hermetic distributions may use dynamically linked interpreters and require libpython.so.1.0 at runtime. The zipapp size optimization incorrectly removed that library from every Linux runtime. Restore the existing versioned-library inclusion and keep the zipapp optimization limited to preserving executable symlinks. --- news/py-zipapp-size.fixed.md | 3 +-- python/private/hermetic_runtime_repo_setup.bzl | 7 +++---- tests/py_zipapp/main.py | 4 ---- tests/py_zipapp/venv_zipapp_test.py | 6 ------ 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/news/py-zipapp-size.fixed.md b/news/py-zipapp-size.fixed.md index a3294ca190..c971d66cc1 100644 --- a/news/py-zipapp-size.fixed.md +++ b/news/py-zipapp-size.fixed.md @@ -1,3 +1,2 @@ (zipapp) Reduced self-contained archive sizes by preserving Python executable -symlinks and omitting shared `libpython` files when the hermetic interpreter -includes Python statically. \ No newline at end of file +symlinks instead of storing each alias as another copy of the interpreter. \ No newline at end of file diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 7677dc8228..20b0324894 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -67,10 +67,9 @@ def define_hermetic_runtime_toolchain_impl( ] files_include += extra_files_glob_include files_exclude = [ - # The hermetic Linux interpreter includes libpython statically. Keep - # shared libraries available through :libpython for explicit users, - # but don't include them in every Python runtime's runfiles. - "lib/libpython*.so*", + # Unused shared libraries. `python` executable and the `:libpython` target + # depend on `libpython{python_version}.so.1.0`. + "lib/libpython{major}.{minor}*.so".format(**version_dict), # static libraries "lib/**/*.a", # tests for the standard libraries. diff --git a/tests/py_zipapp/main.py b/tests/py_zipapp/main.py index 8bd959926c..5770170d2c 100644 --- a/tests/py_zipapp/main.py +++ b/tests/py_zipapp/main.py @@ -4,10 +4,6 @@ def main(): print("Hello from zipapp") try: - import _ssl - - print(f"dep: {_ssl}") - import some_dep print(f"dep: {some_dep}") diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index b906073fce..bd26d533a3 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -84,12 +84,6 @@ def test_zipapp_structure(self): # On Windows, pyvenv.cfg and bin/python3 are generated at runtime. if os.name != "nt": - self.assertFalse( - any( - "/lib/libpython" in name and ".so" in name for name in namelist - ), - "Statically linked Python should not bundle libpython", - ) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") # The venv directory name depends on the target name, so find it From 63244d9429d50b2c1b4d56feea75f741a115cb45 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Thu, 17 Sep 2026 15:28:27 -0700 Subject: [PATCH 05/10] fix(zipapp): omit libpython from static Astral runtimes Astral Python Standalone builds from 20250604 onward include libpython statically in the interpreter, so packaging their shared libpython objects adds substantial unused size to self-contained zipapps. Recognize only official Astral release URLs at or after that build date and exclude libpython shared objects for those runtimes. Unknown, custom, and older distributions retain the existing versioned libraries for dynamically linked interpreters. --- news/py-zipapp-size.fixed.md | 4 +- python/private/BUILD.bazel | 1 + .../private/hermetic_runtime_repo_setup.bzl | 5 +++ python/private/pbs_manifest.bzl | 27 ++++++++++++ python/private/python_repository.bzl | 4 ++ tests/py_zipapp/venv_zipapp_test.py | 6 +++ .../parse_runtime_manifest_tests.bzl | 43 ++++++++++++++++++- 7 files changed, 88 insertions(+), 2 deletions(-) diff --git a/news/py-zipapp-size.fixed.md b/news/py-zipapp-size.fixed.md index c971d66cc1..508882b9da 100644 --- a/news/py-zipapp-size.fixed.md +++ b/news/py-zipapp-size.fixed.md @@ -1,2 +1,4 @@ (zipapp) Reduced self-contained archive sizes by preserving Python executable -symlinks instead of storing each alias as another copy of the interpreter. \ No newline at end of file +symlinks instead of storing each alias as another copy of the interpreter, and +by omitting shared `libpython` files from recognized statically linked Astral +runtime builds. diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index b4ff84f14a..b88fb724a4 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -780,6 +780,7 @@ bzl_library( srcs = ["python_repository.bzl"], deps = [ ":auth", + ":pbs_manifest", ":repo_utils", ":text_util", "//python:versions", diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324894..d61a040fe6 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -29,6 +29,7 @@ def define_hermetic_runtime_toolchain_impl( name, extra_files_glob_include, extra_files_glob_exclude, + interpreter_has_static_libpython = False, python_version, python_bin, coverage_tool): @@ -45,6 +46,8 @@ def define_hermetic_runtime_toolchain_impl( binaries). extra_files_glob_exclude: {type}`list[str]` additional glob exclude patterns for the target runtime files. + interpreter_has_static_libpython: {type}`bool` whether the interpreter + includes libpython statically. python_version: {type}`str` The Python version, in `major.minor.micro` format. python_bin: {type}`str` The path to the Python binary within the @@ -78,6 +81,8 @@ def define_hermetic_runtime_toolchain_impl( # During pyc creation, temp files named *.pyc.NNN are created "**/__pycache__/*.pyc.*", ] + if interpreter_has_static_libpython: + files_exclude.append("lib/libpython*.so*") files_exclude += extra_files_glob_exclude native.filegroup( diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl index 86434dc0ea..44dbf27108 100644 --- a/python/private/pbs_manifest.bzl +++ b/python/private/pbs_manifest.bzl @@ -1,5 +1,11 @@ """Helper functions to parse python-build-standalone manifests.""" +_ASTRAL_STATIC_LIBPYTHON_RELEASE = 20250604 +_ASTRAL_RELEASE_URL_PREFIXES = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/", + "https://releases.astral.sh/github/python-build-standalone/releases/download/", +] + def parse_filename(filename): """Parses a python-build-standalone filename (or URL) into its components. @@ -112,6 +118,27 @@ def parse_filename(filename): "vendor": vendor, } +# buildifier: disable=function-docstring-args +# buildifier: disable=function-docstring-return +def is_astral_static_libpython_build(urls, release_filename): + """Whether an Astral build includes libpython statically in its interpreter.""" + parsed = parse_filename(release_filename) + if not parsed: + return False + + build_version = parsed["build_version"] + if ( + not build_version.isdigit() or + int(build_version) < _ASTRAL_STATIC_LIBPYTHON_RELEASE + ): + return False + + return any([ + url.startswith(prefix) + for url in urls + for prefix in _ASTRAL_RELEASE_URL_PREFIXES + ]) + def parse_runtime_manifest(content): """Parses the SHA256SUMS file content into a list of structs. diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 9c44971117..b58c7bbd8b 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -17,6 +17,7 @@ load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY") load(":auth.bzl", "get_auth") +load(":pbs_manifest.bzl", "is_astral_static_libpython_build") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") @@ -162,6 +163,7 @@ def _python_repository_impl(rctx): *python_version_info ) urls = rctx.attr.urls or [rctx.attr.url] + interpreter_has_static_libpython = is_astral_static_libpython_build(urls, release_filename) auth = get_auth(rctx, urls) if INSTALL_ONLY in release_filename: @@ -280,6 +282,7 @@ define_hermetic_runtime_toolchain_impl( name = "define_runtime", extra_files_glob_include = {extra_files_glob_include}, extra_files_glob_exclude = {extra_files_glob_exclude}, + interpreter_has_static_libpython = {interpreter_has_static_libpython}, python_version = {python_version}, python_bin = {python_bin}, coverage_tool = {coverage_tool}, @@ -287,6 +290,7 @@ define_hermetic_runtime_toolchain_impl( """.format( extra_files_glob_exclude = render.list(glob_exclude), extra_files_glob_include = render.list(glob_include), + interpreter_has_static_libpython = str(interpreter_has_static_libpython), python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index bd26d533a3..b906073fce 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -84,6 +84,12 @@ def test_zipapp_structure(self): # On Windows, pyvenv.cfg and bin/python3 are generated at runtime. if os.name != "nt": + self.assertFalse( + any( + "/lib/libpython" in name and ".so" in name for name in namelist + ), + "Statically linked Python should not bundle libpython", + ) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") # The venv directory name depends on the target name, so find it diff --git a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl index 4493576147..9516bb7078 100644 --- a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl +++ b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl @@ -4,7 +4,7 @@ load("@bazel_skylib//lib:structs.bzl", "structs") load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") -load("//python/private:pbs_manifest.bzl", "parse_filename", "parse_runtime_manifest") # buildifier: disable=bzl-visibility +load("//python/private:pbs_manifest.bzl", "is_astral_static_libpython_build", "parse_filename", "parse_runtime_manifest") # buildifier: disable=bzl-visibility _tests = [] @@ -97,6 +97,47 @@ def _test_parse_filename_baseline_impl(env, target): _tests.append(_test_parse_filename_baseline) +def _test_is_astral_static_libpython_build(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_is_astral_static_libpython_build_impl, + ) + +def _test_is_astral_static_libpython_build_impl(env, target): + _ = target # @unused + + github_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/archive.tar.gz" + mirror_url = "https://releases.astral.sh/github/python-build-standalone/releases/download/20250604/archive.tar.gz" + cutoff_filename = "cpython-3.13.4+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + cutoff_filename, + )).equals(True) + env.expect.that_bool(is_astral_static_libpython_build( + [mirror_url], + cutoff_filename, + )).equals(True) + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + "cpython-3.13.3+20250531-x86_64-unknown-linux-gnu-install_only.tar.gz", + )).equals(False) + env.expect.that_bool(is_astral_static_libpython_build( + ["https://example.com/20250604/archive.tar.gz"], + cutoff_filename, + )).equals(False) + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + "custom-python.tar.gz", + )).equals(False) + +_tests.append(_test_is_astral_static_libpython_build) + def _test_parse_runtime_manifest(name): """Sets up the manifest file parsing test. From 2ab0e89e426fa8fc57cb651d6885934ec1152321 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Fri, 18 Sep 2026 13:07:24 -0700 Subject: [PATCH 06/10] fix(zipapp): make libpython detection configurable The static libpython transition started with the 20250517 Astral build, not 20250604. Also allow users to force inclusion or exclusion when auto-detection does not match a mirrored or customized runtime. Expose the auto/include/exclude mode through the Python extension overrides and document the Starlark helper argument types. --- python/private/pbs_manifest.bzl | 3 ++- python/private/python.bzl | 20 +++++++++++++++++ python/private/python_register_toolchains.bzl | 4 ++++ python/private/python_repository.bzl | 12 +++++++++- tests/python/python_tests.bzl | 22 +++++++++++++++++++ .../parse_runtime_manifest_tests.bzl | 8 +++---- tests/support/mocks/python_ext.bzl | 3 +++ 7 files changed, 66 insertions(+), 6 deletions(-) diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl index 44dbf27108..ad019df0a8 100644 --- a/python/private/pbs_manifest.bzl +++ b/python/private/pbs_manifest.bzl @@ -1,6 +1,6 @@ """Helper functions to parse python-build-standalone manifests.""" -_ASTRAL_STATIC_LIBPYTHON_RELEASE = 20250604 +_ASTRAL_STATIC_LIBPYTHON_RELEASE = 20250517 _ASTRAL_RELEASE_URL_PREFIXES = [ "https://github.com/astral-sh/python-build-standalone/releases/download/", "https://releases.astral.sh/github/python-build-standalone/releases/download/", @@ -120,6 +120,7 @@ def parse_filename(filename): # buildifier: disable=function-docstring-args # buildifier: disable=function-docstring-return +# urls: list[str], release_filename: str -> bool def is_astral_static_libpython_build(urls, release_filename): """Whether an Astral build includes libpython statically in its interpreter.""" parsed = parse_filename(release_filename) diff --git a/python/private/python.bzl b/python/private/python.bzl index 9fed9393ae..7b9bf3db19 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -612,6 +612,8 @@ def _process_single_version_overrides(*, tag, _fail = fail, default): kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content if tag.distutils: kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + if tag.libpython and tag.libpython != "auto": + kwargs.setdefault(tag.python_version, {})["libpython"] = tag.libpython def _process_single_version_platform_overrides(*, tag, _fail = fail, default): if not _validate_version(tag.python_version, _fail = _fail): @@ -641,6 +643,8 @@ def _process_single_version_platform_overrides(*, tag, _fail = fail, default): if tag.urls: available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + if tag.libpython and tag.libpython != "auto": + available_versions[tag.python_version].setdefault("libpython", {})[tag.platform] = tag.libpython # If platform is customized, or doesn't exist, (re)define one. if ((tag.target_compatible_with or tag.target_settings or tag.os_name or tag.arch) or @@ -719,6 +723,7 @@ def _process_global_overrides(*, tag, default, _fail = fail): forwarded_attrs = sorted(AUTH_ATTRS) + [ "base_urls", + "libpython", "register_all_versions", ] for key in forwarded_attrs: @@ -1348,6 +1353,11 @@ dependencies are introduced. doc = """Deprecated; do not use. This attribute has no effect.""", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = "Whether to include shared libpython files: auto, include, or exclude.", + values = ["auto", "include", "exclude"], + ), "minor_mapping": attr.string_dict( mandatory = False, doc = """\ @@ -1424,6 +1434,11 @@ class. "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = "Whether to include shared libpython files: auto, include, or exclude.", + values = ["auto", "include", "exclude"], + ), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", @@ -1581,6 +1596,11 @@ Docs for [Registering custom runtimes] ::: """, ), + "libpython": attr.string( + default = "auto", + doc = "Whether to include shared libpython files: auto, include, or exclude.", + values = ["auto", "include", "exclude"], + ), "urls": attr.string_list( mandatory = False, doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl index c6f827be22..770c6a6f3b 100644 --- a/python/private/python_register_toolchains.bzl +++ b/python/private/python_register_toolchains.bzl @@ -40,6 +40,7 @@ def python_register_toolchains( register_toolchains = True, register_coverage_tool = False, set_python_version_constraint = False, + libpython = None, tool_versions = None, platforms = PLATFORMS, minor_mapping = None, @@ -68,6 +69,8 @@ def python_register_toolchains( set_python_version_constraint: {type}`bool` When set to `True`, `target_compatible_with` for the toolchains will include a version constraint. + libpython: {type}`str` controls shared libpython files: `auto`, + `include`, or `exclude`. tool_versions: {type}`dict` contains a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. @@ -145,6 +148,7 @@ def python_register_toolchains( urls = urls, strip_prefix = strip_prefix, coverage_tool = coverage_tool, + libpython = libpython or tool_versions[python_version].get("libpython", {}).get(platform, "auto"), **kwargs ) if register_toolchains: diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index b58c7bbd8b..33a4f3cf90 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -163,7 +163,12 @@ def _python_repository_impl(rctx): *python_version_info ) urls = rctx.attr.urls or [rctx.attr.url] - interpreter_has_static_libpython = is_astral_static_libpython_build(urls, release_filename) + if rctx.attr.libpython == "include": + interpreter_has_static_libpython = False + elif rctx.attr.libpython == "exclude": + interpreter_has_static_libpython = True + else: + interpreter_has_static_libpython = is_astral_static_libpython_build(urls, release_filename) auth = get_auth(rctx, urls) if INSTALL_ONLY in release_filename: @@ -365,6 +370,11 @@ For more information see {attr}`py_runtime.coverage_tool`. doc = "Noop, will be removed in the next major release", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = "Whether to include shared libpython files: auto, include, or exclude.", + values = ["auto", "include", "exclude"], + ), "netrc": attr.string( doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", ), diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index ef7ccf034f..22cd5c26b0 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -419,6 +419,28 @@ def _test_auth_overrides(env): _tests.append(_test_auth_overrides) +def _test_libpython_override(env): + py = parse_modules( + module_ctx = python_ext.mctx( + python_ext.module( + name = "my_module", + is_root = True, + override = [ + python_ext.override(libpython = "exclude"), + ], + toolchain = [python_ext.toolchain(python_version = "3.12")], + ), + _rules_python_module(), + ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), + ) + + env.expect.that_dict(py.config.default).contains_at_least({ + "libpython": "exclude", + }) + +_tests.append(_test_libpython_override) + def _test_add_target_settings(env): py = parse_modules( module_ctx = python_ext.mctx( diff --git a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl index 9516bb7078..8c9176aeb2 100644 --- a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl +++ b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl @@ -111,9 +111,9 @@ def _test_is_astral_static_libpython_build(name): def _test_is_astral_static_libpython_build_impl(env, target): _ = target # @unused - github_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/archive.tar.gz" - mirror_url = "https://releases.astral.sh/github/python-build-standalone/releases/download/20250604/archive.tar.gz" - cutoff_filename = "cpython-3.13.4+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + github_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250517/archive.tar.gz" + mirror_url = "https://releases.astral.sh/github/python-build-standalone/releases/download/20250517/archive.tar.gz" + cutoff_filename = "cpython-3.13.4+20250517-x86_64-unknown-linux-gnu-install_only.tar.gz" env.expect.that_bool(is_astral_static_libpython_build( [github_url], @@ -125,7 +125,7 @@ def _test_is_astral_static_libpython_build_impl(env, target): )).equals(True) env.expect.that_bool(is_astral_static_libpython_build( [github_url], - "cpython-3.13.3+20250531-x86_64-unknown-linux-gnu-install_only.tar.gz", + "cpython-3.13.3+20250516-x86_64-unknown-linux-gnu-install_only.tar.gz", )).equals(False) env.expect.that_bool(is_astral_static_libpython_build( ["https://example.com/20250604/archive.tar.gz"], diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl index 4c68e87e50..1cba0ad212 100644 --- a/tests/support/mocks/python_ext.bzl +++ b/tests/support/mocks/python_ext.bzl @@ -32,6 +32,7 @@ def _override(**kwargs): "available_python_versions": [], "base_urls": ["https://github.com/astral-sh/python-build-standalone/releases/download"], "ignore_root_user_error": True, + "libpython": "auto", "minor_mapping": {}, "register_all_versions": False, "runtime_manifest_sha": "", @@ -53,6 +54,7 @@ def _single_version_override(**kwargs): attrs = { "distutils": None, "distutils_content": "", + "libpython": "auto", "patch_strip": 0, "patches": [], "python_version": "", @@ -77,6 +79,7 @@ def _single_version_platform_override(**kwargs): "strip_prefix": "python", "target_compatible_with": [], "target_settings": [], + "libpython": "auto", "urls": [], } attrs.update(kwargs) From 8303a1fcf16f48e35f1da6465a76e5f410ae6f91 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Fri, 18 Sep 2026 13:10:12 -0700 Subject: [PATCH 07/10] docs(zipapp): document libpython configuration Document the auto, include, and exclude modes exposed by the Python extension, including the 20250517 Astral cutoff and fallback behavior for unknown runtimes. Also standardize indentation in the generated hermetic runtime BUILD definition. --- docs/toolchains.md | 10 ++++++++++ python/private/python.bzl | 30 +++++++++++++++++++++++++--- python/private/python_repository.bzl | 12 +++++------ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index de495d5122..3ddce7b97e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -240,12 +240,22 @@ existing attributes: via {attr}`python.override.minor_mapping`. * Per-version control of the coverage tool used using {attr}`python.single_version_platform_override.coverage_tool`. +* Control of shared `libpython` files using the `libpython` attribute on + {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override`, or + {bzl:obj}`python.single_version_platform_override`. * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. * Adding additional Python versions dynamically from a manifest file or URL via {attr}`python.override.add_runtime_manifest_files` or {attr}`python.override.add_runtime_manifest_urls`. +The `libpython` attribute accepts `auto`, `include`, or `exclude`, and defaults +to `auto`. In automatic mode, shared `libpython` files are excluded only for +recognized Astral Python Standalone builds from `20250517` onward, which are +known to statically link `libpython` into the interpreter. Unknown, custom, and +older runtimes retain the shared libraries. Use `include` or `exclude` to +override this behavior when mirroring or customizing a runtime. + ### Registering custom runtimes Because the python-build-standalone project has _thousands_ of prebuilt runtimes diff --git a/python/private/python.bzl b/python/private/python.bzl index 7b9bf3db19..8050b8f4fb 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -1355,7 +1355,15 @@ dependencies are introduced. ), "libpython": attr.string( default = "auto", - doc = "Whether to include shared libpython files: auto, include, or exclude.", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", values = ["auto", "include", "exclude"], ), "minor_mapping": attr.string_dict( @@ -1436,7 +1444,15 @@ class. ), "libpython": attr.string( default = "auto", - doc = "Whether to include shared libpython files: auto, include, or exclude.", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", values = ["auto", "include", "exclude"], ), "patch_strip": attr.int( @@ -1598,7 +1614,15 @@ Docs for [Registering custom runtimes] ), "libpython": attr.string( default = "auto", - doc = "Whether to include shared libpython files: auto, include, or exclude.", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", values = ["auto", "include", "exclude"], ), "urls": attr.string_list( diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 33a4f3cf90..3fd0928a01 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -284,13 +284,13 @@ load("@rules_python//python/private:hermetic_runtime_repo_setup.bzl", "define_he package(default_visibility = ["//visibility:public"]) define_hermetic_runtime_toolchain_impl( - name = "define_runtime", - extra_files_glob_include = {extra_files_glob_include}, - extra_files_glob_exclude = {extra_files_glob_exclude}, + name = "define_runtime", + extra_files_glob_include = {extra_files_glob_include}, + extra_files_glob_exclude = {extra_files_glob_exclude}, interpreter_has_static_libpython = {interpreter_has_static_libpython}, - python_version = {python_version}, - python_bin = {python_bin}, - coverage_tool = {coverage_tool}, + python_version = {python_version}, + python_bin = {python_bin}, + coverage_tool = {coverage_tool}, ) """.format( extra_files_glob_exclude = render.list(glob_exclude), From c77035b2ec43f297dc5baa451add8387b5cb09c6 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Fri, 18 Sep 2026 13:38:38 -0700 Subject: [PATCH 08/10] style(python): sort libpython override attributes --- python/private/python.bzl | 26 +++++++++++++------------- tests/support/mocks/python_ext.bzl | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 8050b8f4fb..c68433fade 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -1598,6 +1598,19 @@ Docs for [Registering custom runtimes] ::: """, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", + values = ["auto", "include", "exclude"], + ), "target_settings": attr.string_list( doc = """ The `target_setings` values to use for the toolchain definition. @@ -1612,19 +1625,6 @@ Docs for [Registering custom runtimes] ::: """, ), - "libpython": attr.string( - default = "auto", - doc = """Whether to include shared libpython files. - -Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized -Astral Python Standalone builds from 20250517 onward exclude the shared -libraries; other runtimes retain them. - -:::{versionadded} VERSION_NEXT_PATCH -::: -""", - values = ["auto", "include", "exclude"], - ), "urls": attr.string_list( mandatory = False, doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl index 1cba0ad212..419afe8188 100644 --- a/tests/support/mocks/python_ext.bzl +++ b/tests/support/mocks/python_ext.bzl @@ -70,6 +70,7 @@ def _single_version_platform_override(**kwargs): attrs = { "arch": "", "coverage_tool": None, + "libpython": "auto", "os_name": "", "patch_strip": 0, "patches": [], @@ -79,7 +80,6 @@ def _single_version_platform_override(**kwargs): "strip_prefix": "python", "target_compatible_with": [], "target_settings": [], - "libpython": "auto", "urls": [], } attrs.update(kwargs) From f575790577f0f99e5b127d7045c891877c9f8914 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Fri, 18 Sep 2026 13:48:58 -0700 Subject: [PATCH 09/10] style(python): sort platform libpython attribute --- python/private/python.bzl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index c68433fade..c7ca018102 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -1524,6 +1524,19 @@ The coverage tool to be used for a particular Python interpreter. This can overr `rules_python` defaults. """, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", + values = ["auto", "include", "exclude"], + ), "os_name": attr.string( doc = """ The host OS the runtime is compatible with. @@ -1598,19 +1611,6 @@ Docs for [Registering custom runtimes] ::: """, ), - "libpython": attr.string( - default = "auto", - doc = """Whether to include shared libpython files. - -Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized -Astral Python Standalone builds from 20250517 onward exclude the shared -libraries; other runtimes retain them. - -:::{versionadded} VERSION_NEXT_PATCH -::: -""", - values = ["auto", "include", "exclude"], - ), "target_settings": attr.string_list( doc = """ The `target_setings` values to use for the toolchain definition. From c6d3fe34fdaeb27a4dcbf494cb923879058e102d Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Sat, 19 Sep 2026 11:51:54 -0700 Subject: [PATCH 10/10] Trigger ReadTheDocs rebuild