From ffbe7c1825c95363a1a25544fa1d285c0f5c0d1f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:36:43 +0900 Subject: [PATCH 1/9] feat(pypi): download and extract wheels once --- news/3978.fixed.md | 5 ++ python/private/BUILD.bazel | 11 +-- python/private/internal_dev_deps.bzl | 12 ++- python/private/pypi/extension.bzl | 33 +++++-- python/private/pypi/hub_builder.bzl | 56 ++---------- python/private/pypi/whl_library.bzl | 88 ++++++++++++++++++- python/private/pypi/whl_library_targets.bzl | 38 ++++---- python/private/repo_utils.bzl | 68 ++++++++++++++ tests/integration/whl_library/MODULE.bazel | 1 - tests/pypi/hub_builder/hub_builder_tests.bzl | 2 +- .../whl_library_targets_tests.bzl | 1 + 11 files changed, 226 insertions(+), 89 deletions(-) create mode 100644 news/3978.fixed.md diff --git a/news/3978.fixed.md b/news/3978.fixed.md new file mode 100644 index 0000000000..fe297fa0ee --- /dev/null +++ b/news/3978.fixed.md @@ -0,0 +1,5 @@ +(pip) Wrap `pip_parse` and `unified_workspace_hub_repo` calls in the +WORKSPACE file with `maybe` from `@bazel_tools//tools/build_defs/repo:utils.bzl` +so that extracted wheel dependencies are reused and not re-created when the +WORKSPACE is evaluated multiple times. This mirrors the `maybe` pattern already +used in the bzlmod `pip` extension and in the generated `install_deps` macro. diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index a5f4d2da3e..33fa3a4cd1 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -929,6 +929,12 @@ bzl_library( deps = [":visibility"], ) +bzl_library( + name = "repo_utils", + srcs = ["repo_utils.bzl"], + deps = [":text_util"], +) + bzl_library( name = "bzlmod_enabled", srcs = ["bzlmod_enabled.bzl"], @@ -994,11 +1000,6 @@ bzl_library( srcs = ["py_runtime_info.bzl"], ) -bzl_library( - name = "repo_utils", - srcs = ["repo_utils.bzl"], -) - bzl_library( name = "runtimes_manifest_workspace", srcs = ["runtimes_manifest_workspace.bzl"], diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 5bbee232f9..926ee37268 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -14,12 +14,15 @@ """Module extension for internal dev_dependency=True setup.""" load("@bazel_ci_rules//:rbe_repo.bzl", "rbe_preconfig") -load("//python/private/pypi:whl_library.bzl", "whl_library") +load("//python/private:repo_utils.bzl", "repo_utils") +load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library") load("//tests/support/whl_from_dir:whl_from_dir_repo.bzl", "whl_from_dir_repo") load(":runtime_env_repo.bzl", "runtime_env_repo") def _internal_dev_deps_impl(mctx): _ = mctx # @unused + maybe = repo_utils.maybe({}) + whl_library = lambda **kwargs: _whl_library(maybe = maybe, **kwargs) # Creates a default toolchain config for RBE. # Use this as is if you are using the rbe_ubuntu16_04 container, @@ -118,6 +121,7 @@ def _internal_dev_deps_impl(mctx): root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", requirement = "pkg[optional]", + maybe = maybe, # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", @@ -126,6 +130,7 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_optional_dep", root = "//tests/pypi/whl_library/testdata/optional_dep:BUILD.bazel", output = "optional_dep-1.0-any-none-any.whl", + maybe = maybe, requirement = "optional_dep", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", @@ -149,15 +154,16 @@ def _internal_dev_deps_impl(mctx): }, ) -def _whl_library_from_dir(*, name, output, root, **kwargs): +def _whl_library_from_dir(*, name, output, root, maybe, **kwargs): whl_from_dir_repo( name = "{}_whl".format(name), root = root, output = output, ) - whl_library( + _whl_library( name = name, whl_file = "@{}_whl//:{}".format(name, output), + maybe = maybe, **kwargs ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 412b03c21f..b607c2fdd8 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -22,6 +22,7 @@ load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python") load("//python/private:repo_utils.bzl", "repo_utils") +load("//python/private:text_util.bzl", "render") load(":hub_builder.bzl", "hub_builder") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") @@ -463,10 +464,27 @@ You cannot use both the additive_build_content and additive_build_content_file a out = hub.build() for whl_name, lib in out.whl_libraries.items(): + # NOTE @aignas 2026-07-04: if the same wheel is downloaded from multiple + # indexes, this will fail, forcing the user to actually download the wheel + # from the same and deterministic location. This is usually the case for + # public wheels and users should setup the defaults.index_url to correct + # fall-back in rules_python we should handle the default index to substitute + # any index-url in requirements pointing to the public PyPI mirrors. if whl_name in whl_libraries: - fail("'{}' already in created".format(whl_name)) - else: - whl_libraries[whl_name] = lib + existing = whl_libraries[whl_name] + + diff = repo_utils.diff_dict(existing, lib) + if diff: + fail("'{}' already in created:\n{}".format( + whl_name, + "\n".join([ + " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) + for key, value in diff.items() + if value + ]), + )) + + whl_libraries[whl_name] = lib exposed_packages[hub.name] = out.exposed_packages extra_aliases[hub.name] = out.extra_aliases @@ -611,8 +629,13 @@ def _pip_impl(module_ctx): # Build all of the wheel modifications if the tag class is called. _whl_mods_impl(mods.whl_mods) + registered = {} for name, args in mods.whl_libraries.items(): - whl_library(name = name, **args) + whl_library( + name = name, + maybe = repo_utils.maybe(registered), + **args + ) for hub_name, whl_map in mods.hub_whl_map.items(): hub_repository( @@ -991,7 +1014,7 @@ a string `"{os}_{arch}"` as the value here. You could also use `"{os}_{arch}_fre """, ), "uv_lock": attr.label( - doc = """ + doc = """\ (label, optional): A label pointing to the uv.lock file. If provided, the uv.lock file will be used as the primary source for package metadata. diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index d7974d39ad..256cef83b9 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -87,6 +87,7 @@ def hub_builder( # Functions to download according to the config # dict[str python_version, callable] _get_index_urls = {}, + _default_index_url = {}, # Tells whether to use the downloader for a package. # dict[str python_version, dict[str package_name, bool use_downloader]] _use_downloader = {}, @@ -256,44 +257,6 @@ def _add_extra_aliases(self, extra_hub_aliases): {alias: True for alias in aliases}, ) -def _diff_dict(first, second): - """A simple utility to shallow compare dictionaries. - - Args: - first: The first dictionary to compare. - second: The second dictionary to compare. - - Returns: - A dictionary containing the differences, with keys "common", "different", - "extra", and "missing", or None if the dictionaries are identical. - """ - missing = {} - extra = { - key: value - for key, value in second.items() - if key not in first - } - common = {} - different = {} - - for key, value in first.items(): - if key not in second: - missing[key] = value - elif value == second[key]: - common[key] = value - else: - different[key] = (value, second[key]) - - if missing or extra or different: - return { - "common": common, - "different": different, - "extra": extra, - "missing": missing, - } - else: - return None - def _add_whl_library(self, *, python_version, whl, repo): """Add a whl_library and kwargs to call it with for the hub. @@ -309,16 +272,10 @@ def _add_whl_library(self, *, python_version, whl, repo): # disallow building from sdist. return - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if - # we are using pipstar and we are downloading the wheel using the downloader - # - # However, for that we should first have a different way to reference closures with - # extras. For example, if some package depends on `foo[extra]` and another depends on - # `foo`, we should have 2 py_library targets. repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) if repo_name in self._whl_libraries: - diff = _diff_dict(self._whl_libraries[repo_name], repo.args) + diff = repo_utils.diff_dict(self._whl_libraries[repo_name], repo.args) if diff: self._logger.fail(lambda: ( "Attempting to create a duplicate library {repo_name} for {whl_name} with different arguments. Already existing declaration has:\n".format( @@ -348,24 +305,25 @@ def _add_whl_library(self, *, python_version, whl, repo): ### end of setters, below we have various functions to implement the public methods def _set_get_index_urls(self, mctx, pip_attr): + python_version = pip_attr.python_version + # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is # active. Without this, an unsubstituted template like ``$RULES_PYTHON_PIP_INDEX_URL`` # is treated as truthy and the mode is forced on, even when the env var # would expand to the empty string. - default_index_url = envsubst( + self._default_index_url[python_version] = envsubst( pip_attr.experimental_index_url, pip_attr.envsubst, mctx.getenv, ) or self._config.index_url default_extra_index_urls = pip_attr.experimental_extra_index_urls or [] - if not default_index_url: + if not self._default_index_url[python_version]: # parallel_download is set to True by default, so we are not checking/validating it # here return False - python_version = pip_attr.python_version self._use_downloader.setdefault(python_version, {}).update({ normalize_name(s): False for s in pip_attr.simpleapi_skip @@ -373,7 +331,7 @@ def _set_get_index_urls(self, mctx, pip_attr): self._get_index_urls[python_version] = lambda ctx, distributions, *, index_url = None, extra_index_urls = None: self._simpleapi_download_fn( ctx, attr = struct( - index_url = (index_url or default_index_url).rstrip("/"), + index_url = (index_url or self._default_index_url[python_version]).rstrip("/"), extra_index_urls = [ x.rstrip("/") for x in (extra_index_urls or default_extra_index_urls) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 53edd62efb..35441901b2 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,6 +14,7 @@ "" +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") @@ -28,6 +29,7 @@ load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":urllib.bzl", "urllib") load(":whl_extract.bzl", "whl_extract") load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") +load(":whl_repo_name.bzl", "whl_repo_name") _CPPFLAGS = "CPPFLAGS" _COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" @@ -736,7 +738,7 @@ def _whl_deps_library_impl(rctx): repo = rctx.attr.repo or ( str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None ), - extras = requirement(rctx.attr.requirement).extras, + extras = rctx.attr.extras, ) rctx.file("BUILD.bazel", build_file_contents) @@ -748,9 +750,10 @@ whl_deps_library = repository_rule( "dep_template", "group_deps", "group_name", - "requirement", + "repo_prefix", ] } | { + "extras": attr.string_list(doc = "The extras to configure for this repo."), "metadata": attr.string( doc = """ The subset of the METADATA contents that is needed for generation of the dependencies. @@ -773,7 +776,7 @@ Does not depend on any python. environ = [REPO_DEBUG_ENV_VAR], ) -def whl_library(name, repo = None, **kwargs): +def whl_library(name, repo = None, maybe = maybe, **kwargs): """Create a whl_library. This proxies to one of the underlying implementations: @@ -783,6 +786,9 @@ def whl_library(name, repo = None, **kwargs): Args: name: {type}`str` The name of the repo. repo: Unused, will be dropped in the next major release. + maybe: This is the repo rule that is used in WORKSPACE mode and in the extension eval to + dedupe some of the invocations. This has to be overridden on bzlmod using + {obj}`repo_utils.bzlmod_maybe`. **kwargs: The args passed to the underlying implementation. Returns: @@ -794,6 +800,80 @@ def whl_library(name, repo = None, **kwargs): urls = kwargs.get("urls", []) filename = kwargs.get("filename") if whl_file or (urls and filename and filename.endswith(".whl")): - whl_archive(name = name, **kwargs) + filename = filename or Label(whl_file).name + if "annotation" in kwargs or "whl_patches" in kwargs: + # No reuse of the whl_library because there is an annotation here + whl_archive(name = name, **kwargs) + return + + extract_args = { + k: v + for k, v in kwargs.items() + if k not in { + # TODO @aignas 2026-08-01: what about python_interpreter and python_interpreter_target + "config_load": None, + "dep_template": None, + } + } + if "index_url" in extract_args: + # TODO @aignas 2026-08-01: figure out where we should do the fix here. + extract_args["index_url"] = extract_args["index_url"].strip("/") + + # The extras do not affect the extraction, so normalize the requirement + # to allow the same wheel to be extracted only once. + extract_args["requirement"] = _without_extras(extract_args["requirement"]) + extract_repo_name = "w_{}".format( + whl_repo_name(filename, kwargs.get("sha256")), + ) + + maybe( + whl_archive, + name = extract_repo_name, + **extract_args + ) + + req = requirement(kwargs["requirement"]) + + deps_args = { + k: kwargs.get(k) + for k in [ + "config_load", + "dep_template", + "group_deps", + "group_name", + ] + if kwargs.get(k) != None + } | { + # TODO @aignas 2026-08-01: add extras only if the list is non-empty + "extras": req.extras, + "metadata_file": "@{}//:metadata.json".format(extract_repo_name), + } + whl_deps_library( + name = name, + **deps_args + ) else: + # No reuse of the whl_library because we have args that force the extraction of the whl + # in the hub context. If we have whl-only pipstar extraction, then we can reuse the + # extracted sources. pip_archive(name = name, **kwargs) + +def _without_extras(requirement_line): + """Remove the extras from a requirement line. + + The extras do not affect which wheel is downloaded and extracted, so they + can be removed to allow the same wheel to be extracted only once even when + it is referenced with different extras. + + Args: + requirement_line: {type}`str` the requirement line, e.g. + `foo[bar]==1.0 --hash=sha256:...`. + + Returns: + The requirement line without the extras, e.g. `foo==1.0 --hash=sha256:...`. + """ + name_and_version, _, extras = requirement_line.partition("[") + if not extras: + return requirement_line + _, _, rest = extras.partition("]") + return name_and_version + rest diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 737a739145..5b19736d0a 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -99,7 +99,6 @@ def whl_library_targets( visibility: {type}`list[str]` The visibility of the targets. **kwargs: Extra args passed to the {obj}`whl_library_deps_targets` and {obj}`whl_library_srcs`. """ - create_extra_targets = bool(requires_dist or group_name) and dep_template whl_library_srcs( name = name, sdist_filename = sdist_filename, @@ -113,27 +112,22 @@ def whl_library_targets( copy_executables = copy_executables, enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs, namespace_package_files = namespace_package_files, - # If there are no dependencies, then let's create the targets with public labels. - # Note, we are not supporting grouping the packages in this case, but that is fine. - whl_name = WHEEL_FILE if create_extra_targets else WHEEL_FILE_PUBLIC_LABEL, - pkg_name = PY_SRCS_LABEL if create_extra_targets else PY_LIBRARY_PUBLIC_LABEL, **kwargs ) - if create_extra_targets: - whl_library_deps_targets( - name = name, - metadata_name = metadata_name, - requires_dist = requires_dist, - group_deps = group_deps, # only needed if requires_dist is present - extras = extras, # only needed if requires_dist is present - include = include, # only needed if requires_dist is present - group_name = group_name, # only needed if requires_dist is present - dep_template = dep_template, # only needed if requires_dist is present - repo = None, # set aliases in the same repo - aliases = {}, - **kwargs - ) + whl_library_deps_targets( + name = name, + metadata_name = metadata_name, + requires_dist = requires_dist, + group_deps = group_deps, # only needed if requires_dist is present + extras = extras, # only needed if requires_dist is present + include = include, # only needed if requires_dist is present + group_name = group_name, # only needed if requires_dist is present + dep_template = dep_template, # only needed if requires_dist is present + repo = None, # set aliases in the same repo + aliases = {}, + **kwargs + ) def whl_library_srcs( *, @@ -348,8 +342,8 @@ def whl_library_deps_targets( repo, aliases = None, metadata_name, - requires_dist, - extras, + requires_dist = [], + extras = [], include = [], group_deps = [], group_name = None, @@ -490,6 +484,8 @@ def whl_library_deps_targets( name = py_library_label, # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], + # Never pre-compile, because we don't have real sources here. + precompile = "disabled", deps = _deps( # We include as deps, so that `PyInfo` and friends (e.g. `pyi_srcs`) get # propagated. Just passing the target as `srcs` is not enough to propagate diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl index 0e83fda28c..dc3d4c4ae8 100644 --- a/python/private/repo_utils.bzl +++ b/python/private/repo_utils.bzl @@ -17,9 +17,75 @@ This code should only be loaded and used during the repository phase. """ +load(":text_util.bzl", "render") + REPO_DEBUG_ENV_VAR = "RULES_PYTHON_REPO_DEBUG" REPO_VERBOSITY_ENV_VAR = "RULES_PYTHON_REPO_DEBUG_VERBOSITY" +def _maybe(store): + def _inner(repo_rule, *, name, **kwargs): + existing = store.get(name) + if existing: + diff = _diff_dict(existing, kwargs) + if not diff: + return + + fail( + "Attempting to create a duplicate rule {name} with different arguments. Already existing declaration has:\n".format( + name = name, + ) + "\n".join([ + " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) + for key, value in diff.items() + if value + ]), + ) + + store[name] = dict(kwargs) # make a copy + repo_rule(name = name, **kwargs) + + return _inner + +def _diff_dict(first, second, *, ignore_keys = {}): + """A simple utility to shallow compare dictionaries. + + Args: + first: The first dictionary to compare. + second: The second dictionary to compare. + ignore_keys: A set of keys to ignore during comparison. + + Returns: + A dictionary containing the differences, with keys "common", "different", + "extra", and "missing", or None if the dictionaries are identical. + """ + missing = {} + extra = { + key: value + for key, value in second.items() + if key not in first and key not in ignore_keys + } + common = {} + different = {} + + for key, value in first.items(): + if key in ignore_keys: + continue + elif key not in second: + missing[key] = value + elif value == second[key]: + common[key] = value + else: + different[key] = (value, second[key]) + + if missing or extra or different: + return { + "common": common, + "different": different, + "extra": extra, + "missing": missing, + } + else: + return None + def _is_repo_debug_enabled(mrctx): """Tells if debbugging output is requested during repo operatiosn. @@ -587,6 +653,7 @@ def _rename(mrctx, src, dest): repo_utils = struct( # keep sorted + diff_dict = _diff_dict, execute_checked = _execute_checked, execute_checked_stdout = _execute_checked_stdout, execute_unchecked = _execute_unchecked, @@ -596,6 +663,7 @@ repo_utils = struct( is_repo_debug_enabled = _is_repo_debug_enabled, logger = _logger, maybe_fix_permissions = _maybe_fix_permissions, + maybe = _maybe, mkdir = _mkdir, norm_path = _norm_path, relative_to = _relative_to, diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index 39a579f79e..9dad3b7d9d 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -53,5 +53,4 @@ whl_deps_library( name = "whl_deps_library", dep_template = "@integration_test//:{name}_{target}", metadata_file = "@whl_archive//:metadata.json", - requirement = "requests", ) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 4651342dd4..958dbca1ca 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1523,7 +1523,7 @@ def _test_err_duplicate_repos(env): env.expect.that_dict(logs).keys().contains_exactly(["rules_python:unit-test FAIL:"]) env.expect.that_collection(logs["rules_python:unit-test FAIL:"]).contains_exactly([ """\ -Attempting to create a duplicate library pypi_315_foo for foo with different arguments. Already existing declaration has: +Attempting to create a duplicate library pypi_315_foo with different arguments. Already existing declaration has: common: { "dep_template": "@pypi//{name}:{target}", "config_load": "@pypi//:config.bzl", diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index 0a059c9d6a..6bb1666d68 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -173,6 +173,7 @@ def _test_whl_library_deps_targets(env): ":is_include_bar_baz_true": ["@pypi//bar_baz:pkg"], "//conditions:default": [], }), + "precompile": "disabled", "tags": [], "visibility": ["//visibility:public"], }) # buildifier: @unsorted-dict-items From 6bb2bb8eafb637fab76192e0923336cb3ce2771c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:48:32 +0900 Subject: [PATCH 2/9] wip --- python/private/pypi/whl_library.bzl | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 35441901b2..12258140eb 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,7 +14,6 @@ "" -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") @@ -776,7 +775,19 @@ Does not depend on any python. environ = [REPO_DEBUG_ENV_VAR], ) -def whl_library(name, repo = None, maybe = maybe, **kwargs): +# TODO @aignas 2026-08-01: add unit tests which would ensure that the right repos are created with +# the right params for each branch. Store the unit tests in +# //tests/pypi/whl_library/whl_library_tests.bzl and follow conventions. +def whl_library( + name, + repo = None, + maybe = lambda fn, **kwargs: fn(**kwargs), + rules = struct( + whl_deps_library = whl_deps_library, + pip_archive = pip_archive, + whl_archive = whl_archive, + ), + **kwargs): """Create a whl_library. This proxies to one of the underlying implementations: @@ -789,6 +800,7 @@ def whl_library(name, repo = None, maybe = maybe, **kwargs): maybe: This is the repo rule that is used in WORKSPACE mode and in the extension eval to dedupe some of the invocations. This has to be overridden on bzlmod using {obj}`repo_utils.bzlmod_maybe`. + rules: Used in tests for mocking. **kwargs: The args passed to the underlying implementation. Returns: @@ -827,7 +839,7 @@ def whl_library(name, repo = None, maybe = maybe, **kwargs): ) maybe( - whl_archive, + rules.whl_archive, name = extract_repo_name, **extract_args ) @@ -848,7 +860,7 @@ def whl_library(name, repo = None, maybe = maybe, **kwargs): "extras": req.extras, "metadata_file": "@{}//:metadata.json".format(extract_repo_name), } - whl_deps_library( + rules.whl_deps_library( name = name, **deps_args ) @@ -856,7 +868,7 @@ def whl_library(name, repo = None, maybe = maybe, **kwargs): # No reuse of the whl_library because we have args that force the extraction of the whl # in the hub context. If we have whl-only pipstar extraction, then we can reuse the # extracted sources. - pip_archive(name = name, **kwargs) + rules.pip_archive(name = name, **kwargs) def _without_extras(requirement_line): """Remove the extras from a requirement line. From 1e65b80de9bc1f620d39ab4d2773475aea485588 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:49:51 +0900 Subject: [PATCH 3/9] fixup the BUILD.bazel file --- python/private/pypi/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 09ba5ef135..84b59d829c 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -470,6 +470,7 @@ bzl_library( ":urllib", ":whl_extract", ":whl_metadata", + ":whl_repo_name", "//python/private:auth", "//python/private:envsubst", "//python/private:is_standalone_interpreter", From 8dc24c87bba56cbe4bf346e5697b1d3e3cd240f8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:05:12 +0900 Subject: [PATCH 4/9] wip --- python/private/pypi/whl_library.bzl | 13 +- tests/pypi/whl_library/BUILD.bazel | 12 +- tests/pypi/whl_library/whl_library_tests.bzl | 238 +++++++++++++++++++ 3 files changed, 246 insertions(+), 17 deletions(-) create mode 100644 tests/pypi/whl_library/whl_library_tests.bzl diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 12258140eb..1008f8347d 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -775,9 +775,6 @@ Does not depend on any python. environ = [REPO_DEBUG_ENV_VAR], ) -# TODO @aignas 2026-08-01: add unit tests which would ensure that the right repos are created with -# the right params for each branch. Store the unit tests in -# //tests/pypi/whl_library/whl_library_tests.bzl and follow conventions. def whl_library( name, repo = None, @@ -822,13 +819,15 @@ def whl_library( k: v for k, v in kwargs.items() if k not in { - # TODO @aignas 2026-08-01: what about python_interpreter and python_interpreter_target + # These attributes are specific to pip_archive and are not + # accepted by whl_archive. "config_load": None, "dep_template": None, + "python_interpreter": None, + "python_interpreter_target": None, } } if "index_url" in extract_args: - # TODO @aignas 2026-08-01: figure out where we should do the fix here. extract_args["index_url"] = extract_args["index_url"].strip("/") # The extras do not affect the extraction, so normalize the requirement @@ -856,10 +855,10 @@ def whl_library( ] if kwargs.get(k) != None } | { - # TODO @aignas 2026-08-01: add extras only if the list is non-empty - "extras": req.extras, "metadata_file": "@{}//:metadata.json".format(extract_repo_name), } + if req.extras: + deps_args["extras"] = req.extras rules.whl_deps_library( name = name, **deps_args diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index cade0d2b8e..65db520c6c 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -1,11 +1,3 @@ -load("//python:py_test.bzl", "py_test") -load("//tests/support:support.bzl", "SUPPORTS_BZLMOD") +load(":whl_library_tests.bzl", "whl_library_test_suite") -py_test( - name = "whl_library_extras_test", - srcs = ["whl_library_extras_test.py"], - target_compatible_with = SUPPORTS_BZLMOD, - deps = [ - "@whl_library_extras_direct_dep//:pkg", - ], -) +whl_library_test_suite(name = "whl_library_tests") diff --git a/tests/pypi/whl_library/whl_library_tests.bzl b/tests/pypi/whl_library/whl_library_tests.bzl new file mode 100644 index 0000000000..271ac2661c --- /dev/null +++ b/tests/pypi/whl_library/whl_library_tests.bzl @@ -0,0 +1,238 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Tests for //python/private/pypi:whl_library.bzl" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") + +# buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_library.bzl", "whl_library") + +_tests = [] + +_state = {"calls": []} + +def _record(rule_name, kwargs): + _state["calls"].append(struct( + rule = rule_name, + kwargs = dict(kwargs), + )) + return kwargs.get("name") + +def _reset_calls(): + _state["calls"] = [] + +def _mock_rules(): + return struct( + whl_archive = lambda **kwargs: _record("whl_archive", kwargs), + pip_archive = lambda **kwargs: _record("pip_archive", kwargs), + whl_deps_library = lambda **kwargs: _record("whl_deps_library", kwargs), + ) + +def _whl_archive_calls(): + return [c for c in _state["calls"] if c.rule == "whl_archive"] + +def _whl_deps_library_calls(): + return [c for c in _state["calls"] if c.rule == "whl_deps_library"] + +def _pip_archive_calls(): + return [c for c in _state["calls"] if c.rule == "pip_archive"] + +def _test_whl_file_reuse_path(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = "@some_wheels//:pkg-1.0-py3-none-any.whl", + requirement = "pkg==1.0", + ) + env.expect.that_int(len(_whl_archive_calls())).equals(1) + env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + "name": "w_pkg_1_0_py3_none_any", + "requirement": "pkg==1.0", + "whl_file": "@some_wheels//:pkg-1.0-py3-none-any.whl", + }) + env.expect.that_int(len(_whl_deps_library_calls())).equals(1) + env.expect.that_dict(_whl_deps_library_calls()[0].kwargs).contains_exactly({ + "metadata_file": "@w_pkg_1_0_py3_none_any//:metadata.json", + "name": "test_repo", + }) + +_tests.append(_test_whl_file_reuse_path) + +def _test_urls_reuse_path(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg==1.0", + config_load = "@pypi//:config.bzl", + ) + env.expect.that_int(len(_whl_archive_calls())).equals(1) + env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + "filename": "pkg-1.0-py3-none-any.whl", + "name": "w_pkg_1_0_py3_none_any", + "requirement": "pkg==1.0", + "urls": ["https://example.com/pkg-1.0-py3-none-any.whl"], + }) + env.expect.that_int(len(_whl_deps_library_calls())).equals(1) + env.expect.that_dict(_whl_deps_library_calls()[0].kwargs).contains_exactly({ + "config_load": "@pypi//:config.bzl", + "metadata_file": "@w_pkg_1_0_py3_none_any//:metadata.json", + "name": "test_repo", + }) + +_tests.append(_test_urls_reuse_path) + +def _test_annotation_no_reuse(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg[extra]==1.0", + annotation = "@some_repo//:pkg.annotation.json", + ) + env.expect.that_int(len(_whl_archive_calls())).equals(1) + env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + "annotation": "@some_repo//:pkg.annotation.json", + "filename": "pkg-1.0-py3-none-any.whl", + "name": "test_repo", + "requirement": "pkg[extra]==1.0", + "urls": ["https://example.com/pkg-1.0-py3-none-any.whl"], + }) + env.expect.that_int(len(_whl_deps_library_calls())).equals(0) + +_tests.append(_test_annotation_no_reuse) + +def _test_whl_patches_no_reuse(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg==1.0", + whl_patches = {"//patches:foo.patch": "json"}, + ) + env.expect.that_int(len(_whl_archive_calls())).equals(1) + env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + "filename": "pkg-1.0-py3-none-any.whl", + "name": "test_repo", + "requirement": "pkg==1.0", + "urls": ["https://example.com/pkg-1.0-py3-none-any.whl"], + "whl_patches": {"//patches:foo.patch": "json"}, + }) + env.expect.that_int(len(_whl_deps_library_calls())).equals(0) + +_tests.append(_test_whl_patches_no_reuse) + +def _test_pip_archive_path(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + requirement = "pkg==1.0", + config_load = "@pypi//:config.bzl", + ) + env.expect.that_int(len(_pip_archive_calls())).equals(1) + env.expect.that_dict(_pip_archive_calls()[0].kwargs).contains_exactly({ + "config_load": "@pypi//:config.bzl", + "name": "test_repo", + "requirement": "pkg==1.0", + }) + env.expect.that_int(len(_whl_archive_calls())).equals(0) + +_tests.append(_test_pip_archive_path) + +def _test_extras_not_passed_when_empty(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg==1.0", + config_load = "@pypi//:config.bzl", + ) + env.expect.that_int(len(_whl_deps_library_calls())).equals(1) + env.expect.that_dict(_whl_deps_library_calls()[0].kwargs).not_exists("extras") + +_tests.append(_test_extras_not_passed_when_empty) + +def _test_extras_passed_when_non_empty(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg[foo,bar]==1.0", + config_load = "@pypi//:config.bzl", + ) + env.expect.that_int(len(_whl_deps_library_calls())).equals(1) + env.expect.that_list(_whl_deps_library_calls()[0].kwargs["extras"]).contains_exactly(["foo", "bar"]) + +_tests.append(_test_extras_passed_when_non_empty) + +def _test_python_interpreter_excluded(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg==1.0", + config_load = "@pypi//:config.bzl", + python_interpreter = "python3", + python_interpreter_target = "@python//:python", + ) + env.expect.that_int(len(_whl_archive_calls())).equals(1) + env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("python_interpreter") + env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("python_interpreter_target") + +_tests.append(_test_python_interpreter_excluded) + +def _test_config_load_excluded_from_extract_args(env): + _reset_calls() + whl_library( + name = "test_repo", + rules = _mock_rules(), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg==1.0", + config_load = "@pypi//:config.bzl", + dep_template = "@pypi%{name}//:{{target}}", + ) + env.expect.that_int(len(_whl_archive_calls())).equals(1) + env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("config_load") + env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("dep_template") + +_tests.append(_test_config_load_excluded_from_extract_args) + +def whl_library_test_suite(name): + test_suite( + name = name, + basic_tests = _tests, + ) From 7ee42f21e1488cca44c9a04b19951a15a3cd01eb Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:13:32 +0900 Subject: [PATCH 5/9] fix tests --- python/private/pypi/whl_library.bzl | 2 +- tests/pypi/whl_library/whl_library_tests.bzl | 156 +++++++++++-------- 2 files changed, 90 insertions(+), 68 deletions(-) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 1008f8347d..35f923e200 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -812,7 +812,7 @@ def whl_library( filename = filename or Label(whl_file).name if "annotation" in kwargs or "whl_patches" in kwargs: # No reuse of the whl_library because there is an annotation here - whl_archive(name = name, **kwargs) + rules.whl_archive(name = name, **kwargs) return extract_args = { diff --git a/tests/pypi/whl_library/whl_library_tests.bzl b/tests/pypi/whl_library/whl_library_tests.bzl index 271ac2661c..84a72237ee 100644 --- a/tests/pypi/whl_library/whl_library_tests.bzl +++ b/tests/pypi/whl_library/whl_library_tests.bzl @@ -12,7 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Tests for //python/private/pypi:whl_library.bzl" +"""Tests for //python/private/pypi:whl_library.bzl + +These tests verify the dispatching logic in the ``whl_library`` function, +ensuring the correct underlying rule is invoked (directly or via ``maybe``) +with the right arguments for each code path. + +The ``rules`` parameter of ``whl_library`` is used to inject mock rule functions +that record their calls instead of creating actual repository rules. +""" load("@rules_testing//lib:test_suite.bzl", "test_suite") @@ -21,50 +29,46 @@ load("//python/private/pypi:whl_library.bzl", "whl_library") _tests = [] -_state = {"calls": []} +def _mock_rules(calls): + def _whl_archive(**kwargs): + calls.append(struct(rule = "whl_archive", kwargs = dict(kwargs))) + return kwargs.get("name") -def _record(rule_name, kwargs): - _state["calls"].append(struct( - rule = rule_name, - kwargs = dict(kwargs), - )) - return kwargs.get("name") + def _pip_archive(**kwargs): + calls.append(struct(rule = "pip_archive", kwargs = dict(kwargs))) + return kwargs.get("name") -def _reset_calls(): - _state["calls"] = [] + def _whl_deps_library(**kwargs): + calls.append(struct(rule = "whl_deps_library", kwargs = dict(kwargs))) + return kwargs.get("name") -def _mock_rules(): return struct( - whl_archive = lambda **kwargs: _record("whl_archive", kwargs), - pip_archive = lambda **kwargs: _record("pip_archive", kwargs), - whl_deps_library = lambda **kwargs: _record("whl_deps_library", kwargs), + whl_archive = _whl_archive, + pip_archive = _pip_archive, + whl_deps_library = _whl_deps_library, ) -def _whl_archive_calls(): - return [c for c in _state["calls"] if c.rule == "whl_archive"] - -def _whl_deps_library_calls(): - return [c for c in _state["calls"] if c.rule == "whl_deps_library"] - -def _pip_archive_calls(): - return [c for c in _state["calls"] if c.rule == "pip_archive"] +def _calls_for(calls, rule_name): + return [c for c in calls if c.rule == rule_name] def _test_whl_file_reuse_path(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = "@some_wheels//:pkg-1.0-py3-none-any.whl", requirement = "pkg==1.0", ) - env.expect.that_int(len(_whl_archive_calls())).equals(1) - env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(1) + env.expect.that_dict(whl_archive_calls[0].kwargs).contains_exactly({ "name": "w_pkg_1_0_py3_none_any", "requirement": "pkg==1.0", "whl_file": "@some_wheels//:pkg-1.0-py3-none-any.whl", }) - env.expect.that_int(len(_whl_deps_library_calls())).equals(1) - env.expect.that_dict(_whl_deps_library_calls()[0].kwargs).contains_exactly({ + deps_calls = _calls_for(calls, "whl_deps_library") + env.expect.that_int(len(deps_calls)).equals(1) + env.expect.that_dict(deps_calls[0].kwargs).contains_exactly({ "metadata_file": "@w_pkg_1_0_py3_none_any//:metadata.json", "name": "test_repo", }) @@ -72,25 +76,28 @@ def _test_whl_file_reuse_path(env): _tests.append(_test_whl_file_reuse_path) def _test_urls_reuse_path(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", requirement = "pkg==1.0", config_load = "@pypi//:config.bzl", ) - env.expect.that_int(len(_whl_archive_calls())).equals(1) - env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(1) + env.expect.that_dict(whl_archive_calls[0].kwargs).contains_exactly({ "filename": "pkg-1.0-py3-none-any.whl", "name": "w_pkg_1_0_py3_none_any", "requirement": "pkg==1.0", "urls": ["https://example.com/pkg-1.0-py3-none-any.whl"], + "whl_file": None, }) - env.expect.that_int(len(_whl_deps_library_calls())).equals(1) - env.expect.that_dict(_whl_deps_library_calls()[0].kwargs).contains_exactly({ + deps_calls = _calls_for(calls, "whl_deps_library") + env.expect.that_int(len(deps_calls)).equals(1) + env.expect.that_dict(deps_calls[0].kwargs).contains_exactly({ "config_load": "@pypi//:config.bzl", "metadata_file": "@w_pkg_1_0_py3_none_any//:metadata.json", "name": "test_repo", @@ -99,106 +106,117 @@ def _test_urls_reuse_path(env): _tests.append(_test_urls_reuse_path) def _test_annotation_no_reuse(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", requirement = "pkg[extra]==1.0", annotation = "@some_repo//:pkg.annotation.json", ) - env.expect.that_int(len(_whl_archive_calls())).equals(1) - env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(1) + env.expect.that_dict(whl_archive_calls[0].kwargs).contains_exactly({ "annotation": "@some_repo//:pkg.annotation.json", "filename": "pkg-1.0-py3-none-any.whl", "name": "test_repo", "requirement": "pkg[extra]==1.0", "urls": ["https://example.com/pkg-1.0-py3-none-any.whl"], + "whl_file": None, }) - env.expect.that_int(len(_whl_deps_library_calls())).equals(0) + env.expect.that_int(len(_calls_for(calls, "whl_deps_library"))).equals(0) _tests.append(_test_annotation_no_reuse) def _test_whl_patches_no_reuse(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", requirement = "pkg==1.0", whl_patches = {"//patches:foo.patch": "json"}, ) - env.expect.that_int(len(_whl_archive_calls())).equals(1) - env.expect.that_dict(_whl_archive_calls()[0].kwargs).contains_exactly({ + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(1) + env.expect.that_dict(whl_archive_calls[0].kwargs).contains_exactly({ "filename": "pkg-1.0-py3-none-any.whl", "name": "test_repo", "requirement": "pkg==1.0", "urls": ["https://example.com/pkg-1.0-py3-none-any.whl"], + "whl_file": None, "whl_patches": {"//patches:foo.patch": "json"}, }) - env.expect.that_int(len(_whl_deps_library_calls())).equals(0) + env.expect.that_int(len(_calls_for(calls, "whl_deps_library"))).equals(0) _tests.append(_test_whl_patches_no_reuse) def _test_pip_archive_path(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), requirement = "pkg==1.0", config_load = "@pypi//:config.bzl", ) - env.expect.that_int(len(_pip_archive_calls())).equals(1) - env.expect.that_dict(_pip_archive_calls()[0].kwargs).contains_exactly({ + pip_archive_calls = _calls_for(calls, "pip_archive") + env.expect.that_int(len(pip_archive_calls)).equals(1) + env.expect.that_dict(pip_archive_calls[0].kwargs).contains_exactly({ "config_load": "@pypi//:config.bzl", "name": "test_repo", "requirement": "pkg==1.0", }) - env.expect.that_int(len(_whl_archive_calls())).equals(0) + env.expect.that_int(len(_calls_for(calls, "whl_archive"))).equals(0) _tests.append(_test_pip_archive_path) def _test_extras_not_passed_when_empty(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", requirement = "pkg==1.0", config_load = "@pypi//:config.bzl", ) - env.expect.that_int(len(_whl_deps_library_calls())).equals(1) - env.expect.that_dict(_whl_deps_library_calls()[0].kwargs).not_exists("extras") + deps_calls = _calls_for(calls, "whl_deps_library") + env.expect.that_int(len(deps_calls)).equals(1) + env.expect.that_collection( + deps_calls[0].kwargs.keys(), + ).contains_none_of(["extras"]) _tests.append(_test_extras_not_passed_when_empty) def _test_extras_passed_when_non_empty(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", requirement = "pkg[foo,bar]==1.0", config_load = "@pypi//:config.bzl", ) - env.expect.that_int(len(_whl_deps_library_calls())).equals(1) - env.expect.that_list(_whl_deps_library_calls()[0].kwargs["extras"]).contains_exactly(["foo", "bar"]) + deps_calls = _calls_for(calls, "whl_deps_library") + env.expect.that_int(len(deps_calls)).equals(1) + env.expect.that_collection( + deps_calls[0].kwargs["extras"], + ).contains_exactly(["foo", "bar"]) _tests.append(_test_extras_passed_when_non_empty) def _test_python_interpreter_excluded(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", @@ -207,17 +225,19 @@ def _test_python_interpreter_excluded(env): python_interpreter = "python3", python_interpreter_target = "@python//:python", ) - env.expect.that_int(len(_whl_archive_calls())).equals(1) - env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("python_interpreter") - env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("python_interpreter_target") + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(1) + env.expect.that_collection( + whl_archive_calls[0].kwargs.keys(), + ).contains_none_of(["python_interpreter", "python_interpreter_target"]) _tests.append(_test_python_interpreter_excluded) def _test_config_load_excluded_from_extract_args(env): - _reset_calls() + calls = [] whl_library( name = "test_repo", - rules = _mock_rules(), + rules = _mock_rules(calls), whl_file = None, urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], filename = "pkg-1.0-py3-none-any.whl", @@ -225,9 +245,11 @@ def _test_config_load_excluded_from_extract_args(env): config_load = "@pypi//:config.bzl", dep_template = "@pypi%{name}//:{{target}}", ) - env.expect.that_int(len(_whl_archive_calls())).equals(1) - env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("config_load") - env.expect.that_dict(_whl_archive_calls()[0].kwargs).not_exists("dep_template") + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(1) + env.expect.that_collection( + whl_archive_calls[0].kwargs.keys(), + ).contains_none_of(["config_load", "dep_template"]) _tests.append(_test_config_load_excluded_from_extract_args) From 3deb0deb58bfc6b70915adf58ba51dfd8d329726 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:25:10 +0900 Subject: [PATCH 6/9] fix: update err_duplicate_repos test to match new error message --- tests/pypi/hub_builder/hub_builder_tests.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 958dbca1ca..4651342dd4 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1523,7 +1523,7 @@ def _test_err_duplicate_repos(env): env.expect.that_dict(logs).keys().contains_exactly(["rules_python:unit-test FAIL:"]) env.expect.that_collection(logs["rules_python:unit-test FAIL:"]).contains_exactly([ """\ -Attempting to create a duplicate library pypi_315_foo with different arguments. Already existing declaration has: +Attempting to create a duplicate library pypi_315_foo for foo with different arguments. Already existing declaration has: common: { "dep_template": "@pypi//{name}:{target}", "config_load": "@pypi//:config.bzl", From 1cf26c575f5f4414fa73f190dbaf9da007c237c9 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 1 Aug 2026 22:31:23 +0000 Subject: [PATCH 7/9] fix(pypi): prevent NoneType error in whl_repo_name and propagate repo_prefix Prevent a NoneType error in whl_repo_name when digest is None after PR #3974 by conditionally checking if digest before partitioning. Also propagate repo_prefix to whl_deps_library in whl_library. --- python/private/pypi/BUILD.bazel | 12 ++++++------ python/private/pypi/whl_library.bzl | 1 + python/private/pypi/whl_repo_name.bzl | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 55ec0ebeb2..c16ee49076 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -523,6 +523,12 @@ bzl_library( deps = ["//python/private:envsubst"], ) +bzl_library( + name = "index_sources", + srcs = ["index_sources.bzl"], + deps = [":hash"], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], @@ -543,12 +549,6 @@ bzl_library( srcs = ["hash.bzl"], ) -bzl_library( - name = "index_sources", - srcs = ["index_sources.bzl"], - deps = [":hash"], -) - bzl_library( name = "labels", srcs = ["labels.bzl"], diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 69ed00e7fc..48e2a8822f 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -868,6 +868,7 @@ def whl_library( "dep_template", "group_deps", "group_name", + "repo_prefix", ] if kwargs.get(k) != None } | { diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl index 739ff2a4a1..a0c693acbf 100644 --- a/python/private/pypi/whl_repo_name.bzl +++ b/python/private/pypi/whl_repo_name.bzl @@ -34,7 +34,8 @@ def whl_repo_name(filename, digest, *target_platforms): # Strip the `:` prefix so that the name only contains the hex digest # and stays the same as it has always been for bare sha256 values. - digest = digest.rpartition(":")[2] + if digest: + digest = digest.rpartition(":")[2] parts = [] From f6344381f63cc97c0574c1cc0f00be059454dfc3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 1 Aug 2026 22:46:04 +0000 Subject: [PATCH 8/9] test(pypi): add unit test for wheel extraction reuse across different extras Add _test_wheel_reuse_across_different_extras in whl_library_tests.bzl to verify that multiple whl_library calls with different extras reuse the same shared extraction repository. --- tests/pypi/whl_library/whl_library_tests.bzl | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/pypi/whl_library/whl_library_tests.bzl b/tests/pypi/whl_library/whl_library_tests.bzl index 84a72237ee..d2a8c275e1 100644 --- a/tests/pypi/whl_library/whl_library_tests.bzl +++ b/tests/pypi/whl_library/whl_library_tests.bzl @@ -253,6 +253,44 @@ def _test_config_load_excluded_from_extract_args(env): _tests.append(_test_config_load_excluded_from_extract_args) +def _test_wheel_reuse_across_different_extras(env): + calls = [] + whl_library( + name = "pkg_foo_repo", + rules = _mock_rules(calls), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg[foo]==1.0", + ) + whl_library( + name = "pkg_bar_repo", + rules = _mock_rules(calls), + whl_file = None, + urls = ["https://example.com/pkg-1.0-py3-none-any.whl"], + filename = "pkg-1.0-py3-none-any.whl", + requirement = "pkg[bar]==1.0", + ) + whl_archive_calls = _calls_for(calls, "whl_archive") + env.expect.that_int(len(whl_archive_calls)).equals(2) + env.expect.that_str(whl_archive_calls[0].kwargs["name"]).equals("w_pkg_1_0_py3_none_any") + env.expect.that_str(whl_archive_calls[1].kwargs["name"]).equals("w_pkg_1_0_py3_none_any") + + deps_calls = _calls_for(calls, "whl_deps_library") + env.expect.that_int(len(deps_calls)).equals(2) + env.expect.that_dict(deps_calls[0].kwargs).contains_exactly({ + "extras": ["foo"], + "metadata_file": "@w_pkg_1_0_py3_none_any//:metadata.json", + "name": "pkg_foo_repo", + }) + env.expect.that_dict(deps_calls[1].kwargs).contains_exactly({ + "extras": ["bar"], + "metadata_file": "@w_pkg_1_0_py3_none_any//:metadata.json", + "name": "pkg_bar_repo", + }) + +_tests.append(_test_wheel_reuse_across_different_extras) + def whl_library_test_suite(name): test_suite( name = name, From e6956a93f244e2b7c770218d3e4bcd054d6d0463 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 1 Aug 2026 22:49:28 +0000 Subject: [PATCH 9/9] test(venv): update expected repo names in test_optimized_grouping_pkgutil_whls Update site-packages repository path prefixes to match the shared wheel extraction repos introduced by PR #3856. --- .../app_files_building_tests.bzl | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl b/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl index 663ab6963e..1349052578 100644 --- a/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl +++ b/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl @@ -452,51 +452,51 @@ def _test_optimized_grouping_pkgutil_whls_impl(env, target): # Entries from pkgutil_ns1 _venv_symlink( "nspkg/__init__.py", - link_to_path = "+internal_dev_deps+pkgutil_nspkg1/site-packages/nspkg/__init__.py", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/nspkg/__init__.py", files = [ - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/nspkg/__init__.py", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/nspkg/__init__.py", ], ), _venv_symlink( "nspkg/one", - link_to_path = "+internal_dev_deps+pkgutil_nspkg1/site-packages/nspkg/one", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/nspkg/one", files = [ - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/nspkg/one/a.txt", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/nspkg/one/a.txt", ], ), _venv_symlink( "pkgutil_nspkg1-1.0.dist-info", - link_to_path = "+internal_dev_deps+pkgutil_nspkg1/site-packages/pkgutil_nspkg1-1.0.dist-info", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/pkgutil_nspkg1-1.0.dist-info", files = [ - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/pkgutil_nspkg1-1.0.dist-info/INSTALLER", - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/pkgutil_nspkg1-1.0.dist-info/METADATA", - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/pkgutil_nspkg1-1.0.dist-info/RECORD", - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/pkgutil_nspkg1-1.0.dist-info/WHEEL", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/pkgutil_nspkg1-1.0.dist-info/INSTALLER", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/pkgutil_nspkg1-1.0.dist-info/METADATA", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/pkgutil_nspkg1-1.0.dist-info/RECORD", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/pkgutil_nspkg1-1.0.dist-info/WHEEL", ], ), # Entries from pkgutil_ns2 _venv_symlink( "nspkg/__init__.py", - link_to_path = "+internal_dev_deps+pkgutil_nspkg2/site-packages/nspkg/__init__.py", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/nspkg/__init__.py", files = [ - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/nspkg/__init__.py", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/nspkg/__init__.py", ], ), _venv_symlink( "nspkg/two", - link_to_path = "+internal_dev_deps+pkgutil_nspkg2/site-packages/nspkg/two", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/nspkg/two", files = [ - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/nspkg/two/b.txt", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/nspkg/two/b.txt", ], ), _venv_symlink( "pkgutil_nspkg2-1.0.dist-info", - link_to_path = "+internal_dev_deps+pkgutil_nspkg2/site-packages/pkgutil_nspkg2-1.0.dist-info", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/pkgutil_nspkg2-1.0.dist-info", files = [ - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/pkgutil_nspkg2-1.0.dist-info/INSTALLER", - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/pkgutil_nspkg2-1.0.dist-info/METADATA", - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/pkgutil_nspkg2-1.0.dist-info/RECORD", - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/pkgutil_nspkg2-1.0.dist-info/WHEEL", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/pkgutil_nspkg2-1.0.dist-info/INSTALLER", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/pkgutil_nspkg2-1.0.dist-info/METADATA", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/pkgutil_nspkg2-1.0.dist-info/RECORD", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/pkgutil_nspkg2-1.0.dist-info/WHEEL", ], ), ]) @@ -507,16 +507,16 @@ def _test_optimized_grouping_pkgutil_whls_impl(env, target): env.expect.that_collection(_venv_symlinks_from_entries(conflicts[0])).contains_exactly([ _venv_symlink( "nspkg/__init__.py", - link_to_path = "+internal_dev_deps+pkgutil_nspkg1/site-packages/nspkg/__init__.py", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/nspkg/__init__.py", files = [ - "../+internal_dev_deps+pkgutil_nspkg1/site-packages/nspkg/__init__.py", + "../+internal_dev_deps+w_pkgutil_nspkg1_1_0_any_none_any/site-packages/nspkg/__init__.py", ], ), _venv_symlink( "nspkg/__init__.py", - link_to_path = "+internal_dev_deps+pkgutil_nspkg2/site-packages/nspkg/__init__.py", + link_to_path = "+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/nspkg/__init__.py", files = [ - "../+internal_dev_deps+pkgutil_nspkg2/site-packages/nspkg/__init__.py", + "../+internal_dev_deps+w_pkgutil_nspkg2_1_0_any_none_any/site-packages/nspkg/__init__.py", ], ), ])