From a48d05fc1fdbc58513ccc0c5e525ffef15af0a8f Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 12:31:19 +0200 Subject: [PATCH 1/6] apply black --- .../project/component_tests/check_config.py | 22 ++++++++++++++----- .../component_tests/run_and_check_output.py | 16 ++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py b/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py index 9bfbf67..a15230b 100644 --- a/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py +++ b/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py @@ -122,13 +122,18 @@ def check_config(config: dict) -> None: print("Check that .namespace is defined", flush=True) assert config.get("namespace"), ".namespace is not defined" - print("Check that .info.type is 'method', 'control_method', or 'metric'", flush=True) + print( + "Check that .info.type is 'method', 'control_method', or 'metric'", flush=True + ) expected_types = ["method", "control_method", "metric"] assert ( comp_type in expected_types ), f".info.type is '{comp_type}' but should be one of: {', '.join(expected_types)}" - print("Check component metadata fields (name, label, summary, description)", flush=True) + print( + "Check component metadata fields (name, label, summary, description)", + flush=True, + ) if comp_type == "metric": metric_infos = info.get("metrics", []) assert metric_infos, ".info.metrics is not defined" @@ -138,7 +143,10 @@ def check_config(config: dict) -> None: check_info(info, config, comp_type=comp_type) if "preferred_normalization" in info: - print("Check that .info.preferred_normalization is a valid normalization method", flush=True) + print( + "Check that .info.preferred_normalization is a valid normalization method", + flush=True, + ) norm_methods = [ "log_cpm", "log_cp10k", @@ -155,7 +163,9 @@ def check_config(config: dict) -> None: ) if "variants" in info: - print("Check that .info.variants only references valid argument names", flush=True) + print( + "Check that .info.variants only references valid argument names", flush=True + ) arg_names = [arg["clean_name"] for arg in config["all_arguments"]] + [ "preferred_normalization" ] @@ -185,7 +195,9 @@ def check_config(config: dict) -> None: ) if not is_nextflow_workflow: - print("Check that the Nextflow runner has time, mem, and cpu labels", flush=True) + print( + "Check that the Nextflow runner has time, mem, and cpu labels", flush=True + ) assert nextflow_runner.get( "directives" ), "directives not a field in nextflow runner" diff --git a/packages/python/openproblems/src/openproblems/project/component_tests/run_and_check_output.py b/packages/python/openproblems/src/openproblems/project/component_tests/run_and_check_output.py index 68b8f76..7a41202 100644 --- a/packages/python/openproblems/src/openproblems/project/component_tests/run_and_check_output.py +++ b/packages/python/openproblems/src/openproblems/project/component_tests/run_and_check_output.py @@ -22,8 +22,12 @@ def check_input_files(arguments: list) -> None: for arg in arguments: if arg["type"] == "file" and arg["direction"] == "input" and arg["required"]: expected_path = arg.get("value") - assert expected_path is not None, f"Input argument '{arg['name']}' is missing a value" - assert not arg["must_exist"] or path.exists(expected_path), f"Input file '{expected_path}' does not exist" + assert ( + expected_path is not None + ), f"Input argument '{arg['name']}' is missing a value" + assert not arg["must_exist"] or path.exists( + expected_path + ), f"Input file '{expected_path}' does not exist" def check_output_files(arguments: list) -> None: @@ -34,8 +38,12 @@ def check_output_files(arguments: list) -> None: for arg in arguments: if arg["type"] == "file" and arg["direction"] == "output" and arg["required"]: expected_path = arg.get("value") - assert expected_path is not None, f"Output argument '{arg['name']}' is missing a value" - assert not arg["must_exist"] or path.exists(expected_path), f"Output file '{expected_path}' does not exist" + assert ( + expected_path is not None + ), f"Output argument '{arg['name']}' is missing a value" + assert not arg["must_exist"] or path.exists( + expected_path + ), f"Output file '{expected_path}' does not exist" print(">> Validating the contents and format of output files", flush=True) for arg in arguments: From fba827993a0fbc996a32c36e29ddca1e6bd2959a Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 12:02:03 +0200 Subject: [PATCH 2/6] check the formatting instead of applying it `tox -e lint` ran `black .`, which reformats the files in the runner and exits 0, so the lint job could not fail. Two files were unformatted on main while CI was green. Run `black` yourself, or `tox -e lint -- .`, to apply the changes. --- packages/python/openproblems/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/openproblems/tox.ini b/packages/python/openproblems/tox.ini index cc9316f..79bc91c 100644 --- a/packages/python/openproblems/tox.ini +++ b/packages/python/openproblems/tox.ini @@ -17,7 +17,7 @@ description = run linters skip_install = true deps = black==22.12 -commands = black {posargs:.} +commands = black {posargs:--check --diff .} [testenv:type] description = run type checks From 228a46cfcf8df403b388b2612c417299722d8b56 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 12:17:50 +0200 Subject: [PATCH 3/6] fix the type annotations mypy complains about * `deep_merge()` and `process_nested_yaml()` annotated their arguments as `any`, the builtin function, rather than `typing.Any` * `process_nested_yaml()`: accept a `project_path` of `None`, which is what `read_nested_yaml()` hands it when the yaml lives outside a viash project * `check_links()`: the values of `.links` are urls, not lists of urls --- .../openproblems/project/component_tests/check_config.py | 4 +--- .../src/openproblems/project/read_nested_yaml.py | 8 +++++--- .../openproblems/src/openproblems/utils/deep_merge.py | 5 ++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py b/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py index a15230b..e5a663c 100644 --- a/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py +++ b/packages/python/openproblems/src/openproblems/project/component_tests/check_config.py @@ -60,9 +60,7 @@ def check_references(references: Dict[str, Union[str, List[str]]]) -> None: assert re.match(r"^@.*{.*", b), f"Invalid bibtex format: {b}" -def check_links( - links: Dict[str, Union[str, List[str]]], required: List[str] = [] -) -> None: +def check_links(links: Dict[str, str], required: List[str] = []) -> None: links = links or {} for expected_link in required: diff --git a/packages/python/openproblems/src/openproblems/project/read_nested_yaml.py b/packages/python/openproblems/src/openproblems/project/read_nested_yaml.py index f3f13fd..e8f392a 100644 --- a/packages/python/openproblems/src/openproblems/project/read_nested_yaml.py +++ b/packages/python/openproblems/src/openproblems/project/read_nested_yaml.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Any + def read_nested_yaml(path: str, project_path: str | None = None) -> dict: """ @@ -35,8 +37,8 @@ def read_nested_yaml(path: str, project_path: str | None = None) -> dict: def process_nested_yaml( - data: any, root_data: dict, path: str, project_path: str -) -> dict: + data: Any, root_data: dict, path: str, project_path: str | None +) -> Any: """ Process the merge keys in a YAML @@ -62,7 +64,7 @@ def process_nested_yaml( for k, v in data.items() } - new_data = {} + new_data: Any = {} if "__merge__" in processed_data and not isinstance( processed_data["__merge__"], dict ): diff --git a/packages/python/openproblems/src/openproblems/utils/deep_merge.py b/packages/python/openproblems/src/openproblems/utils/deep_merge.py index 56ecd53..0f89a0e 100644 --- a/packages/python/openproblems/src/openproblems/utils/deep_merge.py +++ b/packages/python/openproblems/src/openproblems/utils/deep_merge.py @@ -1,4 +1,7 @@ -def deep_merge(obj1: any, obj2: any) -> dict: +from typing import Any + + +def deep_merge(obj1: Any, obj2: Any) -> Any: """Recursively merge two dictionaries or lists. Args: From 2f4c1304747be46e3841b2e374bff0c6923d89f1 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 12:17:50 +0200 Subject: [PATCH 4/6] teach mypy about the optional imports The component tests import anndata, pandas, requests and friends inside the function that needs them, so that a task repo only has to provide the ones its own components use. None of them ship type information. --- packages/python/openproblems/pyproject.toml | 14 ++++++++++++++ packages/python/openproblems/tox.ini | 1 + 2 files changed, 15 insertions(+) diff --git a/packages/python/openproblems/pyproject.toml b/packages/python/openproblems/pyproject.toml index ef68eb7..c414c49 100644 --- a/packages/python/openproblems/pyproject.toml +++ b/packages/python/openproblems/pyproject.toml @@ -51,3 +51,17 @@ exclude = ["tests*"] [tool.setuptools_scm] root = "../../.." + +# the component tests import these inside the function that needs them, so that +# a task repo only has to provide the ones its own components use +[[tool.mypy.overrides]] +module = [ + "anndata.*", + "networkx.*", + "pandas.*", + "requests.*", + "spatialdata.*", + "urllib3.*", + "yaml.*", +] +ignore_missing_imports = true diff --git a/packages/python/openproblems/tox.ini b/packages/python/openproblems/tox.ini index 79bc91c..7dab9b7 100644 --- a/packages/python/openproblems/tox.ini +++ b/packages/python/openproblems/tox.ini @@ -23,5 +23,6 @@ commands = black {posargs:--check --diff .} description = run type checks deps = mypy>=0.991 + pytest>=7 commands = mypy {posargs:src tests} From 06638713297dae4cf35f0e76dd3671e1ebd61c76 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 12:20:48 +0200 Subject: [PATCH 5/6] run the type checks in ci `tox.ini` has had a `type` env since the start, but nothing ran it: the test job skips every env that is not `py`, and the lint job only runs `lint`. It had accumulated 25 errors. --- .github/workflows/python-packages.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/python-packages.yaml b/.github/workflows/python-packages.yaml index 5c92165..df69087 100644 --- a/.github/workflows/python-packages.yaml +++ b/.github/workflows/python-packages.yaml @@ -31,6 +31,28 @@ jobs: run: | cd packages/python/${{ matrix.package }} tox -e lint + + type: + runs-on: ubuntu-latest + strategy: + matrix: + package: [openproblems] + + steps: + - uses: actions/checkout@v7 + + - name: Using Python 3.10 + uses: actions/setup-python@v7 + with: + python-version: "3.10" + + - name: Install tox + run: python -m pip install tox + + - name: Run tox type + run: | + cd packages/python/${{ matrix.package }} + tox -e type test: runs-on: ubuntu-latest From 192dcb6dc6e40895507afbeb432c1708e84cb2a5 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 12:45:02 +0200 Subject: [PATCH 6/6] update changelog --- packages/python/openproblems/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/python/openproblems/CHANGELOG.md b/packages/python/openproblems/CHANGELOG.md index d312304..a4b9ad0 100644 --- a/packages/python/openproblems/CHANGELOG.md +++ b/packages/python/openproblems/CHANGELOG.md @@ -24,6 +24,12 @@ * `check_config`: Skip the Nextflow resource label check for components whose script is itself a Nextflow workflow. Viash renders those as a workflow rather than a process, so the labels would have no effect. +* `tox -e lint` now checks the formatting rather than applying it, so the lint job can actually fail. + Run `black` yourself, or `tox -e lint -- .`, to apply the changes. + +* The `type` environment is no longer skipped in CI, and the 25 type errors it had accumulated + are fixed. + ## BUG FIXES * `read_task_metadata`: Order the task graph topologically instead of by a breadth-first search from a single root.