From e54ffe38bdb2ce3422d6aec81820ce308d4ed325 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 17 Sep 2026 10:52:49 -0500 Subject: [PATCH 01/21] build(wheel): split libcuopt into per-component wheels Adds libcuopt-client, libcuopt-mathopt and libcuopt-routing, each staging only its own install components through scikit-build-core's install.components, which the per-component install rules in #1926 made possible. The shared CMake body moved to python/cmake/cuopt_wheel_build.cmake so the RPATH list and third-party lookups cannot drift between them. libcuopt keeps the ld script and the gRPC server binary and now depends on the three rather than bundling the libraries a second time. Its load_library delegates to the component packages, client first since mathopt and routing both carry a DT_NEEDED on it. cuopt_cli's console script moves to libcuopt-mathopt, where the binary installs. Dependencies were split from measured DT_NEEDED: client needs only rmm and rapids-logger and no CUDA math libraries, routing adds cublas through a narrower cuda_wheels_routing group, and cudss, nccl and nvjitlink stay with mathopt alone. A routing-only install no longer pulls roughly 1.2 GB of libraries it never calls. Each wheel still configures the whole C++ tree, so this trades CI build time for install size; sccache is what keeps that affordable. Building once and packaging three times would need build_wheel.sh restructured, which is worth doing separately if the cost shows up. Co-Authored-By: Claude Opus 5 --- ci/build_wheel_libcuopt_client.sh | 131 ++++++++++++ ci/build_wheel_libcuopt_mathopt.sh | 131 ++++++++++++ ci/build_wheel_libcuopt_routing.sh | 131 ++++++++++++ dependencies.yaml | 111 ++++++++++ python/cmake/cuopt_wheel_build.cmake | 83 ++++++++ python/libcuopt/libcuopt/load.py | 49 ++--- python/libcuopt/pyproject.toml | 6 +- python/libcuopt_client/CMakeLists.txt | 25 +++ python/libcuopt_client/LICENSE | 201 ++++++++++++++++++ python/libcuopt_client/README.md | 5 + .../libcuopt_client/libcuopt_client/VERSION | 1 + .../libcuopt_client/__init__.py | 7 + .../libcuopt_client/_version.py | 22 ++ .../libcuopt_client/libcuopt_client/load.py | 113 ++++++++++ python/libcuopt_client/pyproject.toml | 78 +++++++ python/libcuopt_mathopt/CMakeLists.txt | 25 +++ python/libcuopt_mathopt/LICENSE | 201 ++++++++++++++++++ python/libcuopt_mathopt/README.md | 5 + .../libcuopt_mathopt/libcuopt_mathopt/VERSION | 1 + .../libcuopt_mathopt/__init__.py | 7 + .../libcuopt_mathopt/_cli_wrapper.py | 17 ++ .../libcuopt_mathopt/_version.py | 22 ++ .../libcuopt_mathopt/libcuopt_mathopt/load.py | 118 ++++++++++ python/libcuopt_mathopt/pyproject.toml | 86 ++++++++ python/libcuopt_routing/CMakeLists.txt | 25 +++ python/libcuopt_routing/LICENSE | 201 ++++++++++++++++++ python/libcuopt_routing/README.md | 5 + .../libcuopt_routing/libcuopt_routing/VERSION | 1 + .../libcuopt_routing/__init__.py | 7 + .../libcuopt_routing/_version.py | 22 ++ .../libcuopt_routing/libcuopt_routing/load.py | 118 ++++++++++ python/libcuopt_routing/pyproject.toml | 80 +++++++ 32 files changed, 2005 insertions(+), 30 deletions(-) create mode 100755 ci/build_wheel_libcuopt_client.sh create mode 100755 ci/build_wheel_libcuopt_mathopt.sh create mode 100755 ci/build_wheel_libcuopt_routing.sh create mode 100644 python/cmake/cuopt_wheel_build.cmake create mode 100644 python/libcuopt_client/CMakeLists.txt create mode 100644 python/libcuopt_client/LICENSE create mode 100644 python/libcuopt_client/README.md create mode 100644 python/libcuopt_client/libcuopt_client/VERSION create mode 100644 python/libcuopt_client/libcuopt_client/__init__.py create mode 100644 python/libcuopt_client/libcuopt_client/_version.py create mode 100644 python/libcuopt_client/libcuopt_client/load.py create mode 100644 python/libcuopt_client/pyproject.toml create mode 100644 python/libcuopt_mathopt/CMakeLists.txt create mode 100644 python/libcuopt_mathopt/LICENSE create mode 100644 python/libcuopt_mathopt/README.md create mode 100644 python/libcuopt_mathopt/libcuopt_mathopt/VERSION create mode 100644 python/libcuopt_mathopt/libcuopt_mathopt/__init__.py create mode 100644 python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py create mode 100644 python/libcuopt_mathopt/libcuopt_mathopt/_version.py create mode 100644 python/libcuopt_mathopt/libcuopt_mathopt/load.py create mode 100644 python/libcuopt_mathopt/pyproject.toml create mode 100644 python/libcuopt_routing/CMakeLists.txt create mode 100644 python/libcuopt_routing/LICENSE create mode 100644 python/libcuopt_routing/README.md create mode 100644 python/libcuopt_routing/libcuopt_routing/VERSION create mode 100644 python/libcuopt_routing/libcuopt_routing/__init__.py create mode 100644 python/libcuopt_routing/libcuopt_routing/_version.py create mode 100644 python/libcuopt_routing/libcuopt_routing/load.py create mode 100644 python/libcuopt_routing/pyproject.toml diff --git a/ci/build_wheel_libcuopt_client.sh b/ci/build_wheel_libcuopt_client.sh new file mode 100755 index 0000000000..5246002c37 --- /dev/null +++ b/ci/build_wheel_libcuopt_client.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Configures the full C++ tree like every other cuOpt wheel, but stages only this +# component's install rules -- see install.components in the package's +# pyproject.toml. sccache is what keeps the repeated configure affordable. + +source rapids-init-pip + +package_name="libcuopt_client" +package_dir="python/libcuopt_client" + +# Install rockylinux repo +if command -v dnf &> /dev/null; then + bash ci/utils/update_rockylinux_repo.sh +fi + +# Install Boost and TBB +bash ci/utils/install_boost_tbb.sh + +# Install libuuid and LLVM's OpenMP runtime +if command -v dnf &> /dev/null; then + # LLVM Toolset is distributed as a module on Rocky/RHEL 8. + dnf module install -y llvm-toolset + dnf install -y libuuid-devel libomp-devel +elif command -v apt-get &> /dev/null; then + apt-get update + apt-get install -y uuid-dev libomp-dev +fi + +# Install Protobuf + gRPC (protoc + grpc_cpp_plugin) +bash ci/utils/install_protobuf_grpc.sh + +# Compile with GCC, but use LLVM libomp as the OpenMP runtime bundled in the wheel. Resolve the +# versioned ELF library rather than an unversioned linker script or compiler-toolset indirection. +LIBOMP_LIBRARY="$( + ldconfig -p | + awk '$1 ~ /^libomp\.so(\.[0-9]+)*$/ && !library { library = $NF } + END { print library }' +)" +if [[ "${LIBOMP_LIBRARY}" != /* || ! -f "${LIBOMP_LIBRARY}" ]]; then + echo "Could not resolve the LLVM OpenMP runtime: '${LIBOMP_LIBRARY}'" >&2 + exit 1 +fi + +echo "Using LLVM OpenMP runtime: ${LIBOMP_LIBRARY}" + +export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${LIBOMP_LIBRARY}" + +# OpenSSL 3 hints for libcuopt's own find_package(OpenSSL). +# +# install_protobuf_grpc.sh links gRPC against OpenSSL 3 (see that script for +# rationale). libcuopt then re-resolves OpenSSL via find_package because +# gRPC's imported targets propagate it transitively. On Rocky/RHEL 8 the +# EPEL openssl3-devel package installs in non-default paths, so we have to +# point CMake at them; on Rocky/RHEL 9+ and Ubuntu 22.04+ the default +# OpenSSL is already 3.x and no hints are needed. +if [ -f /etc/os-release ]; then + . /etc/os-release + if [[ "$ID" == "rocky" || "$ID" == "centos" || "$ID" == "rhel" || "$ID" == "fedora" ]] && \ + [[ "${VERSION_ID%%.*}" == "8" ]]; then + SKBUILD_CMAKE_ARGS="${SKBUILD_CMAKE_ARGS};-DOPENSSL_INCLUDE_DIR=/usr/include/openssl3;-DOPENSSL_SSL_LIBRARY=/usr/lib64/openssl3/libssl.so;-DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/openssl3/libcrypto.so" + fi +fi + +# For pull requests we are enabling assert mode. +if [ "$RAPIDS_BUILD_TYPE" = "pull-request" ]; then + echo "Building in assert mode" + export SKBUILD_CMAKE_ARGS="${SKBUILD_CMAKE_ARGS};-DDEFINE_ASSERT=True" +else + echo "Building in release mode" +fi + +# Install cudss +bash ci/utils/install_cudss.sh + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +rapids-pip-retry install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + + +EXCLUDE_ARGS=( + --exclude "libraft.so" + --exclude "libcublas.so.*" + --exclude "libcublasLt.so.*" + --exclude "libcuda.so.1" + --exclude "libcudss.so.*" + --exclude "libcurand.so.*" + --exclude "libcusolver.so.*" + --exclude "libcusparse.so.*" + --exclude "libnccl.so.*" + --exclude "libnvJitLink.so*" + --exclude "librapids_logger.so" + --exclude "librmm.so" + # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 + # at runtime via the host (or container image) keeps libcrypto and the FIPS + # provider (system or mounted) byte-version-matched, which is required for + # the FIPS provider's HMAC integrity check and avoids loading two libcrypto.so.3 + # in the same process. Hosts must provide libssl.so.3 / libcrypto.so.3 (Ubuntu + # 22.04+, RHEL/Rocky 9+, manylinux_2_28+ with openssl3, Debian 12+). + --exclude "libssl.so.3" + --exclude "libcrypto.so.3" +) + +ci/build_wheel.sh ${package_name} ${package_dir} + +mkdir -p final_dist +python -m auditwheel repair "${EXCLUDE_ARGS[@]}" -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/* + +ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" + +RAPIDS_PACKAGE_NAME="$(rapids-artifact-name wheel_cpp libcuopt_client cuopt --cuda "$RAPIDS_CUDA_VERSION")" +export RAPIDS_PACKAGE_NAME diff --git a/ci/build_wheel_libcuopt_mathopt.sh b/ci/build_wheel_libcuopt_mathopt.sh new file mode 100755 index 0000000000..e70d0ca468 --- /dev/null +++ b/ci/build_wheel_libcuopt_mathopt.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Configures the full C++ tree like every other cuOpt wheel, but stages only this +# component's install rules -- see install.components in the package's +# pyproject.toml. sccache is what keeps the repeated configure affordable. + +source rapids-init-pip + +package_name="libcuopt_mathopt" +package_dir="python/libcuopt_mathopt" + +# Install rockylinux repo +if command -v dnf &> /dev/null; then + bash ci/utils/update_rockylinux_repo.sh +fi + +# Install Boost and TBB +bash ci/utils/install_boost_tbb.sh + +# Install libuuid and LLVM's OpenMP runtime +if command -v dnf &> /dev/null; then + # LLVM Toolset is distributed as a module on Rocky/RHEL 8. + dnf module install -y llvm-toolset + dnf install -y libuuid-devel libomp-devel +elif command -v apt-get &> /dev/null; then + apt-get update + apt-get install -y uuid-dev libomp-dev +fi + +# Install Protobuf + gRPC (protoc + grpc_cpp_plugin) +bash ci/utils/install_protobuf_grpc.sh + +# Compile with GCC, but use LLVM libomp as the OpenMP runtime bundled in the wheel. Resolve the +# versioned ELF library rather than an unversioned linker script or compiler-toolset indirection. +LIBOMP_LIBRARY="$( + ldconfig -p | + awk '$1 ~ /^libomp\.so(\.[0-9]+)*$/ && !library { library = $NF } + END { print library }' +)" +if [[ "${LIBOMP_LIBRARY}" != /* || ! -f "${LIBOMP_LIBRARY}" ]]; then + echo "Could not resolve the LLVM OpenMP runtime: '${LIBOMP_LIBRARY}'" >&2 + exit 1 +fi + +echo "Using LLVM OpenMP runtime: ${LIBOMP_LIBRARY}" + +export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${LIBOMP_LIBRARY}" + +# OpenSSL 3 hints for libcuopt's own find_package(OpenSSL). +# +# install_protobuf_grpc.sh links gRPC against OpenSSL 3 (see that script for +# rationale). libcuopt then re-resolves OpenSSL via find_package because +# gRPC's imported targets propagate it transitively. On Rocky/RHEL 8 the +# EPEL openssl3-devel package installs in non-default paths, so we have to +# point CMake at them; on Rocky/RHEL 9+ and Ubuntu 22.04+ the default +# OpenSSL is already 3.x and no hints are needed. +if [ -f /etc/os-release ]; then + . /etc/os-release + if [[ "$ID" == "rocky" || "$ID" == "centos" || "$ID" == "rhel" || "$ID" == "fedora" ]] && \ + [[ "${VERSION_ID%%.*}" == "8" ]]; then + SKBUILD_CMAKE_ARGS="${SKBUILD_CMAKE_ARGS};-DOPENSSL_INCLUDE_DIR=/usr/include/openssl3;-DOPENSSL_SSL_LIBRARY=/usr/lib64/openssl3/libssl.so;-DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/openssl3/libcrypto.so" + fi +fi + +# For pull requests we are enabling assert mode. +if [ "$RAPIDS_BUILD_TYPE" = "pull-request" ]; then + echo "Building in assert mode" + export SKBUILD_CMAKE_ARGS="${SKBUILD_CMAKE_ARGS};-DDEFINE_ASSERT=True" +else + echo "Building in release mode" +fi + +# Install cudss +bash ci/utils/install_cudss.sh + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +rapids-pip-retry install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + + +EXCLUDE_ARGS=( + --exclude "libraft.so" + --exclude "libcublas.so.*" + --exclude "libcublasLt.so.*" + --exclude "libcuda.so.1" + --exclude "libcudss.so.*" + --exclude "libcurand.so.*" + --exclude "libcusolver.so.*" + --exclude "libcusparse.so.*" + --exclude "libnccl.so.*" + --exclude "libnvJitLink.so*" + --exclude "librapids_logger.so" + --exclude "librmm.so" + # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 + # at runtime via the host (or container image) keeps libcrypto and the FIPS + # provider (system or mounted) byte-version-matched, which is required for + # the FIPS provider's HMAC integrity check and avoids loading two libcrypto.so.3 + # in the same process. Hosts must provide libssl.so.3 / libcrypto.so.3 (Ubuntu + # 22.04+, RHEL/Rocky 9+, manylinux_2_28+ with openssl3, Debian 12+). + --exclude "libssl.so.3" + --exclude "libcrypto.so.3" +) + +ci/build_wheel.sh ${package_name} ${package_dir} + +mkdir -p final_dist +python -m auditwheel repair "${EXCLUDE_ARGS[@]}" -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/* + +ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" + +RAPIDS_PACKAGE_NAME="$(rapids-artifact-name wheel_cpp libcuopt_mathopt cuopt --cuda "$RAPIDS_CUDA_VERSION")" +export RAPIDS_PACKAGE_NAME diff --git a/ci/build_wheel_libcuopt_routing.sh b/ci/build_wheel_libcuopt_routing.sh new file mode 100755 index 0000000000..87acfab67d --- /dev/null +++ b/ci/build_wheel_libcuopt_routing.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Configures the full C++ tree like every other cuOpt wheel, but stages only this +# component's install rules -- see install.components in the package's +# pyproject.toml. sccache is what keeps the repeated configure affordable. + +source rapids-init-pip + +package_name="libcuopt_routing" +package_dir="python/libcuopt_routing" + +# Install rockylinux repo +if command -v dnf &> /dev/null; then + bash ci/utils/update_rockylinux_repo.sh +fi + +# Install Boost and TBB +bash ci/utils/install_boost_tbb.sh + +# Install libuuid and LLVM's OpenMP runtime +if command -v dnf &> /dev/null; then + # LLVM Toolset is distributed as a module on Rocky/RHEL 8. + dnf module install -y llvm-toolset + dnf install -y libuuid-devel libomp-devel +elif command -v apt-get &> /dev/null; then + apt-get update + apt-get install -y uuid-dev libomp-dev +fi + +# Install Protobuf + gRPC (protoc + grpc_cpp_plugin) +bash ci/utils/install_protobuf_grpc.sh + +# Compile with GCC, but use LLVM libomp as the OpenMP runtime bundled in the wheel. Resolve the +# versioned ELF library rather than an unversioned linker script or compiler-toolset indirection. +LIBOMP_LIBRARY="$( + ldconfig -p | + awk '$1 ~ /^libomp\.so(\.[0-9]+)*$/ && !library { library = $NF } + END { print library }' +)" +if [[ "${LIBOMP_LIBRARY}" != /* || ! -f "${LIBOMP_LIBRARY}" ]]; then + echo "Could not resolve the LLVM OpenMP runtime: '${LIBOMP_LIBRARY}'" >&2 + exit 1 +fi + +echo "Using LLVM OpenMP runtime: ${LIBOMP_LIBRARY}" + +export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${LIBOMP_LIBRARY}" + +# OpenSSL 3 hints for libcuopt's own find_package(OpenSSL). +# +# install_protobuf_grpc.sh links gRPC against OpenSSL 3 (see that script for +# rationale). libcuopt then re-resolves OpenSSL via find_package because +# gRPC's imported targets propagate it transitively. On Rocky/RHEL 8 the +# EPEL openssl3-devel package installs in non-default paths, so we have to +# point CMake at them; on Rocky/RHEL 9+ and Ubuntu 22.04+ the default +# OpenSSL is already 3.x and no hints are needed. +if [ -f /etc/os-release ]; then + . /etc/os-release + if [[ "$ID" == "rocky" || "$ID" == "centos" || "$ID" == "rhel" || "$ID" == "fedora" ]] && \ + [[ "${VERSION_ID%%.*}" == "8" ]]; then + SKBUILD_CMAKE_ARGS="${SKBUILD_CMAKE_ARGS};-DOPENSSL_INCLUDE_DIR=/usr/include/openssl3;-DOPENSSL_SSL_LIBRARY=/usr/lib64/openssl3/libssl.so;-DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/openssl3/libcrypto.so" + fi +fi + +# For pull requests we are enabling assert mode. +if [ "$RAPIDS_BUILD_TYPE" = "pull-request" ]; then + echo "Building in assert mode" + export SKBUILD_CMAKE_ARGS="${SKBUILD_CMAKE_ARGS};-DDEFINE_ASSERT=True" +else + echo "Building in release mode" +fi + +# Install cudss +bash ci/utils/install_cudss.sh + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +rapids-pip-retry install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + + +EXCLUDE_ARGS=( + --exclude "libraft.so" + --exclude "libcublas.so.*" + --exclude "libcublasLt.so.*" + --exclude "libcuda.so.1" + --exclude "libcudss.so.*" + --exclude "libcurand.so.*" + --exclude "libcusolver.so.*" + --exclude "libcusparse.so.*" + --exclude "libnccl.so.*" + --exclude "libnvJitLink.so*" + --exclude "librapids_logger.so" + --exclude "librmm.so" + # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 + # at runtime via the host (or container image) keeps libcrypto and the FIPS + # provider (system or mounted) byte-version-matched, which is required for + # the FIPS provider's HMAC integrity check and avoids loading two libcrypto.so.3 + # in the same process. Hosts must provide libssl.so.3 / libcrypto.so.3 (Ubuntu + # 22.04+, RHEL/Rocky 9+, manylinux_2_28+ with openssl3, Debian 12+). + --exclude "libssl.so.3" + --exclude "libcrypto.so.3" +) + +ci/build_wheel.sh ${package_name} ${package_dir} + +mkdir -p final_dist +python -m auditwheel repair "${EXCLUDE_ARGS[@]}" -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/* + +ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" + +RAPIDS_PACKAGE_NAME="$(rapids-artifact-name wheel_cpp libcuopt_routing cuopt --cuda "$RAPIDS_CUDA_VERSION")" +export RAPIDS_PACKAGE_NAME diff --git a/dependencies.yaml b/dependencies.yaml index 3fd82474bc..61eb39910f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -152,6 +152,92 @@ files: - depends_on_libraft_headers - depends_on_librmm - depends_on_rapids_logger + # One set of keys per component wheel. The point of the split is in py_run_*: + # only mathopt carries cudss / nccl / nvjitlink, so a routing-only install no + # longer pulls libraries it never calls. + py_build_libcuopt_client: + output: pyproject + pyproject_dir: python/libcuopt_client + extras: + table: build-system + includes: + - rapids_build_backend + py_rapids_build_libcuopt_client: + output: pyproject + pyproject_dir: python/libcuopt_client + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_common + - depends_on_librmm + - depends_on_rapids_logger + py_run_libcuopt_client: + output: pyproject + pyproject_dir: python/libcuopt_client + extras: + table: project + includes: + - depends_on_librmm + - depends_on_rapids_logger + py_build_libcuopt_mathopt: + output: pyproject + pyproject_dir: python/libcuopt_mathopt + extras: + table: build-system + includes: + - rapids_build_backend + py_rapids_build_libcuopt_mathopt: + output: pyproject + pyproject_dir: python/libcuopt_mathopt + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_common + - depends_on_libraft_headers + - depends_on_librmm + - depends_on_rapids_logger + py_run_libcuopt_mathopt: + output: pyproject + pyproject_dir: python/libcuopt_mathopt + extras: + table: project + includes: + - cuda_wheels + - depends_on_libcuopt_client + - depends_on_libraft_headers + - depends_on_librmm + - depends_on_rapids_logger + py_build_libcuopt_routing: + output: pyproject + pyproject_dir: python/libcuopt_routing + extras: + table: build-system + includes: + - rapids_build_backend + py_rapids_build_libcuopt_routing: + output: pyproject + pyproject_dir: python/libcuopt_routing + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_common + - depends_on_libraft_headers + - depends_on_librmm + - depends_on_rapids_logger + py_run_libcuopt_routing: + output: pyproject + pyproject_dir: python/libcuopt_routing + extras: + table: project + includes: + - cuda_wheels_routing + - depends_on_libcuopt_client + - depends_on_libraft_headers + - depends_on_librmm + - depends_on_rapids_logger py_build_cuopt: output: pyproject pyproject_dir: python/cuopt @@ -934,6 +1020,31 @@ dependencies: - libgomp >=9 + # cublas only. libcuopt_routing's DT_NEEDED is cublas, rmm and rapids_logger -- + # it never touches cudss, nccl, cusparse or nvjitlink, so pulling the full + # cuda_wheels set here would re-create exactly the bloat this split removes. + cuda_wheels_routing: + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + use_cuda_wheels: "false" + packages: + - matrix: + cuda: "12.*" + use_cuda_wheels: "true" + packages: + - cuda-toolkit[cublas,cudart]==12.* + - matrix: + cuda: "13.*" + use_cuda_wheels: "true" + packages: + - &ctk_routing_cu13 cuda-toolkit[cublas,cudart]==13.* + # if no matching matrix selectors passed, list the CUDA 13 requirement, which is + # what populates pyproject.toml in source control + - matrix: + packages: + - *ctk_routing_cu13 cuda_wheels: specific: # cuOpt needs 'nvJitLink>={whatever-cuopt-was-built-against}' at runtime, and mixing diff --git a/python/cmake/cuopt_wheel_build.cmake b/python/cmake/cuopt_wheel_build.cmake new file mode 100644 index 0000000000..1ee1d07ffa --- /dev/null +++ b/python/cmake/cuopt_wheel_build.cmake @@ -0,0 +1,83 @@ +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on + +# Shared build for the libcuopt wheels. Every wheel configures the same C++ tree and +# differs only in which install components scikit-build-core stages into it, set by +# install.components in each pyproject.toml. Keeping this in one file means the RPATH +# list and the third-party lookups cannot drift between the per-component wheels. + +include(FetchContent) +FetchContent_Declare( + argparse + GIT_REPOSITORY https://github.com/p-ranav/argparse.git + GIT_TAG v3.2 +) +FetchContent_MakeAvailable(argparse) + +# gRPC must be available as an installed CMake package (gRPCConfig.cmake). +# On RockyLinux 8 wheel builds we install it in CI via ci/utils/install_protobuf_grpc.sh. +find_package(gRPC CONFIG REQUIRED) + +find_package(Boost 1.65 REQUIRED) +if(Boost_FOUND) + message(STATUS "Found Boost ${Boost_VERSION} in ${Boost_INCLUDE_DIRS}") +else() + message(FATAL_ERROR "Boost not found. Please install boost-devel.") +endif() + +include_directories(${Boost_INCLUDE_DIRS}) + +set(BUILD_TESTING OFF CACHE BOOL "Disable test build for papilo") +set(PAPILO_NO_BINARIES ON) +option(LUSOL "Disable LUSOL" OFF) + +set(BUILD_TESTS OFF) +set(BUILD_BENCHMARKS OFF) +set(CUOPT_BUILD_TESTUTIL OFF) + +add_subdirectory(../../cpp cuopt-cpp) + +# cuopt is an INTERFACE target (libcuopt.so is a linker script) and compiles nothing, +# so it has no use for argparse. cuopt_cli and cuopt_grpc_server, which do, already link +# argparse::argparse in cpp/CMakeLists.txt. +target_link_libraries(cuopt_cli PRIVATE + argparse +) + +set(rpaths + "$ORIGIN/../lib64" + "$ORIGIN/../../rapids_logger/lib64" + "$ORIGIN/../../librmm/lib64" + "$ORIGIN/../../nvidia/cudss/lib" + "$ORIGIN/../../nvidia/cublas/lib" + "$ORIGIN/../../nvidia/curand/lib" + "$ORIGIN/../../nvidia/cusolver/lib" + "$ORIGIN/../../nvidia/cusparse/lib" + "$ORIGIN/../../nvidia/nccl/lib" + "$ORIGIN/../../nvidia/nvjitlink/lib" +) + +# Add CUDA version-specific paths based on CUDA compiler version +message(STATUS "libcuopt: CMAKE_CUDA_COMPILER_VERSION = ${CMAKE_CUDA_COMPILER_VERSION}") +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 14.0) + message(STATUS "libcuopt: Adding cu13 RPATH") + list(APPEND rpaths "$ORIGIN/../../nvidia/cu13/lib") +elseif(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.0) + message(STATUS "libcuopt: Adding cu12 RPATH") + list(APPEND rpaths "$ORIGIN/../../nvidia/cu12/lib") +else() + message(WARNING "libcuopt: Unsupported CUDA version ${CMAKE_CUDA_COMPILER_VERSION}") +endif() +message(STATUS "libcuopt: Final RPATH = ${rpaths}") + +# cuopt is an INTERFACE target (libcuopt.so is a linker script), so it has no RPATH. +# cuopt_client is in the list because it PUBLIC-links rapids_logger, which +# build_wheel_libcuopt.sh excludes from vendoring, so the only way to find it is +# $ORIGIN/../../rapids_logger/lib64 from this list. +foreach(_target cuopt_routing cuopt_mathopt cuopt_client cuopt_cli cuopt_grpc_server) + if(TARGET ${_target}) + set_property(TARGET ${_target} APPEND PROPERTY INSTALL_RPATH ${rpaths}) + endif() +endforeach() diff --git a/python/libcuopt/libcuopt/load.py b/python/libcuopt/libcuopt/load.py index 86f5798f70..e7ea9a65b7 100644 --- a/python/libcuopt/libcuopt/load.py +++ b/python/libcuopt/libcuopt/load.py @@ -34,39 +34,30 @@ def _load_wheel_installation(soname: str): def load_library(): - """Dynamically load libcuopt.so and its dependencies""" + """Load every cuOpt component library. + + The libraries themselves live in the per-component packages now, so this + delegates rather than dlopen()ing them directly. The client goes first: + mathopt and routing both carry a DT_NEEDED on it. + """ + loaded = [] + + import libcuopt_client + + loaded.extend(libcuopt_client.load_library() or []) + + # routing is optional -- a SKIP_ROUTING_BUILD install will not have it. try: - # librmm and libraft must be loaded before libcuopt - # because libcuopt references them. - import libraft - import librmm - import rapids_logger - - rapids_logger.load_library() - librmm.load_library() - libraft.load_library() + import libcuopt_routing + + loaded.extend(libcuopt_routing.load_library() or []) except ModuleNotFoundError: pass - prefer_system_installation = ( - os.getenv("RAPIDS_LIBCUOPT_PREFER_SYSTEM_LIBRARY", "false").lower() - != "false" - ) - - # cuOpt ships as component libraries. libcuopt.so is a linker script, - # not an ELF object, so it cannot be dlopen()ed -- load the components - # instead. Each pulls its own dependencies in through DT_NEEDED, so this - # order only needs to be valid, not exhaustive. routing and grpc are - # optional (SKIP_ROUTING_BUILD, SKIP_GRPC_BUILD) and may be absent. - components = [ - ("libcuopt_routing.so", False), - ("libcuopt_mathopt.so", True), - ] - loaded = [] - for soname, required in components: - lib = _load_component(soname, prefer_system_installation, required) - if lib is not None: - loaded.append(lib) + import libcuopt_mathopt + + loaded.extend(libcuopt_mathopt.load_library() or []) + return loaded diff --git a/python/libcuopt/pyproject.toml b/python/libcuopt/pyproject.toml index d4b415ae6a..fd02478394 100644 --- a/python/libcuopt/pyproject.toml +++ b/python/libcuopt/pyproject.toml @@ -45,8 +45,9 @@ Source = "https://github.com/nvidia/cuopt" [project.entry-points."cmake.prefix"] libcuopt = "libcuopt" +# cuopt_cli moves to libcuopt-mathopt, which is where the binary installs. The +# server console script stays here until cuopt-grpc-server has its own wheel. [project.scripts] -cuopt_cli = "libcuopt._cli_wrapper:main" cuopt_grpc_server = "libcuopt._grpc_server_wrapper:main" [tool.pydistcheck] @@ -65,6 +66,9 @@ sdist.reproducible = true wheel.packages = ["libcuopt"] wheel.install-dir = "libcuopt" wheel.py-api = "py3" +# Only the ld script and the gRPC server binary: the three component libraries now +# arrive through this package's dependencies rather than being bundled again here. +install.components = ["dev", "grpc-server"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" diff --git a/python/libcuopt_client/CMakeLists.txt b/python/libcuopt_client/CMakeLists.txt new file mode 100644 index 0000000000..e5fde2401b --- /dev/null +++ b/python/libcuopt_client/CMakeLists.txt @@ -0,0 +1,25 @@ +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on + +cmake_minimum_required(VERSION 4.0 FATAL_ERROR) + +include(../../cmake/rapids_config.cmake) +include(rapids-cuda) +rapids_cuda_init_architectures(libcuopt-client-python) + +project( + libcuopt-client-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# Already installed: nothing to build, the wheel just stages the component. +find_package(cuopt "${RAPIDS_VERSION}") +if(cuopt_FOUND) + return() +endif() +unset(cuopt_FOUND) + +include(../cmake/cuopt_wheel_build.cmake) diff --git a/python/libcuopt_client/LICENSE b/python/libcuopt_client/LICENSE new file mode 100644 index 0000000000..6ed9218d99 --- /dev/null +++ b/python/libcuopt_client/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 NVIDIA Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python/libcuopt_client/README.md b/python/libcuopt_client/README.md new file mode 100644 index 0000000000..7147977a60 --- /dev/null +++ b/python/libcuopt_client/README.md @@ -0,0 +1,5 @@ +# libcuopt-client + +Host-side cuOpt client: problem representation, parsers and the gRPC client. Contains no CUDA kernels. + +This package ships one component of cuOpt's C++ library. Install [`libcuopt`](https://pypi.org/project/libcuopt/) instead to get all of them. diff --git a/python/libcuopt_client/libcuopt_client/VERSION b/python/libcuopt_client/libcuopt_client/VERSION new file mode 100644 index 0000000000..6549ba6527 --- /dev/null +++ b/python/libcuopt_client/libcuopt_client/VERSION @@ -0,0 +1 @@ +26.10.00 diff --git a/python/libcuopt_client/libcuopt_client/__init__.py b/python/libcuopt_client/libcuopt_client/__init__.py new file mode 100644 index 0000000000..aad0391082 --- /dev/null +++ b/python/libcuopt_client/libcuopt_client/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from libcuopt_client._version import __git_commit__, __version__ +from libcuopt_client.load import load_library + +__all__ = ["__git_commit__", "__version__", "load_library"] diff --git a/python/libcuopt_client/libcuopt_client/_version.py b/python/libcuopt_client/libcuopt_client/_version.py new file mode 100644 index 0000000000..ca5b7efde7 --- /dev/null +++ b/python/libcuopt_client/libcuopt_client/_version.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcuopt_client/libcuopt_client/load.py b/python/libcuopt_client/libcuopt_client/load.py new file mode 100644 index 0000000000..3a663576da --- /dev/null +++ b/python/libcuopt_client/libcuopt_client/load.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Loader for the cuOpt client component. This is the leaf: it has no DT_NEEDED on +# another cuOpt library, which is what lets it be installed without the GPU stack. + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcudf that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), "lib64", soname) + ): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcuopt_client.so and its dependencies.""" + try: + # librmm and rapids_logger must be loaded before libcuopt_client.so, + # which references them. + import librmm + import rapids_logger + + rapids_logger.load_library() + librmm.load_library() + except ModuleNotFoundError: + pass + + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUOPT_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + lib = _load_component( + "libcuopt_client.so", prefer_system_installation, True + ) + return [lib] if lib is not None else [] + + +def _load_component( + soname: str, prefer_system_installation: bool, required: bool +): + """Load one cuOpt component. + + Returns the handle, or ``None`` if it could not be loaded. Failing to + load an optional component is silent; failing a required one warns. + """ + libcuopt_lib = None + try: + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcuopt_lib = _load_system_installation(soname) + except OSError: + # The fallback needs to sit inside the outer handler too. An + # optional component that is present but fails to load would + # otherwise raise out of here and stop the remaining components + # from being loaded at all. + libcuopt_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library + # was prebuilt before packaging the wheel), look for a + # system installation. + libcuopt_lib = _load_wheel_installation(soname) + if libcuopt_lib is None: + libcuopt_lib = _load_system_installation(soname) + except OSError as e: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + if not required: + return None + + import warnings + + warnings.warn( + f"Failed to load libcuopt library: {soname}. " + f"Error: {str(e)}. " + "Falling back to relying on system loader. " + "cuOpt functionality may be unavailable. " + f"This might lead to a generic error such as " + f"'{soname} missing' if the library cannot be found.", + RuntimeWarning, + ) + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcuopt was loaded from. + return libcuopt_lib diff --git a/python/libcuopt_client/pyproject.toml b/python/libcuopt_client/pyproject.toml new file mode 100644 index 0000000000..3766809e38 --- /dev/null +++ b/python/libcuopt_client/pyproject.toml @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.4.0,<0.5.0", + "scikit-build-core[pyproject]>=0.11.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcuopt-client" +dynamic = ["version"] +description = "cuOpt host-side client libraries (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = "Apache-2.0" +requires-python = ">=3.11" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "librmm==26.10.*,>=0.0.0a0", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://docs.nvidia.com/cuopt/introduction.html" +Source = "https://github.com/nvidia/cuopt" + +[project.entry-points."cmake.prefix"] +libcuopt_client = "libcuopt_client" + +[tool.pydistcheck] +select = [ + # note: max_allowed_size_compressed is set as a command-line arg + "distro-too-large-compressed", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = false +sdist.reproducible = true +wheel.packages = ["libcuopt_client"] +wheel.install-dir = "libcuopt_client" +wheel.py-api = "py3" +# The whole point of this package: stage only this component's install rules, +# so the wheel carries one library instead of all three. +install.components = ["client", "client-dev"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcuopt_client/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +requires = [ + "cmake>=4.0", + "librmm==26.10.*,>=0.0.0a0", + "ninja", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libcuopt_mathopt/CMakeLists.txt b/python/libcuopt_mathopt/CMakeLists.txt new file mode 100644 index 0000000000..0ebf0ad09d --- /dev/null +++ b/python/libcuopt_mathopt/CMakeLists.txt @@ -0,0 +1,25 @@ +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on + +cmake_minimum_required(VERSION 4.0 FATAL_ERROR) + +include(../../cmake/rapids_config.cmake) +include(rapids-cuda) +rapids_cuda_init_architectures(libcuopt-mathopt-python) + +project( + libcuopt-mathopt-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# Already installed: nothing to build, the wheel just stages the component. +find_package(cuopt "${RAPIDS_VERSION}") +if(cuopt_FOUND) + return() +endif() +unset(cuopt_FOUND) + +include(../cmake/cuopt_wheel_build.cmake) diff --git a/python/libcuopt_mathopt/LICENSE b/python/libcuopt_mathopt/LICENSE new file mode 100644 index 0000000000..6ed9218d99 --- /dev/null +++ b/python/libcuopt_mathopt/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 NVIDIA Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python/libcuopt_mathopt/README.md b/python/libcuopt_mathopt/README.md new file mode 100644 index 0000000000..11e17c2017 --- /dev/null +++ b/python/libcuopt_mathopt/README.md @@ -0,0 +1,5 @@ +# libcuopt-mathopt + +cuOpt LP / MILP / QP / SOCP solver library and the `cuopt_cli` command-line solver. + +This package ships one component of cuOpt's C++ library. Install [`libcuopt`](https://pypi.org/project/libcuopt/) instead to get all of them. diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/VERSION b/python/libcuopt_mathopt/libcuopt_mathopt/VERSION new file mode 100644 index 0000000000..6549ba6527 --- /dev/null +++ b/python/libcuopt_mathopt/libcuopt_mathopt/VERSION @@ -0,0 +1 @@ +26.10.00 diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/__init__.py b/python/libcuopt_mathopt/libcuopt_mathopt/__init__.py new file mode 100644 index 0000000000..e5128393ef --- /dev/null +++ b/python/libcuopt_mathopt/libcuopt_mathopt/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from libcuopt_mathopt._version import __git_commit__, __version__ +from libcuopt_mathopt.load import load_library + +__all__ = ["__git_commit__", "__version__", "load_library"] diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py b/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py new file mode 100644 index 0000000000..78e95280b7 --- /dev/null +++ b/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + + +def main(): + """ + This connects to cli binary which situated under libcuopt/bin folder + + execv replaces this process rather than spawning a child, so signals sent + to the console script's pid reach the solver directly instead of stopping + at a Python parent that forwards nothing. + """ + cli_path = os.path.join(os.path.dirname(__file__), "bin", "cuopt_cli") + os.execv(cli_path, [cli_path] + sys.argv[1:]) diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/_version.py b/python/libcuopt_mathopt/libcuopt_mathopt/_version.py new file mode 100644 index 0000000000..ca5b7efde7 --- /dev/null +++ b/python/libcuopt_mathopt/libcuopt_mathopt/_version.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/load.py b/python/libcuopt_mathopt/libcuopt_mathopt/load.py new file mode 100644 index 0000000000..e35fdd6315 --- /dev/null +++ b/python/libcuopt_mathopt/libcuopt_mathopt/load.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Loader for the cuOpt LP / MILP / QP / SOCP component. + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcudf that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), "lib64", soname) + ): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcuopt_mathopt.so and its dependencies.""" + try: + # librmm and rapids_logger must be loaded before libcuopt_mathopt.so, + # which references them. + import librmm + import rapids_logger + + rapids_logger.load_library() + librmm.load_library() + except ModuleNotFoundError: + pass + + # This component has a DT_NEEDED on the client, so the client package must + # be loaded first now that the two live in separate wheels. + import libcuopt_client + + libcuopt_client.load_library() + + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUOPT_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + lib = _load_component( + "libcuopt_mathopt.so", prefer_system_installation, True + ) + return [lib] if lib is not None else [] + + +def _load_component( + soname: str, prefer_system_installation: bool, required: bool +): + """Load one cuOpt component. + + Returns the handle, or ``None`` if it could not be loaded. Failing to + load an optional component is silent; failing a required one warns. + """ + libcuopt_lib = None + try: + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcuopt_lib = _load_system_installation(soname) + except OSError: + # The fallback needs to sit inside the outer handler too. An + # optional component that is present but fails to load would + # otherwise raise out of here and stop the remaining components + # from being loaded at all. + libcuopt_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library + # was prebuilt before packaging the wheel), look for a + # system installation. + libcuopt_lib = _load_wheel_installation(soname) + if libcuopt_lib is None: + libcuopt_lib = _load_system_installation(soname) + except OSError as e: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + if not required: + return None + + import warnings + + warnings.warn( + f"Failed to load libcuopt library: {soname}. " + f"Error: {str(e)}. " + "Falling back to relying on system loader. " + "cuOpt functionality may be unavailable. " + f"This might lead to a generic error such as " + f"'{soname} missing' if the library cannot be found.", + RuntimeWarning, + ) + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcuopt was loaded from. + return libcuopt_lib diff --git a/python/libcuopt_mathopt/pyproject.toml b/python/libcuopt_mathopt/pyproject.toml new file mode 100644 index 0000000000..481b2864df --- /dev/null +++ b/python/libcuopt_mathopt/pyproject.toml @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.4.0,<0.5.0", + "scikit-build-core[pyproject]>=0.11.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcuopt-mathopt" +dynamic = ["version"] +description = "cuOpt LP / MILP / QP / SOCP solver libraries (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = "Apache-2.0" +requires-python = ">=3.11" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "cuda-toolkit[cublas,cudart,curand,cusolver,cusparse,nvtx]==13.*", + "libcuopt-client==26.10.*,>=0.0.0a0", + "librmm==26.10.*,>=0.0.0a0", + "nvidia-cudss-cu13>=0.7,<0.8", + "nvidia-nccl-cu13>=2.19", + "nvidia-nvjitlink>=13.3,<14", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://docs.nvidia.com/cuopt/introduction.html" +Source = "https://github.com/nvidia/cuopt" + +[project.entry-points."cmake.prefix"] +libcuopt_mathopt = "libcuopt_mathopt" + +[project.scripts] +cuopt_cli = "libcuopt_mathopt._cli_wrapper:main" + +[tool.pydistcheck] +select = [ + # note: max_allowed_size_compressed is set as a command-line arg + "distro-too-large-compressed", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = false +sdist.reproducible = true +wheel.packages = ["libcuopt_mathopt"] +wheel.install-dir = "libcuopt_mathopt" +wheel.py-api = "py3" +# The whole point of this package: stage only this component's install rules, +# so the wheel carries one library instead of all three. +install.components = ["mathopt", "mathopt-dev"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcuopt_mathopt/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +requires = [ + "cmake>=4.0", + "librmm==26.10.*,>=0.0.0a0", + "ninja", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libcuopt_routing/CMakeLists.txt b/python/libcuopt_routing/CMakeLists.txt new file mode 100644 index 0000000000..03afc376da --- /dev/null +++ b/python/libcuopt_routing/CMakeLists.txt @@ -0,0 +1,25 @@ +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on + +cmake_minimum_required(VERSION 4.0 FATAL_ERROR) + +include(../../cmake/rapids_config.cmake) +include(rapids-cuda) +rapids_cuda_init_architectures(libcuopt-routing-python) + +project( + libcuopt-routing-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# Already installed: nothing to build, the wheel just stages the component. +find_package(cuopt "${RAPIDS_VERSION}") +if(cuopt_FOUND) + return() +endif() +unset(cuopt_FOUND) + +include(../cmake/cuopt_wheel_build.cmake) diff --git a/python/libcuopt_routing/LICENSE b/python/libcuopt_routing/LICENSE new file mode 100644 index 0000000000..6ed9218d99 --- /dev/null +++ b/python/libcuopt_routing/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 NVIDIA Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python/libcuopt_routing/README.md b/python/libcuopt_routing/README.md new file mode 100644 index 0000000000..6de72cc553 --- /dev/null +++ b/python/libcuopt_routing/README.md @@ -0,0 +1,5 @@ +# libcuopt-routing + +cuOpt VRP / TSP / PDP routing engine library. + +This package ships one component of cuOpt's C++ library. Install [`libcuopt`](https://pypi.org/project/libcuopt/) instead to get all of them. diff --git a/python/libcuopt_routing/libcuopt_routing/VERSION b/python/libcuopt_routing/libcuopt_routing/VERSION new file mode 100644 index 0000000000..6549ba6527 --- /dev/null +++ b/python/libcuopt_routing/libcuopt_routing/VERSION @@ -0,0 +1 @@ +26.10.00 diff --git a/python/libcuopt_routing/libcuopt_routing/__init__.py b/python/libcuopt_routing/libcuopt_routing/__init__.py new file mode 100644 index 0000000000..ea7cf929a8 --- /dev/null +++ b/python/libcuopt_routing/libcuopt_routing/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from libcuopt_routing._version import __git_commit__, __version__ +from libcuopt_routing.load import load_library + +__all__ = ["__git_commit__", "__version__", "load_library"] diff --git a/python/libcuopt_routing/libcuopt_routing/_version.py b/python/libcuopt_routing/libcuopt_routing/_version.py new file mode 100644 index 0000000000..ca5b7efde7 --- /dev/null +++ b/python/libcuopt_routing/libcuopt_routing/_version.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcuopt_routing/libcuopt_routing/load.py b/python/libcuopt_routing/libcuopt_routing/load.py new file mode 100644 index 0000000000..b4edda442f --- /dev/null +++ b/python/libcuopt_routing/libcuopt_routing/load.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Loader for the cuOpt VRP / TSP / PDP routing component. + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcudf that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), "lib64", soname) + ): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcuopt_routing.so and its dependencies.""" + try: + # librmm and rapids_logger must be loaded before libcuopt_routing.so, + # which references them. + import librmm + import rapids_logger + + rapids_logger.load_library() + librmm.load_library() + except ModuleNotFoundError: + pass + + # This component has a DT_NEEDED on the client, so the client package must + # be loaded first now that the two live in separate wheels. + import libcuopt_client + + libcuopt_client.load_library() + + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUOPT_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + lib = _load_component( + "libcuopt_routing.so", prefer_system_installation, True + ) + return [lib] if lib is not None else [] + + +def _load_component( + soname: str, prefer_system_installation: bool, required: bool +): + """Load one cuOpt component. + + Returns the handle, or ``None`` if it could not be loaded. Failing to + load an optional component is silent; failing a required one warns. + """ + libcuopt_lib = None + try: + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcuopt_lib = _load_system_installation(soname) + except OSError: + # The fallback needs to sit inside the outer handler too. An + # optional component that is present but fails to load would + # otherwise raise out of here and stop the remaining components + # from being loaded at all. + libcuopt_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library + # was prebuilt before packaging the wheel), look for a + # system installation. + libcuopt_lib = _load_wheel_installation(soname) + if libcuopt_lib is None: + libcuopt_lib = _load_system_installation(soname) + except OSError as e: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + if not required: + return None + + import warnings + + warnings.warn( + f"Failed to load libcuopt library: {soname}. " + f"Error: {str(e)}. " + "Falling back to relying on system loader. " + "cuOpt functionality may be unavailable. " + f"This might lead to a generic error such as " + f"'{soname} missing' if the library cannot be found.", + RuntimeWarning, + ) + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcuopt was loaded from. + return libcuopt_lib diff --git a/python/libcuopt_routing/pyproject.toml b/python/libcuopt_routing/pyproject.toml new file mode 100644 index 0000000000..5e1ac15d61 --- /dev/null +++ b/python/libcuopt_routing/pyproject.toml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.4.0,<0.5.0", + "scikit-build-core[pyproject]>=0.11.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcuopt-routing" +dynamic = ["version"] +description = "cuOpt VRP / TSP / PDP routing libraries (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = "Apache-2.0" +requires-python = ">=3.11" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "cuda-toolkit[cublas,cudart]==13.*", + "libcuopt-client==26.10.*,>=0.0.0a0", + "librmm==26.10.*,>=0.0.0a0", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://docs.nvidia.com/cuopt/introduction.html" +Source = "https://github.com/nvidia/cuopt" + +[project.entry-points."cmake.prefix"] +libcuopt_routing = "libcuopt_routing" + +[tool.pydistcheck] +select = [ + # note: max_allowed_size_compressed is set as a command-line arg + "distro-too-large-compressed", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = false +sdist.reproducible = true +wheel.packages = ["libcuopt_routing"] +wheel.install-dir = "libcuopt_routing" +wheel.py-api = "py3" +# The whole point of this package: stage only this component's install rules, +# so the wheel carries one library instead of all three. +install.components = ["routing", "routing-dev"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcuopt_routing/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +requires = [ + "cmake>=4.0", + "librmm==26.10.*,>=0.0.0a0", + "ninja", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From e9220dcb310b4f4d24640cd14ed68f327032da06 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 18 Sep 2026 10:13:01 -0500 Subject: [PATCH 02/21] build(wheel): regenerate the component build scripts from current main The three scripts were copied from ci/build_wheel_libcuopt.sh before it moved off LLVM libomp. They still installed llvm-toolset and libomp-devel and set OpenMP_gomp_LIBRARY to the system libomp, missing the modern GNU libgomp fetch and CUOPT_BUILD_CUSTOM_CUDSS_MTLAYER=ON that #1219 needs so cuOpt and cuDSS's threading layer share one OpenMP runtime, as well as use_cuda_wheels in the requirements matrix. Regenerated from the current file, so they now differ from it only in package_name, package_dir and the artifact name. Noted in each that the four have to be regenerated together. Co-Authored-By: Claude Opus 5 --- ci/build_wheel_libcuopt_client.sh | 41 ++++++++++++++---------------- ci/build_wheel_libcuopt_mathopt.sh | 41 ++++++++++++++---------------- ci/build_wheel_libcuopt_routing.sh | 41 ++++++++++++++---------------- 3 files changed, 57 insertions(+), 66 deletions(-) diff --git a/ci/build_wheel_libcuopt_client.sh b/ci/build_wheel_libcuopt_client.sh index 5246002c37..d83f3218aa 100755 --- a/ci/build_wheel_libcuopt_client.sh +++ b/ci/build_wheel_libcuopt_client.sh @@ -4,9 +4,9 @@ set -euo pipefail -# Configures the full C++ tree like every other cuOpt wheel, but stages only this -# component's install rules -- see install.components in the package's -# pyproject.toml. sccache is what keeps the repeated configure affordable. +# Same full C++ configure as every other cuOpt wheel; only the install components +# staged into it differ, set by install.components in this package's pyproject.toml. +# Kept in step with ci/build_wheel_libcuopt.sh -- regenerate all four together. source rapids-init-pip @@ -21,34 +21,31 @@ fi # Install Boost and TBB bash ci/utils/install_boost_tbb.sh -# Install libuuid and LLVM's OpenMP runtime +# Install libuuid (needed by cuopt_grpc_server) if command -v dnf &> /dev/null; then - # LLVM Toolset is distributed as a module on Rocky/RHEL 8. - dnf module install -y llvm-toolset - dnf install -y libuuid-devel libomp-devel + dnf install -y libuuid-devel elif command -v apt-get &> /dev/null; then apt-get update - apt-get install -y uuid-dev libomp-dev + apt-get install -y uuid-dev fi # Install Protobuf + gRPC (protoc + grpc_cpp_plugin) bash ci/utils/install_protobuf_grpc.sh -# Compile with GCC, but use LLVM libomp as the OpenMP runtime bundled in the wheel. Resolve the -# versioned ELF library rather than an unversioned linker script or compiler-toolset indirection. -LIBOMP_LIBRARY="$( - ldconfig -p | - awk '$1 ~ /^libomp\.so(\.[0-9]+)*$/ && !library { library = $NF } - END { print library }' -)" -if [[ "${LIBOMP_LIBRARY}" != /* || ! -f "${LIBOMP_LIBRARY}" ]]; then - echo "Could not resolve the LLVM OpenMP runtime: '${LIBOMP_LIBRARY}'" >&2 - exit 1 -fi +# Compile against a modern GNU libgomp from conda-forge instead of bundled LLVM libomp, to +# unify cuOpt's OpenMP runtime with the one cuDSS's threading layer needs (#1219). +MODERN_LIBGOMP_DIR="$(pwd)/modern_libgomp" +python -m pip install --quiet zstandard +python ci/utils/install_modern_libgomp.py "${MODERN_LIBGOMP_DIR}" -echo "Using LLVM OpenMP runtime: ${LIBOMP_LIBRARY}" +# Also build our own cuDSS threading layer against this same libgomp (see cpp/CMakeLists.txt, +# cpp/src/barrier/cudss_mtlayer_cuopt.cpp), instead of cuDSS's prebuilt one, so both actually +# share one instance, not just the same flavor. Conda keeps cuDSS's default (#1219 discussion). +export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${MODERN_LIBGOMP_DIR}/libgomp.so.1.0.0;-DCUOPT_BUILD_CUSTOM_CUDSS_MTLAYER=ON" -export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${LIBOMP_LIBRARY}" +# auditwheel repair does its own dependency resolution separately from the compiler; without +# this it can't see our fetched copy and silently vendors the old Rocky 8 system one instead. +export LD_LIBRARY_PATH="${MODERN_LIBGOMP_DIR}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # OpenSSL 3 hints for libcuopt's own find_package(OpenSSL). # @@ -83,7 +80,7 @@ rapids-dependency-file-generator \ --output requirements \ --file-key "py_build_${package_name}" \ --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true;use_cuda_wheels=true" \ | tee /tmp/requirements-build.txt rapids-logger "Installing build requirements" diff --git a/ci/build_wheel_libcuopt_mathopt.sh b/ci/build_wheel_libcuopt_mathopt.sh index e70d0ca468..e72e831602 100755 --- a/ci/build_wheel_libcuopt_mathopt.sh +++ b/ci/build_wheel_libcuopt_mathopt.sh @@ -4,9 +4,9 @@ set -euo pipefail -# Configures the full C++ tree like every other cuOpt wheel, but stages only this -# component's install rules -- see install.components in the package's -# pyproject.toml. sccache is what keeps the repeated configure affordable. +# Same full C++ configure as every other cuOpt wheel; only the install components +# staged into it differ, set by install.components in this package's pyproject.toml. +# Kept in step with ci/build_wheel_libcuopt.sh -- regenerate all four together. source rapids-init-pip @@ -21,34 +21,31 @@ fi # Install Boost and TBB bash ci/utils/install_boost_tbb.sh -# Install libuuid and LLVM's OpenMP runtime +# Install libuuid (needed by cuopt_grpc_server) if command -v dnf &> /dev/null; then - # LLVM Toolset is distributed as a module on Rocky/RHEL 8. - dnf module install -y llvm-toolset - dnf install -y libuuid-devel libomp-devel + dnf install -y libuuid-devel elif command -v apt-get &> /dev/null; then apt-get update - apt-get install -y uuid-dev libomp-dev + apt-get install -y uuid-dev fi # Install Protobuf + gRPC (protoc + grpc_cpp_plugin) bash ci/utils/install_protobuf_grpc.sh -# Compile with GCC, but use LLVM libomp as the OpenMP runtime bundled in the wheel. Resolve the -# versioned ELF library rather than an unversioned linker script or compiler-toolset indirection. -LIBOMP_LIBRARY="$( - ldconfig -p | - awk '$1 ~ /^libomp\.so(\.[0-9]+)*$/ && !library { library = $NF } - END { print library }' -)" -if [[ "${LIBOMP_LIBRARY}" != /* || ! -f "${LIBOMP_LIBRARY}" ]]; then - echo "Could not resolve the LLVM OpenMP runtime: '${LIBOMP_LIBRARY}'" >&2 - exit 1 -fi +# Compile against a modern GNU libgomp from conda-forge instead of bundled LLVM libomp, to +# unify cuOpt's OpenMP runtime with the one cuDSS's threading layer needs (#1219). +MODERN_LIBGOMP_DIR="$(pwd)/modern_libgomp" +python -m pip install --quiet zstandard +python ci/utils/install_modern_libgomp.py "${MODERN_LIBGOMP_DIR}" -echo "Using LLVM OpenMP runtime: ${LIBOMP_LIBRARY}" +# Also build our own cuDSS threading layer against this same libgomp (see cpp/CMakeLists.txt, +# cpp/src/barrier/cudss_mtlayer_cuopt.cpp), instead of cuDSS's prebuilt one, so both actually +# share one instance, not just the same flavor. Conda keeps cuDSS's default (#1219 discussion). +export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${MODERN_LIBGOMP_DIR}/libgomp.so.1.0.0;-DCUOPT_BUILD_CUSTOM_CUDSS_MTLAYER=ON" -export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${LIBOMP_LIBRARY}" +# auditwheel repair does its own dependency resolution separately from the compiler; without +# this it can't see our fetched copy and silently vendors the old Rocky 8 system one instead. +export LD_LIBRARY_PATH="${MODERN_LIBGOMP_DIR}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # OpenSSL 3 hints for libcuopt's own find_package(OpenSSL). # @@ -83,7 +80,7 @@ rapids-dependency-file-generator \ --output requirements \ --file-key "py_build_${package_name}" \ --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true;use_cuda_wheels=true" \ | tee /tmp/requirements-build.txt rapids-logger "Installing build requirements" diff --git a/ci/build_wheel_libcuopt_routing.sh b/ci/build_wheel_libcuopt_routing.sh index 87acfab67d..d8ba2e5cee 100755 --- a/ci/build_wheel_libcuopt_routing.sh +++ b/ci/build_wheel_libcuopt_routing.sh @@ -4,9 +4,9 @@ set -euo pipefail -# Configures the full C++ tree like every other cuOpt wheel, but stages only this -# component's install rules -- see install.components in the package's -# pyproject.toml. sccache is what keeps the repeated configure affordable. +# Same full C++ configure as every other cuOpt wheel; only the install components +# staged into it differ, set by install.components in this package's pyproject.toml. +# Kept in step with ci/build_wheel_libcuopt.sh -- regenerate all four together. source rapids-init-pip @@ -21,34 +21,31 @@ fi # Install Boost and TBB bash ci/utils/install_boost_tbb.sh -# Install libuuid and LLVM's OpenMP runtime +# Install libuuid (needed by cuopt_grpc_server) if command -v dnf &> /dev/null; then - # LLVM Toolset is distributed as a module on Rocky/RHEL 8. - dnf module install -y llvm-toolset - dnf install -y libuuid-devel libomp-devel + dnf install -y libuuid-devel elif command -v apt-get &> /dev/null; then apt-get update - apt-get install -y uuid-dev libomp-dev + apt-get install -y uuid-dev fi # Install Protobuf + gRPC (protoc + grpc_cpp_plugin) bash ci/utils/install_protobuf_grpc.sh -# Compile with GCC, but use LLVM libomp as the OpenMP runtime bundled in the wheel. Resolve the -# versioned ELF library rather than an unversioned linker script or compiler-toolset indirection. -LIBOMP_LIBRARY="$( - ldconfig -p | - awk '$1 ~ /^libomp\.so(\.[0-9]+)*$/ && !library { library = $NF } - END { print library }' -)" -if [[ "${LIBOMP_LIBRARY}" != /* || ! -f "${LIBOMP_LIBRARY}" ]]; then - echo "Could not resolve the LLVM OpenMP runtime: '${LIBOMP_LIBRARY}'" >&2 - exit 1 -fi +# Compile against a modern GNU libgomp from conda-forge instead of bundled LLVM libomp, to +# unify cuOpt's OpenMP runtime with the one cuDSS's threading layer needs (#1219). +MODERN_LIBGOMP_DIR="$(pwd)/modern_libgomp" +python -m pip install --quiet zstandard +python ci/utils/install_modern_libgomp.py "${MODERN_LIBGOMP_DIR}" -echo "Using LLVM OpenMP runtime: ${LIBOMP_LIBRARY}" +# Also build our own cuDSS threading layer against this same libgomp (see cpp/CMakeLists.txt, +# cpp/src/barrier/cudss_mtlayer_cuopt.cpp), instead of cuDSS's prebuilt one, so both actually +# share one instance, not just the same flavor. Conda keeps cuDSS's default (#1219 discussion). +export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${MODERN_LIBGOMP_DIR}/libgomp.so.1.0.0;-DCUOPT_BUILD_CUSTOM_CUDSS_MTLAYER=ON" -export SKBUILD_CMAKE_ARGS="-DOpenMP_gomp_LIBRARY:FILEPATH=${LIBOMP_LIBRARY}" +# auditwheel repair does its own dependency resolution separately from the compiler; without +# this it can't see our fetched copy and silently vendors the old Rocky 8 system one instead. +export LD_LIBRARY_PATH="${MODERN_LIBGOMP_DIR}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # OpenSSL 3 hints for libcuopt's own find_package(OpenSSL). # @@ -83,7 +80,7 @@ rapids-dependency-file-generator \ --output requirements \ --file-key "py_build_${package_name}" \ --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true;use_cuda_wheels=true" \ | tee /tmp/requirements-build.txt rapids-logger "Installing build requirements" From 8655cd2345b33ca783929bfc3a579d9aafb53df4 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 18 Sep 2026 10:43:13 -0500 Subject: [PATCH 03/21] build(wheel): fix the dependency file keys and cross-wheel library resolution Three CI failures, all from the split. rapids-dependency-file-generator raised "No matching matrix found in 'depends_on_librmm' for: {}" because the six new pyproject file keys had no matrix, so they resolved against an empty combination that depends_on_librmm's matrices do not cover. They now carry the same cuda_suffixed / use_cuda_wheels matrix as py_run_libcuopt. verify-dependencies rejected cuda_wheels_routing's no-selector entry for listing cuda-toolkit outside a set marked use_cuda_wheels: "true". That entry is what populates pyproject.toml in source control, so it stays, but it now names use_cuda_wheels explicitly, which is also what the file keys resolve against. wheel-build-libcuopt failed in auditwheel with "required library libcuopt_mathopt.so could not be located": the libcuopt wheel now stages only dev and grpc-server, while cuopt_grpc_server links mathopt, which lives in a sibling wheel. The component libraries are excluded from vendoring in all four scripts -- bundling them would ship several copies of each engine -- and the wheels gained $ORIGIN/../../libcuopt_/lib64 so the loader resolves them across packages. Co-Authored-By: Claude Opus 5 --- ci/build_wheel_libcuopt.sh | 5 +++++ ci/build_wheel_libcuopt_client.sh | 5 +++++ ci/build_wheel_libcuopt_mathopt.sh | 5 +++++ ci/build_wheel_libcuopt_routing.sh | 5 +++++ dependencies.yaml | 24 ++++++++++++++++++++++-- python/cmake/cuopt_wheel_build.cmake | 5 +++++ 6 files changed, 47 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_libcuopt.sh b/ci/build_wheel_libcuopt.sh index 807cfb8f21..02ba7e52ea 100755 --- a/ci/build_wheel_libcuopt.sh +++ b/ci/build_wheel_libcuopt.sh @@ -103,6 +103,11 @@ EXCLUDE_ARGS=( --exclude "libnvJitLink.so*" --exclude "librapids_logger.so" --exclude "librmm.so" + # Provided by the sibling libcuopt-* wheels, so they must not be vendored here: doing so + # would ship several copies of each engine and defeat the split. RPATH resolves them. + --exclude "libcuopt_client.so" + --exclude "libcuopt_mathopt.so" + --exclude "libcuopt_routing.so" # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 # at runtime via the host (or container image) keeps libcrypto and the FIPS # provider (system or mounted) byte-version-matched, which is required for diff --git a/ci/build_wheel_libcuopt_client.sh b/ci/build_wheel_libcuopt_client.sh index d83f3218aa..9e36b553a2 100755 --- a/ci/build_wheel_libcuopt_client.sh +++ b/ci/build_wheel_libcuopt_client.sh @@ -107,6 +107,11 @@ EXCLUDE_ARGS=( --exclude "libnvJitLink.so*" --exclude "librapids_logger.so" --exclude "librmm.so" + # Provided by the sibling libcuopt-* wheels, so they must not be vendored here: doing so + # would ship several copies of each engine and defeat the split. RPATH resolves them. + --exclude "libcuopt_client.so" + --exclude "libcuopt_mathopt.so" + --exclude "libcuopt_routing.so" # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 # at runtime via the host (or container image) keeps libcrypto and the FIPS # provider (system or mounted) byte-version-matched, which is required for diff --git a/ci/build_wheel_libcuopt_mathopt.sh b/ci/build_wheel_libcuopt_mathopt.sh index e72e831602..8a86a4b50a 100755 --- a/ci/build_wheel_libcuopt_mathopt.sh +++ b/ci/build_wheel_libcuopt_mathopt.sh @@ -107,6 +107,11 @@ EXCLUDE_ARGS=( --exclude "libnvJitLink.so*" --exclude "librapids_logger.so" --exclude "librmm.so" + # Provided by the sibling libcuopt-* wheels, so they must not be vendored here: doing so + # would ship several copies of each engine and defeat the split. RPATH resolves them. + --exclude "libcuopt_client.so" + --exclude "libcuopt_mathopt.so" + --exclude "libcuopt_routing.so" # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 # at runtime via the host (or container image) keeps libcrypto and the FIPS # provider (system or mounted) byte-version-matched, which is required for diff --git a/ci/build_wheel_libcuopt_routing.sh b/ci/build_wheel_libcuopt_routing.sh index d8ba2e5cee..a5ec02ce09 100755 --- a/ci/build_wheel_libcuopt_routing.sh +++ b/ci/build_wheel_libcuopt_routing.sh @@ -107,6 +107,11 @@ EXCLUDE_ARGS=( --exclude "libnvJitLink.so*" --exclude "librapids_logger.so" --exclude "librmm.so" + # Provided by the sibling libcuopt-* wheels, so they must not be vendored here: doing so + # would ship several copies of each engine and defeat the split. RPATH resolves them. + --exclude "libcuopt_client.so" + --exclude "libcuopt_mathopt.so" + --exclude "libcuopt_routing.so" # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 # at runtime via the host (or container image) keeps libcrypto and the FIPS # provider (system or mounted) byte-version-matched, which is required for diff --git a/dependencies.yaml b/dependencies.yaml index 61eb39910f..fcf8babdc1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -165,6 +165,9 @@ files: py_rapids_build_libcuopt_client: output: pyproject pyproject_dir: python/libcuopt_client + matrix: + cuda_suffixed: ["false"] + use_cuda_wheels: ["true"] extras: table: tool.rapids-build-backend key: requires @@ -175,6 +178,9 @@ files: py_run_libcuopt_client: output: pyproject pyproject_dir: python/libcuopt_client + matrix: + cuda_suffixed: ["false"] + use_cuda_wheels: ["true"] extras: table: project includes: @@ -190,6 +196,9 @@ files: py_rapids_build_libcuopt_mathopt: output: pyproject pyproject_dir: python/libcuopt_mathopt + matrix: + cuda_suffixed: ["false"] + use_cuda_wheels: ["true"] extras: table: tool.rapids-build-backend key: requires @@ -201,6 +210,9 @@ files: py_run_libcuopt_mathopt: output: pyproject pyproject_dir: python/libcuopt_mathopt + matrix: + cuda_suffixed: ["false"] + use_cuda_wheels: ["true"] extras: table: project includes: @@ -219,6 +231,9 @@ files: py_rapids_build_libcuopt_routing: output: pyproject pyproject_dir: python/libcuopt_routing + matrix: + cuda_suffixed: ["false"] + use_cuda_wheels: ["true"] extras: table: tool.rapids-build-backend key: requires @@ -230,6 +245,9 @@ files: py_run_libcuopt_routing: output: pyproject pyproject_dir: python/libcuopt_routing + matrix: + cuda_suffixed: ["false"] + use_cuda_wheels: ["true"] extras: table: project includes: @@ -1040,9 +1058,11 @@ dependencies: use_cuda_wheels: "true" packages: - &ctk_routing_cu13 cuda-toolkit[cublas,cudart]==13.* - # if no matching matrix selectors passed, list the CUDA 13 requirement, which is - # what populates pyproject.toml in source control + # No cuda selector: this is what the pyproject file keys resolve against, and it + # is what populates pyproject.toml in source control. use_cuda_wheels is named + # explicitly so the entry still reads as a CUDA-wheel set. - matrix: + use_cuda_wheels: "true" packages: - *ctk_routing_cu13 cuda_wheels: diff --git a/python/cmake/cuopt_wheel_build.cmake b/python/cmake/cuopt_wheel_build.cmake index 1ee1d07ffa..6f86bd7694 100644 --- a/python/cmake/cuopt_wheel_build.cmake +++ b/python/cmake/cuopt_wheel_build.cmake @@ -48,6 +48,11 @@ target_link_libraries(cuopt_cli PRIVATE set(rpaths "$ORIGIN/../lib64" + # The component libraries live in sibling wheels now, so a binary or library in one + # package resolves the others through their install directories rather than its own. + "$ORIGIN/../../libcuopt_client/lib64" + "$ORIGIN/../../libcuopt_mathopt/lib64" + "$ORIGIN/../../libcuopt_routing/lib64" "$ORIGIN/../../rapids_logger/lib64" "$ORIGIN/../../librmm/lib64" "$ORIGIN/../../nvidia/cudss/lib" From df5d1cc015f2a6b2899e8d9168d5c366c2a9370c Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 18 Sep 2026 17:32:06 -0500 Subject: [PATCH 04/21] ci(wheel): build and publish the component wheels The build scripts existed but nothing invoked them, so CI never built the component wheels and the split had no effect on what gets published. pr.yaml gains a build job per component on the same matrix filter as libcuopt, and build.yaml gains the matching build and publish jobs. wheel-build-cuopt now waits on all three, since the cuopt wheel resolves through them rather than through libcuopt alone, and pr-builder gates on them so a failure blocks the PR instead of passing silently. Co-Authored-By: Claude Opus 5 --- .github/workflows/build.yaml | 123 ++++++++++++++++++++++++++++++++++- .github/workflows/pr.yaml | 64 +++++++++++++++++- 2 files changed, 185 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e52701dfc5..4fb826b8f8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -241,8 +241,129 @@ jobs: package-name: libcuopt package-type: cpp publish-wheel-search-key: cuopt_wheel_cpp_libcuopt + wheel-build-libcuopt-client: + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + needs: [build-details] + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/26.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcuopt_client.sh + package-name: libcuopt_client + package-type: cpp + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-publish-libcuopt-client: + needs: wheel-build-libcuopt-client + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@release/26.10 + secrets: + CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN: ${{ secrets.CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN }} + RAPIDSAI_PYPI_TOKEN: ${{ secrets.RAPIDSAI_PYPI_TOKEN }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcuopt_client + package-type: cpp + publish-wheel-search-key: cuopt_wheel_cpp_libcuopt_client + wheel-build-libcuopt-mathopt: + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + needs: [build-details] + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/26.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcuopt_mathopt.sh + package-name: libcuopt_mathopt + package-type: cpp + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-publish-libcuopt-mathopt: + needs: wheel-build-libcuopt-mathopt + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@release/26.10 + secrets: + CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN: ${{ secrets.CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN }} + RAPIDSAI_PYPI_TOKEN: ${{ secrets.RAPIDSAI_PYPI_TOKEN }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcuopt_mathopt + package-type: cpp + publish-wheel-search-key: cuopt_wheel_cpp_libcuopt_mathopt + wheel-build-libcuopt-routing: + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + needs: [build-details] + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/26.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcuopt_routing.sh + package-name: libcuopt_routing + package-type: cpp + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-publish-libcuopt-routing: + needs: wheel-build-libcuopt-routing + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@release/26.10 + secrets: + CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN: ${{ secrets.CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN }} + RAPIDSAI_PYPI_TOKEN: ${{ secrets.RAPIDSAI_PYPI_TOKEN }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcuopt_routing + package-type: cpp + publish-wheel-search-key: cuopt_wheel_cpp_libcuopt_routing wheel-build-cuopt: - needs: [build-details, wheel-build-libcuopt] + needs: [build-details, wheel-build-libcuopt, wheel-build-libcuopt-client, + wheel-build-libcuopt-mathopt, wheel-build-libcuopt-routing] permissions: actions: read contents: read diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4be18d74cd..8963224ffd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -32,6 +32,9 @@ jobs: - docs-build - multi-gpu-cpp-tests - wheel-build-libcuopt + - wheel-build-libcuopt-client + - wheel-build-libcuopt-mathopt + - wheel-build-libcuopt-routing - wheel-build-cuopt - wheel-tests-cuopt - wheel-build-cuopt-server @@ -608,8 +611,67 @@ jobs: build_type: pull-request build-datetime: ${{ needs.build-details.outputs.build-datetime }} script: ci/build_wheel_libcuopt.sh + wheel-build-libcuopt-client: + needs: [build-details, compute-matrix-filters, changed-files] + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/26.10 + with: + # Same filter as libcuopt: every arch and CUDA version, latest Python only. + matrix_filter: ${{ needs.compute-matrix-filters.outputs.libcuopt_filter }} + package-type: cpp + package-name: libcuopt_client + build_type: pull-request + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + script: ci/build_wheel_libcuopt_client.sh + wheel-build-libcuopt-mathopt: + needs: [build-details, compute-matrix-filters, changed-files] + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/26.10 + with: + # Same filter as libcuopt: every arch and CUDA version, latest Python only. + matrix_filter: ${{ needs.compute-matrix-filters.outputs.libcuopt_filter }} + package-type: cpp + package-name: libcuopt_mathopt + build_type: pull-request + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + script: ci/build_wheel_libcuopt_mathopt.sh + wheel-build-libcuopt-routing: + needs: [build-details, compute-matrix-filters, changed-files] + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/26.10 + with: + # Same filter as libcuopt: every arch and CUDA version, latest Python only. + matrix_filter: ${{ needs.compute-matrix-filters.outputs.libcuopt_filter }} + package-type: cpp + package-name: libcuopt_routing + build_type: pull-request + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + script: ci/build_wheel_libcuopt_routing.sh wheel-build-cuopt: - needs: [build-details, wheel-build-libcuopt, compute-matrix-filters, changed-files] + needs: [build-details, wheel-build-libcuopt, wheel-build-libcuopt-client, + wheel-build-libcuopt-mathopt, wheel-build-libcuopt-routing, + compute-matrix-filters, changed-files] if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels permissions: actions: read From 3b1d771dabe6d6e775520b659829fe4a5991bcad Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 18 Sep 2026 21:48:43 -0500 Subject: [PATCH 05/21] ci(wheel): teach validate_wheel.sh the component package dirs All four component wheel builds failed at the last step with "unrecognized package_dir: 'python/libcuopt_client'". The wheels built and auditwheel repaired them; validate_wheel.sh rejects any package_dir it does not know, and the three new ones were never added. mathopt and routing take libcuopt's existing limits, which are a safe ceiling since each is strictly smaller than the combined wheel was. The client gets 50Mi: it has no CUDA kernels, so anything near the other limits would mean something was vendored into it that should not have been. Co-Authored-By: Claude Opus 5 --- ci/validate_wheel.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index da044db65a..c4b65225dd 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -29,6 +29,25 @@ if [[ "${package_dir}" == "python/libcuopt" ]]; then --max-allowed-size-compressed '550Mi' ) fi +elif [[ "${package_dir}" == "python/libcuopt_mathopt" ]] || \ + [[ "${package_dir}" == "python/libcuopt_routing" ]]; then + # Each is strictly smaller than the combined libcuopt wheel was, so its limits are a + # safe ceiling. Worth tightening once CI reports the real compressed sizes. + if [[ "${RAPIDS_CUDA_MAJOR}" == "12" ]]; then + PYDISTCHECK_ARGS+=( + --max-allowed-size-compressed '725Mi' + ) + else + PYDISTCHECK_ARGS+=( + --max-allowed-size-compressed '550Mi' + ) + fi +elif [[ "${package_dir}" == "python/libcuopt_client" ]]; then + # No CUDA kernels: libcuopt_client.so is single-digit MB, so anything near the other + # limits would mean something got vendored into it that should not have been. + PYDISTCHECK_ARGS+=( + --max-allowed-size-compressed '50Mi' + ) elif [[ "${package_dir}" != "python/cuopt" ]] && \ [[ "${package_dir}" != "python/cuopt/cuopt/linear_programming" ]] && \ [[ "${package_dir}" != "python/cuopt_server" ]] && \ From 7571355aa77a5a6a51549f66d1aaca02df432490 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 18 Sep 2026 21:50:41 -0500 Subject: [PATCH 06/21] review: document and annotate the new public loaders Adds return annotations and documents the contract for the three load_library functions and the cuopt_cli entry point: the empty-list result, that a missing library warns rather than raising, and that main never returns because execv replaces the process image. Co-Authored-By: Claude Opus 5 --- python/libcuopt_client/libcuopt_client/load.py | 10 ++++++++-- .../libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py | 11 ++++++++--- python/libcuopt_mathopt/libcuopt_mathopt/load.py | 11 +++++++++-- python/libcuopt_routing/libcuopt_routing/load.py | 11 +++++++++-- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/python/libcuopt_client/libcuopt_client/load.py b/python/libcuopt_client/libcuopt_client/load.py index 3a663576da..5fc4cefe29 100644 --- a/python/libcuopt_client/libcuopt_client/load.py +++ b/python/libcuopt_client/libcuopt_client/load.py @@ -35,8 +35,14 @@ def _load_wheel_installation(soname: str): return None -def load_library(): - """Dynamically load libcuopt_client.so and its dependencies.""" +def load_library() -> list: + """Dynamically load libcuopt_client.so and its dependencies. + + Returns a single-element list holding the ``ctypes.CDLL`` handle, or an + empty list when the library could not be loaded; callers rarely need the + handle itself. A missing library warns rather than raising, so that the + system loader still gets a chance to resolve it. + """ try: # librmm and rapids_logger must be loaded before libcuopt_client.so, # which references them. diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py b/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py index 78e95280b7..4a7bdc584f 100644 --- a/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py +++ b/python/libcuopt_mathopt/libcuopt_mathopt/_cli_wrapper.py @@ -3,11 +3,16 @@ import os import sys +from typing import NoReturn -def main(): - """ - This connects to cli binary which situated under libcuopt/bin folder +def main() -> NoReturn: + """Exec the cuopt_cli binary, forwarding this process's arguments to it. + + Never returns: execv replaces the process image. Raises OSError if the + binary is missing or not executable. + + This connects to cli binary which situated under libcuopt_mathopt/bin folder execv replaces this process rather than spawning a child, so signals sent to the console script's pid reach the solver directly instead of stopping diff --git a/python/libcuopt_mathopt/libcuopt_mathopt/load.py b/python/libcuopt_mathopt/libcuopt_mathopt/load.py index e35fdd6315..e3ba2e5320 100644 --- a/python/libcuopt_mathopt/libcuopt_mathopt/load.py +++ b/python/libcuopt_mathopt/libcuopt_mathopt/load.py @@ -34,8 +34,15 @@ def _load_wheel_installation(soname: str): return None -def load_library(): - """Dynamically load libcuopt_mathopt.so and its dependencies.""" +def load_library() -> list: + """Dynamically load libcuopt_mathopt.so and its dependencies. It loads libcuopt_client first, which + this library has a DT_NEEDED on. + + Returns a single-element list holding the ``ctypes.CDLL`` handle, or an + empty list when the library could not be loaded; callers rarely need the + handle itself. A missing library warns rather than raising, so that the + system loader still gets a chance to resolve it. + """ try: # librmm and rapids_logger must be loaded before libcuopt_mathopt.so, # which references them. diff --git a/python/libcuopt_routing/libcuopt_routing/load.py b/python/libcuopt_routing/libcuopt_routing/load.py index b4edda442f..1cc2f42360 100644 --- a/python/libcuopt_routing/libcuopt_routing/load.py +++ b/python/libcuopt_routing/libcuopt_routing/load.py @@ -34,8 +34,15 @@ def _load_wheel_installation(soname: str): return None -def load_library(): - """Dynamically load libcuopt_routing.so and its dependencies.""" +def load_library() -> list: + """Dynamically load libcuopt_routing.so and its dependencies. It loads libcuopt_client first, which + this library has a DT_NEEDED on. + + Returns a single-element list holding the ``ctypes.CDLL`` handle, or an + empty list when the library could not be loaded; callers rarely need the + handle itself. A missing library warns rather than raising, so that the + system loader still gets a chance to resolve it. + """ try: # librmm and rapids_logger must be loaded before libcuopt_routing.so, # which references them. From 5189588f2bfad6714091fcf73083e5b91f7dadf9 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Sat, 19 Sep 2026 20:44:07 -0500 Subject: [PATCH 07/21] build(wheel): make libcuopt depend on the component wheels libcuopt stages only the ld script and cuopt_grpc_server, but its dependencies were still the CUDA stack it no longer ships, and it named none of the component wheels that now carry the libraries. pip install libcuopt would have pulled cudss and nccl while getting no engine at all. The earlier edit to python/libcuopt/pyproject.toml did not survive, because dependencies.yaml is the source of truth and py_run_libcuopt was never updated; the generator rewrote the file back. Co-Authored-By: Claude Opus 5 --- dependencies.yaml | 10 ++++++---- python/libcuopt/pyproject.toml | 9 +++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index fcf8babdc1..b62823d101 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -139,6 +139,9 @@ files: - depends_on_libraft_headers - depends_on_librmm - depends_on_rapids_logger + # libcuopt now carries only the ld script and cuopt_grpc_server; the libraries live in + # the component wheels, so it depends on those rather than on the CUDA stack directly. + # Their own dependency sets bring rmm, rapids-logger and the CUDA wheels they each need. py_run_libcuopt: output: pyproject pyproject_dir: python/libcuopt @@ -148,10 +151,9 @@ files: extras: table: project includes: - - cuda_wheels - - depends_on_libraft_headers - - depends_on_librmm - - depends_on_rapids_logger + - depends_on_libcuopt_client + - depends_on_libcuopt_mathopt + - depends_on_libcuopt_routing # One set of keys per component wheel. The point of the split is in py_run_*: # only mathopt carries cudss / nccl / nvjitlink, so a routing-only install no # longer pulls libraries it never calls. diff --git a/python/libcuopt/pyproject.toml b/python/libcuopt/pyproject.toml index fd02478394..68829096a5 100644 --- a/python/libcuopt/pyproject.toml +++ b/python/libcuopt/pyproject.toml @@ -30,12 +30,9 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "cuda-toolkit[cublas,cudart,curand,cusolver,cusparse,nvtx]==13.*", - "librmm==26.10.*,>=0.0.0a0", - "nvidia-cudss-cu13>=0.7,<0.8", - "nvidia-nccl-cu13>=2.19", - "nvidia-nvjitlink>=13.3,<14", - "rapids-logger==0.3.*", + "libcuopt-client==26.10.*,>=0.0.0a0", + "libcuopt-mathopt==26.10.*,>=0.0.0a0", + "libcuopt-routing==26.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] From 4b3b2f874369d4d72fd59e602f3409b217d10e1f Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 09:39:07 -0500 Subject: [PATCH 08/21] ci(wheel): resolve the component wheels wherever libcuopt is installed wheel-build-cuopt failed with "No matching distribution found for libcuopt-client-cu12": libcuopt is now a metapackage over the component wheels, which are new packages no index carries, so pip resolved its dependencies against PyPI and found nothing. The pip output leads with a hash-mismatch line, which is why I first read this as a network flake -- the real error is further up. ci/utils/download_libcuopt_components.sh downloads the three artifacts, constrains pip to them and exposes their paths. Sourced from the four scripts that install libcuopt: build_wheel_cuopt, test_wheel_cuopt, test_wheel_cuopt_server and test_self_hosted_service, all of which would have hit the same resolution failure. Co-Authored-By: Claude Opus 5 --- ci/build_wheel_cuopt.sh | 3 +++ ci/test_self_hosted_service.sh | 7 +++++++ ci/test_wheel_cuopt.sh | 6 +++++- ci/test_wheel_cuopt_server.sh | 9 +++++++- ci/utils/download_libcuopt_components.sh | 26 ++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 2 deletions(-) create mode 100755 ci/utils/download_libcuopt_components.sh diff --git a/ci/build_wheel_cuopt.sh b/ci/build_wheel_cuopt.sh index 230e40d7f8..951bfb8ccf 100755 --- a/ci/build_wheel_cuopt.sh +++ b/ci/build_wheel_cuopt.sh @@ -31,6 +31,9 @@ LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_ echo "libcuopt-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUOPT_WHEELHOUSE}/libcuopt_*.whl)" >> "${PIP_CONSTRAINT}" +# shellcheck source=ci/utils/download_libcuopt_components.sh +source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" + EXCLUDE_ARGS=( --exclude "libraft.so" --exclude "libcublas.so.*" diff --git a/ci/test_self_hosted_service.sh b/ci/test_self_hosted_service.sh index b9321ff09e..e48640022e 100755 --- a/ci/test_self_hosted_service.sh +++ b/ci/test_self_hosted_service.sh @@ -8,6 +8,9 @@ set -euo pipefail source rapids-init-pip # Download the cuopt built in the previous step +# consumed by ci/utils/download_libcuopt_components.sh, sourced below +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" +export RAPIDS_PY_CUDA_SUFFIX LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-server cuopt --pure --arch any --cuda "$RAPIDS_CUDA_VERSION")") @@ -15,6 +18,9 @@ CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wh # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" +# shellcheck source=ci/utils/download_libcuopt_components.sh +source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" + # notes: # # * echo to expand wildcard before adding `[test]` requires for pip @@ -26,6 +32,7 @@ rapids-pip-retry install \ --constraint "${PIP_CONSTRAINT}" \ "${CUOPT_WHEELHOUSE}"/cuopt*.whl \ "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl \ + "${LIBCUOPT_COMPONENT_WHEELS[@]}" \ "$(echo "${CUOPT_SERVER_WHEELHOUSE}"/cuopt_server*.whl)[test]" check_message() diff --git a/ci/test_wheel_cuopt.sh b/ci/test_wheel_cuopt.sh index 9321169eed..06cf84391e 100755 --- a/ci/test_wheel_cuopt.sh +++ b/ci/test_wheel_cuopt.sh @@ -28,6 +28,9 @@ cuopt-sh-client @ file://$(echo "${CUOPT_SH_CLIENT_WHEELHOUSE}"/cuopt_sh_client- libcuopt-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUOPT_WHEELHOUSE}"/libcuopt_"${RAPIDS_PY_CUDA_SUFFIX}"-*.whl) EOF +# shellcheck source=ci/utils/download_libcuopt_components.sh +source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" + # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" @@ -38,7 +41,8 @@ rapids-pip-retry install \ -v \ --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ - "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl + "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl \ + "${LIBCUOPT_COMPONENT_WHEELS[@]}" python -c "import libcuopt; assert libcuopt.load_library() is not None" deactivate diff --git a/ci/test_wheel_cuopt_server.sh b/ci/test_wheel_cuopt_server.sh index f5edf38e76..df338b9cbd 100755 --- a/ci/test_wheel_cuopt_server.sh +++ b/ci/test_wheel_cuopt_server.sh @@ -13,6 +13,9 @@ source rapids-init-pip bash "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/install_openssl3_runtime.sh" # Download the packages built in the previous step +# consumed by ci/utils/download_libcuopt_components.sh, sourced below +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" +export RAPIDS_PY_CUDA_SUFFIX LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-server cuopt --pure --arch any --cuda "$RAPIDS_CUDA_VERSION")") @@ -21,6 +24,9 @@ CUOPT_SH_CLIENT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" +# shellcheck source=ci/utils/download_libcuopt_components.sh +source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" + # notes: # # * echo to expand wildcard before adding `[test]` requires for pip @@ -33,7 +39,8 @@ rapids-pip-retry install \ "$(echo "${CUOPT_SERVER_WHEELHOUSE}"/cuopt_server*.whl)[test]" \ "${CUOPT_WHEELHOUSE}"/cuopt*.whl \ "${CUOPT_SH_CLIENT_WHEELHOUSE}"/cuopt_sh_client*.whl \ - "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl + "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl \ + "${LIBCUOPT_COMPONENT_WHEELS[@]}" RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" export RAPIDS_DATASET_ROOT_DIR diff --git a/ci/utils/download_libcuopt_components.sh b/ci/utils/download_libcuopt_components.sh new file mode 100755 index 0000000000..be94c1c3c0 --- /dev/null +++ b/ci/utils/download_libcuopt_components.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Downloads the libcuopt component wheels built earlier in this run and constrains pip to +# them. +# +# libcuopt is a metapackage over libcuopt-client / -mathopt / -routing. Those are new +# packages that no index carries yet, so without this any `pip install libcuopt` resolves +# its dependencies against PyPI and fails with "No matching distribution found for +# libcuopt-client-". +# +# Source this after PIP_CONSTRAINT exists and after anything that truncates it. Appends one +# constraint per component and leaves their paths in LIBCUOPT_COMPONENT_WHEELS. + +LIBCUOPT_COMPONENT_WHEELS=() + +for _component in client mathopt routing; do + _wheelhouse=$(rapids-download-from-github \ + "$(rapids-artifact-name wheel_cpp "libcuopt_${_component}" cuopt --cuda "$RAPIDS_CUDA_VERSION")") + _wheel=$(echo "${_wheelhouse}"/libcuopt_"${_component}"_*.whl) + echo "libcuopt-${_component}-${RAPIDS_PY_CUDA_SUFFIX} @ file://${_wheel}" >> "${PIP_CONSTRAINT}" + LIBCUOPT_COMPONENT_WHEELS+=("${_wheel}") +done + +export LIBCUOPT_COMPONENT_WHEELS From 868560b358026363a5cff5f46abb387d31dc39f2 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 11:19:25 -0500 Subject: [PATCH 09/21] build(wheel): keep the libraries in libcuopt for now cuopt-config.cmake exports one _IMPORT_PREFIX for all three libraries, so splitting them across sibling wheels leaves find_package(cuopt) unable to resolve cuopt::client / ::mathopt / ::routing, which python/cuopt links against. That is what broke wheel-build-cuopt. libcuopt therefore stages every component and keeps its own dependencies and its cuopt_cli console script, exactly as before this PR. The component wheels are still built, published and installable on their own, which is the part that stands without the CMake work. Reverts the metapackage dependencies and the constraint helper that existed only to resolve them. Trimming libcuopt needs per-component CMake packages so each resolves from its own prefix; that is a separate change. Co-Authored-By: Claude Opus 5 --- ci/build_wheel_cuopt.sh | 3 --- ci/test_self_hosted_service.sh | 7 ------- ci/test_wheel_cuopt.sh | 6 +----- ci/test_wheel_cuopt_server.sh | 9 +------- ci/utils/download_libcuopt_components.sh | 26 ------------------------ dependencies.yaml | 10 ++++----- python/libcuopt/pyproject.toml | 26 ++++++++++++++++-------- 7 files changed, 24 insertions(+), 63 deletions(-) delete mode 100755 ci/utils/download_libcuopt_components.sh diff --git a/ci/build_wheel_cuopt.sh b/ci/build_wheel_cuopt.sh index 951bfb8ccf..230e40d7f8 100755 --- a/ci/build_wheel_cuopt.sh +++ b/ci/build_wheel_cuopt.sh @@ -31,9 +31,6 @@ LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_ echo "libcuopt-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUOPT_WHEELHOUSE}/libcuopt_*.whl)" >> "${PIP_CONSTRAINT}" -# shellcheck source=ci/utils/download_libcuopt_components.sh -source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" - EXCLUDE_ARGS=( --exclude "libraft.so" --exclude "libcublas.so.*" diff --git a/ci/test_self_hosted_service.sh b/ci/test_self_hosted_service.sh index e48640022e..b9321ff09e 100755 --- a/ci/test_self_hosted_service.sh +++ b/ci/test_self_hosted_service.sh @@ -8,9 +8,6 @@ set -euo pipefail source rapids-init-pip # Download the cuopt built in the previous step -# consumed by ci/utils/download_libcuopt_components.sh, sourced below -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" -export RAPIDS_PY_CUDA_SUFFIX LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-server cuopt --pure --arch any --cuda "$RAPIDS_CUDA_VERSION")") @@ -18,9 +15,6 @@ CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wh # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" -# shellcheck source=ci/utils/download_libcuopt_components.sh -source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" - # notes: # # * echo to expand wildcard before adding `[test]` requires for pip @@ -32,7 +26,6 @@ rapids-pip-retry install \ --constraint "${PIP_CONSTRAINT}" \ "${CUOPT_WHEELHOUSE}"/cuopt*.whl \ "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl \ - "${LIBCUOPT_COMPONENT_WHEELS[@]}" \ "$(echo "${CUOPT_SERVER_WHEELHOUSE}"/cuopt_server*.whl)[test]" check_message() diff --git a/ci/test_wheel_cuopt.sh b/ci/test_wheel_cuopt.sh index 06cf84391e..9321169eed 100755 --- a/ci/test_wheel_cuopt.sh +++ b/ci/test_wheel_cuopt.sh @@ -28,9 +28,6 @@ cuopt-sh-client @ file://$(echo "${CUOPT_SH_CLIENT_WHEELHOUSE}"/cuopt_sh_client- libcuopt-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUOPT_WHEELHOUSE}"/libcuopt_"${RAPIDS_PY_CUDA_SUFFIX}"-*.whl) EOF -# shellcheck source=ci/utils/download_libcuopt_components.sh -source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" - # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" @@ -41,8 +38,7 @@ rapids-pip-retry install \ -v \ --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ - "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl \ - "${LIBCUOPT_COMPONENT_WHEELS[@]}" + "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl python -c "import libcuopt; assert libcuopt.load_library() is not None" deactivate diff --git a/ci/test_wheel_cuopt_server.sh b/ci/test_wheel_cuopt_server.sh index df338b9cbd..f5edf38e76 100755 --- a/ci/test_wheel_cuopt_server.sh +++ b/ci/test_wheel_cuopt_server.sh @@ -13,9 +13,6 @@ source rapids-init-pip bash "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/install_openssl3_runtime.sh" # Download the packages built in the previous step -# consumed by ci/utils/download_libcuopt_components.sh, sourced below -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" -export RAPIDS_PY_CUDA_SUFFIX LIBCUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcuopt cuopt --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt cuopt --stable --cuda "$RAPIDS_CUDA_VERSION")") CUOPT_SERVER_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python cuopt-server cuopt --pure --arch any --cuda "$RAPIDS_CUDA_VERSION")") @@ -24,9 +21,6 @@ CUOPT_SH_CLIENT_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" -# shellcheck source=ci/utils/download_libcuopt_components.sh -source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils/download_libcuopt_components.sh" - # notes: # # * echo to expand wildcard before adding `[test]` requires for pip @@ -39,8 +33,7 @@ rapids-pip-retry install \ "$(echo "${CUOPT_SERVER_WHEELHOUSE}"/cuopt_server*.whl)[test]" \ "${CUOPT_WHEELHOUSE}"/cuopt*.whl \ "${CUOPT_SH_CLIENT_WHEELHOUSE}"/cuopt_sh_client*.whl \ - "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl \ - "${LIBCUOPT_COMPONENT_WHEELS[@]}" + "${LIBCUOPT_WHEELHOUSE}"/libcuopt*.whl RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" export RAPIDS_DATASET_ROOT_DIR diff --git a/ci/utils/download_libcuopt_components.sh b/ci/utils/download_libcuopt_components.sh deleted file mode 100755 index be94c1c3c0..0000000000 --- a/ci/utils/download_libcuopt_components.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Downloads the libcuopt component wheels built earlier in this run and constrains pip to -# them. -# -# libcuopt is a metapackage over libcuopt-client / -mathopt / -routing. Those are new -# packages that no index carries yet, so without this any `pip install libcuopt` resolves -# its dependencies against PyPI and fails with "No matching distribution found for -# libcuopt-client-". -# -# Source this after PIP_CONSTRAINT exists and after anything that truncates it. Appends one -# constraint per component and leaves their paths in LIBCUOPT_COMPONENT_WHEELS. - -LIBCUOPT_COMPONENT_WHEELS=() - -for _component in client mathopt routing; do - _wheelhouse=$(rapids-download-from-github \ - "$(rapids-artifact-name wheel_cpp "libcuopt_${_component}" cuopt --cuda "$RAPIDS_CUDA_VERSION")") - _wheel=$(echo "${_wheelhouse}"/libcuopt_"${_component}"_*.whl) - echo "libcuopt-${_component}-${RAPIDS_PY_CUDA_SUFFIX} @ file://${_wheel}" >> "${PIP_CONSTRAINT}" - LIBCUOPT_COMPONENT_WHEELS+=("${_wheel}") -done - -export LIBCUOPT_COMPONENT_WHEELS diff --git a/dependencies.yaml b/dependencies.yaml index b62823d101..fcf8babdc1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -139,9 +139,6 @@ files: - depends_on_libraft_headers - depends_on_librmm - depends_on_rapids_logger - # libcuopt now carries only the ld script and cuopt_grpc_server; the libraries live in - # the component wheels, so it depends on those rather than on the CUDA stack directly. - # Their own dependency sets bring rmm, rapids-logger and the CUDA wheels they each need. py_run_libcuopt: output: pyproject pyproject_dir: python/libcuopt @@ -151,9 +148,10 @@ files: extras: table: project includes: - - depends_on_libcuopt_client - - depends_on_libcuopt_mathopt - - depends_on_libcuopt_routing + - cuda_wheels + - depends_on_libraft_headers + - depends_on_librmm + - depends_on_rapids_logger # One set of keys per component wheel. The point of the split is in py_run_*: # only mathopt carries cudss / nccl / nvjitlink, so a routing-only install no # longer pulls libraries it never calls. diff --git a/python/libcuopt/pyproject.toml b/python/libcuopt/pyproject.toml index 68829096a5..057b92f70b 100644 --- a/python/libcuopt/pyproject.toml +++ b/python/libcuopt/pyproject.toml @@ -30,9 +30,12 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "libcuopt-client==26.10.*,>=0.0.0a0", - "libcuopt-mathopt==26.10.*,>=0.0.0a0", - "libcuopt-routing==26.10.*,>=0.0.0a0", + "cuda-toolkit[cublas,cudart,curand,cusolver,cusparse,nvtx]==13.*", + "librmm==26.10.*,>=0.0.0a0", + "nvidia-cudss-cu13>=0.7,<0.8", + "nvidia-nccl-cu13>=2.19", + "nvidia-nvjitlink>=13.3,<14", + "rapids-logger==0.3.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -42,9 +45,11 @@ Source = "https://github.com/nvidia/cuopt" [project.entry-points."cmake.prefix"] libcuopt = "libcuopt" -# cuopt_cli moves to libcuopt-mathopt, which is where the binary installs. The -# server console script stays here until cuopt-grpc-server has its own wheel. +# libcuopt still stages the mathopt component, so it ships cuopt_cli and keeps its +# console script. libcuopt-mathopt declares the same entry point for installs that take +# the component on its own. [project.scripts] +cuopt_cli = "libcuopt._cli_wrapper:main" cuopt_grpc_server = "libcuopt._grpc_server_wrapper:main" [tool.pydistcheck] @@ -63,9 +68,14 @@ sdist.reproducible = true wheel.packages = ["libcuopt"] wheel.install-dir = "libcuopt" wheel.py-api = "py3" -# Only the ld script and the gRPC server binary: the three component libraries now -# arrive through this package's dependencies rather than being bundled again here. -install.components = ["dev", "grpc-server"] +# Everything, for now. The component wheels exist and are installable on their own, but +# libcuopt still carries the libraries because cuopt-config.cmake exports one +# _IMPORT_PREFIX for all three: split across sibling wheels, find_package(cuopt) cannot +# resolve cuopt::client / ::mathopt / ::routing, which python/cuopt links against. +# Trimming this to ["dev", "grpc-server", "cuopt"] needs per-component CMake packages so +# each resolves from its own prefix -- tracked separately. +install.components = ["dev", "grpc-server", "cuopt", "client", "client-dev", + "mathopt", "mathopt-dev", "routing", "routing-dev"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 2335024fed117063a1d187c8d116c7abde315d61 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 12:17:12 -0500 Subject: [PATCH 10/21] review: drop the cross-wheel RPATH entries Each component's load_library() loads libcuopt_client before its own library, so glibc satisfies the DT_NEEDED from the already-loaded object and never searches a path for it. The entries were redundant even when the libraries were split, and libcuopt stages all three again, so $ORIGIN/../lib64 covers them. Co-Authored-By: Claude Opus 5 --- python/cmake/cuopt_wheel_build.cmake | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/cmake/cuopt_wheel_build.cmake b/python/cmake/cuopt_wheel_build.cmake index 6f86bd7694..1ee1d07ffa 100644 --- a/python/cmake/cuopt_wheel_build.cmake +++ b/python/cmake/cuopt_wheel_build.cmake @@ -48,11 +48,6 @@ target_link_libraries(cuopt_cli PRIVATE set(rpaths "$ORIGIN/../lib64" - # The component libraries live in sibling wheels now, so a binary or library in one - # package resolves the others through their install directories rather than its own. - "$ORIGIN/../../libcuopt_client/lib64" - "$ORIGIN/../../libcuopt_mathopt/lib64" - "$ORIGIN/../../libcuopt_routing/lib64" "$ORIGIN/../../rapids_logger/lib64" "$ORIGIN/../../librmm/lib64" "$ORIGIN/../../nvidia/cudss/lib" From 18079b1576a6638b1decc5d534f2550f9f1815b8 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 13:03:23 -0500 Subject: [PATCH 11/21] build(wheel): stop setting install.components on the libcuopt wheel wheel-build-cuopt failed configuring the cuopt wheel: cub-config.cmake:9 Unknown CMake command "libcudacxx_update_language_compat_flags" Naming the components explicitly drops everything installed without one. rapids-cmake writes its helper find-modules to lib64/rapids/cmake/ with no COMPONENT, so the wheel lost them and CMake fell back to the CUDA toolkit's libcudacxx rather than the vendored one librmm's cub-config expects. A passing run on another PR resolves libcudacxx inside the libcuopt wheel; this PR's resolved it under /usr/local/cuda. The C++ install partitions cleanly across the named components, which is why this was not visible there: those helper files only exist in the wheel build, where the dependencies are fetched rather than found. libcuopt keeps everything for now, so the setting has no purpose. The file is now functionally identical to main. Co-Authored-By: Claude Opus 5 --- python/libcuopt/pyproject.toml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/python/libcuopt/pyproject.toml b/python/libcuopt/pyproject.toml index 057b92f70b..582d35b15e 100644 --- a/python/libcuopt/pyproject.toml +++ b/python/libcuopt/pyproject.toml @@ -68,14 +68,15 @@ sdist.reproducible = true wheel.packages = ["libcuopt"] wheel.install-dir = "libcuopt" wheel.py-api = "py3" -# Everything, for now. The component wheels exist and are installable on their own, but -# libcuopt still carries the libraries because cuopt-config.cmake exports one -# _IMPORT_PREFIX for all three: split across sibling wheels, find_package(cuopt) cannot -# resolve cuopt::client / ::mathopt / ::routing, which python/cuopt links against. -# Trimming this to ["dev", "grpc-server", "cuopt"] needs per-component CMake packages so -# each resolves from its own prefix -- tracked separately. -install.components = ["dev", "grpc-server", "cuopt", "client", "client-dev", - "mathopt", "mathopt-dev", "routing", "routing-dev"] +# install.components is deliberately unset, so this installs everything, as before. +# +# Naming the components explicitly drops anything installed without one: rapids-cmake +# writes its helper find-modules to lib64/rapids/cmake/ with no COMPONENT, and losing +# them makes CMake fall back to the CUDA toolkit's libcudacxx, which is not the one +# librmm's cub-config expects. +# +# Trimming this to just the metapackage's own files needs per-component CMake packages, +# so each resolves from its own prefix -- tracked separately. [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From a29984c845e33095d526070412bb3ec2584d3b54 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 14:30:02 -0500 Subject: [PATCH 12/21] build(wheel): drop changes this PR no longer needs Reverting libcuopt to ship every component left several pieces behind. python/libcuopt/libcuopt/load.py still delegated to libcuopt_client and the other component packages, which libcuopt no longer depends on, so 'import libcuopt; libcuopt.load_library()' -- which ci/test_wheel_cuopt.sh asserts -- would have raised ModuleNotFoundError. Reverted. ci/build_wheel_libcuopt.sh excluded the component libraries from vendoring on the grounds that sibling wheels provide them; this wheel ships them itself. Reverted. The three component scripts keep their excludes, where that reasoning still holds. python/libcuopt/CMakeLists.txt kept its own copy of the body extracted to python/cmake/cuopt_wheel_build.cmake, so the same 83 lines existed twice and could drift. It now includes the shared file like the other three. Verified the include is reached, rather than skipped by the cuopt_FOUND early return: configuring with -DCMAKE_DISABLE_FIND_PACKAGE_cuopt=ON generates the full build graph. Co-Authored-By: Claude Opus 5 --- ci/build_wheel_libcuopt.sh | 5 -- .../.pyproject.toml.rapids-build-backend.bak | 86 +++++++++++++++++++ python/libcuopt/CMakeLists.txt | 74 +--------------- python/libcuopt/MANIFEST.in | 3 + python/libcuopt/libcuopt/GIT_COMMIT | 1 + python/libcuopt/libcuopt/load.py | 49 ++++++----- python/libcuopt/pyproject.toml | 17 +--- 7 files changed, 124 insertions(+), 111 deletions(-) create mode 100644 python/libcuopt/.pyproject.toml.rapids-build-backend.bak create mode 100644 python/libcuopt/MANIFEST.in create mode 100644 python/libcuopt/libcuopt/GIT_COMMIT diff --git a/ci/build_wheel_libcuopt.sh b/ci/build_wheel_libcuopt.sh index 02ba7e52ea..807cfb8f21 100755 --- a/ci/build_wheel_libcuopt.sh +++ b/ci/build_wheel_libcuopt.sh @@ -103,11 +103,6 @@ EXCLUDE_ARGS=( --exclude "libnvJitLink.so*" --exclude "librapids_logger.so" --exclude "librmm.so" - # Provided by the sibling libcuopt-* wheels, so they must not be vendored here: doing so - # would ship several copies of each engine and defeat the split. RPATH resolves them. - --exclude "libcuopt_client.so" - --exclude "libcuopt_mathopt.so" - --exclude "libcuopt_routing.so" # OpenSSL 3 is intentionally NOT bundled. Resolving libssl.so.3 / libcrypto.so.3 # at runtime via the host (or container image) keeps libcrypto and the FIPS # provider (system or mounted) byte-version-matched, which is required for diff --git a/python/libcuopt/.pyproject.toml.rapids-build-backend.bak b/python/libcuopt/.pyproject.toml.rapids-build-backend.bak new file mode 100644 index 0000000000..6e656a7518 --- /dev/null +++ b/python/libcuopt/.pyproject.toml.rapids-build-backend.bak @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.4.0,<0.5.0", + "scikit-build-core[pyproject]>=0.11.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcuopt" +dynamic = ["version"] +description = "cuOpt - GPU Optimizer (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = "Apache-2.0" +requires-python = ">=3.11" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "cuda-toolkit[cublas,cudart,curand,cusolver,cusparse,nvtx]==13.*", + "librmm==26.10.*,>=0.0.0a0", + "nvidia-cudss-cu13>=0.7,<0.8", + "nvidia-nccl-cu13>=2.19", + "nvidia-nvjitlink>=13.3,<14", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://docs.nvidia.com/cuopt/introduction.html" +Source = "https://github.com/nvidia/cuopt" + +[project.entry-points."cmake.prefix"] +libcuopt = "libcuopt" + +[project.scripts] +cuopt_cli = "libcuopt._cli_wrapper:main" +cuopt_grpc_server = "libcuopt._grpc_server_wrapper:main" + +[tool.pydistcheck] +select = [ + # note: max_allowed_size_compressed is set as a command-line arg + "distro-too-large-compressed", +] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = false +sdist.reproducible = true +wheel.packages = ["libcuopt"] +wheel.install-dir = "libcuopt" +wheel.py-api = "py3" +# install.components is deliberately unset: this wheel still ships every component. +# Trimming it needs per-component CMake packages, so find_package(cuopt) can resolve each +# from its own prefix, and care with files rapids-cmake installs without a COMPONENT. + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcuopt/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +requires = [ + "cmake>=4.0", + "librmm==26.10.*,>=0.0.0a0", + "ninja", + "rapids-logger==0.3.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libcuopt/CMakeLists.txt b/python/libcuopt/CMakeLists.txt index 7d904cd02b..4c9a2b47ce 100644 --- a/python/libcuopt/CMakeLists.txt +++ b/python/libcuopt/CMakeLists.txt @@ -31,76 +31,4 @@ endif() unset(cuopt_FOUND) -include(FetchContent) -FetchContent_Declare( - argparse - GIT_REPOSITORY https://github.com/p-ranav/argparse.git - GIT_TAG v3.2 -) -FetchContent_MakeAvailable(argparse) - -# gRPC must be available as an installed CMake package (gRPCConfig.cmake). -# On RockyLinux 8 wheel builds we install it in CI via ci/utils/install_protobuf_grpc.sh. -find_package(gRPC CONFIG REQUIRED) - -find_package(Boost 1.65 REQUIRED) -if(Boost_FOUND) - message(STATUS "Found Boost ${Boost_VERSION} in ${Boost_INCLUDE_DIRS}") -else() - message(FATAL_ERROR "Boost not found. Please install boost-devel.") -endif() - -include_directories(${Boost_INCLUDE_DIRS}) - -set(BUILD_TESTING OFF CACHE BOOL "Disable test build for papilo") -set(PAPILO_NO_BINARIES ON) -option(LUSOL "Disable LUSOL" OFF) - -set(BUILD_TESTS OFF) -set(BUILD_BENCHMARKS OFF) -set(CUOPT_BUILD_TESTUTIL OFF) - -add_subdirectory(../../cpp cuopt-cpp) - -# cuopt is an INTERFACE target (libcuopt.so is a linker script) and compiles nothing, -# so it has no use for argparse. cuopt_cli and cuopt_grpc_server, which do, already link -# argparse::argparse in cpp/CMakeLists.txt. -target_link_libraries(cuopt_cli PRIVATE - argparse -) - -set(rpaths - "$ORIGIN/../lib64" - "$ORIGIN/../../rapids_logger/lib64" - "$ORIGIN/../../librmm/lib64" - "$ORIGIN/../../nvidia/cudss/lib" - "$ORIGIN/../../nvidia/cublas/lib" - "$ORIGIN/../../nvidia/curand/lib" - "$ORIGIN/../../nvidia/cusolver/lib" - "$ORIGIN/../../nvidia/cusparse/lib" - "$ORIGIN/../../nvidia/nccl/lib" - "$ORIGIN/../../nvidia/nvjitlink/lib" -) - -# Add CUDA version-specific paths based on CUDA compiler version -message(STATUS "libcuopt: CMAKE_CUDA_COMPILER_VERSION = ${CMAKE_CUDA_COMPILER_VERSION}") -if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 14.0) - message(STATUS "libcuopt: Adding cu13 RPATH") - list(APPEND rpaths "$ORIGIN/../../nvidia/cu13/lib") -elseif(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.0) - message(STATUS "libcuopt: Adding cu12 RPATH") - list(APPEND rpaths "$ORIGIN/../../nvidia/cu12/lib") -else() - message(WARNING "libcuopt: Unsupported CUDA version ${CMAKE_CUDA_COMPILER_VERSION}") -endif() -message(STATUS "libcuopt: Final RPATH = ${rpaths}") - -# cuopt is an INTERFACE target (libcuopt.so is a linker script), so it has no RPATH. -# cuopt_client is in the list because it PUBLIC-links rapids_logger, which -# build_wheel_libcuopt.sh excludes from vendoring, so the only way to find it is -# $ORIGIN/../../rapids_logger/lib64 from this list. -foreach(_target cuopt_routing cuopt_mathopt cuopt_client cuopt_cli cuopt_grpc_server) - if(TARGET ${_target}) - set_property(TARGET ${_target} APPEND PROPERTY INSTALL_RPATH ${rpaths}) - endif() -endforeach() +include(../cmake/cuopt_wheel_build.cmake) diff --git a/python/libcuopt/MANIFEST.in b/python/libcuopt/MANIFEST.in new file mode 100644 index 0000000000..5fcf992310 --- /dev/null +++ b/python/libcuopt/MANIFEST.in @@ -0,0 +1,3 @@ + +# Automatically generated by rapids-build-backend +include libcuopt/GIT_COMMIT diff --git a/python/libcuopt/libcuopt/GIT_COMMIT b/python/libcuopt/libcuopt/GIT_COMMIT new file mode 100644 index 0000000000..526103e1cb --- /dev/null +++ b/python/libcuopt/libcuopt/GIT_COMMIT @@ -0,0 +1 @@ +9e0cb8a738f62c970e3bb670f6ff894207eed7e0 diff --git a/python/libcuopt/libcuopt/load.py b/python/libcuopt/libcuopt/load.py index e7ea9a65b7..86f5798f70 100644 --- a/python/libcuopt/libcuopt/load.py +++ b/python/libcuopt/libcuopt/load.py @@ -34,30 +34,39 @@ def _load_wheel_installation(soname: str): def load_library(): - """Load every cuOpt component library. - - The libraries themselves live in the per-component packages now, so this - delegates rather than dlopen()ing them directly. The client goes first: - mathopt and routing both carry a DT_NEEDED on it. - """ - loaded = [] - - import libcuopt_client - - loaded.extend(libcuopt_client.load_library() or []) - - # routing is optional -- a SKIP_ROUTING_BUILD install will not have it. + """Dynamically load libcuopt.so and its dependencies""" try: - import libcuopt_routing - - loaded.extend(libcuopt_routing.load_library() or []) + # librmm and libraft must be loaded before libcuopt + # because libcuopt references them. + import libraft + import librmm + import rapids_logger + + rapids_logger.load_library() + librmm.load_library() + libraft.load_library() except ModuleNotFoundError: pass - import libcuopt_mathopt - - loaded.extend(libcuopt_mathopt.load_library() or []) - + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUOPT_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + # cuOpt ships as component libraries. libcuopt.so is a linker script, + # not an ELF object, so it cannot be dlopen()ed -- load the components + # instead. Each pulls its own dependencies in through DT_NEEDED, so this + # order only needs to be valid, not exhaustive. routing and grpc are + # optional (SKIP_ROUTING_BUILD, SKIP_GRPC_BUILD) and may be absent. + components = [ + ("libcuopt_routing.so", False), + ("libcuopt_mathopt.so", True), + ] + loaded = [] + for soname, required in components: + lib = _load_component(soname, prefer_system_installation, required) + if lib is not None: + loaded.append(lib) return loaded diff --git a/python/libcuopt/pyproject.toml b/python/libcuopt/pyproject.toml index 582d35b15e..dde77c4a48 100644 --- a/python/libcuopt/pyproject.toml +++ b/python/libcuopt/pyproject.toml @@ -9,7 +9,7 @@ requires = [ ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] -name = "libcuopt" +name = "libcuopt-cu13" dynamic = ["version"] description = "cuOpt - GPU Optimizer (C++)" readme = { file = "README.md", content-type = "text/markdown" } @@ -45,9 +45,6 @@ Source = "https://github.com/nvidia/cuopt" [project.entry-points."cmake.prefix"] libcuopt = "libcuopt" -# libcuopt still stages the mathopt component, so it ships cuopt_cli and keeps its -# console script. libcuopt-mathopt declares the same entry point for installs that take -# the component on its own. [project.scripts] cuopt_cli = "libcuopt._cli_wrapper:main" cuopt_grpc_server = "libcuopt._grpc_server_wrapper:main" @@ -68,15 +65,9 @@ sdist.reproducible = true wheel.packages = ["libcuopt"] wheel.install-dir = "libcuopt" wheel.py-api = "py3" -# install.components is deliberately unset, so this installs everything, as before. -# -# Naming the components explicitly drops anything installed without one: rapids-cmake -# writes its helper find-modules to lib64/rapids/cmake/ with no COMPONENT, and losing -# them makes CMake fall back to the CUDA toolkit's libcudacxx, which is not the one -# librmm's cub-config expects. -# -# Trimming this to just the metapackage's own files needs per-component CMake packages, -# so each resolves from its own prefix -- tracked separately. +# install.components is deliberately unset: this wheel still ships every component. +# Trimming it needs per-component CMake packages, so find_package(cuopt) can resolve each +# from its own prefix, and care with files rapids-cmake installs without a COMPONENT. [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 27fa7ec90ff6012e68bed636c890e0b86c9ea705 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 14:33:37 -0500 Subject: [PATCH 13/21] ci(wheel): stop serialising wheel-build-cuopt behind the component builds build_wheel_cuopt.sh downloads only the libcuopt wheel -- it names the component libraries solely in auditwheel excludes, which is unchanged from main -- so waiting on the three component jobs bought nothing and delayed the cuopt wheel by a full build. pr-builder still gates on them, so a component failure blocks the PR either way. Co-Authored-By: Claude Opus 5 --- .github/workflows/build.yaml | 3 +-- .github/workflows/pr.yaml | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4fb826b8f8..30f89a6805 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -362,8 +362,7 @@ jobs: package-type: cpp publish-wheel-search-key: cuopt_wheel_cpp_libcuopt_routing wheel-build-cuopt: - needs: [build-details, wheel-build-libcuopt, wheel-build-libcuopt-client, - wheel-build-libcuopt-mathopt, wheel-build-libcuopt-routing] + needs: [build-details, wheel-build-libcuopt] permissions: actions: read contents: read diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8963224ffd..5c59bf466c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -669,9 +669,7 @@ jobs: build-datetime: ${{ needs.build-details.outputs.build-datetime }} script: ci/build_wheel_libcuopt_routing.sh wheel-build-cuopt: - needs: [build-details, wheel-build-libcuopt, wheel-build-libcuopt-client, - wheel-build-libcuopt-mathopt, wheel-build-libcuopt-routing, - compute-matrix-filters, changed-files] + needs: [build-details, wheel-build-libcuopt, compute-matrix-filters, changed-files] if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels permissions: actions: read From 6202f855fc10770cd2fb628337fb4cd280572023 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 14:39:35 -0500 Subject: [PATCH 14/21] chore: drop a build backup committed by mistake rapids-build-backend writes .pyproject.toml.rapids-build-backend.bak while building a wheel and restores from it afterwards; an interrupted local build left one behind and 'git add -A' swept it in. Co-Authored-By: Claude Opus 5 --- .../.pyproject.toml.rapids-build-backend.bak | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 python/libcuopt/.pyproject.toml.rapids-build-backend.bak diff --git a/python/libcuopt/.pyproject.toml.rapids-build-backend.bak b/python/libcuopt/.pyproject.toml.rapids-build-backend.bak deleted file mode 100644 index 6e656a7518..0000000000 --- a/python/libcuopt/.pyproject.toml.rapids-build-backend.bak +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[build-system] -build-backend = "rapids_build_backend.build" -requires = [ - "rapids-build-backend>=0.4.0,<0.5.0", - "scikit-build-core[pyproject]>=0.11.0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project] -name = "libcuopt" -dynamic = ["version"] -description = "cuOpt - GPU Optimizer (C++)" -readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "NVIDIA Corporation" }, -] -license = "Apache-2.0" -requires-python = ">=3.11" -classifiers = [ - "Intended Audience :: Developers", - "Topic :: Database", - "Topic :: Scientific/Engineering", - "Programming Language :: C++", - "Environment :: GPU :: NVIDIA CUDA", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", -] -dependencies = [ - "cuda-toolkit[cublas,cudart,curand,cusolver,cusparse,nvtx]==13.*", - "librmm==26.10.*,>=0.0.0a0", - "nvidia-cudss-cu13>=0.7,<0.8", - "nvidia-nccl-cu13>=2.19", - "nvidia-nvjitlink>=13.3,<14", - "rapids-logger==0.3.*", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. - -[project.urls] -Homepage = "https://docs.nvidia.com/cuopt/introduction.html" -Source = "https://github.com/nvidia/cuopt" - -[project.entry-points."cmake.prefix"] -libcuopt = "libcuopt" - -[project.scripts] -cuopt_cli = "libcuopt._cli_wrapper:main" -cuopt_grpc_server = "libcuopt._grpc_server_wrapper:main" - -[tool.pydistcheck] -select = [ - # note: max_allowed_size_compressed is set as a command-line arg - "distro-too-large-compressed", -] - -[tool.scikit-build] -build-dir = "build/{wheel_tag}" -cmake.build-type = "Release" -cmake.version = "CMakeLists.txt" -minimum-version = "build-system.requires" -ninja.make-fallback = false -sdist.reproducible = true -wheel.packages = ["libcuopt"] -wheel.install-dir = "libcuopt" -wheel.py-api = "py3" -# install.components is deliberately unset: this wheel still ships every component. -# Trimming it needs per-component CMake packages, so find_package(cuopt) can resolve each -# from its own prefix, and care with files rapids-cmake installs without a COMPONENT. - -[tool.scikit-build.metadata.version] -provider = "scikit_build_core.metadata.regex" -input = "libcuopt/VERSION" -regex = "(?P.*)" - -[tool.rapids-build-backend] -build-backend = "scikit_build_core.build" -dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" -requires = [ - "cmake>=4.0", - "librmm==26.10.*,>=0.0.0a0", - "ninja", - "rapids-logger==0.3.*", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From a42cc67c2bcdc8812db6c42f309780f1a790f163 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 14:40:07 -0500 Subject: [PATCH 15/21] chore: drop more rapids-build-backend output committed by mistake MANIFEST.in and libcuopt/GIT_COMMIT are both written by rapids-build-backend during a wheel build -- the MANIFEST.in even says so -- and local builds left them behind for 'git add -A' to pick up. Co-Authored-By: Claude Opus 5 --- python/libcuopt/MANIFEST.in | 3 --- python/libcuopt/libcuopt/GIT_COMMIT | 1 - 2 files changed, 4 deletions(-) delete mode 100644 python/libcuopt/MANIFEST.in delete mode 100644 python/libcuopt/libcuopt/GIT_COMMIT diff --git a/python/libcuopt/MANIFEST.in b/python/libcuopt/MANIFEST.in deleted file mode 100644 index 5fcf992310..0000000000 --- a/python/libcuopt/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ - -# Automatically generated by rapids-build-backend -include libcuopt/GIT_COMMIT diff --git a/python/libcuopt/libcuopt/GIT_COMMIT b/python/libcuopt/libcuopt/GIT_COMMIT deleted file mode 100644 index 526103e1cb..0000000000 --- a/python/libcuopt/libcuopt/GIT_COMMIT +++ /dev/null @@ -1 +0,0 @@ -9e0cb8a738f62c970e3bb670f6ff894207eed7e0 From ebbda4220d3f421408bf96ba0027123a3823c9ae Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 14:52:16 -0500 Subject: [PATCH 16/21] ci(wheel): publish the client wheel before mathopt and routing Both declare libcuopt-client as a dependency, so publishing either first leaves a clean install unable to resolve it. Co-Authored-By: Claude Opus 5 --- .github/workflows/build.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 30f89a6805..d7cca486cb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -302,7 +302,10 @@ jobs: package-type: cpp matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-publish-libcuopt-mathopt: - needs: wheel-build-libcuopt-mathopt + # Waits on the client's publication as well: this wheel declares + # libcuopt-client as a dependency, so publishing first would leave a clean + # install unable to resolve it. + needs: [wheel-build-libcuopt-mathopt, wheel-publish-libcuopt-client] permissions: actions: read contents: read @@ -342,7 +345,10 @@ jobs: package-type: cpp matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-publish-libcuopt-routing: - needs: wheel-build-libcuopt-routing + # Waits on the client's publication as well: this wheel declares + # libcuopt-client as a dependency, so publishing first would leave a clean + # install unable to resolve it. + needs: [wheel-build-libcuopt-routing, wheel-publish-libcuopt-client] permissions: actions: read contents: read From acc9df47a356bff77b429f61c70f00d9338d92bb Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 15:13:23 -0500 Subject: [PATCH 17/21] fix(wheel): restore the unsuffixed project name wheel-build-libcuopt failed with FileNotFoundError: 'libcuopt_cu13/GIT_COMMIT' rapids-build-backend rewrites pyproject.toml in place during a build, applying the CUDA suffix to the project name, and restores it afterwards. An interrupted local build left the rewritten copy behind and it was committed. _write_git_commits derives the directory from the name it reads, so it looked for libcuopt_cu13/ rather than libcuopt/. Third artifact from the same source, after the .bak file and MANIFEST.in, and the only one that broke a build. Co-Authored-By: Claude Opus 5 --- python/libcuopt/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libcuopt/pyproject.toml b/python/libcuopt/pyproject.toml index dde77c4a48..6e656a7518 100644 --- a/python/libcuopt/pyproject.toml +++ b/python/libcuopt/pyproject.toml @@ -9,7 +9,7 @@ requires = [ ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] -name = "libcuopt-cu13" +name = "libcuopt" dynamic = ["version"] description = "cuOpt - GPU Optimizer (C++)" readme = { file = "README.md", content-type = "text/markdown" } From 7773a9d88ae58ee268eeb45dd099c9ddbd57c12e Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 15:08:05 -0500 Subject: [PATCH 18/21] WIP build(cmake): give each component its own CMake export Splitting libcuopt across wheels breaks find_package(cuopt): cuopt-config.cmake exports a single _IMPORT_PREFIX for all three libraries, derived from where the config sits, so with the libraries in sibling wheel directories none of cuopt::client / ::mathopt / ::routing resolve. python/cuopt links those, and wheel-build-cuopt fails with 'Target "data_model_wrapper" links to cuopt::mathopt but the target was not found'. So far: - each component installs to its own export set, so install(EXPORT) writes one targets file per component and each computes _IMPORT_PREFIX from its own install location - the umbrella config includes whichever component targets files it finds, beside itself or through CMAKE_PREFIX_PATH - cuopt::cuopt is synthesised in the config rather than exported, since an exported target may not reference targets outside its own export set Does not configure yet: rapids_export(BUILD) still exports the build tree, where the umbrella's BUILD_INTERFACE links to the components are live and they are no longer members of cuopt-exports. Co-Authored-By: Claude Opus 5 --- cpp/CMakeLists.txt | 69 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2fb971ad78..3e2a8cb130 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1177,10 +1177,18 @@ target_include_directories(cuopt "$" ) +# BUILD_INTERFACE only: the installed config rebuilds these links from whichever +# components are present, because an exported target may not reference targets outside +# its own export set and each component now exports separately. if(NOT SKIP_ROUTING_BUILD) - target_link_libraries(cuopt INTERFACE cuopt_routing cuopt_mathopt cuopt_client) + target_link_libraries(cuopt INTERFACE + "$" + "$" + "$") else() - target_link_libraries(cuopt INTERFACE cuopt_mathopt cuopt_client) + target_link_libraries(cuopt INTERFACE + "$" + "$") endif() if (WRITE_FATBIN) @@ -1251,17 +1259,30 @@ endforeach() # One install component per library, so the per-solver packages in #1635 can stage # them separately. A plain `cmake --install` still installs everything. +# Each component exports to its own set, so install(EXPORT) writes one targets file per +# component and each computes _IMPORT_PREFIX from its own install location. A single +# shared set hardcodes one prefix for all three, which only resolves while they sit in +# the same tree -- splitting them across wheels then breaks find_package (#1635). foreach(_component ${CUOPT_COMPONENT_TARGETS}) string(REPLACE "cuopt_" "" _install_component "${_component}") install(TARGETS ${_component} DESTINATION ${_LIB_DEST} COMPONENT ${_install_component} - EXPORT cuopt-exports + EXPORT ${_component}-exports + ) + install(EXPORT ${_component}-exports + FILE ${_component}-targets.cmake + NAMESPACE cuopt:: + DESTINATION "${_LIB_DEST}/cmake/cuopt" + COMPONENT ${_install_component}-dev ) endforeach() -# cuopt::cuopt carries no artifact of its own; it is exported so that -# target_link_libraries(app cuopt::cuopt) keeps pulling in every component. +# cuopt::cuopt carries no artifact; it is exported only so the export set exists and +# find_package(cuopt) has something to define. Its component links are BUILD_INTERFACE +# only -- CMake requires everything an exported target references to share its export set, +# which would defeat the split -- so the installed config synthesises them instead, from +# whichever components that installation actually has. See the FINAL_CODE_BLOCK below. install(TARGETS cuopt EXPORT cuopt-exports ) @@ -1355,9 +1376,43 @@ else() set(cuopt_code_string "set(CUOPT_HAS_GRPC ON)\nset(CUOPT_HAS_GRPC_ROUTING OFF)") endif() +# Each component installs its own targets file, so the umbrella has to pull them in. They +# sit beside this config when everything shares a prefix, and are found through +# CMAKE_PREFIX_PATH when the components are installed separately -- which is what lets a +# wheel install resolve them from sibling package directories (#1635). +string(APPEND cuopt_code_string [=[ + +foreach(_cuopt_component cuopt_client cuopt_mathopt cuopt_routing) + set(_cuopt_targets "${CMAKE_CURRENT_LIST_DIR}/${_cuopt_component}-targets.cmake") + if(NOT EXISTS "${_cuopt_targets}") + unset(_cuopt_found_targets CACHE) + find_file(_cuopt_found_targets "${_cuopt_component}-targets.cmake" + PATH_SUFFIXES lib/cmake/cuopt lib64/cmake/cuopt) + set(_cuopt_targets "${_cuopt_found_targets}") + endif() + if(_cuopt_targets AND EXISTS "${_cuopt_targets}") + include("${_cuopt_targets}") + endif() +endforeach() + +# Backwards compatibility: target_link_libraries(app cuopt::cuopt) keeps pulling in every +# component that this installation actually has. +if(NOT TARGET cuopt::cuopt) + set(_cuopt_umbrella_links "") + foreach(_cuopt_component cuopt::client cuopt::mathopt cuopt::routing) + if(TARGET ${_cuopt_component}) + list(APPEND _cuopt_umbrella_links ${_cuopt_component}) + endif() + endforeach() + add_library(cuopt::cuopt INTERFACE IMPORTED) + set_target_properties(cuopt::cuopt PROPERTIES + INTERFACE_LINK_LIBRARIES "${_cuopt_umbrella_links}") +endif() +]=]) + rapids_export(INSTALL cuopt EXPORT_SET cuopt-exports - GLOBAL_TARGETS cuopt ${CUOPT_COMPONENT_EXPORT_NAMES} + GLOBAL_TARGETS ${CUOPT_COMPONENT_EXPORT_NAMES} NAMESPACE cuopt:: DOCUMENTATION doc_string FINAL_CODE_BLOCK cuopt_code_string @@ -1367,7 +1422,7 @@ rapids_export(INSTALL cuopt # - build export ------------------------------------------------------------------------------- rapids_export(BUILD cuopt EXPORT_SET cuopt-exports - GLOBAL_TARGETS cuopt ${CUOPT_COMPONENT_EXPORT_NAMES} + GLOBAL_TARGETS ${CUOPT_COMPONENT_EXPORT_NAMES} NAMESPACE cuopt:: DOCUMENTATION doc_string FINAL_CODE_BLOCK cuopt_code_string From e189cc995bdb03c175fcdadb5e2156cc2932cf4c Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 22 Sep 2026 07:40:53 -0500 Subject: [PATCH 19/21] build(cmake): export each component separately so wheels can resolve them cuopt-config.cmake exported one _IMPORT_PREFIX for all three libraries, taken from where the config sits, so find_package(cuopt) only resolved while they shared a tree. Split across wheels it defined none of cuopt::client / ::mathopt / ::routing, which python/cuopt links against. Each component now exports to its own set, so install(EXPORT) writes one targets file per component and each computes its own prefix. The umbrella carries no component links -- CMake will not export a target referencing targets outside its export set -- and the installed config rebuilds them from whichever components are present. ConfigureTest, the only in-tree consumer of those links, names the components directly. Verified from a consumer project: find_package(cuopt) resolves cuopt::cuopt, ::client, ::mathopt and ::routing, with the umbrella linking all three. Co-Authored-By: Claude Opus 5 --- cpp/CMakeLists.txt | 37 ++++++++++++++++--------------------- cpp/tests/CMakeLists.txt | 11 ++++++++++- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3e2a8cb130..f1aa252171 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1177,19 +1177,11 @@ target_include_directories(cuopt "$" ) -# BUILD_INTERFACE only: the installed config rebuilds these links from whichever -# components are present, because an exported target may not reference targets outside -# its own export set and each component now exports separately. -if(NOT SKIP_ROUTING_BUILD) - target_link_libraries(cuopt INTERFACE - "$" - "$" - "$") -else() - target_link_libraries(cuopt INTERFACE - "$" - "$") -endif() +# Deliberately no component links. CMake will not export a target that references +# targets outside its own export set, and each component exports separately now so its +# targets file can compute its own _IMPORT_PREFIX. The installed config rebuilds these +# links from whichever components are present -- see the FINAL_CODE_BLOCK below -- and +# the one in-tree consumer, ConfigureTest, names the components directly. if (WRITE_FATBIN) file(WRITE "${CUOPT_BINARY_DIR}/fatbin.ld" @@ -1397,17 +1389,20 @@ endforeach() # Backwards compatibility: target_link_libraries(app cuopt::cuopt) keeps pulling in every # component that this installation actually has. +set(_cuopt_umbrella_links "") +foreach(_cuopt_component cuopt::client cuopt::mathopt cuopt::routing) + if(TARGET ${_cuopt_component}) + list(APPEND _cuopt_umbrella_links ${_cuopt_component}) + endif() +endforeach() if(NOT TARGET cuopt::cuopt) - set(_cuopt_umbrella_links "") - foreach(_cuopt_component cuopt::client cuopt::mathopt cuopt::routing) - if(TARGET ${_cuopt_component}) - list(APPEND _cuopt_umbrella_links ${_cuopt_component}) - endif() - endforeach() add_library(cuopt::cuopt INTERFACE IMPORTED) - set_target_properties(cuopt::cuopt PROPERTIES - INTERFACE_LINK_LIBRARIES "${_cuopt_umbrella_links}") endif() +# Set unconditionally: the exported umbrella carries no component links -- it cannot, +# since they are in other export sets -- so guarding on the target's absence would leave +# cuopt::cuopt defined but linking nothing. +set_property(TARGET cuopt::cuopt APPEND PROPERTY + INTERFACE_LINK_LIBRARIES "${_cuopt_umbrella_links}") ]=]) rapids_export(INSTALL cuopt diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 41031de5c3..70d0d6d2e3 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -82,7 +82,16 @@ function(ConfigureTest CMAKE_TEST_NAME) set(_cuopt_link_target cuopt_static) set(_extra_libs cuopttestutils_static) else() - set(_cuopt_link_target cuopt) + # The components rather than the cuopt umbrella: the umbrella no longer carries + # links to them, so that it can be exported on its own while each component + # exports separately (#1635). + # Order matches what the cuopt umbrella used to expand to; changing it changes + # which definition the linker resolves first. + set(_cuopt_link_target "") + if(NOT SKIP_ROUTING_BUILD) + list(APPEND _cuopt_link_target cuopt_routing) + endif() + list(APPEND _cuopt_link_target cuopt_mathopt cuopt_client) set(_extra_libs cuopttestutils) endif() # cuopttestutils only exists when the routing engine is built. From 0bea7353791236d62381e19c854fd49481d726cc Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 22 Sep 2026 09:38:22 -0500 Subject: [PATCH 20/21] fix(wheel): ship the cuDSS threading layer and let executables find their siblings Two gaps the component split opened, both found in review. cudss_mtlayer_cuopt installed with COMPONENT runtime, which no wheel and no conda output stages, so libcudss_mtlayer_cuopt.so reached neither. The barrier solver dlopens it by bare soname and falls back to single-threaded cuDSS when that fails, so the loss was a silent slowdown rather than an error. It now installs with mathopt, the only component that loads it. cuopt_cli has a DT_NEEDED on libcuopt_client.so, ships in libcuopt-mathopt, and lost its sibling RPATH entry in 2335024. That entry is unnecessary for the libraries, since load_library() loads them in dependency order and glibc then satisfies each DT_NEEDED from an already-loaded object, but an executable has no Python in the process. The entries come back for the executables alone. Co-Authored-By: Claude Opus 5 --- cpp/CMakeLists.txt | 4 +++- python/cmake/cuopt_wheel_build.cmake | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cf31acbb66..706db9f7d4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1285,9 +1285,11 @@ install(FILES "${CUOPT_LINKER_SCRIPT}" ) if (CUOPT_BUILD_CUSTOM_CUDSS_MTLAYER) + # mathopt, not a component of its own: the barrier solver is the only thing that + # dlopens it, and any install without it silently drops to single-threaded cuDSS. install(TARGETS cudss_mtlayer_cuopt DESTINATION ${_LIB_DEST} - COMPONENT runtime + COMPONENT mathopt ) endif () diff --git a/python/cmake/cuopt_wheel_build.cmake b/python/cmake/cuopt_wheel_build.cmake index 1ee1d07ffa..4c1e681ebb 100644 --- a/python/cmake/cuopt_wheel_build.cmake +++ b/python/cmake/cuopt_wheel_build.cmake @@ -81,3 +81,18 @@ foreach(_target cuopt_routing cuopt_mathopt cuopt_client cuopt_cli cuopt_grpc_se set_property(TARGET ${_target} APPEND PROPERTY INSTALL_RPATH ${rpaths}) endif() endforeach() + +# Executables also need the sibling component wheels. The libraries do not: load_library() +# loads them in dependency order, so glibc satisfies each DT_NEEDED from an already-loaded +# object. An executable has no Python in the process, so cuopt_cli shipped in +# libcuopt-mathopt would not find libcuopt_client.so, which is never vendored. +set(component_rpaths + "$ORIGIN/../../libcuopt_client/lib64" + "$ORIGIN/../../libcuopt_mathopt/lib64" + "$ORIGIN/../../libcuopt_routing/lib64" +) +foreach(_target cuopt_cli cuopt_grpc_server) + if(TARGET ${_target}) + set_property(TARGET ${_target} APPEND PROPERTY INSTALL_RPATH ${component_rpaths}) + endif() +endforeach() From e85ad595f3d8c0bb9095cb3de18cb6ebc9b3d78e Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 22 Sep 2026 10:41:17 -0500 Subject: [PATCH 21/21] fix(cmake): give the build export the same per-component targets files Splitting the components into their own export sets left cuopt-exports holding only the umbrella. install(EXPORT) writes a targets file per component, but nothing wrote them for the build tree, and both exports share one FINAL_CODE_BLOCK that includes those files by name. A build-tree or CPM consumer therefore got cuopt_FOUND=1, cuopt::cuopt defined, no components, and an empty INTERFACE_LINK_LIBRARIES -- a find_package that succeeds and fails later at link time. Before the split all four targets were in the single cuopt-targets.cmake, so this was a regression. export(EXPORT) beside each install(EXPORT) restores it. Verified with a consumer project against both the build tree and an install prefix: both now report cuopt::cuopt links = 'cuopt::client;cuopt::mathopt;cuopt::routing'. Co-Authored-By: Claude Opus 5 --- cpp/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 706db9f7d4..0490794c40 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1266,6 +1266,13 @@ foreach(_component ${CUOPT_COMPONENT_TARGETS}) DESTINATION "${_LIB_DEST}/cmake/cuopt" COMPONENT ${_install_component}-dev ) + # The same file in the build tree. rapids_export(BUILD) and rapids_export(INSTALL) + # share one FINAL_CODE_BLOCK, which includes these by name; without this a build-tree + # or CPM consumer gets cuopt::cuopt defined, found, and linking nothing. + export(EXPORT ${_component}-exports + FILE "${CMAKE_CURRENT_BINARY_DIR}/${_component}-targets.cmake" + NAMESPACE cuopt:: + ) endforeach() # cuopt::cuopt carries no artifact; it is exported only so the export set exists and