From b1d3ced36bf6269e304d5e2c609c9f53c207e335 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 10:36:02 -0500 Subject: [PATCH 1/6] Bundle the self-contained Java classifier JAR in the official Docker images Neither ci/docker/Dockerfile nor Dockerfile.ubi installed any Java support at all. The existing dynamic-link Java build (ci/build_java.sh) can't help here either: it needs a conda environment with libcuopt's headers, and these images are pip-based (no headers, just the compiled libcuopt.so). The self-contained classifier JAR from java-static-build (#1817) is the right fit instead: it embeds libcuopt statically, so nothing has to link against this image's own pip-installed libcuopt at all. build.yaml already builds one per ARCH x CUDA-major combination on every branch/nightly run, in the same overall workflow run that (via build_test_publish_images.yaml) builds these images, so build_images.yaml downloads it as a same-run artifact rather than needing any new publish channel. Verified against a real build: built the actual Dockerfile with a real classifier JAR pulled from a recent successful main run, then compiled and ran a small Java program against the resulting image's cuopt.jar. Found and fixed a real gap doing that -- the JNI library failed with UnsatisfiedLinkError: libcublas.so.13, because this image's CUDA runtime comes from pip-installed nvidia-cublas-cu13 (under .../dist-packages/nvidia/cu13/lib), not /usr/local/cuda/targets/.../lib the way the RAPIDS CI image the JAR was built and tested against has it. libcuopt.so itself doesn't hit this (it carries its own $ORIGIN-relative RPATH into that same pip directory), but the JNI library has no such RPATH, so LD_LIBRARY_PATH now includes it explicitly. Confirmed the fix: the program got past native library loading and all the way to an actual cuOptSolve call (which then hit CUOPT_VALIDATION_ERROR only because this local test had no GPU passed through -- a test-environment limitation, not a packaging bug). --- .github/workflows/build.yaml | 1 + .github/workflows/build_images.yaml | 18 ++++++++++++++++++ ci/docker/Dockerfile | 29 ++++++++++++++++++++++++++++- ci/docker/Dockerfile.ubi | 14 ++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e52701dfc5..255b400fd6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -634,6 +634,7 @@ jobs: build-images: needs: - build-details + - java-static-build - wheel-publish-cuopt - wheel-publish-cuopt-server - wheel-publish-cuopt-sh-client diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index 4b7881e806..d746feb686 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -84,6 +84,23 @@ jobs: run: | echo "CUDA_SHORT=$(echo "$CUDA_VER" | sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+/\1/')" >> $GITHUB_OUTPUT echo "PYTHON_SHORT=$(echo "$PYTHON_VER" | sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+/\1/')" >> $GITHUB_OUTPUT + echo "CUDA_MAJOR=$(echo "$CUDA_VER" | cut -d. -f1)" >> $GITHUB_OUTPUT + # The self-contained Java classifier JAR (see java-static-build in build.yaml, #1817) is + # built once per ARCH x CUDA-major combination in the same build.yaml run this workflow is + # always called from (build.yaml -> build_test_publish_images.yaml -> build_images.yaml), + # so it is available here as a same-run artifact -- no separate download/auth setup needed, + # unlike cross-run fetches (rapids-download-from-github). build-images (build.yaml) depends + # on java-static-build, so it is guaranteed to exist by the time this step runs. + - name: Download the Java classifier JAR for this image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: cuopt_java_${{ matrix.ARCH }}_cu${{ steps.trim.outputs.CUDA_MAJOR }} + path: ./java-classifier-download + - name: Stage the classifier JAR into the Docker build context + run: | + . java/cuopt/ci/java_classifier.sh + jar="$(cuopt_java_resolve_artifact_jar ./java-classifier-download)" + cp "${jar}" ./ci/docker/context/cuopt.jar - name: Build image and push to DockerHub and NGC uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: @@ -97,6 +114,7 @@ jobs: sbom: false build-args: | CUDA_VER=${{ inputs.CUDA_VER }} + CUDA_MAJOR=${{ steps.trim.outputs.CUDA_MAJOR }} PYTHON_SHORT_VER=${{ steps.trim.outputs.PYTHON_SHORT }} CUOPT_VER=${{ inputs.CUOPT_VER }} LINUX_VER=${{ inputs.LINUX_VER }} diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 5c6e4c3b39..e3a2df5b90 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -89,14 +89,35 @@ RUN \ && apt-get purge -y --autoremove gcc \ && rm -rf /var/lib/apt/lists/* +# A JRE for the self-contained Java classifier JAR (#1817): the JAR embeds libcuopt (statically +# linked) and its own companion libraries, so nothing here needs to link against the +# pip-installed libcuopt at all -- a plain runtime, not a full JDK, is enough. headless: no X11 +# dependency, matching this image having no display. Version matches +# java/cuopt/pom.xml's maven.compiler.release. +RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-17-jre-headless \ + && rm -rf /var/lib/apt/lists/* + FROM install-env AS cuopt-final ARG PYTHON_SHORT_VER +# Major CUDA version only (e.g. "13"), not the full CUDA_VER: Dockerfile ENV/ARG substitution +# has no shell-style ${VAR%%.*} parameter expansion, so this is computed once in +# build_images.yaml (which already needs it for the classifier JAR artifact name) and passed +# through as its own build-arg rather than recomputed here. +ARG CUDA_MAJOR # Make cuopt_grpc_server, cuopt_cli, and shared libraries available to all processes # (profile.d scripts are only sourced by login shells; ENV works for all containers) ENV PATH="/usr/local/cuda/bin:/usr/bin:/usr/local/bin:/usr/local/nvidia/bin/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/bin:${PATH}" -ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/wsl/lib:/usr/lib/wsl/lib/libnvidia-container:/usr/lib/nvidia:/usr/lib/nvidia-current:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/lib/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/rapids_logger/lib64:${LD_LIBRARY_PATH}" +# .../nvidia/cu/lib holds the pip-installed CUDA runtime (libcublas, libcusparse, ...) +# this image actually ships -- there is no /usr/local/cuda/targets install here at all (a plain +# `-base` CUDA image, not `-runtime`/`-devel`). libcuopt.so itself doesn't need this on +# LD_LIBRARY_PATH: it carries its own $ORIGIN-relative RPATH into that same directory. The Java +# classifier JAR's JNI library (#1817) has no such RPATH -- it was built for a RAPIDS CI image +# where /usr/local/cuda/targets is fully populated -- so without this it fails with +# UnsatisfiedLinkError: libcublas.so.: cannot open shared object file. +ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/wsl/lib:/usr/lib/wsl/lib/libnvidia-container:/usr/lib/nvidia:/usr/lib/nvidia-current:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/lib/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/rapids_logger/lib64:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/nvidia/cu${CUDA_MAJOR}/lib:${LD_LIBRARY_PATH}" # Directory creation, permissions RUN mkdir -p /opt/cuopt && \ @@ -119,6 +140,12 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC). COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ +# The self-contained Java classifier JAR matching this image's CUDA major version and +# architecture (see build_images.yaml, which downloads it from java-static-build before this +# build starts). Nothing else needs installing to use it: `java -cp cuopt.jar ...` or add it as +# a local Maven/Gradle file dependency. +COPY ./cuopt.jar /opt/cuopt/java/cuopt.jar + # Entrypoint supports server selection: # Default: Python REST server # CUOPT_SERVER_TYPE=grpc: gRPC server (uses CUOPT_SERVER_PORT, CUOPT_GPU_COUNT) diff --git a/ci/docker/Dockerfile.ubi b/ci/docker/Dockerfile.ubi index 5668d7864f..f73b7e341a 100644 --- a/ci/docker/Dockerfile.ubi +++ b/ci/docker/Dockerfile.ubi @@ -61,6 +61,14 @@ RUN \ dnf clean all && \ ln -sf /usr/bin/python3.14 /usr/bin/python +# A JRE for the self-contained Java classifier JAR (#1817): the JAR embeds libcuopt (statically +# linked) and its own companion libraries, so nothing here needs to link against the +# pip-installed libcuopt at all -- a plain runtime, not a full JDK, is enough. headless: no X11 +# dependency, matching this image having no display. Version matches +# java/cuopt/pom.xml's maven.compiler.release. +RUN dnf install -y java-17-openjdk-headless \ + && dnf clean all + FROM install-env AS cuopt-final # Register python via alternatives so the system alternatives database is @@ -110,6 +118,12 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC) COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ +# The self-contained Java classifier JAR matching this image's CUDA major version and +# architecture (see build_images.yaml, which downloads it from java-static-build before this +# build starts). Nothing else needs installing to use it: `java -cp cuopt.jar ...` or add it as +# a local Maven/Gradle file dependency. +COPY ./cuopt.jar /opt/cuopt/java/cuopt.jar + COPY ./entrypoint.sh /opt/cuopt/entrypoint.sh ENTRYPOINT ["/opt/cuopt/entrypoint.sh"] CMD ["python", "-m", "cuopt_server.cuopt_service"] From a80ed7287ffd965401c26568abf5055c26207a30 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 13:10:13 -0500 Subject: [PATCH 2/6] Revert "Bundle the self-contained Java classifier JAR in the official Docker images" This reverts commit b1d3ced36bf6269e304d5e2c609c9f53c207e335. --- .github/workflows/build.yaml | 1 - .github/workflows/build_images.yaml | 18 ------------------ ci/docker/Dockerfile | 29 +---------------------------- ci/docker/Dockerfile.ubi | 14 -------------- 4 files changed, 1 insertion(+), 61 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 255b400fd6..e52701dfc5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -634,7 +634,6 @@ jobs: build-images: needs: - build-details - - java-static-build - wheel-publish-cuopt - wheel-publish-cuopt-server - wheel-publish-cuopt-sh-client diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index d746feb686..4b7881e806 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -84,23 +84,6 @@ jobs: run: | echo "CUDA_SHORT=$(echo "$CUDA_VER" | sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+/\1/')" >> $GITHUB_OUTPUT echo "PYTHON_SHORT=$(echo "$PYTHON_VER" | sed -E 's/([0-9]+\.[0-9]+)\.[0-9]+/\1/')" >> $GITHUB_OUTPUT - echo "CUDA_MAJOR=$(echo "$CUDA_VER" | cut -d. -f1)" >> $GITHUB_OUTPUT - # The self-contained Java classifier JAR (see java-static-build in build.yaml, #1817) is - # built once per ARCH x CUDA-major combination in the same build.yaml run this workflow is - # always called from (build.yaml -> build_test_publish_images.yaml -> build_images.yaml), - # so it is available here as a same-run artifact -- no separate download/auth setup needed, - # unlike cross-run fetches (rapids-download-from-github). build-images (build.yaml) depends - # on java-static-build, so it is guaranteed to exist by the time this step runs. - - name: Download the Java classifier JAR for this image - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: cuopt_java_${{ matrix.ARCH }}_cu${{ steps.trim.outputs.CUDA_MAJOR }} - path: ./java-classifier-download - - name: Stage the classifier JAR into the Docker build context - run: | - . java/cuopt/ci/java_classifier.sh - jar="$(cuopt_java_resolve_artifact_jar ./java-classifier-download)" - cp "${jar}" ./ci/docker/context/cuopt.jar - name: Build image and push to DockerHub and NGC uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: @@ -114,7 +97,6 @@ jobs: sbom: false build-args: | CUDA_VER=${{ inputs.CUDA_VER }} - CUDA_MAJOR=${{ steps.trim.outputs.CUDA_MAJOR }} PYTHON_SHORT_VER=${{ steps.trim.outputs.PYTHON_SHORT }} CUOPT_VER=${{ inputs.CUOPT_VER }} LINUX_VER=${{ inputs.LINUX_VER }} diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index e3a2df5b90..5c6e4c3b39 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -89,35 +89,14 @@ RUN \ && apt-get purge -y --autoremove gcc \ && rm -rf /var/lib/apt/lists/* -# A JRE for the self-contained Java classifier JAR (#1817): the JAR embeds libcuopt (statically -# linked) and its own companion libraries, so nothing here needs to link against the -# pip-installed libcuopt at all -- a plain runtime, not a full JDK, is enough. headless: no X11 -# dependency, matching this image having no display. Version matches -# java/cuopt/pom.xml's maven.compiler.release. -RUN apt-get update \ - && apt-get install -y --no-install-recommends openjdk-17-jre-headless \ - && rm -rf /var/lib/apt/lists/* - FROM install-env AS cuopt-final ARG PYTHON_SHORT_VER -# Major CUDA version only (e.g. "13"), not the full CUDA_VER: Dockerfile ENV/ARG substitution -# has no shell-style ${VAR%%.*} parameter expansion, so this is computed once in -# build_images.yaml (which already needs it for the classifier JAR artifact name) and passed -# through as its own build-arg rather than recomputed here. -ARG CUDA_MAJOR # Make cuopt_grpc_server, cuopt_cli, and shared libraries available to all processes # (profile.d scripts are only sourced by login shells; ENV works for all containers) ENV PATH="/usr/local/cuda/bin:/usr/bin:/usr/local/bin:/usr/local/nvidia/bin/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/bin:${PATH}" -# .../nvidia/cu/lib holds the pip-installed CUDA runtime (libcublas, libcusparse, ...) -# this image actually ships -- there is no /usr/local/cuda/targets install here at all (a plain -# `-base` CUDA image, not `-runtime`/`-devel`). libcuopt.so itself doesn't need this on -# LD_LIBRARY_PATH: it carries its own $ORIGIN-relative RPATH into that same directory. The Java -# classifier JAR's JNI library (#1817) has no such RPATH -- it was built for a RAPIDS CI image -# where /usr/local/cuda/targets is fully populated -- so without this it fails with -# UnsatisfiedLinkError: libcublas.so.: cannot open shared object file. -ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/wsl/lib:/usr/lib/wsl/lib/libnvidia-container:/usr/lib/nvidia:/usr/lib/nvidia-current:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/lib/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/rapids_logger/lib64:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/nvidia/cu${CUDA_MAJOR}/lib:${LD_LIBRARY_PATH}" +ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/wsl/lib:/usr/lib/wsl/lib/libnvidia-container:/usr/lib/nvidia:/usr/lib/nvidia-current:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/lib/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/rapids_logger/lib64:${LD_LIBRARY_PATH}" # Directory creation, permissions RUN mkdir -p /opt/cuopt && \ @@ -140,12 +119,6 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC). COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ -# The self-contained Java classifier JAR matching this image's CUDA major version and -# architecture (see build_images.yaml, which downloads it from java-static-build before this -# build starts). Nothing else needs installing to use it: `java -cp cuopt.jar ...` or add it as -# a local Maven/Gradle file dependency. -COPY ./cuopt.jar /opt/cuopt/java/cuopt.jar - # Entrypoint supports server selection: # Default: Python REST server # CUOPT_SERVER_TYPE=grpc: gRPC server (uses CUOPT_SERVER_PORT, CUOPT_GPU_COUNT) diff --git a/ci/docker/Dockerfile.ubi b/ci/docker/Dockerfile.ubi index f73b7e341a..5668d7864f 100644 --- a/ci/docker/Dockerfile.ubi +++ b/ci/docker/Dockerfile.ubi @@ -61,14 +61,6 @@ RUN \ dnf clean all && \ ln -sf /usr/bin/python3.14 /usr/bin/python -# A JRE for the self-contained Java classifier JAR (#1817): the JAR embeds libcuopt (statically -# linked) and its own companion libraries, so nothing here needs to link against the -# pip-installed libcuopt at all -- a plain runtime, not a full JDK, is enough. headless: no X11 -# dependency, matching this image having no display. Version matches -# java/cuopt/pom.xml's maven.compiler.release. -RUN dnf install -y java-17-openjdk-headless \ - && dnf clean all - FROM install-env AS cuopt-final # Register python via alternatives so the system alternatives database is @@ -118,12 +110,6 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC) COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ -# The self-contained Java classifier JAR matching this image's CUDA major version and -# architecture (see build_images.yaml, which downloads it from java-static-build before this -# build starts). Nothing else needs installing to use it: `java -cp cuopt.jar ...` or add it as -# a local Maven/Gradle file dependency. -COPY ./cuopt.jar /opt/cuopt/java/cuopt.jar - COPY ./entrypoint.sh /opt/cuopt/entrypoint.sh ENTRYPOINT ["/opt/cuopt/entrypoint.sh"] CMD ["python", "-m", "cuopt_server.cuopt_service"] From 20648c76d3c3af820b37a3f3b3d7a6bdf2397042 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 15:03:25 -0500 Subject: [PATCH 3/6] Build Java bindings dynamically against the pip-installed libcuopt The self-contained classifier JAR approach (reverted in the prior commit) statically links libcuopt and its dependencies (TBB, NCCL, cuDSS, rmm, rapids_logger) into the JNI library, duplicating ~600MB of libraries the image already carries for the Python side via pip. Instead, compile libcuopt_jni.so inside each image against the libcuopt.so already pip-installed there: CUOPT_RUNTIME_LIBRARY_DIR bakes libcuopt's location into cuopt_jni's own RPATH, so no LD_LIBRARY_PATH changes are needed -- libcuopt.so resolves the rest of its dependency chain itself via its existing $ORIGIN-relative RPATH. The result is a 55KB jar + ~250KB .so per image instead of a ~600MB embedded copy. Verified locally end-to-end on both Dockerfile (Ubuntu) and Dockerfile.ubi (RHEL/UBI10): built each image, compiled and ran a small Java program against cuopt.jar with -Dcuopt.native.dir, and confirmed native library loading succeeds fully (no UnsatisfiedLinkError) and the call chain reaches cuOptSolve, which reports CUOPT_VALIDATION_ERROR only because this host's driver doesn't support the images' CUDA 13.3 requirement -- the same signature seen when verifying the (now-abandoned) static approach. UBI10's AppStream repo has no java-17-openjdk package; use java-21-openjdk (an LTS release satisfying build_native.sh's JDK 17+ requirement) instead. --- .github/workflows/build_images.yaml | 18 +++++++++ ci/docker/Dockerfile | 41 ++++++++++++++++++++ ci/docker/Dockerfile.ubi | 35 +++++++++++++++++ ci/docker/build_java_dynamic.sh | 58 +++++++++++++++++++++++++++++ ci/docker/build_java_dynamic_ubi.sh | 54 +++++++++++++++++++++++++++ 5 files changed, 206 insertions(+) create mode 100644 ci/docker/build_java_dynamic.sh create mode 100644 ci/docker/build_java_dynamic_ubi.sh diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index 4b7881e806..7503edfcc1 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -61,6 +61,24 @@ jobs: run: | git rev-parse HEAD > ./ci/docker/context/COMMIT_SHA git log -n1 --pretty='%ct' > ./ci/docker/context/COMMIT_TIME + # The Java bindings are built dynamically inside the image (both Dockerfile and + # Dockerfile.ubi, which share this staged context), against the libcuopt.so already + # pip-installed there -- rather than embedding a separate statically-linked copy + # (java-static-build, #1817), which would duplicate libcuopt/TBB/NCCL/cuDSS/rmm/ + # rapids_logger this image already carries for the Python side. That needs java/cuopt's + # own sources and two cpp/ directories the pip wheel doesn't ship: cpp/include (for headers + # newer than what's pip-installed, kept in step with this checkout) and cpp/src (internal + # headers cuopt_jni.cpp reaches, e.g. pdlp/cuopt_c_internal.hpp, never part of the + # public/pip include tree at all). + - name: Stage Java sources for the in-image dynamic build + run: | + mkdir -p ./ci/docker/context/repo + cp -r ./java ./ci/docker/context/repo/java + mkdir -p ./ci/docker/context/repo/cpp + cp -r ./cpp/include ./ci/docker/context/repo/cpp/include + cp -r ./cpp/src ./ci/docker/context/repo/cpp/src + cp ./ci/docker/build_java_dynamic.sh ./ci/docker/context/build_java_dynamic.sh + cp ./ci/docker/build_java_dynamic_ubi.sh ./ci/docker/context/build_java_dynamic_ubi.sh - name: Login to NGC uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 5c6e4c3b39..6454e0ea13 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -89,10 +89,46 @@ RUN \ && apt-get purge -y --autoremove gcc \ && rm -rf /var/lib/apt/lists/* +# Builds the Java bindings' JNI library dynamically against the libcuopt.so already +# pip-installed in install-env, instead of embedding a separate statically-linked copy (#1817), +# which would duplicate libcuopt/TBB/NCCL/cuDSS/rmm/rapids_logger this image already carries for +# the Python side. Its own stage: the build toolchain (JDK, cmake, ninja, g++) never needs to +# persist into cuopt-final, which only needs a JRE to run the result. See build_java_dynamic.sh. +FROM install-env AS java-build + +ARG PYTHON_SHORT_VER + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + openjdk-17-jdk \ + build-essential \ + cmake \ + && rm -rf /var/lib/apt/lists/* + +# find_package(CUDAToolkit REQUIRED) (java/cuopt/CMakeLists.txt) needs nvcc on PATH to detect +# the toolkit, even though nothing here actually compiles a .cu file (this project is CXX-only) +# -- this image's own base is `-base`, which has no CUDA toolkit at all, only the runtime driver +# API. Reuse the devel image already pulled for cuda-headers instead of installing the whole +# toolkit a second time via apt. +COPY --from=cuda-headers /usr/local/cuda /usr/local/cuda +ENV PATH="/usr/local/cuda/bin:${PATH}" + +COPY ./repo /build/repo +COPY ./build_java_dynamic.sh /build/build_java_dynamic.sh +RUN bash /build/build_java_dynamic.sh /build/repo "${PYTHON_SHORT_VER}" /opt/cuopt/java + FROM install-env AS cuopt-final ARG PYTHON_SHORT_VER +# A JRE to run the Java bindings (#1817): the JNI library was already built and linked (in the +# java-build stage) against libcuopt.so already installed here, so nothing beyond a runtime is +# needed -- headless: no X11 dependency, matching this image having no display. Version matches +# java/cuopt/pom.xml's maven.compiler.release. +RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-17-jre-headless \ + && rm -rf /var/lib/apt/lists/* + # Make cuopt_grpc_server, cuopt_cli, and shared libraries available to all processes # (profile.d scripts are only sourced by login shells; ENV works for all containers) ENV PATH="/usr/local/cuda/bin:/usr/bin:/usr/local/bin:/usr/local/nvidia/bin/:/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt/bin:${PATH}" @@ -119,6 +155,11 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC). COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ +# cuopt.jar (plain classes, no embedded native resources) and libcuopt_jni.so, built against +# this image's own libcuopt.so in the java-build stage. Use with, e.g.: +# java -Dcuopt.native.dir=/opt/cuopt/java -cp /opt/cuopt/java/cuopt.jar ... +COPY --from=java-build /opt/cuopt/java /opt/cuopt/java + # Entrypoint supports server selection: # Default: Python REST server # CUOPT_SERVER_TYPE=grpc: gRPC server (uses CUOPT_SERVER_PORT, CUOPT_GPU_COUNT) diff --git a/ci/docker/Dockerfile.ubi b/ci/docker/Dockerfile.ubi index 5668d7864f..83620c1e0a 100644 --- a/ci/docker/Dockerfile.ubi +++ b/ci/docker/Dockerfile.ubi @@ -61,8 +61,38 @@ RUN \ dnf clean all && \ ln -sf /usr/bin/python3.14 /usr/bin/python +# Builds the Java bindings' JNI library dynamically against the libcuopt.so already +# pip-installed in install-env, instead of embedding a separate statically-linked copy (#1817), +# which would duplicate libcuopt/TBB/NCCL/cuDSS/rmm/rapids_logger this image already carries for +# the Python side. Its own stage: the build toolchain (JDK, cmake, gcc-c++) never needs to persist +# into cuopt-final, which only needs a JRE to run the result. See build_java_dynamic.sh. +FROM install-env AS java-build + +# UBI10's AppStream repo only ships java-21-openjdk and java-25-openjdk, not 17 -- 21 is an LTS +# release satisfying build_native.sh's "JDK 17+" requirement, and javac --release 17 (used below) +# is supported by any JDK >= 17. +RUN dnf install -y java-21-openjdk-devel cmake gcc-c++ make && dnf clean all + +# find_package(CUDAToolkit REQUIRED) (java/cuopt/CMakeLists.txt) needs nvcc on PATH to detect +# the toolkit, even though nothing here actually compiles a .cu file (this project is CXX-only) +# -- this image's own base is `-base`, which has no CUDA toolkit at all, only the runtime driver +# API. Reuse the devel image already pulled for cuda-headers instead of installing the whole +# toolkit a second time via dnf. +COPY --from=cuda-headers /usr/local/cuda /usr/local/cuda +ENV PATH="/usr/local/cuda/bin:${PATH}" + +COPY ./repo /build/repo +COPY ./build_java_dynamic_ubi.sh /build/build_java_dynamic_ubi.sh +RUN bash /build/build_java_dynamic_ubi.sh /build/repo /opt/cuopt/java + FROM install-env AS cuopt-final +# A JRE to run the Java bindings (#1817): the JNI library was already built and linked (in the +# java-build stage) against libcuopt.so already installed here, so nothing beyond a runtime is +# needed -- headless: no X11 dependency, matching this image having no display. Version matches +# java/cuopt/pom.xml's maven.compiler.release. +RUN dnf install -y java-21-openjdk-headless && dnf clean all + # Register python via alternatives so the system alternatives database is # consistent with the /usr/bin/python symlink created in install-env. # NOTE: do NOT register python3 → python3.14 here: /usr/bin/dnf uses @@ -110,6 +140,11 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC) COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ +# cuopt.jar (plain classes, no embedded native resources) and libcuopt_jni.so, built against +# this image's own libcuopt.so in the java-build stage. Use with, e.g.: +# java -Dcuopt.native.dir=/opt/cuopt/java -cp /opt/cuopt/java/cuopt.jar ... +COPY --from=java-build /opt/cuopt/java /opt/cuopt/java + COPY ./entrypoint.sh /opt/cuopt/entrypoint.sh ENTRYPOINT ["/opt/cuopt/entrypoint.sh"] CMD ["python", "-m", "cuopt_server.cuopt_service"] diff --git a/ci/docker/build_java_dynamic.sh b/ci/docker/build_java_dynamic.sh new file mode 100644 index 0000000000..27c01e68ee --- /dev/null +++ b/ci/docker/build_java_dynamic.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Builds the Java bindings' native library (libcuopt_jni.so) against the libcuopt.so already +# pip-installed in this image, and compiles the plain (non-classifier) Java classes with javac +# rather than Maven -- java/cuopt/pom.xml has no non-test dependency at all, so there is nothing +# for Maven to resolve, and skipping it avoids adding a Maven Central/mirror network dependency +# to every nightly/branch image build. Run from the repo root staged under REPO_ROOT (see +# build_images.yaml); writes cuopt.jar and libcuopt_jni.so to OUT_DIR. + +set -euo pipefail + +REPO_ROOT="${1:?missing repo root}" +PYTHON_SHORT_VER="${2:?missing python short version, e.g. 3.14}" +OUT_DIR="${3:?missing output directory}" + +CUOPT_SITE_PACKAGES="/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt" +if [[ ! -f "${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" ]]; then + echo "libcuopt.so not found under ${CUOPT_SITE_PACKAGES}/lib64; is libcuopt pip-installed yet?" >&2 + exit 1 +fi + +# build_native.sh (java/cuopt/scripts) already knows how to point the JNI CMake build at an +# arbitrary cuOpt install; only the paths differ from its conda-prefix default. libcuopt.so +# resolves everything else (TBB, NCCL, cuDSS, rmm, rapids_logger, cublas, ...) itself through its +# own $ORIGIN-relative RPATH (see its rpath entries), so cuopt_jni.so only has to find libcuopt.so +# -- CUOPT_RUNTIME_LIBRARY_DIR bakes that into its own RPATH, no LD_LIBRARY_PATH needed. +export CUOPT_PREFIX="${CUOPT_SITE_PACKAGES}" +export CUOPT_LIBRARY="${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" +export CUOPT_RUNTIME_LIBRARY_DIR="${CUOPT_SITE_PACKAGES}/lib64" +# raft's headers ship bundled inside libcuopt's own include tree (dist-packages/libcuopt/include +# /raft), but rmm and rapids_logger are their own separate pip packages with their own include +# directories -- unlike a conda install, where CUOPT_PREFIX/include/rapids covers all three +# (see the CUOPT_PREFIX/include/rapids handling in java/cuopt/CMakeLists.txt, which does not +# apply here). +PIP_SITE_PACKAGES="$(dirname "${CUOPT_SITE_PACKAGES}")" +export CUOPT_EXTRA_INCLUDE_DIRS="${REPO_ROOT}/cpp/include;${REPO_ROOT}/cpp/src;${PIP_SITE_PACKAGES}/librmm/include;${PIP_SITE_PACKAGES}/rapids_logger/include" +export CUOPT_JAVA_NATIVE_BUILD_DIR="${REPO_ROOT}/java/cuopt/build/native" + +cd "${REPO_ROOT}" +bash java/cuopt/scripts/build_native.sh + +GEN_SRC_DIR="${REPO_ROOT}/java/cuopt/target/generated-sources/cuopt" +bash java/cuopt/scripts/generate_constants.sh \ + "${REPO_ROOT}/cpp/include/cuopt/mathematical_optimization/constants.h" \ + "${GEN_SRC_DIR}" + +CLASSES_DIR="$(mktemp -d)" +mapfile -t JAVA_SOURCES < <(find "${REPO_ROOT}/java/cuopt/src/main/java" "${GEN_SRC_DIR}" -name '*.java') +javac -d "${CLASSES_DIR}" --release 17 "${JAVA_SOURCES[@]}" + +mkdir -p "${OUT_DIR}" +jar cf "${OUT_DIR}/cuopt.jar" -C "${CLASSES_DIR}" . +cp "${CUOPT_JAVA_NATIVE_BUILD_DIR}/libcuopt_jni.so" "${OUT_DIR}/" +rm -rf "${CLASSES_DIR}" + +echo "Wrote ${OUT_DIR}/cuopt.jar and ${OUT_DIR}/libcuopt_jni.so" diff --git a/ci/docker/build_java_dynamic_ubi.sh b/ci/docker/build_java_dynamic_ubi.sh new file mode 100644 index 0000000000..0e705322d5 --- /dev/null +++ b/ci/docker/build_java_dynamic_ubi.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# UBI10 counterpart to build_java_dynamic.sh: same idea (build libcuopt_jni.so against the +# already pip-installed libcuopt.so, compile plain classes with javac, no Maven), but RHEL's +# platlib/purelib split puts libcuopt.so under lib64/python3.14/site-packages rather than +# Debian's single dist-packages tree (see Dockerfile.ubi's cuopt-final stage for the same +# lib64-vs-lib distinction on the Python side). Run from the repo root staged under REPO_ROOT +# (see build_images.yaml); writes cuopt.jar and libcuopt_jni.so to OUT_DIR. + +set -euo pipefail + +REPO_ROOT="${1:?missing repo root}" +OUT_DIR="${2:?missing output directory}" + +CUOPT_SITE_PACKAGES="/usr/local/lib64/python3.14/site-packages/libcuopt" +if [[ ! -f "${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" ]]; then + echo "libcuopt.so not found under ${CUOPT_SITE_PACKAGES}/lib64; is libcuopt pip-installed yet?" >&2 + exit 1 +fi + +# See build_java_dynamic.sh for why only CUOPT_LIBRARY/CUOPT_RUNTIME_LIBRARY_DIR need pointing +# at libcuopt.so directly: it resolves TBB/NCCL/cuDSS/rmm/rapids_logger/cublas/... itself via its +# own $ORIGIN-relative RPATH, so cuopt_jni.so only has to find libcuopt.so. +export CUOPT_PREFIX="${CUOPT_SITE_PACKAGES}" +export CUOPT_LIBRARY="${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" +export CUOPT_RUNTIME_LIBRARY_DIR="${CUOPT_SITE_PACKAGES}/lib64" +# rmm and rapids_logger are separate pip packages with their own include directories, also under +# lib64/python3.14/site-packages on UBI (platlib), unlike a conda install where +# CUOPT_PREFIX/include/rapids covers all three (java/cuopt/CMakeLists.txt's CUOPT_PREFIX/include +# /rapids handling does not apply here). +PIP_SITE_PACKAGES="$(dirname "${CUOPT_SITE_PACKAGES}")" +export CUOPT_EXTRA_INCLUDE_DIRS="${REPO_ROOT}/cpp/include;${REPO_ROOT}/cpp/src;${PIP_SITE_PACKAGES}/librmm/include;${PIP_SITE_PACKAGES}/rapids_logger/include" +export CUOPT_JAVA_NATIVE_BUILD_DIR="${REPO_ROOT}/java/cuopt/build/native" + +cd "${REPO_ROOT}" +bash java/cuopt/scripts/build_native.sh + +GEN_SRC_DIR="${REPO_ROOT}/java/cuopt/target/generated-sources/cuopt" +bash java/cuopt/scripts/generate_constants.sh \ + "${REPO_ROOT}/cpp/include/cuopt/mathematical_optimization/constants.h" \ + "${GEN_SRC_DIR}" + +CLASSES_DIR="$(mktemp -d)" +mapfile -t JAVA_SOURCES < <(find "${REPO_ROOT}/java/cuopt/src/main/java" "${GEN_SRC_DIR}" -name '*.java') +javac -d "${CLASSES_DIR}" --release 17 "${JAVA_SOURCES[@]}" + +mkdir -p "${OUT_DIR}" +jar cf "${OUT_DIR}/cuopt.jar" -C "${CLASSES_DIR}" . +cp "${CUOPT_JAVA_NATIVE_BUILD_DIR}/libcuopt_jni.so" "${OUT_DIR}/" +rm -rf "${CLASSES_DIR}" + +echo "Wrote ${OUT_DIR}/cuopt.jar and ${OUT_DIR}/libcuopt_jni.so" From 709b69468467d364215a36199fbc821ac3c54935 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 15:42:18 -0500 Subject: [PATCH 4/6] Trim comments in the Docker Java build changes Keep only the load-bearing why, not the full rationale already covered in the PR description. --- .github/workflows/build_images.yaml | 11 ++--------- ci/docker/Dockerfile | 22 +++++----------------- ci/docker/Dockerfile.ubi | 26 ++++++-------------------- ci/docker/build_java_dynamic.sh | 24 ++++++++---------------- ci/docker/build_java_dynamic_ubi.sh | 18 +++++------------- 5 files changed, 26 insertions(+), 75 deletions(-) diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index 7503edfcc1..649919ffff 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -61,15 +61,8 @@ jobs: run: | git rev-parse HEAD > ./ci/docker/context/COMMIT_SHA git log -n1 --pretty='%ct' > ./ci/docker/context/COMMIT_TIME - # The Java bindings are built dynamically inside the image (both Dockerfile and - # Dockerfile.ubi, which share this staged context), against the libcuopt.so already - # pip-installed there -- rather than embedding a separate statically-linked copy - # (java-static-build, #1817), which would duplicate libcuopt/TBB/NCCL/cuDSS/rmm/ - # rapids_logger this image already carries for the Python side. That needs java/cuopt's - # own sources and two cpp/ directories the pip wheel doesn't ship: cpp/include (for headers - # newer than what's pip-installed, kept in step with this checkout) and cpp/src (internal - # headers cuopt_jni.cpp reaches, e.g. pdlp/cuopt_c_internal.hpp, never part of the - # public/pip include tree at all). + # cpp/include and cpp/src aren't in the pip wheel; cuopt_jni.cpp needs both (the latter for + # internal headers like pdlp/cuopt_c_internal.hpp) to build the JNI library in-image. - name: Stage Java sources for the in-image dynamic build run: | mkdir -p ./ci/docker/context/repo diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 6454e0ea13..ce54fdb441 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -89,11 +89,7 @@ RUN \ && apt-get purge -y --autoremove gcc \ && rm -rf /var/lib/apt/lists/* -# Builds the Java bindings' JNI library dynamically against the libcuopt.so already -# pip-installed in install-env, instead of embedding a separate statically-linked copy (#1817), -# which would duplicate libcuopt/TBB/NCCL/cuDSS/rmm/rapids_logger this image already carries for -# the Python side. Its own stage: the build toolchain (JDK, cmake, ninja, g++) never needs to -# persist into cuopt-final, which only needs a JRE to run the result. See build_java_dynamic.sh. +# Own stage so the build toolchain doesn't persist into cuopt-final. See build_java_dynamic.sh. FROM install-env AS java-build ARG PYTHON_SHORT_VER @@ -105,11 +101,8 @@ RUN apt-get update \ cmake \ && rm -rf /var/lib/apt/lists/* -# find_package(CUDAToolkit REQUIRED) (java/cuopt/CMakeLists.txt) needs nvcc on PATH to detect -# the toolkit, even though nothing here actually compiles a .cu file (this project is CXX-only) -# -- this image's own base is `-base`, which has no CUDA toolkit at all, only the runtime driver -# API. Reuse the devel image already pulled for cuda-headers instead of installing the whole -# toolkit a second time via apt. +# CMake's find_package(CUDAToolkit REQUIRED) needs nvcc on PATH; reuse cuda-headers instead of +# installing the toolkit again via apt. COPY --from=cuda-headers /usr/local/cuda /usr/local/cuda ENV PATH="/usr/local/cuda/bin:${PATH}" @@ -121,10 +114,7 @@ FROM install-env AS cuopt-final ARG PYTHON_SHORT_VER -# A JRE to run the Java bindings (#1817): the JNI library was already built and linked (in the -# java-build stage) against libcuopt.so already installed here, so nothing beyond a runtime is -# needed -- headless: no X11 dependency, matching this image having no display. Version matches -# java/cuopt/pom.xml's maven.compiler.release. +# Headless JRE to run the Java bindings; version matches java/cuopt/pom.xml's maven.compiler.release. RUN apt-get update \ && apt-get install -y --no-install-recommends openjdk-17-jre-headless \ && rm -rf /var/lib/apt/lists/* @@ -155,9 +145,7 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC). COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ -# cuopt.jar (plain classes, no embedded native resources) and libcuopt_jni.so, built against -# this image's own libcuopt.so in the java-build stage. Use with, e.g.: -# java -Dcuopt.native.dir=/opt/cuopt/java -cp /opt/cuopt/java/cuopt.jar ... +# cuopt.jar + libcuopt_jni.so built in java-build against this image's own libcuopt.so. COPY --from=java-build /opt/cuopt/java /opt/cuopt/java # Entrypoint supports server selection: diff --git a/ci/docker/Dockerfile.ubi b/ci/docker/Dockerfile.ubi index 83620c1e0a..baf0c414c7 100644 --- a/ci/docker/Dockerfile.ubi +++ b/ci/docker/Dockerfile.ubi @@ -61,23 +61,14 @@ RUN \ dnf clean all && \ ln -sf /usr/bin/python3.14 /usr/bin/python -# Builds the Java bindings' JNI library dynamically against the libcuopt.so already -# pip-installed in install-env, instead of embedding a separate statically-linked copy (#1817), -# which would duplicate libcuopt/TBB/NCCL/cuDSS/rmm/rapids_logger this image already carries for -# the Python side. Its own stage: the build toolchain (JDK, cmake, gcc-c++) never needs to persist -# into cuopt-final, which only needs a JRE to run the result. See build_java_dynamic.sh. +# Own stage so the build toolchain doesn't persist into cuopt-final. See build_java_dynamic_ubi.sh. FROM install-env AS java-build -# UBI10's AppStream repo only ships java-21-openjdk and java-25-openjdk, not 17 -- 21 is an LTS -# release satisfying build_native.sh's "JDK 17+" requirement, and javac --release 17 (used below) -# is supported by any JDK >= 17. +# No java-17-openjdk on UBI10 AppStream (only 21, 25); 21 satisfies the JDK 17+ requirement. RUN dnf install -y java-21-openjdk-devel cmake gcc-c++ make && dnf clean all -# find_package(CUDAToolkit REQUIRED) (java/cuopt/CMakeLists.txt) needs nvcc on PATH to detect -# the toolkit, even though nothing here actually compiles a .cu file (this project is CXX-only) -# -- this image's own base is `-base`, which has no CUDA toolkit at all, only the runtime driver -# API. Reuse the devel image already pulled for cuda-headers instead of installing the whole -# toolkit a second time via dnf. +# CMake's find_package(CUDAToolkit REQUIRED) needs nvcc on PATH; reuse cuda-headers instead of +# installing the toolkit again via dnf. COPY --from=cuda-headers /usr/local/cuda /usr/local/cuda ENV PATH="/usr/local/cuda/bin:${PATH}" @@ -87,10 +78,7 @@ RUN bash /build/build_java_dynamic_ubi.sh /build/repo /opt/cuopt/java FROM install-env AS cuopt-final -# A JRE to run the Java bindings (#1817): the JNI library was already built and linked (in the -# java-build stage) against libcuopt.so already installed here, so nothing beyond a runtime is -# needed -- headless: no X11 dependency, matching this image having no display. Version matches -# java/cuopt/pom.xml's maven.compiler.release. +# Headless JRE to run the Java bindings; version matches java/cuopt/pom.xml's maven.compiler.release. RUN dnf install -y java-21-openjdk-headless && dnf clean all # Register python via alternatives so the system alternatives database is @@ -140,9 +128,7 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC) COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ -# cuopt.jar (plain classes, no embedded native resources) and libcuopt_jni.so, built against -# this image's own libcuopt.so in the java-build stage. Use with, e.g.: -# java -Dcuopt.native.dir=/opt/cuopt/java -cp /opt/cuopt/java/cuopt.jar ... +# cuopt.jar + libcuopt_jni.so built in java-build against this image's own libcuopt.so. COPY --from=java-build /opt/cuopt/java /opt/cuopt/java COPY ./entrypoint.sh /opt/cuopt/entrypoint.sh diff --git a/ci/docker/build_java_dynamic.sh b/ci/docker/build_java_dynamic.sh index 27c01e68ee..7df21079e9 100644 --- a/ci/docker/build_java_dynamic.sh +++ b/ci/docker/build_java_dynamic.sh @@ -2,12 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Builds the Java bindings' native library (libcuopt_jni.so) against the libcuopt.so already -# pip-installed in this image, and compiles the plain (non-classifier) Java classes with javac -# rather than Maven -- java/cuopt/pom.xml has no non-test dependency at all, so there is nothing -# for Maven to resolve, and skipping it avoids adding a Maven Central/mirror network dependency -# to every nightly/branch image build. Run from the repo root staged under REPO_ROOT (see -# build_images.yaml); writes cuopt.jar and libcuopt_jni.so to OUT_DIR. +# Builds libcuopt_jni.so against the pip-installed libcuopt.so and compiles the Java classes with +# javac (java/cuopt/pom.xml has no non-test dependency, so Maven isn't needed). Writes cuopt.jar +# and libcuopt_jni.so to OUT_DIR. set -euo pipefail @@ -21,19 +18,14 @@ if [[ ! -f "${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" ]]; then exit 1 fi -# build_native.sh (java/cuopt/scripts) already knows how to point the JNI CMake build at an -# arbitrary cuOpt install; only the paths differ from its conda-prefix default. libcuopt.so -# resolves everything else (TBB, NCCL, cuDSS, rmm, rapids_logger, cublas, ...) itself through its -# own $ORIGIN-relative RPATH (see its rpath entries), so cuopt_jni.so only has to find libcuopt.so -# -- CUOPT_RUNTIME_LIBRARY_DIR bakes that into its own RPATH, no LD_LIBRARY_PATH needed. +# libcuopt.so resolves its own dependencies (TBB, NCCL, cuDSS, rmm, rapids_logger, cublas, ...) +# via RPATH, so cuopt_jni.so only needs to find libcuopt.so itself -- CUOPT_RUNTIME_LIBRARY_DIR +# bakes that into its RPATH too, no LD_LIBRARY_PATH needed. export CUOPT_PREFIX="${CUOPT_SITE_PACKAGES}" export CUOPT_LIBRARY="${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" export CUOPT_RUNTIME_LIBRARY_DIR="${CUOPT_SITE_PACKAGES}/lib64" -# raft's headers ship bundled inside libcuopt's own include tree (dist-packages/libcuopt/include -# /raft), but rmm and rapids_logger are their own separate pip packages with their own include -# directories -- unlike a conda install, where CUOPT_PREFIX/include/rapids covers all three -# (see the CUOPT_PREFIX/include/rapids handling in java/cuopt/CMakeLists.txt, which does not -# apply here). +# rmm and rapids_logger are separate pip packages with their own include dirs (unlike conda, +# where CUOPT_PREFIX/include/rapids covers all three). PIP_SITE_PACKAGES="$(dirname "${CUOPT_SITE_PACKAGES}")" export CUOPT_EXTRA_INCLUDE_DIRS="${REPO_ROOT}/cpp/include;${REPO_ROOT}/cpp/src;${PIP_SITE_PACKAGES}/librmm/include;${PIP_SITE_PACKAGES}/rapids_logger/include" export CUOPT_JAVA_NATIVE_BUILD_DIR="${REPO_ROOT}/java/cuopt/build/native" diff --git a/ci/docker/build_java_dynamic_ubi.sh b/ci/docker/build_java_dynamic_ubi.sh index 0e705322d5..1f9a80fc39 100644 --- a/ci/docker/build_java_dynamic_ubi.sh +++ b/ci/docker/build_java_dynamic_ubi.sh @@ -2,12 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# UBI10 counterpart to build_java_dynamic.sh: same idea (build libcuopt_jni.so against the -# already pip-installed libcuopt.so, compile plain classes with javac, no Maven), but RHEL's -# platlib/purelib split puts libcuopt.so under lib64/python3.14/site-packages rather than -# Debian's single dist-packages tree (see Dockerfile.ubi's cuopt-final stage for the same -# lib64-vs-lib distinction on the Python side). Run from the repo root staged under REPO_ROOT -# (see build_images.yaml); writes cuopt.jar and libcuopt_jni.so to OUT_DIR. +# UBI10 counterpart to build_java_dynamic.sh: same approach, but libcuopt.so lives under +# lib64/python3.14/site-packages rather than Debian's dist-packages tree. set -euo pipefail @@ -20,16 +16,12 @@ if [[ ! -f "${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" ]]; then exit 1 fi -# See build_java_dynamic.sh for why only CUOPT_LIBRARY/CUOPT_RUNTIME_LIBRARY_DIR need pointing -# at libcuopt.so directly: it resolves TBB/NCCL/cuDSS/rmm/rapids_logger/cublas/... itself via its -# own $ORIGIN-relative RPATH, so cuopt_jni.so only has to find libcuopt.so. +# See build_java_dynamic.sh: libcuopt.so resolves its own dependencies via RPATH, so +# cuopt_jni.so only needs CUOPT_RUNTIME_LIBRARY_DIR pointing at it. export CUOPT_PREFIX="${CUOPT_SITE_PACKAGES}" export CUOPT_LIBRARY="${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" export CUOPT_RUNTIME_LIBRARY_DIR="${CUOPT_SITE_PACKAGES}/lib64" -# rmm and rapids_logger are separate pip packages with their own include directories, also under -# lib64/python3.14/site-packages on UBI (platlib), unlike a conda install where -# CUOPT_PREFIX/include/rapids covers all three (java/cuopt/CMakeLists.txt's CUOPT_PREFIX/include -# /rapids handling does not apply here). +# rmm and rapids_logger are separate pip packages with their own include dirs, also under lib64. PIP_SITE_PACKAGES="$(dirname "${CUOPT_SITE_PACKAGES}")" export CUOPT_EXTRA_INCLUDE_DIRS="${REPO_ROOT}/cpp/include;${REPO_ROOT}/cpp/src;${PIP_SITE_PACKAGES}/librmm/include;${PIP_SITE_PACKAGES}/rapids_logger/include" export CUOPT_JAVA_NATIVE_BUILD_DIR="${REPO_ROOT}/java/cuopt/build/native" From 9cc97db6ee63ae03e40308fa951d86f0c2c7deb6 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 21 Sep 2026 15:46:42 -0500 Subject: [PATCH 5/6] Drop remaining self-evident COPY comments --- ci/docker/Dockerfile | 1 - ci/docker/Dockerfile.ubi | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index ce54fdb441..845a6cc7f1 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -145,7 +145,6 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC). COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ -# cuopt.jar + libcuopt_jni.so built in java-build against this image's own libcuopt.so. COPY --from=java-build /opt/cuopt/java /opt/cuopt/java # Entrypoint supports server selection: diff --git a/ci/docker/Dockerfile.ubi b/ci/docker/Dockerfile.ubi index baf0c414c7..7b6cf3c7ab 100644 --- a/ci/docker/Dockerfile.ubi +++ b/ci/docker/Dockerfile.ubi @@ -128,7 +128,6 @@ COPY --from=cuda-libs /usr/local/cuda/lib64/libnvJitLink* /usr/local/cuda/lib64/ # Copy CUDA headers needed for runtime compilation (e.g., CuPy NVRTC) COPY --from=cuda-headers /usr/local/cuda/include/ /usr/local/cuda/include/ -# cuopt.jar + libcuopt_jni.so built in java-build against this image's own libcuopt.so. COPY --from=java-build /opt/cuopt/java /opt/cuopt/java COPY ./entrypoint.sh /opt/cuopt/entrypoint.sh From a92e14ce5e76b3c7a6c9f3532d490ff8159e019c Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 22 Sep 2026 09:41:39 -0500 Subject: [PATCH 6/6] Merge the Ubuntu/UBI Java build scripts into one build_java_dynamic.sh and build_java_dynamic_ubi.sh differed only in how they located libcuopt's pip site-packages dir. Take that path as a direct argument instead, and have each Dockerfile pass its own layout, so there's one script to maintain. --- .github/workflows/build_images.yaml | 1 - ci/docker/Dockerfile | 3 +- ci/docker/Dockerfile.ubi | 7 +++-- ci/docker/build_java_dynamic.sh | 3 +- ci/docker/build_java_dynamic_ubi.sh | 46 ----------------------------- 5 files changed, 7 insertions(+), 53 deletions(-) delete mode 100644 ci/docker/build_java_dynamic_ubi.sh diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index 649919ffff..dd9a0735e0 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -71,7 +71,6 @@ jobs: cp -r ./cpp/include ./ci/docker/context/repo/cpp/include cp -r ./cpp/src ./ci/docker/context/repo/cpp/src cp ./ci/docker/build_java_dynamic.sh ./ci/docker/context/build_java_dynamic.sh - cp ./ci/docker/build_java_dynamic_ubi.sh ./ci/docker/context/build_java_dynamic_ubi.sh - name: Login to NGC uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 845a6cc7f1..8bfa48ed0c 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -108,7 +108,8 @@ ENV PATH="/usr/local/cuda/bin:${PATH}" COPY ./repo /build/repo COPY ./build_java_dynamic.sh /build/build_java_dynamic.sh -RUN bash /build/build_java_dynamic.sh /build/repo "${PYTHON_SHORT_VER}" /opt/cuopt/java +RUN bash /build/build_java_dynamic.sh /build/repo \ + "/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt" /opt/cuopt/java FROM install-env AS cuopt-final diff --git a/ci/docker/Dockerfile.ubi b/ci/docker/Dockerfile.ubi index 7b6cf3c7ab..fb208ca5a8 100644 --- a/ci/docker/Dockerfile.ubi +++ b/ci/docker/Dockerfile.ubi @@ -61,7 +61,7 @@ RUN \ dnf clean all && \ ln -sf /usr/bin/python3.14 /usr/bin/python -# Own stage so the build toolchain doesn't persist into cuopt-final. See build_java_dynamic_ubi.sh. +# Own stage so the build toolchain doesn't persist into cuopt-final. See build_java_dynamic.sh. FROM install-env AS java-build # No java-17-openjdk on UBI10 AppStream (only 21, 25); 21 satisfies the JDK 17+ requirement. @@ -73,8 +73,9 @@ COPY --from=cuda-headers /usr/local/cuda /usr/local/cuda ENV PATH="/usr/local/cuda/bin:${PATH}" COPY ./repo /build/repo -COPY ./build_java_dynamic_ubi.sh /build/build_java_dynamic_ubi.sh -RUN bash /build/build_java_dynamic_ubi.sh /build/repo /opt/cuopt/java +COPY ./build_java_dynamic.sh /build/build_java_dynamic.sh +RUN bash /build/build_java_dynamic.sh /build/repo \ + /usr/local/lib64/python3.14/site-packages/libcuopt /opt/cuopt/java FROM install-env AS cuopt-final diff --git a/ci/docker/build_java_dynamic.sh b/ci/docker/build_java_dynamic.sh index 7df21079e9..d94a77660a 100644 --- a/ci/docker/build_java_dynamic.sh +++ b/ci/docker/build_java_dynamic.sh @@ -9,10 +9,9 @@ set -euo pipefail REPO_ROOT="${1:?missing repo root}" -PYTHON_SHORT_VER="${2:?missing python short version, e.g. 3.14}" +CUOPT_SITE_PACKAGES="${2:?missing path to the pip-installed libcuopt package, e.g. /usr/local/lib/python3.14/dist-packages/libcuopt}" OUT_DIR="${3:?missing output directory}" -CUOPT_SITE_PACKAGES="/usr/local/lib/python${PYTHON_SHORT_VER}/dist-packages/libcuopt" if [[ ! -f "${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" ]]; then echo "libcuopt.so not found under ${CUOPT_SITE_PACKAGES}/lib64; is libcuopt pip-installed yet?" >&2 exit 1 diff --git a/ci/docker/build_java_dynamic_ubi.sh b/ci/docker/build_java_dynamic_ubi.sh deleted file mode 100644 index 1f9a80fc39..0000000000 --- a/ci/docker/build_java_dynamic_ubi.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# UBI10 counterpart to build_java_dynamic.sh: same approach, but libcuopt.so lives under -# lib64/python3.14/site-packages rather than Debian's dist-packages tree. - -set -euo pipefail - -REPO_ROOT="${1:?missing repo root}" -OUT_DIR="${2:?missing output directory}" - -CUOPT_SITE_PACKAGES="/usr/local/lib64/python3.14/site-packages/libcuopt" -if [[ ! -f "${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" ]]; then - echo "libcuopt.so not found under ${CUOPT_SITE_PACKAGES}/lib64; is libcuopt pip-installed yet?" >&2 - exit 1 -fi - -# See build_java_dynamic.sh: libcuopt.so resolves its own dependencies via RPATH, so -# cuopt_jni.so only needs CUOPT_RUNTIME_LIBRARY_DIR pointing at it. -export CUOPT_PREFIX="${CUOPT_SITE_PACKAGES}" -export CUOPT_LIBRARY="${CUOPT_SITE_PACKAGES}/lib64/libcuopt.so" -export CUOPT_RUNTIME_LIBRARY_DIR="${CUOPT_SITE_PACKAGES}/lib64" -# rmm and rapids_logger are separate pip packages with their own include dirs, also under lib64. -PIP_SITE_PACKAGES="$(dirname "${CUOPT_SITE_PACKAGES}")" -export CUOPT_EXTRA_INCLUDE_DIRS="${REPO_ROOT}/cpp/include;${REPO_ROOT}/cpp/src;${PIP_SITE_PACKAGES}/librmm/include;${PIP_SITE_PACKAGES}/rapids_logger/include" -export CUOPT_JAVA_NATIVE_BUILD_DIR="${REPO_ROOT}/java/cuopt/build/native" - -cd "${REPO_ROOT}" -bash java/cuopt/scripts/build_native.sh - -GEN_SRC_DIR="${REPO_ROOT}/java/cuopt/target/generated-sources/cuopt" -bash java/cuopt/scripts/generate_constants.sh \ - "${REPO_ROOT}/cpp/include/cuopt/mathematical_optimization/constants.h" \ - "${GEN_SRC_DIR}" - -CLASSES_DIR="$(mktemp -d)" -mapfile -t JAVA_SOURCES < <(find "${REPO_ROOT}/java/cuopt/src/main/java" "${GEN_SRC_DIR}" -name '*.java') -javac -d "${CLASSES_DIR}" --release 17 "${JAVA_SOURCES[@]}" - -mkdir -p "${OUT_DIR}" -jar cf "${OUT_DIR}/cuopt.jar" -C "${CLASSES_DIR}" . -cp "${CUOPT_JAVA_NATIVE_BUILD_DIR}/libcuopt_jni.so" "${OUT_DIR}/" -rm -rf "${CLASSES_DIR}" - -echo "Wrote ${OUT_DIR}/cuopt.jar and ${OUT_DIR}/libcuopt_jni.so"