Build against cuOpt 26.10 nightly: multi-GPU PDLP, new options, iteration/node reporting

.github/workflows/main-arm64.yml

@@ -28,9 +28,9 @@ jobs:
         run: |
           mkdir -p venvs
           uv venv venvs/cu12
-          uv pip install --python venvs/cu12 --extra-index-url=https://pypi.nvidia.com 'cuopt-cu12==26.8.*' -qq
+          uv pip install --python venvs/cu12 --prerelease=allow --index-strategy unsafe-best-match --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple --extra-index-url=https://pypi.nvidia.com 'libcuopt-cu12==26.10.0a223.post261002122804' -qq
           uv venv venvs/cu13
-          uv pip install --python venvs/cu13 --extra-index-url=https://pypi.nvidia.com 'cuopt-cu13==26.8.*' -qq
+          uv pip install --python venvs/cu13 --prerelease=allow --index-strategy unsafe-best-match --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple --extra-index-url=https://pypi.nvidia.com 'libcuopt-cu13==26.10.0a223.post261002122804' -qq
 
       # Get GAMS (ARM64 version)
       - name: Download and extract latest GAMS distribution
@@ -51,7 +51,7 @@ jobs:
           gcc -Wall gmscuopt.c -o gmscuopt-cu12.out \
             -DCUOPT_VERSION=\"$CUOPT_VERSION\" -DCUOPT_HASH=\"$CUOPT_HASH\" \
             -I $GAMSCAPI $GAMSCAPI/gmomcc.c $GAMSCAPI/optcc.c $GAMSCAPI/gevmcc.c \
-            -I $CUOPT/include $JITLINK/libnvJitLink.so.12 -L $CUOPT/lib64 -lcuopt
+            -I $CUOPT/include $JITLINK/libnvJitLink.so.12 -L $CUOPT/lib64 -lcuopt_mathopt
           patchelf --set-rpath \$ORIGIN gmscuopt-cu12.out
           export CUOPT="venvs/cu13/lib/python3.13/site-packages/libcuopt"
           export JITLINK="venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib"
@@ -60,7 +60,7 @@ jobs:
           gcc -Wall gmscuopt.c -o gmscuopt-cu13.out \
             -DCUOPT_VERSION=\"$CUOPT_VERSION\" -DCUOPT_HASH=\"$CUOPT_HASH\" \
             -I $GAMSCAPI $GAMSCAPI/gmomcc.c $GAMSCAPI/optcc.c $GAMSCAPI/gevmcc.c \
-            -I $CUOPT/include $JITLINK/libnvJitLink.so.13 -L $CUOPT/lib64 -lcuopt
+            -I $CUOPT/include $JITLINK/libnvJitLink.so.13 -L $CUOPT/lib64 -lcuopt_mathopt
           patchelf --set-rpath \$ORIGIN gmscuopt-cu13.out
 
       # Collect dependencies for link and runtime convenience archive
@@ -69,17 +69,18 @@ jobs:
           mkdir release-cu12
           cp gmscuopt-cu12.out release-cu12/gmscuopt.out
           cp assets/* release-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcuopt.so release-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/libraft_cu12.libs/libgomp-*.so* release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_mathopt.so release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_client.so release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libgomp-*.so* release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libtbb-*.so* release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libtbbmalloc-*.so* release-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libomp-*.so* release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcudss_mtlayer_cuopt.so release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libcudart-*.so* release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/rapids_logger/lib64/librapids_logger.so release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/librmm/lib64/librmm.so release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/nccl/lib/libnccl.so* release-cu12/
           mkdir runtime-cu12
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cu12/lib/libcudss.so* runtime-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/nvidia/cu12/lib/libcudss_mtlayer_gomp.so* runtime-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cusolver/lib/libcusolver.so* runtime-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cublas/lib/libcublas.so* runtime-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cublas/lib/libcublasLt.so* runtime-cu12/
@@ -90,17 +91,18 @@ jobs:
           mkdir release-cu13
           cp gmscuopt-cu13.out release-cu13/gmscuopt.out
           cp assets/* release-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcuopt.so release-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/libraft_cu13.libs/libgomp-*.so* release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_mathopt.so release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_client.so release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libgomp-*.so* release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libtbb-*.so* release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libtbbmalloc-*.so* release-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libomp-*.so* release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcudss_mtlayer_cuopt.so release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libcudart-*.so* release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/rapids_logger/lib64/librapids_logger.so release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/librmm/lib64/librmm.so release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/nccl/lib/libnccl.so* release-cu13/
           mkdir runtime-cu13
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcudss.so* runtime-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcudss_mtlayer_gomp.so* runtime-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libnvJitLink.so* runtime-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcublas.so* runtime-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcublasLt.so* runtime-cu13/

.github/workflows/main-x86_64.yml

@@ -28,9 +28,9 @@ jobs:
         run: |
           mkdir -p venvs
           uv venv venvs/cu12
-          uv pip install --python venvs/cu12 --extra-index-url=https://pypi.nvidia.com 'cuopt-cu12==26.8.*' -qq
+          uv pip install --python venvs/cu12 --prerelease=allow --index-strategy unsafe-best-match --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple --extra-index-url=https://pypi.nvidia.com 'libcuopt-cu12==26.10.0a223.post261002122804' -qq
           uv venv venvs/cu13
-          uv pip install --python venvs/cu13 --extra-index-url=https://pypi.nvidia.com 'cuopt-cu13==26.8.*' -qq
+          uv pip install --python venvs/cu13 --prerelease=allow --index-strategy unsafe-best-match --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple --extra-index-url=https://pypi.nvidia.com 'libcuopt-cu13==26.10.0a223.post261002122804' -qq
 
       # Get GAMS
       - name: Download and extract latest GAMS distribution
@@ -51,7 +51,7 @@ jobs:
           gcc -Wall gmscuopt.c -o gmscuopt-cu12.out \
             -DCUOPT_VERSION=\"$CUOPT_VERSION\" -DCUOPT_HASH=\"$CUOPT_HASH\" \
             -I $GAMSCAPI $GAMSCAPI/gmomcc.c $GAMSCAPI/optcc.c $GAMSCAPI/gevmcc.c \
-            -I $CUOPT/include $JITLINK/libnvJitLink.so.12 -L $CUOPT/lib64 -lcuopt
+            -I $CUOPT/include $JITLINK/libnvJitLink.so.12 -L $CUOPT/lib64 -lcuopt_mathopt
           patchelf --set-rpath \$ORIGIN gmscuopt-cu12.out
           export CUOPT="venvs/cu13/lib/python3.13/site-packages/libcuopt"
           export JITLINK="venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib"
@@ -60,7 +60,7 @@ jobs:
           gcc -Wall gmscuopt.c -o gmscuopt-cu13.out \
             -DCUOPT_VERSION=\"$CUOPT_VERSION\" -DCUOPT_HASH=\"$CUOPT_HASH\" \
             -I $GAMSCAPI $GAMSCAPI/gmomcc.c $GAMSCAPI/optcc.c $GAMSCAPI/gevmcc.c \
-            -I $CUOPT/include $JITLINK/libnvJitLink.so.13 -L $CUOPT/lib64 -lcuopt
+            -I $CUOPT/include $JITLINK/libnvJitLink.so.13 -L $CUOPT/lib64 -lcuopt_mathopt
           patchelf --set-rpath \$ORIGIN gmscuopt-cu13.out
 
       # Collect dependencies for link and runtime convenience archive
@@ -69,17 +69,18 @@ jobs:
           mkdir release-cu12
           cp gmscuopt-cu12.out release-cu12/gmscuopt.out
           cp assets/* release-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcuopt.so release-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/libraft_cu12.libs/libgomp-*.so* release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_mathopt.so release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_client.so release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libgomp-*.so* release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libtbb-*.so* release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libtbbmalloc-*.so* release-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libomp-*.so* release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt/lib64/libcudss_mtlayer_cuopt.so release-cu12/
+          cp venvs/cu12/lib/python3.13/site-packages/libcuopt_cu12.libs/libcudart-*.so* release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/rapids_logger/lib64/librapids_logger.so release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/librmm/lib64/librmm.so release-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/nccl/lib/libnccl.so* release-cu12/
           mkdir runtime-cu12
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cu12/lib/libcudss.so* runtime-cu12/
-          cp venvs/cu12/lib/python3.13/site-packages/nvidia/cu12/lib/libcudss_mtlayer_gomp.so* runtime-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cusolver/lib/libcusolver.so* runtime-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cublas/lib/libcublas.so* runtime-cu12/
           cp venvs/cu12/lib/python3.13/site-packages/nvidia/cublas/lib/libcublasLt.so* runtime-cu12/
@@ -90,17 +91,18 @@ jobs:
           mkdir release-cu13
           cp gmscuopt-cu13.out release-cu13/gmscuopt.out
           cp assets/* release-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcuopt.so release-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/libraft_cu13.libs/libgomp-*.so* release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_mathopt.so release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcuopt_client.so release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libgomp-*.so* release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libtbb-*.so* release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libtbbmalloc-*.so* release-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libomp-*.so* release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt/lib64/libcudss_mtlayer_cuopt.so release-cu13/
+          cp venvs/cu13/lib/python3.13/site-packages/libcuopt_cu13.libs/libcudart-*.so* release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/rapids_logger/lib64/librapids_logger.so release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/librmm/lib64/librmm.so release-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/nccl/lib/libnccl.so* release-cu13/
           mkdir runtime-cu13
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcudss.so* runtime-cu13/
-          cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcudss_mtlayer_gomp.so* runtime-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libnvJitLink.so* runtime-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcublas.so* runtime-cu13/
           cp venvs/cu13/lib/python3.13/site-packages/nvidia/cu13/lib/libcublasLt.so* runtime-cu13/

README.md

@@ -14,8 +14,9 @@ Supported model types are LP, MIP, RMIP, QCP, RMIQCP. QCP and RMIQCP models must
 - **CPU architecture:** x86_64, arm64
 - **GAMS:** Version 54 or newer
 - **GAMSPy:** Version 1.12.1 or newer
-- **NVIDIA GPU:** Volta architecture or better
+- **NVIDIA GPU:** Volta architecture or better with CUDA 12, Turing architecture or better with CUDA 13
 - **CUDA Runtime Libraries:** 12 or 13
+- **System libraries:** OpenSSL 3 (`libssl.so.3`, `libcrypto.so.3`) and zlib (`libz.so.1`)
 
 ## Installation using `fetch-cuoptlink.py`
 
@@ -70,9 +71,9 @@ python fetch-cuoptlink.py uninstall -g /opt/gams/gams55.0
 - Make sure [CUDA runtime](https://developer.nvidia.com/cuda-downloads?target_os=Linux) is installed
 - Download and unpack `cuopt-link-release-cu12-{x86_64,arm64}.zip` or `cuopt-link-release-cu13-{x86_64,arm64}.zip` (for CUDA 12 and 13 respectively) from the [releases page](https://github.com/GAMS-dev/cuoptlink-builder/releases):
     - Unpack the contents of `cuopt-link-release-cu*-*.zip` into your GAMS system directory. For GAMSPy, you can find out your system directory by running `gamspy show base`. So for example you can run `unzip -o cuopt-link-release-cu*-*.zip -d $(gamspy show base)`.
-    - **Caution:** This will overwrite any existing `gamsconfig.yaml` file in that directory. The contained `gamsconfig.yaml` contains a `solverConfig` section to make cuOpt available to GAMS.
+    - The archive contains a `gamsconfig_cuopt.yaml` with a `solverConfig` section that makes cuOpt available to GAMS. If the GAMS system directory has no `gamsconfig.yaml` yet, copy it, e.g. `cp gamsconfig_cuopt.yaml gamsconfig.yaml`. Otherwise add its `solverConfig` entry to the existing `gamsconfig.yaml` (`fetch-cuoptlink.py` does this merge automatically).
 
-The neccessary files from the CUDA 12 or 13 runtime can also be downloaded as convenient archive `cu12-runtime-{x86_64,arm64}.zip` or `cu13-runtime-{x86_64,arm64}.zip` from the [releases page](https://github.com/GAMS-dev/cuoptlink-builder/releases).
+The necessary files from the CUDA 12 or 13 runtime can also be downloaded as convenient archive `cu12-runtime-{x86_64,arm64}.zip` or `cu13-runtime-{x86_64,arm64}.zip` from the [releases page](https://github.com/GAMS-dev/cuoptlink-builder/releases).
 
 ## Test the setup
 
@@ -82,16 +83,36 @@ gamslib trnsport
 gams trnsport lp cuopt
 ```
 
+## Multi-GPU PDLP
+
+With cuOpt 26.10 or newer, LPs can be solved with PDLP on several GPUs at once. This requires both `method 1` (PDLP) and `num_gpus -1` (all visible GPUs) or `num_gpus` greater than 1 in the option file:
+```
+* cuopt.opt
+method 1
+num_gpus -1
+```
+```
+gams mymodel lp=cuopt optfile=1
+```
+
+- Without `method 1`, i.e. in the default concurrent mode, `num_gpus 2` instead runs PDLP and barrier in parallel on two GPUs.
+- `multigpu_pdlp_partitioner` selects how the problem is split across the GPUs: `0` auto (default), `1` KaMinPar (better balanced, extra partitioning time), `2` round robin.
+- The GPUs used can be restricted with `CUDA_VISIBLE_DEVICES`, e.g. `CUDA_VISIBLE_DEVICES=0,1 gams mymodel lp=cuopt optfile=1`.
+- Only LPs are supported, and the whole problem currently has to fit into the memory of a single GPU.
+- Starting values (levels and marginals) from GAMS are not passed to multi-GPU PDLP.
+- Convergence can differ from single-GPU PDLP, so some models need noticeably more iterations.
+
 ## Examples
 
 ### Notebooks
 
 - [examples/trnsport_cuopt.ipynb](examples/trnsport_cuopt.ipynb) for CUDA 12 on x86_64
-- [examples/trnsport_cuopt.ipynb](examples/trnsport_cuopt_cu13.ipynb) for CUDA 13 on x86_64
+- [examples/trnsport_cuopt_cu13.ipynb](examples/trnsport_cuopt_cu13.ipynb) for CUDA 13 on x86_64
 
 ### GAMS models
 
 Various GAMS models can be found in subfolder `examples/models` and are used to verify the solver link.
+
 ### Regression tests
 
 The self-checking models in `examples/models/regression_tests` cover dual signs and reduced costs, QP/QCQP marginals, RMIQCP, option handling, error reporting, LP limit points, GMO handling (e.g. `=N=` rows, `requestMarginals=2`), rejection of unsupported features (SOS, semi-integer, MIQCP) and solve/model status mapping. Each model aborts if a result deviates from the reference values (obtained with CPLEX). Run them all against the GAMS system found in your `PATH` (it needs a GPU and the installed solver link):
@@ -101,3 +122,13 @@ examples/models/regression_tests/run_tests.sh
 ```
 
 The script prints `[PASS]` or `[FAIL]` per model and keeps the listing and log file of failed models for inspection. Its exit code is the number of failed models.
+
+### gamslib regression test
+
+`tests/test-gamslib-cuopt.py` solves the gamslib models listed in `tests/baseline.txt` with cuOpt and compares the objective values against a CPLEX baseline. It needs a GPU and the installed solver link:
+
+```
+python3 tests/test-gamslib-cuopt.py -g <GAMS system directory> -j 2   # -g defaults to the GAMS found in PATH
+```
+
+Use `-r` to change the time limit per model (default 120s; an optional fourth column in `tests/baseline.txt` sets a model-specific time limit instead), `-t` for the relative tolerance (default 1e-6) and pass model names to run only a subset. The exit code is 1 if any model mismatches or fails to produce a solution.