From 7c1123a7ce229d41838e2efdadf6290215c1c9c7 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 12:58:09 -0400 Subject: [PATCH 1/8] jenkins: run macos tests consistently, simplify parallelism --- Jenkinsfile | 86 +++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 48 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ad6d6e05fc..6289e52289d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ def props = [ - buildDiscarder(logRotator(numToKeepStr: '20', daysToKeepStr: '30')), + buildDiscarder(logRotator(numToKeepStr: '40', daysToKeepStr: '30')), parameters([ string(defaultValue: '', name: 'math_pr', description: "Leave blank " + "unless testing against a specific math repo pull request, " @@ -9,7 +9,6 @@ def props = [ string(defaultValue: 'nightly', name: 'stanc3_bin_url', description: 'Custom stanc3 binary url'), booleanParam(defaultValue: false, name: 'downsteam', description: 'Run downstream tests from math (was previously downstream_hotfix/downstream_tests [develop])'), - booleanParam(defaultValue: false, name: 'run_tests_all_os', description: 'Run unit and integration tests on all OS.'), booleanParam(defaultValue: false, name: 'compile_all_models', description: 'Run integration tests on the full test model suite.'), booleanParam(defaultValue: false, name: 'run_all', description: 'Pretend all files changes'), ]) @@ -100,7 +99,7 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) } if (runRemainingStages) { - stage('Unit tests') { + stage('Unit & integration tests') { def runUnit = { args -> def local = "CXX=$args.cxx\n$stanc3_bin_url" if (args.local) @@ -115,43 +114,6 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) junit 'test/**/*.xml' } - parallel windows: { - node('windows') { - stage('Windows Headers & Unit') { - checkout scm - bat """$WINSETENV - make -f lib/stan_math/make/standalone math-libs - """ - withEnv(["PATH+TBB=$WORKSPACE\\lib\\stan_math\\lib\\tbb"]) { - runUnit(cxx: WIN_CXX, pre: WINSETENV) - } - } - } - }, linux: { - runPod(image: image, gpus: 1) { - stage('Linux Unit') { - runUnit(cxx: LINUX_CXX, local: """ -STAN_OPENCL=true -OPENCL_PLATFORM_ID=0 -OPENCL_DEVICE_ID=0 -LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib -""") - } - } - }, mac: { - if (!params.downstream && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master") || params.run_tests_all_os) { - node('macos') { - stage('Mac Unit') { - checkout scm - runUnit(cxx: MAC_CXX) - } - } - } - } - } - - stage('Integration') { - // TODO: this was disabled before def integration_tests_flags = params.compile_all_models ? '--no-ignore-models' : '' def runIntegration = { args -> def pre = args.pre ?: '' @@ -182,21 +144,49 @@ LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib } } - parallel linux: { + parallel windowsUnit: { + node('windows') { + stage('Windows Headers & Unit') { + checkout scm + bat """$WINSETENV + make -f lib/stan_math/make/standalone math-libs + """ + withEnv(["PATH+TBB=$WORKSPACE\\lib\\stan_math\\lib\\tbb"]) { + runUnit(cxx: WIN_CXX, pre: WINSETENV) + } + } + } + }, linuxUnit: { + runPod(image: image, gpus: 1) { + stage('Linux Unit') { + runUnit(cxx: LINUX_CXX, local: """ +STAN_OPENCL=true +OPENCL_PLATFORM_ID=0 +OPENCL_DEVICE_ID=0 +LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib +""") + } + } + }, macUnit: { + node('macos') { + stage('Mac Unit') { + checkout scm + runUnit(cxx: MAC_CXX) + } + } + }, linuxInt: { runPod(image: image, checkout: false, cpus: 16, memory: '128Gi') { stage('Integration Linux') { runIntegration(local: "O=0\nCXX=${LINUX_CXX}") } } - }, mac: { - if (!params.downstream && (env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master') || params.run_tests_all_os) { - node('macos') { - stage('Integration Mac') { - runIntegration(local: "O=0\nCXX=${MAC_CXX}") - } + }, macInt: { + node('macos') { + stage('Integration Mac') { + runIntegration(local: "O=0\nCXX=${MAC_CXX}") } } - }, windows: { + }, windowsInt: { node('windows') { stage('Integration Windows') { withEnv(["PATH+TBB=${WORKSPACE}\\cmdstan\\stan\\lib\\stan_math\\lib\\tbb"]) { From 925061bdd0bd9556863ec7816575cc2b6d6d9a0c Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 14:29:04 -0400 Subject: [PATCH 2/8] jenkins: use O=0 on Windows for consistency --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6289e52289d..66c9e617242 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -190,7 +190,7 @@ LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib node('windows') { stage('Integration Windows') { withEnv(["PATH+TBB=${WORKSPACE}\\cmdstan\\stan\\lib\\stan_math\\lib\\tbb"]) { - runIntegration(local: "CXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) + runIntegration(local: "O=0\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) } } } From 321798309012d80bcb72c5cee81d6fa89b5c7713 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 15:05:01 -0400 Subject: [PATCH 3/8] jenkins: remove vestigial PATH+TBB environment variable --- Jenkinsfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 66c9e617242..5408d6f2c82 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -151,9 +151,7 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) bat """$WINSETENV make -f lib/stan_math/make/standalone math-libs """ - withEnv(["PATH+TBB=$WORKSPACE\\lib\\stan_math\\lib\\tbb"]) { - runUnit(cxx: WIN_CXX, pre: WINSETENV) - } + runUnit(cxx: WIN_CXX, pre: WINSETENV) } } }, linuxUnit: { @@ -189,9 +187,7 @@ LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib }, windowsInt: { node('windows') { stage('Integration Windows') { - withEnv(["PATH+TBB=${WORKSPACE}\\cmdstan\\stan\\lib\\stan_math\\lib\\tbb"]) { - runIntegration(local: "O=0\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) - } + runIntegration(local: "O=1\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) } } } From ee69b47839fc5d0f887b44a61be36dd08f7b97c1 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 15:16:54 -0400 Subject: [PATCH 4/8] restore --- Jenkinsfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5408d6f2c82..240d26d12cd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -151,7 +151,9 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) bat """$WINSETENV make -f lib/stan_math/make/standalone math-libs """ - runUnit(cxx: WIN_CXX, pre: WINSETENV) + withEnv(["PATH+TBB=$WORKSPACE\\lib\\stan_math\\lib\\tbb"]) { + runUnit(cxx: WIN_CXX, pre: WINSETENV) + } } } }, linuxUnit: { @@ -187,7 +189,9 @@ LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib }, windowsInt: { node('windows') { stage('Integration Windows') { - runIntegration(local: "O=1\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) + withEnv(["PATH+TBB=${WORKSPACE}\\cmdstan\\stan\\lib\\stan_math\\lib\\tbb"]) { + runIntegration(local: "O=1\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) + } } } } From b53c480e3ed8fdec2e16de9c9823cedd8b21ab14 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 15:51:39 -0400 Subject: [PATCH 5/8] jenkins: fail fast --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 240d26d12cd..70fb0b0e9ca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -108,9 +108,8 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) def pre = args.pre ?: '' batsh(pre + 'make -j$PARALLEL test-headers') batsh(pre + 'python3 runTests.py -j$PARALLEL src/test/unit --make-only') - catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { - batsh(pre + 'python3 runTests.py -j$PARALLEL src/test/unit') - } + batsh(pre + 'python3 runTests.py -j$PARALLEL src/test/unit') + junit 'test/**/*.xml' } @@ -144,7 +143,8 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) } } - parallel windowsUnit: { + parallel failFast: true, + windowsUnit: { node('windows') { stage('Windows Headers & Unit') { checkout scm @@ -190,7 +190,7 @@ LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib node('windows') { stage('Integration Windows') { withEnv(["PATH+TBB=${WORKSPACE}\\cmdstan\\stan\\lib\\stan_math\\lib\\tbb"]) { - runIntegration(local: "O=1\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) + runIntegration(local: "O=2\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) } } } From 7b558b743342e5b617377e85b65e57bec4bd9ed1 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 16:19:17 -0400 Subject: [PATCH 6/8] jenkins: update windows compiler flags --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 70fb0b0e9ca..ff2404029d8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ def image = 'stanorg/ci:v1' def commit def runRemainingStages = false def LINUX_CXX = 'clang++-7 -Werror -Wno-inconsistent-missing-override -Wno-error=return-type -Wno-error=division-by-zero' -def WIN_CXX = 'g++ -Werror -Wno-error=overloaded-virtual -Wno-error=template-id-cdtor -Wno-error=deprecated-declarations -Wno-error=cast-user-defined -Wno-error=unused-value -Wno-error=array-bounds' +def WIN_CXX = 'g++ -Werror -Wno-error=overloaded-virtual -Wno-error=template-id-cdtor -Wno-error=deprecated-declarations -Wno-error=cast-user-defined -Wno-error=unused-value -Wno-error=array-bounds -Wno-error=dangling-reference -Wno-error=return-type -Wno-error=div-by-zero -w -m64 -Wa,-mbig-obj' def MAC_CXX = 'clang++' // -Werror -Wno-inconsistent-missing-override -Wno-unused-but-set-variable def WINSETENV = ''' SET "PATH=%RTOOLS%\\x86_64-w64-mingw32.static.posix\\bin;%RTOOLS%;%RTOOLS%\\usr\\bin;%CONDA%;%PATH%" @@ -190,7 +190,7 @@ LDFLAGS_OPENCL=-L/usr/local/cuda/targets/x86_64-linux/lib node('windows') { stage('Integration Windows') { withEnv(["PATH+TBB=${WORKSPACE}\\cmdstan\\stan\\lib\\stan_math\\lib\\tbb"]) { - runIntegration(local: "O=2\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) + runIntegration(local: "O=0\nCXX=${WIN_CXX}\nPRECOMPILED_HEADERS=true\n", pre: WINSETENV) } } } From b6dadf77e7d5e8a17883bcf303f75a5b995df339 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 16:34:17 -0400 Subject: [PATCH 7/8] jenkins: break up integration calls like unit --- Jenkinsfile | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ff2404029d8..87f1dd2ea14 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,17 +129,14 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) writeFile(file: 'make/local', text: stanc3_bin_url) } writeFile(file: 'cmdstan/make/local', text: args.local+"\n$stanc3_bin_url") - batsh pre + """ - make -C cmdstan -j\$PARALLEL build - python3 ./runPerformanceTests.py -j\$PARALLEL $integration_tests_flags --runs=0 stanc3/test/integration/good - python3 ./runPerformanceTests.py -j\$PARALLEL $integration_tests_flags --runs=0 example-models - """ + batsh(pre + 'make -C cmdstan -j$PARALLEL build') + batsh(pre + "python3 ./runPerformanceTests.py -j\$PARALLEL $integration_tests_flags --runs=0 stanc3/test/integration/good") + batsh(pre + "python3 ./runPerformanceTests.py -j\$PARALLEL $integration_tests_flags --runs=0 example-models") + dir('cmdstan/stan') { - batsh pre + """ - python3 ./runTests.py src/test/integration/compile_standalone_functions_test.cpp - python3 ./runTests.py src/test/integration/standalone_functions_test.cpp - python3 ./runTests.py src/test/integration/multiple_translation_units_test.cpp - """ + batsh(pre + "python3 ./runTests.py src/test/integration/compile_standalone_functions_test.cpp") + batsh(pre + "python3 ./runTests.py src/test/integration/standalone_functions_test.cpp") + batsh(pre + "python3 ./runTests.py src/test/integration/multiple_translation_units_test.cpp") } } From 88c669f4f15f1f53c2b271ebbc02ec25ba929b5a Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Wed, 9 Sep 2026 16:37:31 -0400 Subject: [PATCH 8/8] jenkins: abortPrevious on PRs --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 87f1dd2ea14..1b9e2911971 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,7 +15,7 @@ def props = [ ] if (!params.downstream) { - props <<= disableConcurrentBuilds() + props <<= disableConcurrentBuilds(abortPrevious: env.BRANCH_NAME != 'develop') } properties(props)