From 420aff82c4acaf232128a5d32361f9509de2ad1f Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 24 Mar 2026 13:54:43 -0400 Subject: [PATCH 1/8] feature: add core unit testing to CI --- .gitlab/stage_unit.yml | 29 +++++++++++++++++++++++++++++ core/docker/Dockerfile | 11 ++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.gitlab/stage_unit.yml b/.gitlab/stage_unit.yml index 70e4e5a98..e1536ba73 100644 --- a/.gitlab/stage_unit.yml +++ b/.gitlab/stage_unit.yml @@ -1,6 +1,7 @@ --- include: - local: .gitlab/stage_build.yml + - local: .gitlab/common.yml run-ws-unit-job: # Either the web container needs to be rebuilt or the container needs to be @@ -126,3 +127,31 @@ run-authz-unit-job: - echo "-c 'cd /datafed/source; /opt/datafed/dependencies/bin/cmake --build build --target test'" >> run_globus.sh - chmod +x run_globus.sh - ./run_globus.sh + + +run-core-unit-job: + needs: ["run-core-build-job"] + stage: unit + variables: + PROJECT: "datafed" + COMPONENT: "core" + GIT_STRATEGY: clone + INTERMEDIATE_LAYER_NAME: "build" + tags: + - ci-datafed-core + - docker + script: + - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') + - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + - ./scripts/container_stop.sh -n "core-unit-" -p + - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1") + - | + docker run --rm \ + --name "core-unit-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" \ + --entrypoint bash \ + -t "${REGISTRY}/${PROJECT}/${COMPONENT}-${INTERMEDIATE_LAYER_NAME}-${BRANCH_LOWER}:${CI_COMMIT_SHA}" \ + -c 'cd /datafed/source && \ + /opt/datafed/dependencies/bin/ctest \ + --test-dir build \ + -LE "integration|fixture" \ + --output-on-failure' diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index b1981e9ba..dde9e0e7d 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -43,6 +43,11 @@ COPY ./cmake ${BUILD_DIR}/cmake COPY ./core/docker/entrypoint.sh ${BUILD_DIR}/core/docker/ COPY ./core/server ${BUILD_DIR}/core/server +# --------------------------------------------------------------------------- +# CHANGED: Enable unit and integration tests so the intermediate image +# contains all test binaries. The production 'core' stage below still only +# copies the server binary, so tests never land in the production image. +# --------------------------------------------------------------------------- RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ ${BUILD_DIR}/scripts/generate_datafed.sh && \ ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \ @@ -53,10 +58,14 @@ RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ -DBUILD_DOCS=False \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False \ - -DENABLE_INTEGRATION_TESTS=False + -DENABLE_UNIT_TESTS=True \ + -DENABLE_INTEGRATION_TESTS=True RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install +# =========================================================================== +# Production image — unchanged. Only the server binary is copied in. +# =========================================================================== FROM ${RUNTIME} AS core SHELL ["/bin/bash", "-c"] From 0cead873ed1b4efd169837f0867d9b93b2c867e5 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 24 Mar 2026 16:51:29 -0400 Subject: [PATCH 2/8] refactor: leave off integration tests for first step. --- core/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index dde9e0e7d..eb1ebc36e 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -59,7 +59,7 @@ RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False \ -DENABLE_UNIT_TESTS=True \ - -DENABLE_INTEGRATION_TESTS=True + -DENABLE_INTEGRATION_TESTS=False RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install From 3096a7a5ee39b228c17307bef022d900b491b7d2 Mon Sep 17 00:00:00 2001 From: Joshua S Brown Date: Wed, 25 Mar 2026 06:20:23 -0400 Subject: [PATCH 3/8] Apply suggestions from code review Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --- .gitlab/stage_unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/stage_unit.yml b/.gitlab/stage_unit.yml index e1536ba73..8a246e63e 100644 --- a/.gitlab/stage_unit.yml +++ b/.gitlab/stage_unit.yml @@ -142,7 +142,7 @@ run-core-unit-job: - docker script: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + - echo "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${HARBOR_USER}" --password-stdin - ./scripts/container_stop.sh -n "core-unit-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1") - | From 6be23f4200bb1c1da3c13b741522cfc896fdddb5 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 06:26:01 -0400 Subject: [PATCH 4/8] fix: make sure build container in intermediate stage exists. --- .gitlab/build/force_build_core_image.yml | 4 +++- .gitlab/build/retag_core_image.yml | 4 +++- .gitlab/stage_unit.yml | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab/build/force_build_core_image.yml b/.gitlab/build/force_build_core_image.yml index 1520cd505..cd0ad710b 100644 --- a/.gitlab/build/force_build_core_image.yml +++ b/.gitlab/build/force_build_core_image.yml @@ -14,7 +14,9 @@ build-core: GIT_STRATEGY: clone DOCKER_FILE_PATH: "core/docker/Dockerfile" DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count - BUILD_INTERMEDIATE: "FALSE" + BUILD_INTERMEDIATE: "TRUE" + INTERMEDIATE_TARGET: "core-build" # Name of the layer in the dockerfile + INTERMEDIATE_LAYER_NAME: "build" tags: - ci-datafed-core - docker diff --git a/.gitlab/build/retag_core_image.yml b/.gitlab/build/retag_core_image.yml index 8b719df82..6042b367b 100644 --- a/.gitlab/build/retag_core_image.yml +++ b/.gitlab/build/retag_core_image.yml @@ -13,6 +13,8 @@ retag-image: COMPONENT: "core" GIT_STRATEGY: clone DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count - BUILD_INTERMEDIATE: "FALSE" + BUILD_INTERMEDIATE: "TRUE" + INTERMEDIATE_TARGET: "core-build" # Name of the layer in the dockerfile + INTERMEDIATE_LAYER_NAME: "build" tags: - docker diff --git a/.gitlab/stage_unit.yml b/.gitlab/stage_unit.yml index 8a246e63e..bd14d9e33 100644 --- a/.gitlab/stage_unit.yml +++ b/.gitlab/stage_unit.yml @@ -128,7 +128,6 @@ run-authz-unit-job: - chmod +x run_globus.sh - ./run_globus.sh - run-core-unit-job: needs: ["run-core-build-job"] stage: unit From 24b056954e5dec3e2f224b16159336fb0530ef87 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 06:36:31 -0400 Subject: [PATCH 5/8] feat: enable building of core integration tests --- core/docker/Dockerfile | 2 +- tests/mock/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index eb1ebc36e..dde9e0e7d 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -59,7 +59,7 @@ RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False \ -DENABLE_UNIT_TESTS=True \ - -DENABLE_INTEGRATION_TESTS=False + -DENABLE_INTEGRATION_TESTS=True RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install diff --git a/tests/mock/CMakeLists.txt b/tests/mock/CMakeLists.txt index 0e8b7eeb0..759f52d49 100644 --- a/tests/mock/CMakeLists.txt +++ b/tests/mock/CMakeLists.txt @@ -8,7 +8,7 @@ foreach(PROG file(GLOB ${PROG}_SOURCES ${PROG}*.cpp) add_executable(mock_liveness_${PROG} ${${PROG}_SOURCES}) - target_link_libraries(mock_liveness_${PROG} PUBLIC ${DATAFED_BOOST_LIBRARIES} ${DATAFED_GSSAPI_LIBRARIES} ${DATAFED_GLOBUS_COMMON_LIBRARIES} common ) + target_link_libraries(mock_liveness_${PROG} PUBLIC ${DATAFED_BOOST_LIBRARIES} common ) if(BUILD_SHARED_LIBS) target_compile_definitions(mock_liveness_${PROG} PRIVATE BOOST_TEST_DYN_LINK) endif() From 51ced0ca67b6584287e0364f0ffc5c7816c8d3ec Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 07:01:44 -0400 Subject: [PATCH 6/8] fix: add intermediate to core image check. --- .gitlab/stage_image_check.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/stage_image_check.yml b/.gitlab/stage_image_check.yml index d29b8209f..5cf061748 100644 --- a/.gitlab/stage_image_check.yml +++ b/.gitlab/stage_image_check.yml @@ -18,7 +18,8 @@ check-core-image: variables: PROJECT: "datafed" COMPONENT: "core" - BUILD_INTERMEDIATE: "FALSE" + BUILD_INTERMEDIATE: "TRUE" + INTERMEDIATE_LAYER_NAME: "build" WATCHED_PATHS: "docker scripts core common CMakeLists.txt cmake .gitlab-ci.yml" check-repo-image: From b20add2ad6ff1823dde2bdeedb38ef9b5450e41c Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 07:23:08 -0400 Subject: [PATCH 7/8] fix: add missing test labels --- tests/CMakeLists.txt | 4 ++-- tests/mock/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1677976af..cf8f01d48 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,8 +27,8 @@ if( ENABLE_INTEGRATION_TESTS ) WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/mock_core ) - set_tests_properties(start_mock PROPERTIES FIXTURES_SETUP FIX_MOCK) - set_tests_properties(stop_mock PROPERTIES FIXTURES_CLEANUP FIX_MOCK) + set_tests_properties(start_mock PROPERTIES FIXTURES_SETUP FIX_MOCK LABELS "integration") + set_tests_properties(stop_mock PROPERTIES FIXTURES_CLEANUP FIX_MOCK LABELS "integration")) add_subdirectory(mock_core) add_subdirectory(mock) diff --git a/tests/mock/CMakeLists.txt b/tests/mock/CMakeLists.txt index 0e8b7eeb0..038f16943 100644 --- a/tests/mock/CMakeLists.txt +++ b/tests/mock/CMakeLists.txt @@ -19,6 +19,6 @@ foreach(PROG COMMAND ${CMAKE_CURRENT_BINARY_DIR}/mock_liveness_${PROG} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests/mock_core ) - set_tests_properties(mock_liveness_${PROG} PROPERTIES FIXTURES_REQUIRED FIX_MOCK) + set_tests_properties(mock_liveness_${PROG} PROPERTIES FIXTURES_REQUIRED FIX_MOCK LABELS "integration") endforeach(PROG) From 2cc37f93062cbdad46f79f48cebcf8e9a76137d7 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 25 Mar 2026 07:24:28 -0400 Subject: [PATCH 8/8] refactor: add core build intermediate stage to compose build --- scripts/compose_build_images.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/compose_build_images.sh b/scripts/compose_build_images.sh index 1536367b7..ee771de5f 100755 --- a/scripts/compose_build_images.sh +++ b/scripts/compose_build_images.sh @@ -73,6 +73,13 @@ if [[ "$BUILD_METADATA" == "TRUE" ]]; then --build-arg BASE_IMAGE=$BASE_IMAGE \ -t datafed-runtime:latest fi + docker build -f \ + "${PROJECT_ROOT}/core/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies:latest" \ + --build-arg RUNTIME="datafed-runtime:latest" \ + --target core-build \ + "${PROJECT_ROOT}" \ + -t datafed-core-build:latest docker build -f \ "${PROJECT_ROOT}/core/docker/Dockerfile" \ --build-arg DEPENDENCIES="datafed-dependencies:latest" \