diff --git a/Jenkinsfile b/Jenkinsfile index f197f20..1cd6b27 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!/usr/bin/groovy -/* Copyright (C) 2020 Intel Corporation +/* Copyright (C) 2019-2023 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,7 +38,7 @@ */ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -//@Library(value="pipeline-lib@your_branch") _ - -packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], 'test-tag': 'mfu']) +@Library(value="pipeline-lib@bmurrell/el9") _ +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'el9', 'leap15', 'ubuntu20.04'], + 'test-tag': 'mfu']) diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..167d187 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +mpifileutils (0.11.1-4) focal; urgency=medium + + * Initial packaging + + -- Brian J. Murrell Mon, 30 Jan 2023 08:02:22 -0500 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..8148aea --- /dev/null +++ b/debian/control @@ -0,0 +1,69 @@ +Source: mpifileutils +Maintainer: DAOS +Uploaders: Brian J. Murrell +Section: devel +Priority: optional +Build-Depends: debhelper-compat (= 12), + cmake, + chrpath, + libbz2-dev, + libssl-dev, + uuid-dev, + libattr1-dev, + libdtcmp-mpich-dev, + libcircle-mpich-dev, + libhdf5-vol-daos-mpich-dev, + libhdf5-mpich-dev, + libmpich-dev, + libdaos-dev +Standards-Version: 4.4.1.0 +Vcs-Browser: https://github.com/hpc/mpifileutils +Vcs-Git: https://github.com/hpc/mpifileutils.git +Homepage: https://github.com/hpc/mpifileutils.git + +Package: mpifileutils +Architecture: any +Multi-Arch: foreign +Depends: ${misc:Depends}, + ${shlibs:Depends} +Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based + tools to manage large datasets, which may vary from large directory trees to + large files. High-performance computing users often generate large datasets + with parallel applications that run with many processes (millions in some + cases). However those users are then stuck with single-process tools like cp + and rm to manage their datasets. This suite provides MPI-based tools to handle + typical jobs like copy, remove, and compare for such datasets, providing + speedups of up to 20-30x. It also provides a library that simplifies the + creation of new tools or can be used in applications. + +Package: libmfu4-mpich +Architecture: linux-any +Depends: ${misc:Depends}, ${shlibs:Depends} +Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based + tools to manage large datasets, which may vary from large directory trees to + large files. High-performance computing users often generate large datasets + with parallel applications that run with many processes (millions in some + cases). However those users are then stuck with single-process tools like cp + and rm to manage their datasets. This suite provides MPI-based tools to handle + typical jobs like copy, remove, and compare for such datasets, providing + speedups of up to 20-30x. It also provides a library that simplifies the + creation of new tools or can be used in applications. + . + This package contains the shared libraries for use with MPICH2. + +Package: libmfu-mpich-dev +Section: libdevel +Architecture: linux-any +Depends: libmfu4-mpich (= ${binary:Version}), ${misc:Depends} +Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based + tools to manage large datasets, which may vary from large directory trees to + large files. High-performance computing users often generate large datasets + with parallel applications that run with many processes (millions in some + cases). However those users are then stuck with single-process tools like cp + and rm to manage their datasets. This suite provides MPI-based tools to handle + typical jobs like copy, remove, and compare for such datasets, providing + speedups of up to 20-30x. It also provides a library that simplifies the + creation of new tools or can be used in applications. + . + This package is needed to compile programs against libmfu4-mpich. + It contains the header files and links needed for compiling. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..991be4b --- /dev/null +++ b/debian/copyright @@ -0,0 +1,48 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: mpifileutils +Upstream-Contact: https://github.com/hpc/mpifileutils +Source: https://github.com/hpc/mpifileutils +Copyright: 2018, Lawrence Livermore National Security, LLC + 2006-2007,2011-2015, Los Alamos National Security, LLC + 2013-2015, UT-Battelle, LLC + 2015, DataDirect Networks, Inc. + +Files: * +Copyright: 2018, Lawrence Livermore National Security, LLC + 2006-2007,2011-2015, Los Alamos National Security, LLC + 2013-2015, UT-Battelle, LLC + 2015, DataDirect Networks, Inc. +License: BSD 3-Clause "New" or "Revised" License +BSD 3-Clause License + +Copyright (c) 2018, Lawrence Livermore National Security, LLC +Copyright (c) 2006-2007,2011-2015, Los Alamos National Security, LLC +Copyright (2013-2015) UT-Battelle, LLC +Copyright (c) 2015, DataDirect Networks, Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/debian/libmfu-mpich-dev.install b/debian/libmfu-mpich-dev.install new file mode 100644 index 0000000..a6d8936 --- /dev/null +++ b/debian/libmfu-mpich-dev.install @@ -0,0 +1,2 @@ +usr/include/* +usr/lib/x86_64-linux-gnu/lib*.so \ No newline at end of file diff --git a/debian/libmfu4-mpich.install b/debian/libmfu4-mpich.install new file mode 100644 index 0000000..8f99241 --- /dev/null +++ b/debian/libmfu4-mpich.install @@ -0,0 +1 @@ +usr/lib/x86_64-linux-gnu/lib*.so.* diff --git a/debian/mpifileutils.install b/debian/mpifileutils.install new file mode 100644 index 0000000..abb838e --- /dev/null +++ b/debian/mpifileutils.install @@ -0,0 +1 @@ +usr/bin/* \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..26717de --- /dev/null +++ b/debian/rules @@ -0,0 +1,20 @@ +#! /usr/bin/make -f + +export DH_VERBOSE=1 + +DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) + +%: + dh $@ + +override_dh_auto_configure: + dh_auto_configure -- -DENABLE_DAOS=ON \ + -DENABLE_LIBARCHIVE=OFF \ + -DENABLE_HDF5=ON \ + -DHDF5_INCLUDE_DIRS=/usr/include/hdf5/mpich \ + -DHDF5_LIBRARIES=/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5.so + +override_dh_auto_install-arch: + dh_auto_install + chrpath -d debian/tmp/usr/lib/*/*.so* debian/tmp/usr/bin/* + rm -f debian/tmp/usr/lib/*/*.la diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/mpifileutils.spec b/mpifileutils.spec index 4c19608..8f1e703 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -78,7 +78,7 @@ BuildRequires: libattr-devel %if (0%{?suse_version} >= 1500) -BuildRequires: libfabric1 >= 1.12.0 +BuildRequires: libfabric1 >= 1.12.0, libfabric1 < 1.18 %endif %if (0%{?suse_version} > 0) diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index e51357c..c804819 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -34,18 +34,24 @@ RUN echo "$USER:$PASSWD" | chpasswd # add the user to the mock group so it can run mock RUN usermod -a -G mock $USER +ARG CB0 +RUN dnf -y upgrade && \ + dnf clean all + # Monkey-patch rpmlint until a new release is made with # https://github.com/rpm-software-management/rpmlint/pull/795 in it +# But make sure to patch after dnf upgrade so that an upgraded rpmlint +# RPM doesn't wipe out our patch COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . -RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ - patch -p1 && \ - rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \ +RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ + if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ + if ! patch -p1; then \ + exit 1; \ + fi; \ + rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ + fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ rm -f rpmlint--ignore-unused-rpmlintrc.patch -ARG CB0 -RUN dnf -y upgrade && \ - dnf clean all - # show the release that was built ARG CACHEBUST RUN cat /etc/os-release diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index cf49ac8..ec76bfd 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -7,33 +7,60 @@ FROM ubuntu:20.04 LABEL org.opencontainers.image.authors="daos@daos.groups.io" -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 -ARG REPO_URL="" -ARG REPO_UBUNTU_20_04="" +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl gpg + +ARG REPO_FILE_URL +RUN if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/apt/sources.list.d && \ + curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ + "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ + true > ../sources.list && \ + mv daos_ci-ubuntu20.04-artifactory.list.tmp \ + daos_ci-ubuntu20.04-artifactory.list; \ + fi; \ + cd -; \ + curl -f -O "$REPO_FILE_URL"esad_repo.key; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key; \ + mkdir -p /usr/local/share/keyrings/; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg; \ + rm ./temp-keyring.gpg; \ + url_prefix=https://downloads.linux.hpe.com/SDR/; \ + for url in hpPublicKey2048.pub \ + hpPublicKey2048_key1.pub \ + hpePublicKey2048_key1.pub; do \ + curl -f -O "$url_prefix$url"; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import "$(basename $url)"; \ + done; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ + rm ./temp-keyring.gpg # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ - dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ - make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget cmake valgrind + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools -# rpmdevtools -RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - rpmdevtools +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 # Add build user (to keep chrootbuild happy) ENV USER build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot -RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ - echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ - fi; \ - echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ - chmod 0440 /etc/sudoers.d/build; \ - visudo -c; \ +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ sudo -l -U build diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index e9e0784..6a7f88b 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -47,6 +47,18 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 8 SED_EXPR := 1s/$(DIST)//p endif +ifeq ($(patsubst %epel-9-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 9 +DISTRO_ID := el9 +DISTRO_BASE := EL_9 +ifneq ($(DISTRO_VERSION_EL9),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL9) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 9 +SED_EXPR := 1s/$(DIST)//p +endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 DISTRO_ID := sl15.2 diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 8e3e8d6..b19489e 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -39,6 +39,7 @@ PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') +EL_9_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el9: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') @@ -66,7 +67,7 @@ DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p))$(OTHER_SOURCES) SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) @@ -84,7 +85,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="el8" \ + el*) distro="$(DISTRO_ID)" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -271,6 +272,9 @@ $(DEB_TOP)/$(DEB_DSC): $(CALLING_MAKEFILE) $(DEB_BUILD).tar.$(SRC_EXT) \ cd $(DEB_BUILD); dpkg-buildpackage -S --no-sign --no-check-builddeps $(SRPM): $(SPEC) $(SOURCES) + if [ -f bz-1955184_find-requires ]; then \ + chmod 755 bz-1955184_find-requires; \ + fi rpmbuild -bs $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC) srpm: $(SRPM) @@ -317,20 +321,6 @@ patch: echo "PKG_GIT_COMMIT is not defined" endif -# *_LOCAL_* repos are locally built packages. -ifeq ($(LOCAL_REPOS),true) - ifneq ($(ARTIFACTORY_URL),) - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) - DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not - # convert to artifactory url - DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists - # of values with spaces as environment variables - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) - endif # ifneq ($(ARTIFACTORY_URL),) -endif # ifeq ($(LOCAL_REPOS),true) ifeq ($(ID_LIKE),debian) chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(call distro_map) \ @@ -346,6 +336,8 @@ chrootbuild: $(DEB_TOP)/$(DEB_DSC) DEB_TOP="$(DEB_TOP)" \ DEB_DSC="$(DEB_DSC)" \ DISTRO_ID_OPT="$(DISTRO_ID_OPT)" \ + LOCAL_REPOS='$(LOCAL_REPOS)' \ + ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ packaging/debian_chrootbuild else chrootbuild: $(SRPM) $(CALLING_MAKEFILE) @@ -360,7 +352,7 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) REPO_FILE_URL="$(REPO_FILE_URL)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ - DISTRO_REPOS='$(DISTRO_REPOS)' \ + LOCAL_REPOS='$(LOCAL_REPOS)' \ ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ DISTRO_VERSION="$(DISTRO_VERSION)" \ TARGET="$<" \ @@ -425,6 +417,7 @@ packaging_check: --exclude install \ --exclude packaging \ --exclude utils \ + --exclude .vscode \ -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \ exit 1; \ fi diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index ba82141..cc2cc96 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,17 +2,17 @@ set -uex -# shellcheck disable=SC2153 -IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" +if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then + echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc +fi # shellcheck disable=SC2086 -sudo pbuilder create \ - --extrapackages "gnupg ca-certificates" \ - --othermirror \ - "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME universe|deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME-updates main universe" \ +sudo pbuilder create \ + --extrapackages "gnupg ca-certificates" \ $DISTRO_ID_OPT repo_args="" +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -24,15 +24,32 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do - repo_args="$repo_args|deb ${repo} $VERSION_CODENAME main" + +repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | + sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ + -e 's/signed-by=.*\.gpg/trusted=yes/' | + sed -e ':a; N; $!ba; s/\n/|/g')" +for repo in $JOB_REPOS; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") + repo_args+="|deb ${repo} $VERSION_CODENAME main" done # NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. # After the build is updated to use 22.04, which supports go >= 1.18, it # should no longer be needed. -repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu focal main" +repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" echo "$repo_args" if [ "$repo_args" = "|" ]; then repo_args="" diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 2effdbb..bfb49ed 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -18,6 +18,11 @@ config_opts['module_setup_commands'] = [ ('disable', 'go-toolset') ] EOF +elif [[ $CHROOT_NAME == *epel-9-x86_64 ]]; then + # DO NOT LAND + # waiting for an answer on https://github.com/rpm-software-management/mock/discussions/1078 + ln -sf /etc/mock/almalinux-9-x86_64.cfg "$mock_config_dir" + ln -sf /etc/mock/rocky-9-x86_64.cfg "$mock_config_dir" fi # Use dnf on CentOS 7 @@ -39,8 +44,7 @@ repo_dels=() echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" -if [ -n "${ARTIFACTORY_URL:-}" ] && - [ -n "$DISTRO_REPOS" ]; then +if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") if [ -n "${REPO_FILE_URL:-}" ]; then @@ -66,6 +70,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && fi fi +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -77,20 +82,30 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi - repo_adds+=("--enablerepo $repo:$branch:$build_number") - echo -e "[$repo:$branch:$build_number]\n\ -name=$repo:$branch:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") + echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ +name=$repo:${branch//[@\/]/_}:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_adds+=("--enablerepo $repo_name") - echo -e "[${repo_name//@/_}]\n\ + echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ +baseurl=${repo//\//%2F}\n\ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file"