From fa4ed87beeeb313a18ac12818b67e60280e410bb Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 2 Feb 2023 12:07:04 -0500 Subject: [PATCH 1/7] Build on Ubuntu Skip-PR-comments: true PR-repos-ubuntu20: dtcmp@PR-6:lastBuild mpich@PR-64:lastBuild lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos-mpich@PR-38:lastBuild hdf5@PR-38:lastBuild daos@PR-11205:73 Signed-off-by: Brian J. Murrell --- debian/changelog | 5 ++ debian/control | 65 ++++++++++++++++++++ debian/copyright | 48 +++++++++++++++ debian/libmfu-mpich-dev.install | 2 + debian/libmfu4-mpich.install | 1 + debian/mpifileutils.install | 1 + debian/rules | 20 +++++++ debian/source/format | 1 + packaging/Dockerfile.mockbuild | 20 ++++--- packaging/Dockerfile.ubuntu.20.04 | 63 ++++++++++++++------ packaging/Makefile_packaging.mk | 98 +++++++++++++++---------------- packaging/debian_chrootbuild | 23 +++++--- packaging/rpm_chrootbuild | 3 +- 13 files changed, 262 insertions(+), 88 deletions(-) create mode 100644 debian/changelog create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/libmfu-mpich-dev.install create mode 100644 debian/libmfu4-mpich.install create mode 100644 debian/mpifileutils.install create mode 100755 debian/rules create mode 100644 debian/source/format diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..167d187 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +mpifileutils (0.11.1-4) focal; urgency=medium + + * Initial packaging + + -- Brian J. Murrell Mon, 30 Jan 2023 08:02:22 -0500 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..9024a1a --- /dev/null +++ b/debian/control @@ -0,0 +1,65 @@ +Source: mpifileutils +Maintainer: DAOS +Uploaders: Brian J. Murrell +Section: devel +Priority: optional +Build-Depends: debhelper-compat (= 12), + cmake, + libbz2-dev, + libssl-dev, + uuid-dev, + libattr1-dev, + libdtcmp-mpich-dev, + libcircle-mpich-dev, + libhdf5-vol-daos-mpich-dev +Standards-Version: 4.4.1.0 +Vcs-Browser: https://github.com/hpc/mpifileutils +Vcs-Git: https://github.com/hpc/mpifileutils.git +Homepage: https://github.com/hpc/mpifileutils.git + +Package: mpifileutils +Architecture: any +Multi-Arch: foreign +Depends: ${misc:Depends}, + ${shlibs:Depends} +Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based + tools to manage large datasets, which may vary from large directory trees to + large files. High-performance computing users often generate large datasets + with parallel applications that run with many processes (millions in some + cases). However those users are then stuck with single-process tools like cp + and rm to manage their datasets. This suite provides MPI-based tools to handle + typical jobs like copy, remove, and compare for such datasets, providing + speedups of up to 20-30x. It also provides a library that simplifies the + creation of new tools or can be used in applications. + +Package: libmfu4-mpich +Architecture: linux-any +Depends: ${misc:Depends}, ${shlibs:Depends} +Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based + tools to manage large datasets, which may vary from large directory trees to + large files. High-performance computing users often generate large datasets + with parallel applications that run with many processes (millions in some + cases). However those users are then stuck with single-process tools like cp + and rm to manage their datasets. This suite provides MPI-based tools to handle + typical jobs like copy, remove, and compare for such datasets, providing + speedups of up to 20-30x. It also provides a library that simplifies the + creation of new tools or can be used in applications. + . + This package contains the shared libraries for use with MPICH2. + +Package: libmfu-mpich-dev +Section: libdevel +Architecture: linux-any +Depends: libmfu4-mpich (= ${binary:Version}), ${misc:Depends} +Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based + tools to manage large datasets, which may vary from large directory trees to + large files. High-performance computing users often generate large datasets + with parallel applications that run with many processes (millions in some + cases). However those users are then stuck with single-process tools like cp + and rm to manage their datasets. This suite provides MPI-based tools to handle + typical jobs like copy, remove, and compare for such datasets, providing + speedups of up to 20-30x. It also provides a library that simplifies the + creation of new tools or can be used in applications. + . + This package is needed to compile programs against libmfu4-mpich. + It contains the header files and links needed for compiling. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..991be4b --- /dev/null +++ b/debian/copyright @@ -0,0 +1,48 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: mpifileutils +Upstream-Contact: https://github.com/hpc/mpifileutils +Source: https://github.com/hpc/mpifileutils +Copyright: 2018, Lawrence Livermore National Security, LLC + 2006-2007,2011-2015, Los Alamos National Security, LLC + 2013-2015, UT-Battelle, LLC + 2015, DataDirect Networks, Inc. + +Files: * +Copyright: 2018, Lawrence Livermore National Security, LLC + 2006-2007,2011-2015, Los Alamos National Security, LLC + 2013-2015, UT-Battelle, LLC + 2015, DataDirect Networks, Inc. +License: BSD 3-Clause "New" or "Revised" License +BSD 3-Clause License + +Copyright (c) 2018, Lawrence Livermore National Security, LLC +Copyright (c) 2006-2007,2011-2015, Los Alamos National Security, LLC +Copyright (2013-2015) UT-Battelle, LLC +Copyright (c) 2015, DataDirect Networks, Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/debian/libmfu-mpich-dev.install b/debian/libmfu-mpich-dev.install new file mode 100644 index 0000000..a6d8936 --- /dev/null +++ b/debian/libmfu-mpich-dev.install @@ -0,0 +1,2 @@ +usr/include/* +usr/lib/x86_64-linux-gnu/lib*.so \ No newline at end of file diff --git a/debian/libmfu4-mpich.install b/debian/libmfu4-mpich.install new file mode 100644 index 0000000..8f99241 --- /dev/null +++ b/debian/libmfu4-mpich.install @@ -0,0 +1 @@ +usr/lib/x86_64-linux-gnu/lib*.so.* diff --git a/debian/mpifileutils.install b/debian/mpifileutils.install new file mode 100644 index 0000000..abb838e --- /dev/null +++ b/debian/mpifileutils.install @@ -0,0 +1 @@ +usr/bin/* \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..26717de --- /dev/null +++ b/debian/rules @@ -0,0 +1,20 @@ +#! /usr/bin/make -f + +export DH_VERBOSE=1 + +DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) + +%: + dh $@ + +override_dh_auto_configure: + dh_auto_configure -- -DENABLE_DAOS=ON \ + -DENABLE_LIBARCHIVE=OFF \ + -DENABLE_HDF5=ON \ + -DHDF5_INCLUDE_DIRS=/usr/include/hdf5/mpich \ + -DHDF5_LIBRARIES=/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5.so + +override_dh_auto_install-arch: + dh_auto_install + chrpath -d debian/tmp/usr/lib/*/*.so* debian/tmp/usr/bin/* + rm -f debian/tmp/usr/lib/*/*.la diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index e51357c..c804819 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -34,18 +34,24 @@ RUN echo "$USER:$PASSWD" | chpasswd # add the user to the mock group so it can run mock RUN usermod -a -G mock $USER +ARG CB0 +RUN dnf -y upgrade && \ + dnf clean all + # Monkey-patch rpmlint until a new release is made with # https://github.com/rpm-software-management/rpmlint/pull/795 in it +# But make sure to patch after dnf upgrade so that an upgraded rpmlint +# RPM doesn't wipe out our patch COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . -RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ - patch -p1 && \ - rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \ +RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ + if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ + if ! patch -p1; then \ + exit 1; \ + fi; \ + rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ + fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ rm -f rpmlint--ignore-unused-rpmlintrc.patch -ARG CB0 -RUN dnf -y upgrade && \ - dnf clean all - # show the release that was built ARG CACHEBUST RUN cat /etc/os-release diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index cf49ac8..ec76bfd 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -7,33 +7,60 @@ FROM ubuntu:20.04 LABEL org.opencontainers.image.authors="daos@daos.groups.io" -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 -ARG REPO_URL="" -ARG REPO_UBUNTU_20_04="" +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl gpg + +ARG REPO_FILE_URL +RUN if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/apt/sources.list.d && \ + curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ + "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ + true > ../sources.list && \ + mv daos_ci-ubuntu20.04-artifactory.list.tmp \ + daos_ci-ubuntu20.04-artifactory.list; \ + fi; \ + cd -; \ + curl -f -O "$REPO_FILE_URL"esad_repo.key; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key; \ + mkdir -p /usr/local/share/keyrings/; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg; \ + rm ./temp-keyring.gpg; \ + url_prefix=https://downloads.linux.hpe.com/SDR/; \ + for url in hpPublicKey2048.pub \ + hpPublicKey2048_key1.pub \ + hpePublicKey2048_key1.pub; do \ + curl -f -O "$url_prefix$url"; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import "$(basename $url)"; \ + done; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ + rm ./temp-keyring.gpg # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ - dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ - make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget cmake valgrind + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools -# rpmdevtools -RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - rpmdevtools +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 # Add build user (to keep chrootbuild happy) ENV USER build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot -RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ - echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ - fi; \ - echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ - chmod 0440 /etc/sudoers.d/build; \ - visudo -c; \ +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ sudo -l -U build diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 8e3e8d6..35b0422 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -64,9 +64,15 @@ RPMS = $(eval RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86 DEB_TOP := _topdir/BUILD DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) -SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) -PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +REAL_SOURCE ?= $(eval REAL_SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(REAL_SOURCE) +ifeq ($(ID_LIKE),debian) +ifneq ($(DEB_SOURCE),) +SOURCE ?= $(DEB_SOURCE) +endif +endif +SOURCE ?= $(REAL_SOURCE) +PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) @@ -170,6 +176,7 @@ $(DEB_TARBASE).orig.tar.$(SRC_EXT): $(DEB_BUILD).tar.$(SRC_EXT) ln -f $< $@ deb_detar: $(notdir $(SOURCE)) $(DEB_TARBASE).orig.tar.$(SRC_EXT) + echo "ID_LIKE: $(ID_LIKE)" # Unpack tarball rm -rf ./$(DEB_TOP)/.patched ./$(DEB_TOP)/.detar rm -rf ./$(DEB_BUILD)/* ./$(DEB_BUILD)/.pc ./$(DEB_BUILD)/.libs @@ -181,28 +188,38 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ $(DEB_BUILD)/debian/ mkdir -p ${DEB_BUILD}/debian/patches mkdir -p $(DEB_TOP)/patches - for f in $(PATCHES); do \ - rm -f $(DEB_TOP)/patches/*; \ - if git mailsplit -o$(DEB_TOP)/patches < "$$f" ;then \ - fn=$$(basename "$$f"); \ - for f1 in $(DEB_TOP)/patches/*;do \ - [ -e "$$f1" ] || continue; \ - f1n=$$(basename "$$f1"); \ - echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series ; \ - mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ - done; \ - else \ - fb=$$(basename "$$f"); \ - cp "$$f" $(DEB_BUILD)/debian/patches/ ; \ - echo "$$fb" >> $(DEB_BUILD)/debian/patches/series ; \ - if ! grep -q "^Description:\|^Subject:" "$$f" ;then \ - sed -i '1 iSubject: Auto added patch' \ - "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ - if ! grep -q "^Origin:\|^Author:\|^From:" "$$f" ;then \ - sed -i '1 iOrigin: other' \ - "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ - fi ; \ + if [ -f $(DEB_BUILD)/debian/patches/series ]; then \ + mv $(DEB_BUILD)/debian/patches/series \ + $(DEB_BUILD)/debian/patches/series.orig; \ + fi + for f in $(PATCHES); do \ + rm -f $(DEB_TOP)/patches/*; \ + if git mailsplit -o$(DEB_TOP)/patches < "$$f"; then \ + fn=$$(basename "$$f"); \ + for f1 in $(DEB_TOP)/patches/*;do \ + [ -e "$$f1" ] || continue; \ + f1n=$$(basename "$$f1"); \ + echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series; \ + mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ + done; \ + else \ + fb=$$(basename "$$f"); \ + cp "$$f" $(DEB_BUILD)/debian/patches/; \ + echo "$$fb" >> $(DEB_BUILD)/debian/patches/series; \ + if ! grep -q "^Description:\|^Subject:" "$$f"; then \ + sed -i '1 iSubject: Auto added patch' \ + "$(DEB_BUILD)/debian/patches/$$fb"; \ + fi; \ + if ! grep -q "^Origin:\|^Author:\|^From:" "$$f"; then \ + sed -i '1 iOrigin: other' \ + "$(DEB_BUILD)/debian/patches/$$fb"; \ + fi; \ + fi; \ done + if [ -f $(DEB_BUILD)/debian/patches/series.orig ]; then \ + cat $(DEB_BUILD)/debian/patches/series.orig >> \ + $(DEB_BUILD)/debian/patches/series; \ + fi touch $@ @@ -210,22 +227,11 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ ifeq ($(ID_LIKE),debian) $(DEB_TOP)/.deb_files: $(shell find $(TOPDIR)/debian -type f) deb_detar | \ $(DEB_BUILD)/debian/ - cd $(TOPDIR)/ && \ - find debian -maxdepth 1 -type f -exec cp '{}' '$(BUILD_PREFIX)/$(DEB_BUILD)/{}' ';' - if [ -e $(TOPDIR)/debian/source ]; then \ - cp -r $(TOPDIR)/debian/source $(DEB_BUILD)/debian; fi - if [ -e $(TOPDIR)/debian/local ]; then \ - cp -r $(TOPDIR)/debian/local $(DEB_BUILD)/debian; fi - if [ -e $(TOPDIR)/debian/examples ]; then \ - cp -r $(TOPDIR)/debian/examples $(DEB_BUILD)/debian; fi - if [ -e $(TOPDIR)/debian/upstream ]; then \ - cp -r $(TOPDIR)/debian/upstream $(DEB_BUILD)/debian; fi - if [ -e $(TOPDIR)/debian/tests ]; then \ - cp -r $(TOPDIR)/debian/tests $(DEB_BUILD)/debian; fi + cp -r $(TOPDIR)/debian $(DEB_BUILD)/ rm -f $(DEB_BUILD)/debian/*.ex $(DEB_BUILD)/debian/*.EX rm -f $(DEB_BUILD)/debian/*.orig ifneq ($(GIT_INFO),) - cd $(DEB_BUILD); dch --distribution unstable \ + cd $(DEB_BUILD); dch --distribution unstable \ --newversion $(DEB_PREV_RELEASE)$(GIT_INFO) \ "Git commit information" endif @@ -317,20 +323,6 @@ patch: echo "PKG_GIT_COMMIT is not defined" endif -# *_LOCAL_* repos are locally built packages. -ifeq ($(LOCAL_REPOS),true) - ifneq ($(ARTIFACTORY_URL),) - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) - DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not - # convert to artifactory url - DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists - # of values with spaces as environment variables - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)) - endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) - endif # ifneq ($(ARTIFACTORY_URL),) -endif # ifeq ($(LOCAL_REPOS),true) ifeq ($(ID_LIKE),debian) chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(call distro_map) \ @@ -346,6 +338,8 @@ chrootbuild: $(DEB_TOP)/$(DEB_DSC) DEB_TOP="$(DEB_TOP)" \ DEB_DSC="$(DEB_DSC)" \ DISTRO_ID_OPT="$(DISTRO_ID_OPT)" \ + LOCAL_REPOS='$(LOCAL_REPOS)' \ + ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ packaging/debian_chrootbuild else chrootbuild: $(SRPM) $(CALLING_MAKEFILE) @@ -360,7 +354,7 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) REPO_FILE_URL="$(REPO_FILE_URL)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ - DISTRO_REPOS='$(DISTRO_REPOS)' \ + LOCAL_REPOS='$(LOCAL_REPOS)' \ ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ DISTRO_VERSION="$(DISTRO_VERSION)" \ TARGET="$<" \ diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index ba82141..c0e7123 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,14 +2,13 @@ set -uex -# shellcheck disable=SC2153 -IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" +if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then + echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc +fi # shellcheck disable=SC2086 -sudo pbuilder create \ - --extrapackages "gnupg ca-certificates" \ - --othermirror \ - "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME universe|deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME-updates main universe" \ +sudo pbuilder create \ + --extrapackages "gnupg ca-certificates" \ $DISTRO_ID_OPT repo_args="" @@ -26,13 +25,19 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do fi repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do - repo_args="$repo_args|deb ${repo} $VERSION_CODENAME main" + +repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | + sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ + -e '/daos-stack-daos/d' \ + -e 's/signed-by=.*\.gpg/trusted=yes/' | + sed -e ':a; N; $!ba; s/\n/|/g')" +for repo in $JOB_REPOS; do + repo_args+="|deb ${repo} $VERSION_CODENAME main" done # NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. # After the build is updated to use 22.04, which supports go >= 1.18, it # should no longer be needed. -repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu focal main" +repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" echo "$repo_args" if [ "$repo_args" = "|" ]; then repo_args="" diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 2effdbb..fb6aa00 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -39,8 +39,7 @@ repo_dels=() echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" -if [ -n "${ARTIFACTORY_URL:-}" ] && - [ -n "$DISTRO_REPOS" ]; then +if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") if [ -n "${REPO_FILE_URL:-}" ]; then From 355791c16597e54680f591b37150be4b72453bfa Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 2 Feb 2023 12:19:55 -0500 Subject: [PATCH 2/7] Add ubuntu20 build target Skip-PR-comments: true PR-repos-ubuntu20: dtcmp@PR-6:lastBuild mpich@PR-64:lastBuild lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild hdf5@PR-38:lastBuild daos@PR-11205:73 Required-githooks: true Signed-off-by: Brian J. Murrell --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f197f20..3536e8f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!/usr/bin/groovy -/* Copyright (C) 2020 Intel Corporation +/* Copyright (C) 2019-2023 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -40,5 +40,5 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], 'test-tag': 'mfu']) - +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15', 'ubuntu20.04'], + 'test-tag': 'mfu']) From e7ddc31b20910bc438d8fb633a9397aa0830616d Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 2 Feb 2023 13:09:34 -0500 Subject: [PATCH 3/7] Add BD: libmpich-dev Signed-off-by: Brian J. Murrell Skip-PR-comments: true PR-repos-ubuntu20: dtcmp@PR-6:lastBuild mpich@PR-64:lastBuild lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild hdf5@PR-38:lastBuild daos@PR-11205:73 --- debian/control | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 9024a1a..bb94e72 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,8 @@ Build-Depends: debhelper-compat (= 12), libattr1-dev, libdtcmp-mpich-dev, libcircle-mpich-dev, - libhdf5-vol-daos-mpich-dev + libhdf5-vol-daos-mpich-dev, + libmpich-dev Standards-Version: 4.4.1.0 Vcs-Browser: https://github.com/hpc/mpifileutils Vcs-Git: https://github.com/hpc/mpifileutils.git From d6f3418457f7f38ffd1a705e31a1e5d8cc81b6c5 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 2 Feb 2023 13:24:39 -0500 Subject: [PATCH 4/7] Add BD: libhdf5-mpich-dev Signed-off-by: Brian J. Murrell Skip-PR-comments: true PR-repos-ubuntu20: dtcmp@PR-6:lastBuild mpich@PR-64:lastBuild lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild hdf5@PR-38:lastBuild daos@PR-11205:73 --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index bb94e72..0cea2cc 100644 --- a/debian/control +++ b/debian/control @@ -12,6 +12,7 @@ Build-Depends: debhelper-compat (= 12), libdtcmp-mpich-dev, libcircle-mpich-dev, libhdf5-vol-daos-mpich-dev, + libhdf5-mpich-dev, libmpich-dev Standards-Version: 4.4.1.0 Vcs-Browser: https://github.com/hpc/mpifileutils From b2129b1ba65c8187a7853c16cb63a1a6ae795e08 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 2 Feb 2023 13:39:41 -0500 Subject: [PATCH 5/7] Add BD: libdaos-dev Signed-off-by: Brian J. Murrell Skip-PR-comments: true PR-repos-ubuntu20: dtcmp@PR-6:lastBuild mpich@PR-64:lastBuild lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild hdf5@PR-38:lastBuild daos@PR-11205:73 --- debian/control | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 0cea2cc..e1ca2e8 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,8 @@ Build-Depends: debhelper-compat (= 12), libcircle-mpich-dev, libhdf5-vol-daos-mpich-dev, libhdf5-mpich-dev, - libmpich-dev + libmpich-dev, + libdaos-dev Standards-Version: 4.4.1.0 Vcs-Browser: https://github.com/hpc/mpifileutils Vcs-Git: https://github.com/hpc/mpifileutils.git From ff32351d06c602db518b5ed40697041db9f09d0c Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 2 Feb 2023 14:04:21 -0500 Subject: [PATCH 6/7] Add BD: chrpath Signed-off-by: Brian J. Murrell Skip-PR-comments: true PR-repos-ubuntu20: dtcmp@PR-6:lastBuild mpich@PR-64:lastBuild lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild hdf5@PR-38:lastBuild daos@PR-11205:73 --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index e1ca2e8..8148aea 100644 --- a/debian/control +++ b/debian/control @@ -5,6 +5,7 @@ Section: devel Priority: optional Build-Depends: debhelper-compat (= 12), cmake, + chrpath, libbz2-dev, libssl-dev, uuid-dev, From 00e6adee9b4e761a9814902643fa3240aa3c5e2d Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 30 May 2023 14:12:39 -0400 Subject: [PATCH 7/7] Build on EL9 Skip-packaging-check: true Skip-PR-comments: true Repo-files-PR: PR-16 PR-repos-ubuntu20: hdf5@PR-38:8 PR-repos: dtcmp@PR-6:lastBuild mpich@PR-64 lwgrp@PR-6:lastBuild libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild hdf5@PR-39:lastBuild daos@PR-12176:40 isa-l@PR-11 isa-l_crypto@PR-9 mercury@PR-102 Required-githooks: true Signed-off-by: Brian J. Murrell --- Jenkinsfile | 4 +- mpifileutils.spec | 2 +- packaging/Makefile_distro_vars.mk | 12 +++++ packaging/Makefile_packaging.mk | 87 +++++++++++++++---------------- packaging/debian_chrootbuild | 14 ++++- packaging/rpm_chrootbuild | 28 +++++++--- 6 files changed, 93 insertions(+), 54 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3536e8f..1cd6b27 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ */ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -//@Library(value="pipeline-lib@your_branch") _ +@Library(value="pipeline-lib@bmurrell/el9") _ -packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15', 'ubuntu20.04'], +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'el9', 'leap15', 'ubuntu20.04'], 'test-tag': 'mfu']) diff --git a/mpifileutils.spec b/mpifileutils.spec index 4c19608..8f1e703 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -78,7 +78,7 @@ BuildRequires: libattr-devel %if (0%{?suse_version} >= 1500) -BuildRequires: libfabric1 >= 1.12.0 +BuildRequires: libfabric1 >= 1.12.0, libfabric1 < 1.18 %endif %if (0%{?suse_version} > 0) diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index e9e0784..6a7f88b 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -47,6 +47,18 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 8 SED_EXPR := 1s/$(DIST)//p endif +ifeq ($(patsubst %epel-9-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 9 +DISTRO_ID := el9 +DISTRO_BASE := EL_9 +ifneq ($(DISTRO_VERSION_EL9),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL9) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 9 +SED_EXPR := 1s/$(DIST)//p +endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 DISTRO_ID := sl15.2 diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 35b0422..b19489e 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -39,6 +39,7 @@ PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') +EL_9_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el9: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') @@ -64,15 +65,9 @@ RPMS = $(eval RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86 DEB_TOP := _topdir/BUILD DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) -REAL_SOURCE ?= $(eval REAL_SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(REAL_SOURCE) -ifeq ($(ID_LIKE),debian) -ifneq ($(DEB_SOURCE),) -SOURCE ?= $(DEB_SOURCE) -endif -endif -SOURCE ?= $(REAL_SOURCE) -PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) +PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p))$(OTHER_SOURCES) SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) @@ -90,7 +85,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="el8" \ + el*) distro="$(DISTRO_ID)" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -176,7 +171,6 @@ $(DEB_TARBASE).orig.tar.$(SRC_EXT): $(DEB_BUILD).tar.$(SRC_EXT) ln -f $< $@ deb_detar: $(notdir $(SOURCE)) $(DEB_TARBASE).orig.tar.$(SRC_EXT) - echo "ID_LIKE: $(ID_LIKE)" # Unpack tarball rm -rf ./$(DEB_TOP)/.patched ./$(DEB_TOP)/.detar rm -rf ./$(DEB_BUILD)/* ./$(DEB_BUILD)/.pc ./$(DEB_BUILD)/.libs @@ -188,38 +182,28 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ $(DEB_BUILD)/debian/ mkdir -p ${DEB_BUILD}/debian/patches mkdir -p $(DEB_TOP)/patches - if [ -f $(DEB_BUILD)/debian/patches/series ]; then \ - mv $(DEB_BUILD)/debian/patches/series \ - $(DEB_BUILD)/debian/patches/series.orig; \ - fi - for f in $(PATCHES); do \ - rm -f $(DEB_TOP)/patches/*; \ - if git mailsplit -o$(DEB_TOP)/patches < "$$f"; then \ - fn=$$(basename "$$f"); \ - for f1 in $(DEB_TOP)/patches/*;do \ - [ -e "$$f1" ] || continue; \ - f1n=$$(basename "$$f1"); \ - echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series; \ - mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ - done; \ - else \ - fb=$$(basename "$$f"); \ - cp "$$f" $(DEB_BUILD)/debian/patches/; \ - echo "$$fb" >> $(DEB_BUILD)/debian/patches/series; \ - if ! grep -q "^Description:\|^Subject:" "$$f"; then \ - sed -i '1 iSubject: Auto added patch' \ - "$(DEB_BUILD)/debian/patches/$$fb"; \ - fi; \ - if ! grep -q "^Origin:\|^Author:\|^From:" "$$f"; then \ - sed -i '1 iOrigin: other' \ - "$(DEB_BUILD)/debian/patches/$$fb"; \ - fi; \ - fi; \ + for f in $(PATCHES); do \ + rm -f $(DEB_TOP)/patches/*; \ + if git mailsplit -o$(DEB_TOP)/patches < "$$f" ;then \ + fn=$$(basename "$$f"); \ + for f1 in $(DEB_TOP)/patches/*;do \ + [ -e "$$f1" ] || continue; \ + f1n=$$(basename "$$f1"); \ + echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series ; \ + mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ + done; \ + else \ + fb=$$(basename "$$f"); \ + cp "$$f" $(DEB_BUILD)/debian/patches/ ; \ + echo "$$fb" >> $(DEB_BUILD)/debian/patches/series ; \ + if ! grep -q "^Description:\|^Subject:" "$$f" ;then \ + sed -i '1 iSubject: Auto added patch' \ + "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ + if ! grep -q "^Origin:\|^Author:\|^From:" "$$f" ;then \ + sed -i '1 iOrigin: other' \ + "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ + fi ; \ done - if [ -f $(DEB_BUILD)/debian/patches/series.orig ]; then \ - cat $(DEB_BUILD)/debian/patches/series.orig >> \ - $(DEB_BUILD)/debian/patches/series; \ - fi touch $@ @@ -227,11 +211,22 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ ifeq ($(ID_LIKE),debian) $(DEB_TOP)/.deb_files: $(shell find $(TOPDIR)/debian -type f) deb_detar | \ $(DEB_BUILD)/debian/ - cp -r $(TOPDIR)/debian $(DEB_BUILD)/ + cd $(TOPDIR)/ && \ + find debian -maxdepth 1 -type f -exec cp '{}' '$(BUILD_PREFIX)/$(DEB_BUILD)/{}' ';' + if [ -e $(TOPDIR)/debian/source ]; then \ + cp -r $(TOPDIR)/debian/source $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/local ]; then \ + cp -r $(TOPDIR)/debian/local $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/examples ]; then \ + cp -r $(TOPDIR)/debian/examples $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/upstream ]; then \ + cp -r $(TOPDIR)/debian/upstream $(DEB_BUILD)/debian; fi + if [ -e $(TOPDIR)/debian/tests ]; then \ + cp -r $(TOPDIR)/debian/tests $(DEB_BUILD)/debian; fi rm -f $(DEB_BUILD)/debian/*.ex $(DEB_BUILD)/debian/*.EX rm -f $(DEB_BUILD)/debian/*.orig ifneq ($(GIT_INFO),) - cd $(DEB_BUILD); dch --distribution unstable \ + cd $(DEB_BUILD); dch --distribution unstable \ --newversion $(DEB_PREV_RELEASE)$(GIT_INFO) \ "Git commit information" endif @@ -277,6 +272,9 @@ $(DEB_TOP)/$(DEB_DSC): $(CALLING_MAKEFILE) $(DEB_BUILD).tar.$(SRC_EXT) \ cd $(DEB_BUILD); dpkg-buildpackage -S --no-sign --no-check-builddeps $(SRPM): $(SPEC) $(SOURCES) + if [ -f bz-1955184_find-requires ]; then \ + chmod 755 bz-1955184_find-requires; \ + fi rpmbuild -bs $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC) srpm: $(SRPM) @@ -419,6 +417,7 @@ packaging_check: --exclude install \ --exclude packaging \ --exclude utils \ + --exclude .vscode \ -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \ exit 1; \ fi diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index c0e7123..cc2cc96 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -12,6 +12,7 @@ sudo pbuilder create \ $DISTRO_ID_OPT repo_args="" +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -23,15 +24,26 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ - -e '/daos-stack-daos/d' \ -e 's/signed-by=.*\.gpg/trusted=yes/' | sed -e ':a; N; $!ba; s/\n/|/g')" for repo in $JOB_REPOS; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_args+="|deb ${repo} $VERSION_CODENAME main" done # NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index fb6aa00..bfb49ed 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -18,6 +18,11 @@ config_opts['module_setup_commands'] = [ ('disable', 'go-toolset') ] EOF +elif [[ $CHROOT_NAME == *epel-9-x86_64 ]]; then + # DO NOT LAND + # waiting for an answer on https://github.com/rpm-software-management/mock/discussions/1078 + ln -sf /etc/mock/almalinux-9-x86_64.cfg "$mock_config_dir" + ln -sf /etc/mock/rocky-9-x86_64.cfg "$mock_config_dir" fi # Use dnf on CentOS 7 @@ -65,6 +70,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then fi fi +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -76,20 +82,30 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi - repo_adds+=("--enablerepo $repo:$branch:$build_number") - echo -e "[$repo:$branch:$build_number]\n\ -name=$repo:$branch:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") + echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ +name=$repo:${branch//[@\/]/_}:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_adds+=("--enablerepo $repo_name") - echo -e "[${repo_name//@/_}]\n\ + echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ +baseurl=${repo//\//%2F}\n\ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file"