From 8da659632a30c256ffcd8aa3bf2ccc5a1325d8b5 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Fri, 27 Feb 2026 00:01:47 -0500 Subject: [PATCH 1/5] services/nomad/build/build-rsyncd: handle multilib properly --- services/nomad/build/rsync-post-xfer | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/services/nomad/build/rsync-post-xfer b/services/nomad/build/rsync-post-xfer index 7d70ceef..b2ed8561 100644 --- a/services/nomad/build/rsync-post-xfer +++ b/services/nomad/build/rsync-post-xfer @@ -11,19 +11,19 @@ process_repo() { fi # add to repodata find "$RSYNC_MODULE_PATH"/"$repo" -maxdepth 1 -type f -printf "$TGT_BASE/$TGT/$repo/%f\0" | \ - xargs -r0 xbps-rindex -a + xargs -r0 env XBPS_TARGET_ARCH="$ARCH" xbps-rindex -a # clean old packages from repodata - xbps-rindex -c "$TGT_BASE/$TGT/$repo" + XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo" # remove old packages - xbps-rindex -r "$TGT_BASE/$TGT/$repo" + XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo" # Remove signatures that don't have a corresponding package find "$TGT_BASE/$TGT/$repo" -maxdepth 1 \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) \ -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} + } -export XBPS_TARGET_ARCH="${RSYNC_MODULE_NAME#*-}" +ARCH="${RSYNC_MODULE_NAME#*-}" -case "$XBPS_TARGET_ARCH" in +case "$ARCH" in aarch64*) TGT="aarch64" ;; *-musl) TGT="musl" ;; *) TGT="" ;; @@ -36,11 +36,14 @@ for repo in / /bootstrap /debug /nonfree; do process_repo "$repo" done -if [ "$XBPS_TARGET_ARCH" = 'i686' ]; then - for repo in /multilib /multilib/bootstrap /multilib/nonfree; do - process_repo "$repo" - done -fi +case "$ARCH" in + x86_64|i686) + ARCH=x86_64 + for repo in /multilib /multilib/bootstrap /multilib/nonfree; do + process_repo "$repo" + done + ;; +esac # clean up incoming rm -r "${RSYNC_MODULE_PATH:?}"/* From fd7824600a167c33dc295d2fa4d15b048c7566cc Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Tue, 17 Feb 2026 12:22:53 -0500 Subject: [PATCH 2/5] services/nomad/build/build-rsyncd,buildbot: improve sync/index of packages * to prevent re-syncing packages between each package build, use --remove-source-files * to prevent races with syncing and signing, update the repo as a pre-xfer script and collect the log to show in the builder output --- services/nomad/build/build-rsyncd.nomad | 13 ++++-- services/nomad/build/buildbot.cfg | 35 +++++++++++++-- services/nomad/build/rsync-post-xfer | 51 ---------------------- services/nomad/build/rsync-update-repo | 58 +++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 58 deletions(-) delete mode 100644 services/nomad/build/rsync-post-xfer create mode 100644 services/nomad/build/rsync-update-repo diff --git a/services/nomad/build/build-rsyncd.nomad b/services/nomad/build/build-rsyncd.nomad index 0235dd19..0a8f2621 100644 --- a/services/nomad/build/build-rsyncd.nomad +++ b/services/nomad/build/build-rsyncd.nomad @@ -65,8 +65,8 @@ job "build-rsyncd" { } template { - data = file("rsync-post-xfer") - destination = "local/rsync-post-xfer" + data = file("rsync-update-repo") + destination = "local/rsync-update-repo" perms = "0755" } @@ -114,8 +114,13 @@ EOF [incoming-${template.value}] path = /incoming/${template.value} auth users = buildsync:rw -filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* -post-xfer exec = /local/rsync-post-xfer +filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* - *.log + +[updaterepo-${template.value}] +path = /incoming/${template.value} +auth users = buildsync:r +filter = + update-repo.log - * - .* +early exec = /local/rsync-update-repo EOF destination = "local/rsyncd.conf.d/${template.value}.conf.inc" } diff --git a/services/nomad/build/buildbot.cfg b/services/nomad/build/buildbot.cfg index cc3391fe..cb12745f 100644 --- a/services/nomad/build/buildbot.cfg +++ b/services/nomad/build/buildbot.cfg @@ -10,7 +10,6 @@ from pathlib import Path from twisted.internet import defer from twisted.python import log -from buildbot.process.results import SUCCESS from buildbot.plugins import util, secrets, reporters, worker, schedulers from buildbot.plugins import steps @@ -222,20 +221,45 @@ def make_rsync_cmd(props): return ['bash', '-e', '-c', util.Interpolate(""" echo "=> Syncing packages to the shadow repository..." -rsync -vurk --delete-after \ +rsync -vurk --delete-after --remove-source-files \ --filter='+ */' --filter='+ *.%(prop:target)s.xbps' \ --filter='- .*' --filter='- *' \ --password-file=/secrets/rsync/password /hostdir/binpkgs/ \ {{ range nomadService 1 $allocID "build-rsyncd" -}} rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s {{ end -}} -case "$?" in +RSYNC_EXIT="$?" +echo "=> Cleaning repodatas..." +find /hostdir/binpkgs -name '%(prop:target)s-repodata' -print -delete +case "$RSYNC_EXIT" in 0|23|24) exit 0 ;; *) exit 1 ;; esac """)] +@util.renderer +def make_update_repo_cmd(props): + return ['bash', '-e', '-c', + util.Interpolate(f""" +echo "=> Moving packages to the shadow repository..." +rsync -vurk --delete-after --password-file=/secrets/rsync/password \ +{{ range nomadService 1 $allocID "build-rsyncd" -}} +rsync://buildsync@{{ .Address }}:{{ .Port }}/updaterepo-%(prop:target)s/update-repo.log ./update-repo.log +{{ end -}} +case "$?" in + 0|23|24) + cat ./update-repo.log + exit 0 + ;; + *) + cat ./update-repo.log + exit 1 + ;; +esac +""")] + + @util.renderer def make_clean_cmd(props): return [ @@ -259,6 +283,11 @@ def build_packages(props): logname=f'sync:{p}', haltOnFailure=True, )) + cmds.append(util.ShellArg( + command=make_update_repo_cmd, + logname=f'update-repo:{p}', + haltOnFailure=True, + )) if cmds: cmds.append(util.ShellArg( command=['make', 'clean'], diff --git a/services/nomad/build/rsync-post-xfer b/services/nomad/build/rsync-post-xfer deleted file mode 100644 index b2ed8561..00000000 --- a/services/nomad/build/rsync-post-xfer +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -set -e - -TGT_BASE="/mirror/current" - -process_repo() { - repo="$1" - if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then - return - fi - # add to repodata - find "$RSYNC_MODULE_PATH"/"$repo" -maxdepth 1 -type f -printf "$TGT_BASE/$TGT/$repo/%f\0" | \ - xargs -r0 env XBPS_TARGET_ARCH="$ARCH" xbps-rindex -a - # clean old packages from repodata - XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo" - # remove old packages - XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo" - # Remove signatures that don't have a corresponding package - find "$TGT_BASE/$TGT/$repo" -maxdepth 1 \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) \ - -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} + -} - -ARCH="${RSYNC_MODULE_NAME#*-}" - -case "$ARCH" in - aarch64*) TGT="aarch64" ;; - *-musl) TGT="musl" ;; - *) TGT="" ;; -esac - -# copy files to repo -rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/" - -for repo in / /bootstrap /debug /nonfree; do - process_repo "$repo" -done - -case "$ARCH" in - x86_64|i686) - ARCH=x86_64 - for repo in /multilib /multilib/bootstrap /multilib/nonfree; do - process_repo "$repo" - done - ;; -esac - -# clean up incoming -rm -r "${RSYNC_MODULE_PATH:?}"/* - -exit 0 diff --git a/services/nomad/build/rsync-update-repo b/services/nomad/build/rsync-update-repo new file mode 100644 index 00000000..9d70f4ef --- /dev/null +++ b/services/nomad/build/rsync-update-repo @@ -0,0 +1,58 @@ +#!/bin/sh + +msg() { + printf '\033[32m=> %s\033[0m\n' "$*" +} + +process_repo() { + repo="$1" + msg "Processing repository: $repo" + if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then + msg "Nothing to do" + return + fi + # add to repodata + find "$RSYNC_MODULE_PATH"/"$repo" -mindepth 1 -maxdepth 1 -name '*.xbps' -printf "$TGT_BASE/$TGT/$repo/%f\0" | \ + xargs -r0 env XBPS_TARGET_ARCH="$ARCH" xbps-rindex -a + # clean old packages from repodata + XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo" + # remove old packages + XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo" +} + +run() { + set -e + TGT_BASE="/mirror/current" + ARCH="${RSYNC_MODULE_NAME#*-}" + + case "$ARCH" in + aarch64*) TGT="aarch64" ;; + *-musl) TGT="musl" ;; + *) TGT="" ;; + esac + + # copy files to repo + msg "Copying new packages to the shadow mirror" + rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/" + + for repo in / /bootstrap /debug /nonfree; do + process_repo "$repo" + done + + case "$ARCH" in + x86_64|i686) + ARCH=x86_64 + for repo in /multilib /multilib/bootstrap /multilib/nonfree; do + process_repo "$repo" + done + ;; + esac + + # clean up incoming + msg "Cleaning up" + rm -rvf "${RSYNC_MODULE_PATH:?}"/*.xbps + set +e +} + +# if this exits non-zero, rsync will fail and the log will be lost +run >"$RSYNC_MODULE_PATH"/update-repo.log 2>&1 || true From 1b153e239b89bbc4becb33c467d21954721e6c87 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sat, 14 Mar 2026 11:05:22 -0400 Subject: [PATCH 3/5] services/nomad/build/build-rsyncd: only handle multilib on i686 --- services/nomad/build/rsync-update-repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/nomad/build/rsync-update-repo b/services/nomad/build/rsync-update-repo index 9d70f4ef..a746178e 100644 --- a/services/nomad/build/rsync-update-repo +++ b/services/nomad/build/rsync-update-repo @@ -40,7 +40,7 @@ run() { done case "$ARCH" in - x86_64|i686) + i686) ARCH=x86_64 for repo in /multilib /multilib/bootstrap /multilib/nonfree; do process_repo "$repo" From 84cd90d287e94c5e5859b3dbf9bbce03b87936b2 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sat, 14 Mar 2026 11:05:32 -0400 Subject: [PATCH 4/5] buildbot: keep built packages separate with xbps-src -r, sync multilib properly --- services/nomad/build/buildbot.cfg | 38 +++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/services/nomad/build/buildbot.cfg b/services/nomad/build/buildbot.cfg index cb12745f..00859070 100644 --- a/services/nomad/build/buildbot.cfg +++ b/services/nomad/build/buildbot.cfg @@ -203,6 +203,7 @@ def make_xbps_bulk_cmd(props): '-h', hostdir, '-d', distdir(''), '-m', buildroot(), + '-c', util.Interpolate(f'-r %(prop:target)s-%(prop:buildnumber)s'), '-t', '-s', '-S', ] if props.getProperty('cross') == 'True': @@ -220,28 +221,45 @@ def make_xbps_bulk_cmd(props): def make_rsync_cmd(props): return ['bash', '-e', '-c', util.Interpolate(""" +BUILDREPO="%(prop:target)s-%(prop:buildnumber)s" echo "=> Syncing packages to the shadow repository..." rsync -vurk --delete-after --remove-source-files \ --filter='+ */' --filter='+ *.%(prop:target)s.xbps' \ --filter='- .*' --filter='- *' \ ---password-file=/secrets/rsync/password /hostdir/binpkgs/ \ +--password-file=/secrets/rsync/password /hostdir/binpkgs/$BUILDREPO/ \ {{ range nomadService 1 $allocID "build-rsyncd" -}} rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s {{ end -}} -RSYNC_EXIT="$?" -echo "=> Cleaning repodatas..." -find /hostdir/binpkgs -name '%(prop:target)s-repodata' -print -delete -case "$RSYNC_EXIT" in - 0|23|24) exit 0 ;; +case "$?" in + 0|23|24) ;; *) exit 1 ;; esac +if [ "%(prop:target)s" = i686 ] && [ -d "/hostdir/binpkgs/$BUILDREPO/multilib" ]; then + echo "=> Syncing packages to the shadow repository..." + rsync -vurk --delete-after --remove-source-files \ + --filter='+ */' --filter='+ *.x86_64.xbps' \ + --filter='- .*' --filter='- *' \ + --password-file=/secrets/rsync/password /hostdir/binpkgs/$BUILDREPO/multilib/ \ + {{ range nomadService 1 $allocID "build-rsyncd" -}} + rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s/multilib/ + {{ end -}} + case "$?" in + 0|23|24) ;; + *) exit 1 ;; + esac +fi +echo "=> Cleaning repodatas..." +find /hostdir/binpkgs/$BUILDREPO -name '%(prop:target)s-repodata' -print -delete +if [ "%(prop:target)s" = i686 ] && [ -d "/hostdir/binpkgs/$BUILDREPO/multilib" ]; then + find /hostdir/binpkgs/$BUILDREPO/multilib -name 'x86_64-repodata' -print -delete +fi """)] @util.renderer def make_update_repo_cmd(props): return ['bash', '-e', '-c', - util.Interpolate(f""" + util.Interpolate(""" echo "=> Moving packages to the shadow repository..." rsync -vurk --delete-after --password-file=/secrets/rsync/password \ {{ range nomadService 1 $allocID "build-rsyncd" -}} @@ -262,11 +280,7 @@ esac @util.renderer def make_clean_cmd(props): - return [ - 'find', '/hostdir/binpkgs', '(', - '-name', util.Interpolate('*.%(prop:target)s.xbps'), '-o', '-name', util.Interpolate('%(prop:target)s-repodata'), - ')', '-print', '-delete', - ], + return ['rm', '-vr', util.Interpolate('/hostdir/binpkgs/%(prop:target)s-%(prop:buildnumber)s')] @util.renderer From a623499cb5b39ebb4790f428bac76985bebcd762 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sat, 14 Mar 2026 11:24:00 -0400 Subject: [PATCH 5/5] services/nomad/build/build-rsyncd: ignore existing files when moving from incoming --- services/nomad/build/rsync-update-repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/nomad/build/rsync-update-repo b/services/nomad/build/rsync-update-repo index a746178e..fa59efee 100644 --- a/services/nomad/build/rsync-update-repo +++ b/services/nomad/build/rsync-update-repo @@ -33,7 +33,7 @@ run() { # copy files to repo msg "Copying new packages to the shadow mirror" - rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/" + rsync -va --ignore-existing "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/" for repo in / /bootstrap /debug /nonfree; do process_repo "$repo"