From f9aa17728860f64e963ee40f9a62e075c7c41a8e Mon Sep 17 00:00:00 2001 From: Micael Malta Date: Sat, 10 Jul 2021 12:37:47 -0400 Subject: [PATCH 1/9] Docker apt-cacher --- bin/make-base-vm | 25 ++++++++++++++++++------- libexec/start-target | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/bin/make-base-vm b/bin/make-base-vm index 30e4fbf..d6b3ac5 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -9,6 +9,7 @@ LXC=0 VBOX=0 DOCKER=0 DOCKER_IMAGE_HASH="" +APT_CACHER_DEFAULT=0 usage() { echo "Usage: ${0##*/} [OPTION]..." @@ -24,6 +25,7 @@ usage() { --vbox use VirtualBox instead of kvm --docker use docker instead of kvm --docker-image-hash D digest of the docker image to build from + --enable-apt-cacher enable APT Cacher The MIRROR_HOST environment variable can be used to change the apt-cacher host. It should be something that both the host and the @@ -88,6 +90,10 @@ if [ $# != 0 ] ; then DOCKER=1 shift 1 ;; + --enable-apt-cacher) + APT_CACHER=1 + shift 1 + ;; --docker-image-digest) DOCKER_IMAGE_HASH="$2" shift 2 @@ -103,13 +109,11 @@ if [ $# != 0 ] ; then done fi -if [ $DOCKER = "1" ]; then - MIRROR_DEFAULT=172.17.0.1 -else - MIRROR_DEFAULT=127.0.0.1 -fi +MIRROR_DEFAULT=127.0.0.1 MIRROR_BASE=http://${MIRROR_HOST:-$MIRROR_DEFAULT}:3142 +APT_CACHER=${APT_CACHER:-$APT_CACHER_DEFAULT} + if [ $DISTRO = "ubuntu" ]; then MIRROR=$MIRROR_BASE/archive.ubuntu.com/ubuntu SECURITY_MIRROR=$MIRROR_BASE/security.ubuntu.com/ubuntu @@ -193,12 +197,19 @@ if [ $DOCKER = "1" ]; then base_image="$DISTRO:$SUITE" fi + apt_cacher="" + if [ "$APT_CACHER" = 1 ]; then + apt_cacher="RUN echo 'Acquire::http { Proxy "$MIRROR_BASE"; };' > /etc/apt/apt.conf.d/50cacher" + fi + # Generate the dockerfile cat << EOF > $OUT.Dockerfile FROM $base_image ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "$MIRROR_BASE"; };' > /etc/apt/apt.conf.d/50cacher +# DELETE ESM Files: W: Failed to fetch https://esm.ubuntu.com/ubuntu/dists/trusty-infra-security/main/binary-amd64/Packages Received HTTP code 403 from proxy after CONNECT +RUN [ -f /etc/apt/sources.list.d/*esm*.list ] && rm /etc/apt/sources.list.d/*esm*.list +$apt_cacher RUN apt-get update && apt-get --no-install-recommends -y install $addpkg RUN useradd -ms /bin/bash -U $DISTRO @@ -208,7 +219,7 @@ WORKDIR /home/$DISTRO CMD ["sleep", "infinity"] EOF - docker build --pull -f $OUT.Dockerfile -t $OUT . + docker build --network host --pull -f $OUT.Dockerfile -t $OUT . exit 0 fi diff --git a/libexec/start-target b/libexec/start-target index 0b20f06..f0f5974 100755 --- a/libexec/start-target +++ b/libexec/start-target @@ -37,6 +37,6 @@ case $VMSW in echo "Gitian-${2}" > var/target.vmname ;; DOCKER) - docker run -d --name gitian-target base-$SUFFIX:latest > /dev/null + docker run --network host -d --name gitian-target base-$SUFFIX:latest > /dev/null ;; esac From 0b106189fac5b8d5ea702e0f848aae4411be1b4f Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Thu, 16 Sep 2021 15:48:47 +0200 Subject: [PATCH 2/9] make-base-vm: switch apt cacher option, using cacher enabled by default --- bin/make-base-vm | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/make-base-vm b/bin/make-base-vm index d6b3ac5..623b8f3 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -9,7 +9,7 @@ LXC=0 VBOX=0 DOCKER=0 DOCKER_IMAGE_HASH="" -APT_CACHER_DEFAULT=0 +APT_CACHER=1 usage() { echo "Usage: ${0##*/} [OPTION]..." @@ -25,7 +25,7 @@ usage() { --vbox use VirtualBox instead of kvm --docker use docker instead of kvm --docker-image-hash D digest of the docker image to build from - --enable-apt-cacher enable APT Cacher + --disable-apt-cacher disable APT Cacher The MIRROR_HOST environment variable can be used to change the apt-cacher host. It should be something that both the host and the @@ -90,8 +90,8 @@ if [ $# != 0 ] ; then DOCKER=1 shift 1 ;; - --enable-apt-cacher) - APT_CACHER=1 + --disable-apt-cacher) + APT_CACHER=0 shift 1 ;; --docker-image-digest) @@ -112,8 +112,6 @@ fi MIRROR_DEFAULT=127.0.0.1 MIRROR_BASE=http://${MIRROR_HOST:-$MIRROR_DEFAULT}:3142 -APT_CACHER=${APT_CACHER:-$APT_CACHER_DEFAULT} - if [ $DISTRO = "ubuntu" ]; then MIRROR=$MIRROR_BASE/archive.ubuntu.com/ubuntu SECURITY_MIRROR=$MIRROR_BASE/security.ubuntu.com/ubuntu From f9a993ec2b28dc1a55382542e41c60e5c793b0b9 Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Thu, 16 Sep 2021 22:25:22 +0200 Subject: [PATCH 3/9] make-base-vm: allow disabling apt cacher for lxc & kvm --- bin/make-base-vm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/make-base-vm b/bin/make-base-vm index 623b8f3..efd6677 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -264,6 +264,7 @@ if [ $LXC = "1" ]; then echo "sudo will preserve (some) env flags" preserve_env=yes # if you would want to set false then unset this variable fi + [ $APT_CACHER -eq 0 ] && MIRROR="" env -i LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 DEBOOTSTRAP_DIR="$DEBOOTSTRAP_DIR" sudo ${preserve_env+--preserve-env} debootstrap --arch=$ARCH --include=$addpkg --exclude=$removepkg --components=$components $SUITE $OUT-bootstrap $MIRROR # Fix lxc issue if [ -f $OUT-bootstrap/usr/lib/lxc/lxc-init ] @@ -302,7 +303,8 @@ else libexec/config-bootstrap-fixup rm -rf $OUT - env -i LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 sudo vmbuilder kvm $DISTRO --rootsize $DISKSIZE --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_rsa.pub --ssh-user-key=var/id_rsa.pub --mirror=$MIRROR --security-mirror=$SECURITY_MIRROR --dest=$OUT --flavour=$FLAVOUR --firstboot=`pwd`/target-bin/bootstrap-fixup + [ $APT_CACHER -eq 1 ] && mirror_options="--mirror=$MIRROR --security-mirror=$SECURITY_MIRROR" + env -i LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 sudo vmbuilder kvm $DISTRO --rootsize $DISKSIZE --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_rsa.pub --ssh-user-key=var/id_rsa.pub --dest=$OUT --flavour=$FLAVOUR --firstboot=`pwd`/target-bin/bootstrap-fixup $(echo $mirror_options) mv $OUT/*.qcow2 $OUT.qcow2 rm -rf $OUT # bootstrap-fixup is done on first boot From f58f6812bfc597f3269389dce6d775f31d84455b Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Thu, 16 Sep 2021 22:45:13 +0200 Subject: [PATCH 4/9] apt-cacher: reset default mirror and avoid use of host network with docker --- bin/make-base-vm | 6 +++++- libexec/start-target | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/make-base-vm b/bin/make-base-vm index efd6677..f09ee33 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -109,7 +109,11 @@ if [ $# != 0 ] ; then done fi -MIRROR_DEFAULT=127.0.0.1 +if [ $DOCKER = "1" ]; then + MIRROR_DEFAULT=172.17.0.1 +else + MIRROR_DEFAULT=127.0.0.1 +fi MIRROR_BASE=http://${MIRROR_HOST:-$MIRROR_DEFAULT}:3142 if [ $DISTRO = "ubuntu" ]; then diff --git a/libexec/start-target b/libexec/start-target index f0f5974..0b20f06 100755 --- a/libexec/start-target +++ b/libexec/start-target @@ -37,6 +37,6 @@ case $VMSW in echo "Gitian-${2}" > var/target.vmname ;; DOCKER) - docker run --network host -d --name gitian-target base-$SUFFIX:latest > /dev/null + docker run -d --name gitian-target base-$SUFFIX:latest > /dev/null ;; esac From 68dd7565398d41485f426663becfd2ce30eab44b Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Thu, 16 Sep 2021 23:28:12 +0200 Subject: [PATCH 5/9] apt-cacher: add readme doc for --disable-apt-cacher --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f58c17..a37e6dc 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,13 @@ Also, I had to modify the default /etc/sudoers file to uncomment the `secure_pat ### Ubuntu: +By default and for regular build, `apt-cacher` server is used to cache dependencies. Install following packages to enable it, otherwise use `--disable-apt-cacher` : + + sudo apt-get install apache2 apt-cacher-ng + This pulls in all pre-requisites for KVM building on Ubuntu: - sudo apt-get install git apache2 apt-cacher-ng python-vm-builder ruby qemu-utils + sudo apt-get install git python-vm-builder ruby qemu-utils If you'd like to use LXC mode instead, install it as follows: @@ -67,7 +71,7 @@ On Debian Wheezy you run the same command, but you must first add backports to y sudo port install ruby coreutils export PATH=$PATH:/opt/local/libexec/gnubin # Needed for sha256sum - + ### OSX with Homebrew: brew install ruby coreutils From 7568db8fc9f51f315449b526a40e89819e890f4b Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Thu, 16 Sep 2021 23:40:06 +0200 Subject: [PATCH 6/9] apt-cacher: avoid use of host network with docker --- bin/make-base-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/make-base-vm b/bin/make-base-vm index f09ee33..2a633ca 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -221,7 +221,7 @@ WORKDIR /home/$DISTRO CMD ["sleep", "infinity"] EOF - docker build --network host --pull -f $OUT.Dockerfile -t $OUT . + docker build --pull -f $OUT.Dockerfile -t $OUT . exit 0 fi From 3793b654ed73884f427b5a8825f6c37b2dab79af Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Sat, 18 Sep 2021 22:11:24 +0200 Subject: [PATCH 7/9] apt-cache: avoid use mirror with lxc --- libexec/config-bootstrap-fixup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libexec/config-bootstrap-fixup b/libexec/config-bootstrap-fixup index 61d69b0..f87df1a 100755 --- a/libexec/config-bootstrap-fixup +++ b/libexec/config-bootstrap-fixup @@ -12,4 +12,8 @@ if [ -z "$MIRROR_HOST" ] || [ "$MIRROR_HOST" == "127.0.0.1" ]; then MIRROR_HOST=$GITIAN_HOST_IP fi -sed "s;HOSTIP;$MIRROR_HOST;g" < target-bin/bootstrap-fixup.in > target-bin/bootstrap-fixup +if [ $APT_CACHER = "1" ]; then + sed "s;HOSTIP;$MIRROR_HOST;g" < target-bin/bootstrap-fixup.in > target-bin/bootstrap-fixup +else + sed "s;HOSTIP:3142/;;g" < target-bin/bootstrap-fixup.in > target-bin/bootstrap-fixup +fi From 525e33acdb15a6c7b2b261d06c2d301318e3d753 Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Sun, 19 Sep 2021 23:13:10 +0200 Subject: [PATCH 8/9] apt-cacher: fix configuration error of the Dockerfile --- bin/make-base-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/make-base-vm b/bin/make-base-vm index 2a633ca..a88e1aa 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -201,7 +201,7 @@ if [ $DOCKER = "1" ]; then apt_cacher="" if [ "$APT_CACHER" = 1 ]; then - apt_cacher="RUN echo 'Acquire::http { Proxy "$MIRROR_BASE"; };' > /etc/apt/apt.conf.d/50cacher" + apt_cacher="RUN echo 'Acquire::http { Proxy \"$MIRROR_BASE\"; };' > /etc/apt/apt.conf.d/50cacher" fi # Generate the dockerfile From d8d370de67a6d846c81cebb313af100572d9180f Mon Sep 17 00:00:00 2001 From: AbcSxyZ Date: Mon, 20 Sep 2021 14:46:48 +0200 Subject: [PATCH 9/9] apt-cacher: avoid configuration of bootstrap-fixup.in in make-clean-vm --- libexec/make-clean-vm | 1 - 1 file changed, 1 deletion(-) diff --git a/libexec/make-clean-vm b/libexec/make-clean-vm index 9c5cacd..1ef646b 100755 --- a/libexec/make-clean-vm +++ b/libexec/make-clean-vm @@ -62,7 +62,6 @@ case $VMSW in ;; LXC) cp -a --sparse=always $BASE $OUT - libexec/config-bootstrap-fixup on-target -u root bash < target-bin/bootstrap-fixup ;; VBOX)