Install mission critical dependencies from source, improve dep handling

README.adoc

@@ -70,11 +70,11 @@ sudo ./install_deps.sh
 
 See the `depends` file for the complete list of required packages.
 
-=== Important
+rpi-image-gen builds its own copies of critical host tools so it always has current, fully featured versions and supports architectures the workstation distribution might not package. Installing and running them from a private sysroot keeps the host system untouched. The trade-off is a small one-time build delay, plus additional native dependencies that most developer workstations already carry. The resulting tools live in a reusable sysroot that stays isolated from distribution packages and can be reused for every build.
 
-rpi-image-gen has been developed on Raspberry Pi OS which, at the time of writing, is Debian Bookworm arm64. It heavily favours Debian-based systems and will run on non-arm64 platforms (such as x86_64) via QEMU emulation or inside container environments. However, there is currently no formal support for these non-native environments.
+=== Important
 
-If utilising sparse images in your workflow, e.g. with *rpi-sb-provisioner* (https://github.com/raspberrypi/rpi-sb-provisioner), please refer to the link:docs/provisioning/index.adoc[provisioning documentation] for important advice.
+rpi-image-gen is developed on Raspberry Pi OS and supports builds on Debian Bookworm and Trixie arm64 systems. It expects a Debian-based host and will run on non-arm64 platforms (such as x86_64) via QEMU emulation or in container environments, although those non-native setups aren’t formally supported.
 
 == Documentation
 
@@ -98,6 +98,7 @@ See the `examples/` directory for tips and help
 * `layer/` - Layer library
 * `layer-hooks/` - Common hooks used by layer library
 * `lib/` - Execution helpers and macros, eg CLI handling, reusable constructs
+* `package/` - Build recipes and framework for tools
 * `scripts/` - Dedicated functional hooks, eg for bdebstrap
 * `site/` - Core Python engine classes
 * `templates/` - Templating assets, eg doc generation

bin/vfetch

@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# File fetch with verification and cache support
+# Usage:
+#   [IGconf_sys_cachedir=/path/to/cache] <script> <metadata.file> </path/to/output.file>
+#
+# Metadata - mirrors supported, use consistent name:
+#   <name> <algo:hex> <url or local path>
+#   algo: sha256|sha256sum|sha512|sha512sum
+
+
+msg() {
+   echo -e "$*"
+}
+
+err (){
+   >&2 msg "Error: $*"
+}
+
+die (){
+   [[ -n "$*" ]] && err "$*"
+   exit 1
+}
+
+
+meta="${1:-}"; dest="${2:-}"
+[[ -n "$meta" && -n "$dest" ]] || die "Expected args: <metadata.file> </path/to/output.file>"
+
+
+# Enforce that dest is a file path, not a directory
+if [[ -d "$dest" || "$dest" == */ ]]; then
+   die "Output path must be a file, not a directory"
+fi
+
+
+# Default cache location
+cache_dir="${IGconf_sys_cachedir:-/tmp}"
+mkdir -p -- "$cache_dir"
+
+
+# Output path is always an explicit file path
+out_path="$dest"
+out_dir="$(dirname -- "$out_path")"
+mkdir -p -- "$out_dir"
+
+
+# digest <algo> <file> -> hex
+digest()
+{
+   case "$1" in
+      sha256|sha256sum) sha256sum "$2" | sed 's/[[:space:]].*$//' ;;
+      sha512|sha512sum) sha512sum "$2" | sed 's/[[:space:]].*$//' ;;
+      *) die "Unsupported algo: $1" ;;
+   esac
+}
+
+
+# atomic file copy <src> <dst>
+atomic_install_copy()
+{
+   local src="$1" dst="$2" dstdir tmp
+   dstdir="$(dirname -- "$dst")"
+   tmp="$(mktemp -p "$dstdir" ".fetch.install.XXXXXX")"
+   cp -f -- "$src" "$tmp"
+   mv -f -- "$tmp" "$dst"
+}
+
+
+# Two-pass approach:
+# pass 1 tries cache against any candidate digest
+# pass 2 downloads per candidate
+target_name=""
+
+
+# Pass 1: determine target, verify cache against all candidate digests
+cache_path=""
+
+while IFS= read -r line || [[ -n "$line" ]]; do
+   [[ "$line" =~ ^[[:space:]]*$ ]] && continue
+   [[ "$line" =~ ^[[:space:]]*# ]] && continue
+
+   set -- $line
+   name="${1:-}"; algo_hex="${2:-}"; src="${3:-}"
+   [[ -n "$name" && -n "$algo_hex" && -n "$src" ]] || continue
+
+   if [[ -z "$target_name" ]]; then
+      target_name="$name"
+      cache_path="${cache_dir}/${target_name}"
+   fi
+   [[ "$name" == "$target_name" ]] || continue
+
+   algo="${algo_hex%%:*}"
+   expected="${algo_hex#*:}"
+   expected="${expected,,}"
+   if [[ "$algo" == "$expected" || -z "$expected" ]]; then
+      die "Malformed metadata for $target_name: expected '<algo>:<hex>'"
+   fi
+
+   if [[ -f "$cache_path" ]]; then
+      got="$(digest "$algo" "$cache_path")"
+      if [[ "${got,,}" == "$expected" ]]; then
+         atomic_install_copy "$cache_path" "$out_path"
+         msg "Using verified cache: $cache_path -> $out_path"
+         exit 0
+      fi
+   fi
+done < "${meta}"
+
+[[ -n "${target_name}" ]] || die "No metadata entries in $meta"
+
+
+# Pass 2: download each candidate and verify with its own expected digest
+while IFS= read -r line || [[ -n "${line}" ]]; do
+   [[ "${line}" =~ ^[[:space:]]*$ ]] && continue
+   [[ "${line}" =~ ^[[:space:]]*# ]] && continue
+
+   set -- $line
+   name="${1:-}"; algo_hex="${2:-}"; src="${3:-}"
+   [[ -n "$name" && -n "$algo_hex" && -n "$src" ]] || continue
+   [[ "$name" == "$target_name" ]] || continue
+
+   algo="${algo_hex%%:*}"
+   expected="${algo_hex#*:}"
+   expected="${expected,,}"
+   if [[ "$algo" == "$expected" || -z "$expected" ]]; then
+      die "Malformed metadata for $target_name: expected '<algo>:<hex>'"
+   fi
+
+   tmp_cache="$(mktemp -p "$cache_dir" ".fetch.${target_name}.XXXXXX")"
+   if [[ "$src" == http://* || "$src" == https://* ]]; then
+      if ! curl --fail --location --silent --show-error --retry 3 --retry-delay 2 \
+         --connect-timeout 30 --output "$tmp_cache" "$src"; then
+         rm -f -- "$tmp_cache"
+         continue
+      fi
+   else
+      if ! cp -f -- "$src" "$tmp_cache"; then
+         rm -f -- "$tmp_cache"
+         continue
+      fi
+   fi
+
+   got="$(digest "$algo" "$tmp_cache")"
+   if [[ "${got,,}" != "$expected" ]]; then
+      err "Checksum mismatch for ${target_name} from ${src}"
+      rm -f -- "${tmp_cache}"
+      continue
+   fi
+
+   atomic_install_copy "$tmp_cache" "$out_path"
+   mv -f -- "$tmp_cache" "$cache_path"
+   msg "Fetched: $src -> $out_path cached at $cache_path"
+   exit 0
+done < "${meta}"
+
+
+die "Failed: no verified source succeeded for ${target_name}"

depends

@@ -1,28 +1,44 @@
-realpath:coreutils
-zip
-mkdosfs:dosfstools
-mke2fs:e2fsprogs
-grep
-rsync
-curl
-genimage
-mtools
-mmdebstrap
-bdebstrap
-podman
-zstd
-pv
-newuidmap:uidmap
-python-is-python3
-dbus-user-session
-btrfs-progs
-dctrl-tools
-uuidgen:uuid-runtime
-fdisk
-python3-yaml
-python3-debian
-python3-jsonschema
-# doc gen only
+all:realpath:coreutils
+all:bash
+all:mmdebstrap
+all:podman
+all:newuidmap:uidmap
+all::python-is-python3
+all::dbus-user-session
+
+bootstrap::python3-yaml
+bootstrap::python3-debian
+
+# image build essential
+build:zip
+build:mkdosfs:dosfstools
+build:mke2fs:e2fsprogs
+build:grep
+build:rsync
+build:curl
+build:mtools
+build:zstd
+build:pv
+build:mkfs.btrfs:btrfs-progs
+build::dctrl-tools
+build:uuidgen:uuid-runtime
+build:fdisk
+build::python3-jsonschema
+build::dpkg-dev
+
+# pkg buildsys
+build::python3-pip
+build:make
+build::build-essential
+build:autoconf
+build:automake
+build::libtool
+build:autopoint
+build:flex
+build:gettext
+build:pkg-config
+
+# maintainer only (docgen)
 # python3-markdown
 # asciidoctor
 # python3-jinja2

docs/config/index.adoc

@@ -15,7 +15,7 @@ rpi-image-gen provides a flexible, hierarchical approach to managing build varia
 
 === Variable Naming Convention
 
-All configuration variables in rpi-image-gen use the `IGconf_` prefix followed by a structured naming pattern:
+Excluding environment keys, configuration variables in rpi-image-gen use the `IGconf_` prefix followed by a structured naming pattern:
 
 [source]
 ----
@@ -101,6 +101,9 @@ YAML provides the most flexible configuration format with native support for hie
 include:
   file: base.yaml
 
+env:
+  MYVAR: UNCHANGED
+
 device:
   class: pi5
   storage_type: sd
@@ -120,6 +123,9 @@ INI format provides traditional section-based configuration:
 ----
 !include base.cfg
 
+[env]
+MYVAR = UNCHANGED
+
 [device]
 class = pi5
 storage_type = sd

docs/config/index.html

@@ -234,7 +234,7 @@ <h2 id="_core_architecture"><a class="anchor" href="#_core_architecture"></a><a
 <div class="sect2">
 <h3 id="_variable_naming_convention"><a class="anchor" href="#_variable_naming_convention"></a><a class="link" href="#_variable_naming_convention">Variable Naming Convention</a></h3>
 <div class="paragraph">
-<p>All configuration variables in rpi-image-gen use the <code>IGconf_</code> prefix followed by a structured naming pattern:</p>
+<p>Excluding environment keys, configuration variables in rpi-image-gen use the <code>IGconf_</code> prefix followed by a structured naming pattern:</p>
 </div>
 <div class="listingblock">
 <div class="content">
@@ -357,6 +357,9 @@ <h3 id="_yaml_configuration_format"><a class="anchor" href="#_yaml_configuration
 <pre class="highlight"><code class="language-yaml" data-lang="yaml">include:
   file: base.yaml
 
+env:
+  MYVAR: UNCHANGED
+
 device:
   class: pi5
   storage_type: sd
@@ -378,6 +381,9 @@ <h3 id="_ini_configuration_format"><a class="anchor" href="#_ini_configuration_f
 <div class="content">
 <pre class="highlight"><code class="language-ini" data-lang="ini">!include base.cfg
 
+[env]
+MYVAR = UNCHANGED
+
 [device]
 class = pi5
 storage_type = sd

docs/layer/sys-build-base.html

@@ -233,6 +233,40 @@ <h2>Configuration Variables</h2>
                     </td>
                 </tr>
 
+                <tr>
+                    <td><code>IGconf_sys_workroot</code></td>
+                    <td>The root work directory. Depending on the system(s)
+ being built, this directory can amount to a substantial amount of consumed
+ disk space.</td>
+                    <td>
+
+
+                           <code class="long-default">$(realpath --canonicalize-missing ./work)</code>
+
+
+                    </td>
+                    <td>Non-empty string value</td>
+                    <td>
+                        <a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
+                    </td>
+                </tr>
+
+                <tr>
+                    <td><code>IGconf_sys_cachedir</code></td>
+                    <td>Global cache root directory for all operations.</td>
+                    <td>
+
+
+                           <code class="long-default">${IGconf_sys_workroot}/cache</code>
+
+
+                    </td>
+                    <td>Non-empty string value</td>
+                    <td>
+                        <a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
+                    </td>
+                </tr>
+
                 <tr>
                     <td><code>IGconf_sys_apt_cachedir</code></td>
                     <td>Cache directory for APT operations.
@@ -254,14 +288,30 @@ <h2>Configuration Variables</h2>
                 </tr>
 
                 <tr>
-                    <td><code>IGconf_sys_workroot</code></td>
-                    <td>The root work directory. Depending on the system(s)
- being built, this directory can amount to a substantial amount of consumed
- disk space.</td>
+                    <td><code>IGconf_sys_buildroot</code></td>
+                    <td>Global build root directory for source builds.</td>
                     <td>
 
 
-                           <code class="long-default">$(realpath --canonicalize-missing ./work)</code>
+                           <code class="long-default">${IGconf_sys_workroot}/build</code>
+
+
+                    </td>
+                    <td>Non-empty string value</td>
+                    <td>
+                        <a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
+                    </td>
+                </tr>
+
+                <tr>
+                    <td><code>IGconf_sys_bootstrapdir</code></td>
+                    <td>Directory for bootstrap artefacts (e.g. env, layer
+ order, metadata) used to resume the build in a containerised (or later)
+ invocation.</td>
+                    <td>
+
+
+                           <code class="long-default">${IGconf_sys_workroot}/bootstrap</code>
 
 
                     </td>

docs/provisioning/index.adoc

@@ -46,7 +46,7 @@ IDP requires fastboot to be installed in the client. On Debian systems, this can
 
 [WARNING]
 ====
-Older versions of `genimage` can be problematic when creating sparse images. For example, `genimage` v16 currently shipping in Debian Bookworm does not create usable sparse images of _vfat_ filesystems. Raspberry Pi provisioning tools such as `rpi-sb-provisioner` (https://github.com/raspberrypi/rpi-sb-provisioner) rely on sparse image format, so it is recommended to use the most up-to-date version of `genimage` as possible (e.g. manual build and install to `/usr/local/bin/genimage`).
+Raspberry Pi provisioning tools such as `rpi-sb-provisioner` (https://github.com/raspberrypi/rpi-sb-provisioner) rely on sparse images, so rpi-image-gen ships its own up-to-date `genimage` to avoid compatibility problems seen with older distribution versions.
 ====