Skip to content

Commit c57f6b9

Browse files
learmjlearmj
committed
core: Categorise host build dependencies
Re-work the dependency installation mech to support different categories of host dependencies and validate if a program installed as a dependency can be located post-install. Applying categories to dependencies should be beneficial when introducing containerised execution support. Write bootstrap information to a new output dir declared by the sys base layer. Fixes #106
1 parent e856b76 commit c57f6b9

File tree

5 files changed

+200
-102
lines changed

5 files changed

+200
-102
lines changed

‎depends‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,44 @@
1-
realpath:coreutils
2-
zip
3-
mkdosfs:dosfstools
4-
mke2fs:e2fsprogs
5-
grep
6-
rsync
7-
curl
8-
mtools
9-
mmdebstrap
10-
podman
11-
zstd
12-
pv
13-
newuidmap:uidmap
14-
python-is-python3
15-
dbus-user-session
16-
btrfs-progs
17-
dctrl-tools
18-
uuidgen:uuid-runtime
19-
fdisk
20-
python3-yaml
21-
python3-debian
22-
python3-jsonschema
1+
all:realpath:coreutils
2+
all:bash
3+
all:mmdebstrap
4+
all:podman
5+
all:newuidmap:uidmap
6+
all::python-is-python3
7+
all::dbus-user-session
8+
9+
bootstrap::python3-yaml
10+
bootstrap::python3-debian
11+
12+
# image build essential
13+
build:zip
14+
build:mkdosfs:dosfstools
15+
build:mke2fs:e2fsprogs
16+
build:grep
17+
build:rsync
18+
build:curl
19+
build:mtools
20+
build:zstd
21+
build:pv
22+
build:mkfs.btrfs:btrfs-progs
23+
build::dctrl-tools
24+
build:uuidgen:uuid-runtime
25+
build:fdisk
26+
build::python3-jsonschema
27+
build::dpkg-dev
28+
2329
# pkg buildsys
24-
python3-pip
25-
dpkg-dev
26-
make
27-
build-essential
28-
autoconf
29-
automake
30-
libtool
31-
autopoint
32-
flex
33-
gettext
34-
pkg-config
35-
# doc gen only
30+
build::python3-pip
31+
build:make
32+
build::build-essential
33+
build:autoconf
34+
build:automake
35+
build::libtool
36+
build:autopoint
37+
build:flex
38+
build:gettext
39+
build:pkg-config
40+
41+
# maintainer only (docgen)
3642
# python3-markdown
3743
# asciidoctor
3844
# python3-jinja2

‎docs/layer/sys-build-base.html‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,24 @@ <h2>Configuration Variables</h2>
303303
</td>
304304
</tr>
305305

306+
<tr>
307+
<td><code>IGconf_sys_bootstrapdir</code></td>
308+
<td>Directory for bootstrap artefacts (e.g. env, layer
309+
order, metadata) used to resume the build in a containerised (or later)
310+
invocation.</td>
311+
<td>
312+
313+
314+
<code class="long-default">${IGconf_sys_workroot}/bootstrap</code>
315+
316+
317+
</td>
318+
<td>Non-empty string value</td>
319+
<td>
320+
<a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
321+
</td>
322+
</tr>
323+
306324
</tbody>
307325
</table>
308326

‎layer/base/sys-build-base.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,13 @@
7171
# X-Env-Var-buildroot-Valid: string
7272
# X-Env-Var-buildroot-Set: y
7373
#
74+
# X-Env-Var-bootstrapdir: ${IGconf_sys_workroot}/bootstrap
75+
# X-Env-Var-bootstrapdir-Desc: Directory for bootstrap artefacts (e.g. env, layer
76+
# order, metadata) used to resume the build in a containerised (or later)
77+
# invocation.
78+
# X-Env-Var-bootstrapdir-Required: n
79+
# X-Env-Var-bootstrapdir-Valid: string
80+
# X-Env-Var-bootstrapdir-Set: y
81+
#
7482
# METAEND
7583
---

‎lib/dependencies.sh‎

Lines changed: 123 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,138 @@
11
#!/bin/bash
22

3-
# dependencies_check
4-
# $@ Dependency files to check
3+
# Options:
4+
# --category <name> Category selector
5+
# --install Automatically install missing packages via apt
6+
# <files...> Dependency manifest files to process
57
#
6-
# Each dependency is in the form of a tool to test for, optionally followed by
7-
# a : and the name of a package if the package on a Debian-ish system is not
8-
# named for the tool (i.e., qemu-user-static).
8+
# A manifest declares dependencies in the form of a triplet:
9+
# category:program:package
10+
# The package field is optional and used if the package on a Debian-ish system
11+
# is not named for the program (i.e., qemu-user-static).
912
dependencies_check()
1013
{
11-
local depfile deps missing=() op=$1
12-
13-
if [[ "$op" == install ]] ; then
14-
shift 1
15-
fi
16-
17-
for depfile in "$@"; do
18-
if [[ -e "$depfile" ]]; then
19-
mapfile -t dep_lines < <(grep -v '^[[:space:]]*#' "$depfile" | grep -v '^[[:space:]]*$')
20-
deps="${dep_lines[*]}"
21-
fi
22-
for dep in $deps; do
23-
if ! hash "${dep%:*}" 2>/dev/null; then
24-
if ! dpkg -s "${dep#*:}" > /dev/null 2>&1; then
25-
missing+=("${dep#*:}")
26-
fi
27-
fi
28-
done
29-
done
30-
31-
if [[ "${missing[*]}" ]]; then
32-
echo "Required dependencies not installed"
33-
echo
34-
echo "This can be resolved on Debian systems by installing:"
35-
echo "${missing[@]}"
36-
echo
37-
echo "Script install_deps.sh can be used for this purpose."
38-
echo
39-
40-
if [[ "$op" == install ]] ; then
41-
apt install -y "${missing[@]}"
42-
else
43-
exit 1
44-
fi
45-
fi
14+
local category=all install_mode=0
15+
local -a depfiles=()
16+
local -a missing=()
17+
18+
while [[ $# -gt 0 ]]; do
19+
case "$1" in
20+
--category)
21+
category=$2
22+
shift 2
23+
;;
24+
--install)
25+
install_mode=1
26+
shift
27+
;;
28+
*)
29+
depfiles+=("$1")
30+
shift
31+
;;
32+
esac
33+
done
34+
35+
(( ${#depfiles[@]} )) || return 0
36+
37+
for depfile in "${depfiles[@]}"; do
38+
[[ -r "$depfile" ]] || continue
39+
while IFS= read -r line || [[ -n $line ]]; do
40+
[[ $line =~ ^[[:space:]]*(#|$) ]] && continue
41+
42+
IFS=':' read -r dep_category tool pkg _ <<<"$line"
43+
[[ -n $dep_category ]] || continue
44+
45+
[[ -n $pkg ]] || pkg=$tool
46+
[[ -n $tool ]] || tool=""
47+
48+
if [[ $category != all && $dep_category != all && $dep_category != "$category" ]]; then
49+
continue
50+
fi
51+
52+
if [[ -z $tool ]]; then
53+
# Package-only dependency
54+
dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg")
55+
continue
56+
fi
57+
58+
if command -v "$tool" >/dev/null 2>&1; then
59+
continue # installed and available
60+
fi
61+
62+
if dpkg -s "$pkg" >/dev/null 2>&1; then
63+
echo "Dependency $tool ($pkg) is installed, but not found"
64+
exit 1 # installed but unavailable
65+
fi
66+
67+
missing+=("$pkg") # not installed yet
68+
done < "$depfile"
69+
done
70+
71+
if [[ ${#missing[@]} -gt 0 ]]; then
72+
local -a unique=()
73+
declare -A seen=()
74+
for pkg in "${missing[@]}"; do
75+
[[ -n ${seen[$pkg]:-} ]] && continue
76+
seen[$pkg]=1
77+
unique+=("$pkg")
78+
done
79+
80+
echo "Required dependencies (${category}) not installed"
81+
echo
82+
echo "This can be resolved on Debian systems by installing:"
83+
echo "${unique[@]}"
84+
echo
85+
echo "Script install_deps.sh can be used for this purpose."
86+
echo
87+
88+
if (( install_mode )); then
89+
apt install -y "${unique[@]}"
90+
else
91+
exit 1
92+
fi
93+
fi
4694

4795
# If we're building on a native arm platform, we don't need to check for
4896
# binfmt_misc or require it to be loaded.
4997

50-
binfmt_misc_required=1
51-
52-
case $(uname -m) in
53-
aarch64)
54-
binfmt_misc_required=0
55-
;;
56-
arm*)
57-
binfmt_misc_required=0
58-
;;
59-
esac
60-
61-
if [[ "${binfmt_misc_required}" == "1" ]]; then
62-
if ! grep -q "/proc/sys/fs/binfmt_misc" /proc/mounts; then
63-
echo "Module binfmt_misc not loaded in host"
64-
echo "Please run:"
65-
echo " sudo modprobe binfmt_misc"
66-
exit 1
67-
fi
68-
fi
98+
binfmt_misc_required=1
99+
100+
case $(uname -m) in
101+
aarch64)
102+
binfmt_misc_required=0
103+
;;
104+
arm*)
105+
binfmt_misc_required=0
106+
;;
107+
esac
108+
109+
if [[ "${binfmt_misc_required}" == "1" ]]; then
110+
if ! grep -q "/proc/sys/fs/binfmt_misc" /proc/mounts; then
111+
echo "Module binfmt_misc not loaded in host"
112+
echo "Please run:"
113+
echo " sudo modprobe binfmt_misc"
114+
exit 1
115+
fi
116+
fi
69117
}
70118

71119

72120
dependencies_install()
73121
{
74-
if [ "$(id -u)" != "0" ]; then
75-
>&2 echo "Please run as root to install dependencies."; exit 1
76-
fi
77-
dependencies_check install "$@"
122+
if [ "$(id -u)" != "0" ]; then
123+
>&2 echo "Please run as root to install dependencies."; exit 1
124+
fi
125+
local category=all
126+
while [[ $# -gt 0 ]]; do
127+
case "$1" in
128+
--category)
129+
category=$2
130+
shift 2
131+
;;
132+
*)
133+
break
134+
;;
135+
esac
136+
done
137+
dependencies_check --install --category "$category" "$@"
78138
}

‎rpi-image-gen‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,12 @@ collect_layers()
231231

232232
ctx[LAYER_ORDER]="$layer_order"
233233
ctx[FINALENV]="$finalenv"
234+
235+
# Write bootstrap information
236+
local bdir=$(get_var IGconf_sys_bootstrapdir "${ctx[FINALENV]}") || die
237+
mkdir -p "$bdir"
238+
cp "${ctx[FINALENV]}" "${bdir}/config.env"
239+
cp "${ctx[LAYER_ORDER]}" "${bdir}/layer.order"
234240
}
235241

236242

@@ -551,7 +557,7 @@ main() {
551557
cli_parse_build ctx "$@"
552558
shift $?
553559
path_refresh
554-
dependencies_check "$IGTOP/depends" || exit 1
560+
dependencies_check --category all "$IGTOP/depends" || exit 1
555561
run_stage parameter_assembly "$@"
556562
run_stage collect_layers
557563
run_stage prepare_build_config
@@ -564,7 +570,7 @@ main() {
564570
cli_parse_clean ctx "$@"
565571
shift $?
566572
path_refresh
567-
dependencies_check "$IGTOP/depends" || exit 1
573+
dependencies_check --category all "$IGTOP/depends" || exit 1
568574
run_stage parameter_assembly "$@"
569575
run_stage collect_layers
570576
run_stage clean_worktree
@@ -574,21 +580,21 @@ main() {
574580
cli_parse_layer ctx "$@"
575581
shift $?
576582
path_refresh
577-
dependencies_check "$IGTOP/depends" || exit 1
583+
dependencies_check --category all "$IGTOP/depends" || exit 1
578584
exec "$IGTOP/bin/ig" layer --path "${ctx[LAYER_PATH]}" "$@"
579585
;;
580586

581587
metadata)
582588
cli_parse_metadata ctx "$@"
583589
shift $?
584-
dependencies_check "$IGTOP/depends" || exit 1
590+
dependencies_check --category all "$IGTOP/depends" || exit 1
585591
exec "$IGTOP/bin/ig" metadata "$@"
586592
;;
587593

588594
config)
589595
cli_parse_config ctx "$@"
590596
shift $?
591-
dependencies_check "$IGTOP/depends" || exit 1
597+
dependencies_check --category all "$IGTOP/depends" || exit 1
592598
exec "$IGTOP/bin/ig" config "$@"
593599
;;
594600

0 commit comments

Comments
 (0)