Migrate static configs

Makefile

@@ -45,6 +45,9 @@ dev: | $(config)
 	@+$(call bmake,olddefconfig)
 	@+$(call bmake,all)
 
+migrate-configs:
+	@$(CURDIR)/utils/migrate-configs.sh $(subst :, ,$(BR2_EXTERNAL))
+
 %: | buildroot/Makefile
 	@+$(call bmake,$@)
 
@@ -63,4 +66,4 @@ test:
 buildroot/Makefile:
 	@git submodule update --init
 
-.PHONY: all check coverity dep test cyclonedx list-snippets dev
+.PHONY: all check coverity dep test cyclonedx list-snippets dev migrate-configs

board/aarch64/acer-connect-vero-w6m/rootfs/usr/share/product/acer,connect-vero-w/etc/factory-config.cfg

@@ -428,7 +428,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/bananapi-bpi-r3/rootfs/usr/share/product/bananapi,bpi-r3/etc/factory-config.cfg

@@ -421,7 +421,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/bananapi-bpi-r3/rootfs/usr/share/product/bananapi,bpi-r3mini/etc/factory-config.cfg

@@ -387,7 +387,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/bananapi-bpi-r4/rootfs/usr/share/product/bananapi,bpi-r4-2g5/etc/factory-config.cfg

@@ -345,7 +345,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/bananapi-bpi-r4/rootfs/usr/share/product/bananapi,bpi-r4/etc/factory-config.cfg

@@ -337,7 +337,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/bananapi-bpi-r64/rootfs/usr/share/product/bananapi,bpi-r64/etc/factory-config.cfg

@@ -379,7 +379,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/friendlyarm-nanopi-r2s/rootfs/usr/share/product/friendlyarm,nanopi-r2s/etc/factory-config.cfg

@@ -321,7 +321,7 @@
     ]
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/raspberrypi-rpi64/rootfs/usr/share/product/raspberrypi,4-model-b/etc/factory-config.cfg

@@ -240,7 +240,7 @@
     "infix-system:motd-banner": "Li0tLS0tLS0uCnwgIC4gLiAgfCBJbmZpeCBPUyDigJQgSW1tdXRhYmxlLkZyaWVuZGx5LlNlY3VyZQp8LS4gdiAuLXwgaHR0cHM6Ly9rZXJuZWxraXQub3JnCictJy0tLSctJwo="
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/aarch64/raspberrypi-rpi64/rootfs/usr/share/product/raspberrypi,400/etc/factory-config.cfg

@@ -254,7 +254,7 @@
     "infix-system:motd-banner": "Li0tLS0tLS0uCnwgIC4gLiAgfCBJbmZpeCBPUyDigJQgSW1tdXRhYmxlLkZyaWVuZGx5LlNlY3VyZQp8LS4gdiAuLXwgaHR0cHM6Ly9rZXJuZWxraXQub3JnCictJy0tLSctJwo="
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

board/arm/raspberrypi-rpi2/rootfs/usr/share/product/raspberrypi,2-model-b/etc/factory-config.cfg

@@ -225,7 +225,7 @@
     "infix-system:motd-banner": "Li0tLS0tLS0uCnwgIC4gLiAgfCBJbmZpeCBPUyDigJQgSW1tdXRhYmxlLkZyaWVuZGx5LlNlY3VyZQp8LS4gdiAuLXwgaHR0cHM6Ly9rZXJuZWxraXQub3JnCictJy0tLSctJwo="
   },
   "infix-meta:meta": {
-    "version": "1.7"
+    "version": "1.10"
   },
   "infix-services:mdns": {
     "enabled": true

doc/ChangeLog.md

@@ -109,6 +109,9 @@ All notable changes to the project are documented in this file.
   logins.  Existing configurations are migrated.  NETCONF call-home, NETCONF
   over TLS, and the on-device `netopeer2-cli` tool require the built-in SSH
   server of netopeer2 and are therefore no longer available in default builds
+- Add `make migrate-configs` to bring static configurations, e.g., the
+  per-product `factory-config.cfg` in Infix and in spins, up to date with
+  the current confd version
 
 ### Fixes
 

doc/developers-guide.md

@@ -231,6 +231,17 @@ the startup configuration file as its first argument and must edit it
 in-place.  Scripts are run in lexicographic order, so prefix them with
 a number (e.g. `40-my-change.sh`).
 
+Static configuration files in the tree, e.g., the per-product
+`factory-config.cfg`, must follow suit.  Do not edit their version by
+hand, that skips the syntax changes.  Instead, run them through the
+same migration scripts and review the result with `git diff`:
+
+    make migrate-configs
+
+This covers all `*-config.cfg` files in every br2-external tree listed
+in `BR2_EXTERNAL`.  A spin of Infix can therefore forward the target to
+the Infix `Makefile` to have its own static configurations migrated.
+
 See `src/confd/share/migrate/1.6/40-bridge-port-remove-ip.sh` for a
 worked example, and the [Configuration Migration][upgrade-migration]
 section of the Upgrade documentation for the user-facing side of this

src/confd/share/migrate/1.8/10-keystore-add-gencert.sh

@@ -26,14 +26,15 @@ read_pem() {
     grep -v -- '-----' "$1" | tr -d '\n'
 }
 
-if [ -f "$LEGACY_KEY" ] && [ -f "$LEGACY_CRT" ]; then
+# A static config is shared by all devices, it must not carry a key
+if [ -z "$STATIC_CONFIG" ] && [ -f "$LEGACY_KEY" ] && [ -f "$LEGACY_CRT" ]; then
     priv_key=$(read_pem "$LEGACY_KEY")
     cert_data=$(read_pem "$LEGACY_CRT")
 fi
 
 # Fallback: generate a fresh certificate if legacy files were missing
 # or unreadable, same as keystore.c does on first boot.
-if [ -z "$priv_key" ] || [ -z "$cert_data" ]; then
+if [ -z "$STATIC_CONFIG" ] && { [ -z "$priv_key" ] || [ -z "$cert_data" ]; }; then
     /usr/libexec/infix/mkcert
     if [ -f "$MKCERT_KEY" ] && [ -f "$MKCERT_CRT" ]; then
         priv_key=$(read_pem "$MKCERT_KEY")
@@ -86,4 +87,4 @@ end
 ' "$file" > "$temp" && mv "$temp" "$file"
 
 # Cert/key now live in the keystore, wipe the legacy on-disk copy
-rm -rf "$LEGACY_DIR"
+[ -n "$STATIC_CONFIG" ] || rm -rf "$LEGACY_DIR"

test/case/repo/all.yaml

@@ -1,3 +1,5 @@
 ---
 - case: defconfig.sh
   name: "validate defconfigs"
+- case: config-version.sh
+  name: "validate config versions"

test/case/repo/config-version.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Verify all static .cfg files are at the current confd version
+
+SCRIPT_PATH="$(dirname "$(readlink -f "$0")")"
+TOPDIR="$SCRIPT_PATH/../../.."
+
+confd=$(sed -n 's/^AC_INIT(\[confd\], *\[\([^]]*\)\].*/\1/p' "$TOPDIR/src/confd/configure.ac")
+
+version()
+{
+    jq -r '.["infix-meta:meta"].version' "$1"
+}
+
+check()
+{
+    num=1
+
+    echo "1..$#"
+    for cfg in "$@"; do
+	name=${cfg#"$TOPDIR"/}
+	ver=$(version "$cfg")
+	if [ "$ver" = "$confd" ]; then
+	    echo "ok $num - $name is at confd version $confd"
+	else
+	    echo "not ok $num - Unexpected confd version $ver in $name"
+	fi
+	num=$((num + 1))
+    done
+    echo "# Configurations can be automatically upgraded using 'make migrate-configs'"
+}
+
+# shellcheck disable=SC2046
+check $(find "$TOPDIR/board" -name '*.cfg' | xargs grep -l '"infix-meta:meta"' | LC_ALL=C sort)
+
+exit 0

utils/migrate-configs.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+# Run static .cfg files in br2-external trees through confd migrate.
+
+infix=$(dirname "$(dirname "$(readlink -f "$0")")")
+migrate="$infix/src/confd/bin/migrate"
+scripts="$infix/src/confd/share/migrate"
+
+usage()
+{
+    cat <<EOF
+Usage: $(basename "$0") [-h] [DIR...]
+
+Run all static configuration files, *-config.cfg, found in each DIR
+through the confd migrate script.  The files are edited in place, use
+'git diff' to review the result.
+
+DIR defaults to the br2-external trees listed in \$BR2_EXTERNAL, which
+for a spin includes both Infix and the spin itself.
+EOF
+}
+
+version()
+{
+    jq -r '.["infix-meta:meta"].version // "0.0"' "$1" 2>/dev/null
+}
+
+cleanup()
+{
+    rm -f "$list" "$tmp"
+}
+
+while getopts "h" opt; do
+    case $opt in
+	h)
+	    usage
+	    exit 0
+	    ;;
+	*)
+	    usage >&2
+	    exit 1
+	    ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+if [ $# -eq 0 ]; then
+    # shellcheck disable=SC2046
+    set -- $(echo "$BR2_EXTERNAL" | tr ':' ' ')
+fi
+if [ $# -eq 0 ]; then
+    usage >&2
+    exit 1
+fi
+
+for cmd in jq git; do
+    if ! command -v "$cmd" >/dev/null; then
+	echo "Error: $cmd not found, please install it, e.g., 'sudo apt install $cmd'" >&2
+	exit 1
+    fi
+done
+
+list=$(mktemp)
+tmp=$(mktemp)
+trap cleanup INT HUP TERM EXIT
+
+rc=0
+for dir in "$@"; do
+    if ! git -C "$dir" -c core.quotepath=off ls-files -co --exclude-standard \
+	 -- '*-config.cfg' > "$list" 2>/dev/null; then
+	echo "Error: $dir is not a git repository, skipping." >&2
+	rc=1
+	continue
+    fi
+
+    echo "Migrating static configs in $dir"
+    while IFS= read -r file; do
+	cfg="$dir/$file"
+	[ -f "$cfg" ] || continue
+
+	if ! STATIC_CONFIG=1 FACTORY_CONFIG="$cfg" sh "$migrate" -e -q -s "$scripts" "$cfg" > "$tmp"; then
+	    echo "  $file: failed"
+	    rc=1
+	    continue
+	fi
+
+	# No output, already at latest version
+	if [ ! -s "$tmp" ]; then
+	    echo "  $file: $(version "$cfg"), up to date"
+	    continue
+	fi
+
+	old=$(version "$cfg")
+	cat "$tmp" > "$cfg"
+	echo "  $file: $old -> $(version "$cfg")"
+    done < "$list"
+done
+
+exit $rc