diff --git a/board/common/rootfs/etc/finit.d/10-infix.conf b/board/common/rootfs/etc/finit.d/10-infix.conf index fc65d79fb..de5e9765c 100644 --- a/board/common/rootfs/etc/finit.d/10-infix.conf +++ b/board/common/rootfs/etc/finit.d/10-infix.conf @@ -1,3 +1,7 @@ task name:ixinit [S] \ /usr/libexec/finit/runparts -bp /usr/libexec/infix/init.d \ -- Probing system + +task name:slot-check [2345] \ + /usr/libexec/infix/slot-check \ + -- Checking software partitions diff --git a/board/common/rootfs/etc/watchdogd.conf b/board/common/rootfs/etc/watchdogd.conf index b2c328ff5..28ec74fec 100644 --- a/board/common/rootfs/etc/watchdogd.conf +++ b/board/common/rootfs/etc/watchdogd.conf @@ -62,6 +62,11 @@ device /dev/watchdog { # priority = 98 # script = "/path/to/supervisor-script.sh" #} +supervisor { + enabled = true + priority = 98 + script = "/usr/libexec/infix/supervisor" +} ### Reset reason ####################################################### # The following section controls if/how the reset reason & reset counter diff --git a/board/common/rootfs/usr/libexec/infix/slot-check b/board/common/rootfs/usr/libexec/infix/slot-check new file mode 100755 index 000000000..bed29658c --- /dev/null +++ b/board/common/rootfs/usr/libexec/infix/slot-check @@ -0,0 +1,14 @@ +#!/bin/sh +# Remind the user to upgrade the other partition, issue #1637 + +rauc status --detailed --output-format=json 2>/dev/null | jq -r ' + [.slots | add | .[] | select(.class == "rootfs")] as $slots + | ($slots[] | select(.state == "booted") | .slot_status.bundle.version) as $booted + | $slots[] | select(.state != "booted") + | select(.slot_status.bundle.version != $booted) + | "\(.bootname) \(.slot_status.bundle.version // "unknown") \($booted)"' | +while read -r name version booted; do + msg="NOTE: the $name partition has $version, this is $booted. Use 'upgrade' to update it." + logger -t slot-check -p user.notice "$msg" + printf "\n%s\n" "$msg" | tee -a /etc/banner /etc/issue /etc/issue.net >/dev/null +done diff --git a/board/common/rootfs/usr/libexec/infix/supervisor b/board/common/rootfs/usr/libexec/infix/supervisor new file mode 100755 index 000000000..2a25702d2 --- /dev/null +++ b/board/common/rootfs/usr/libexec/infix/supervisor @@ -0,0 +1,16 @@ +#!/bin/sh +# Called by watchdogd when a supervised process misses its deadline: +# +# $0 supervisor CODE PID LABEL +# +# A hung confd bootstrap goes to fail-secure on next boot, issue #1637. +# Exit non-zero so watchdogd saves the reset reason and resets. A lost +# kick reply is not a hang, exit 0 and watchdogd drops the supervision. + +if [ "$4" = "confd-bootstrap" ]; then + [ "$2" = 5 ] || exit 0 # WDOG_FAILED_TO_MEET_DEADLINE + touch /mnt/aux/startup-config.failed + sync +fi + +exit 1 diff --git a/doc/ChangeLog.md b/doc/ChangeLog.md index d4c21e7f2..04db3a426 100644 --- a/doc/ChangeLog.md +++ b/doc/ChangeLog.md @@ -121,6 +121,19 @@ All notable changes to the project are documented in this file. which are written into the DHCP client service file - Restrict the allowed characters in DHCP server static-host match values - Restrict the allowed characters in a hardware component `name` +- Fix #1637: a startup-config migrated on upgrade was saved to disk at + boot, so the image on the other partition could no longer read it. + The migrated configuration is now only applied to running-config, + use `copy running-config startup-config` to save it. A failed + migration reverts to failure-config. After a downgrade, a newer + startup-config is loaded as-is, unless it uses settings the older + version does not know. A note at login, in `show software`, and on + the WebUI software page shows when the other partition has a + different version +- A startup-config that fails to load now resets the unit, and the next + boot goes straight to failure-config from a clean state, issue #1637. + The same applies if loading startup-config hangs, the system watchdog + then resets the unit - Fix #1619: Raspberry Pi kernel panic when configure Wi-Fi - WebUI: "Save" in the interface editor and "OK" in Add Interface did nothing for Wi-Fi and WireGuard interfaces. The inline "+ New" diff --git a/doc/boot.md b/doc/boot.md index 557fafcf7..c98bf8184 100644 --- a/doc/boot.md +++ b/doc/boot.md @@ -182,9 +182,18 @@ in the second case will cause the device to fail on the next boot. #### Broken startup-config If loading `startup-config` fails for some reason, e.g., invalid JSON -syntax, failed validation against the system's YANG model, or a bug in -the system's `confd` service, the *Fail Secure Mode* is triggered and -`failure-config` is loaded (unless VPD Failure, see above). +syntax, a failed migration or validation against the system's YANG +model, or a bug in the system's `confd` service, the *Fail Secure Mode* +is triggered and `failure-config` is loaded (unless VPD Failure, see +above). The system then marks the boot as failed and resets, so that +the next boot starts from a clean state and goes straight to *Fail +Secure Mode*. The mark is cleared on that boot, so a reboot after it +tries `startup-config` again. + +The same happens if loading `startup-config` hangs, e.g., a service that +never responds. The system watchdog then resets the unit, by default +after 70 seconds, and the next boot goes straight to *Fail Secure +Mode*. > [!TIP] > Please see the [Branding & Releases](branding.md) document for how to diff --git a/doc/upgrade.md b/doc/upgrade.md index 0269a83d9..7b8a3df06 100644 --- a/doc/upgrade.md +++ b/doc/upgrade.md @@ -109,9 +109,13 @@ booted from one partition, an `upgrade` will apply to the other > [!CAUTION] > During boot (step 5), the unit may [migrate](#configuration-migration) -> the startup configuration for any syntax changes. It is therefore -> important that you make sure to upgrade the other partition as well -> after reboot, of course after having verified your setup. +> the startup configuration for any syntax changes. The migrated +> configuration is only applied to `running-config`, and the file on +> disk is kept as-is until you save it. Once saved, the old image on +> the other partition may not be able to read it, so upgrade the other +> partition as well after you have verified your setup. Until then, a +> note at login, in `show software`, and on the WebUI software page +> shows that the other partition has a different version. The CLI example below shows steps 2-5. @@ -414,14 +418,14 @@ The example above illustrated an upgrade from Infix v25.01.0 to v25.03.1. Inbetween these versions, YANG configuration definitions changed slightly (more details given below). -During boot, Infix inspects the `version` meta information within the -startup configuration file to determine if configuration migration is -needed. In this specific case, the configuration file has version +During boot, the system inspects the `version` meta information within +the startup configuration file to determine if configuration migration +is needed. In this specific case, the configuration file has version `1.4` while the booted software expects version `1.5` (the configuration version numbering differs from the Infix image version numbering). The startup configuration is migrated to `1.5` -definitions and stored, while a backup previous startup configuration -is stored in directory `/cfg/backup/`. +definitions and applied to `running-config`, while a backup of the +original startup configuration is stored in directory `/cfg/backup/`.
admin@example:/> dir /cfg/backup/
 /cfg/backup/ directory
@@ -430,8 +434,26 @@ startup-config-1.4.cfg
 admin@example:/>
 
-The modifications made to the startup configuration can be viewed by -comparing the files from the *shell*. An example is shown below. +The file `/cfg/startup-config.cfg` itself is *not* changed. If the new +image fails, the unit can fall back to the old image on the other +partition, and its startup configuration is intact. The migration is +repeated at every boot until the configuration is saved. Until then, a +note at login says the configuration is not saved, and the WebUI shows +unsaved changes, since the `startup-config` datastore reports the old +version. + +When you have verified the unit works as expected, save the migrated +configuration: + +
admin@example:/> copy running-config startup-config
+admin@example:/>
+
+ +If the migration fails, the unit reverts to its [failure config][3]. + +After saving, the modifications made to the startup configuration can +be viewed by comparing the files from the *shell*. An example is shown +below.
admin@example:/> exit
 admin@example:~$ diff /cfg/backup/startup-config-1.4.cfg /cfg/startup-config.cfg
@@ -457,6 +479,10 @@ Downgrading to an earlier version is possible, however, downgrading is
 up with the downgraded version, it may fail to apply the *startup
 config*, and instead apply its [failure config][3].
 
+A startup configuration of a newer version than the downgraded software
+supports is loaded as-is.  It only fails if it uses settings the older
+version does not know.
+
 We consider two cases: downgrading with and without applying a backup
 startup configuration before rebooting.
 
diff --git a/package/confd/Config.in b/package/confd/Config.in
index 38e276929..62aaefef9 100644
--- a/package/confd/Config.in
+++ b/package/confd/Config.in
@@ -6,6 +6,7 @@ config BR2_PACKAGE_CONFD
 	select BR2_PACKAGE_SYSREPO
 	select BR2_PACKAGE_LIBSRX
 	select BR2_PACKAGE_SUPPORT
+	select BR2_PACKAGE_WATCHDOGD
 	help
 	  A plugin to sysrepo that provides the core YANG models used to
 	  manage an Infix based system.  Configuration can be done using
diff --git a/package/confd/confd.mk b/package/confd/confd.mk
index 3c005e233..1394511ca 100644
--- a/package/confd/confd.mk
+++ b/package/confd/confd.mk
@@ -10,7 +10,7 @@ CONFD_SITE = $(BR2_EXTERNAL_INFIX_PATH)/src/confd
 CONFD_LICENSE = BSD-3-Clause
 CONFD_LICENSE_FILES = LICENSE
 CONFD_REDISTRIBUTE = NO
-CONFD_DEPENDENCIES = host-sysrepo sysrepo rousette netopeer2 jansson libite sysrepo libsrx libglib2 libev sysklogd
+CONFD_DEPENDENCIES = host-sysrepo sysrepo rousette netopeer2 jansson libite sysrepo libsrx libglib2 libev sysklogd watchdogd
 CONFD_AUTORECONF = YES
 CONFD_CONF_OPTS += --disable-silent-rules --with-crypt=$(BR2_PACKAGE_CONFD_DEFAULT_CRYPT)
 CONFD_SYSREPO_SHM_PREFIX = sr_buildroot$(subst /,_,$(CONFIG_DIR))_confd
diff --git a/src/confd/configure.ac b/src/confd/configure.ac
index ed1dc8873..a3b701b88 100644
--- a/src/confd/configure.ac
+++ b/src/confd/configure.ac
@@ -127,6 +127,7 @@ PKG_CHECK_MODULES([libyang], [libyang >= 4.2.2])
 PKG_CHECK_MODULES([libsrx],  [libsrx >= 1.0.0])
 PKG_CHECK_MODULES([libsyslog], [libsyslog >= 2.7.0])
 PKG_CHECK_MODULES([libcrypto], [libcrypto])
+PKG_CHECK_MODULES([libwdog], [libwdog >= 4.0])
 
 AC_CHECK_HEADER([ev.h],
     [saved_LIBS="$LIBS"
diff --git a/src/confd/src/Makefile.am b/src/confd/src/Makefile.am
index a9a04db99..6ea4efd2a 100644
--- a/src/confd/src/Makefile.am
+++ b/src/confd/src/Makefile.am
@@ -6,8 +6,8 @@ plugin_LTLIBRARIES      = confd-plugin.la
 
 sbin_PROGRAMS           = confd
 
-confd_CFLAGS            = $(sysrepo_CFLAGS) $(libyang_CFLAGS) $(jansson_CFLAGS) $(libite_CFLAGS) $(libsrx_CFLAGS)
-confd_LDADD             = $(sysrepo_LIBS)   $(libyang_LIBS)   $(jansson_LIBS)   $(libite_LIBS)   $(libsrx_LIBS) $(EV_LIBS) -ldl
+confd_CFLAGS            = $(sysrepo_CFLAGS) $(libyang_CFLAGS) $(jansson_CFLAGS) $(libite_CFLAGS) $(libsrx_CFLAGS) $(libwdog_CFLAGS)
+confd_LDADD             = $(sysrepo_LIBS)   $(libyang_LIBS)   $(jansson_LIBS)   $(libite_LIBS)   $(libsrx_LIBS) $(libwdog_LIBS) $(EV_LIBS) -ldl
 confd_SOURCES           = main.c
 
 confd_plugin_la_LDFLAGS = -module -avoid-version -shared
diff --git a/src/confd/src/core.c b/src/confd/src/core.c
index bd705cce5..92bfafd1f 100644
--- a/src/confd/src/core.c
+++ b/src/confd/src/core.c
@@ -191,6 +191,19 @@ int finit_disablef(const char *fmt, ...)
 }
 
 
+/*
+ * After a migration the startup datastore has the old version of the file
+ * until saved, so stamp every write after bootstrap, issue #1637.
+ */
+static int startup_version(sr_session_ctx_t *session, uint32_t sub_id, const char *model,
+			   const char *xpath, sr_event_t event, unsigned request_id, void *priv)
+{
+	if (event != SR_EV_UPDATE || systemf("runlevel >/dev/null 2>&1"))
+		return SR_ERR_OK;
+
+	return meta_set_version(session);
+}
+
 static int startup_save(sr_session_ctx_t *session, uint32_t sub_id, const char *model,
 			const char *xpath, sr_event_t event, unsigned request_id, void *priv)
 {
@@ -921,6 +934,12 @@ int sr_plugin_init_cb(sr_session_ctx_t *session, void **priv)
 		ERROR("Failed to subscribe to infix-meta");
 		goto err;
 	}
+	rc = sr_module_change_subscribe(confd.startup, "infix-meta", "//.", startup_version, NULL,
+					CB_PRIO_PRIMARY, SR_SUBSCR_UPDATE | SR_SUBSCR_NO_THREAD, &confd.sub);
+	if (rc) {
+		ERROR("Failed to subscribe to infix-meta in startup");
+		goto err;
+	}
 	rc = subscribe_model("ieee1588-ptp-tt", &confd, 0);
 	if (rc) {
 		ERROR("Failed to subscribe to ieee1588-ptp-tt");
diff --git a/src/confd/src/core.h b/src/confd/src/core.h
index 6911107ac..ab244c0ca 100644
--- a/src/confd/src/core.h
+++ b/src/confd/src/core.h
@@ -271,6 +271,7 @@ int factory_rpc_init(struct confd *confd);
 int factory_default_rpc_init(struct confd *confd);
 
 /* meta.c */
+int meta_set_version(sr_session_ctx_t *session);
 int meta_change_cb(sr_session_ctx_t *session, struct lyd_node *config, struct lyd_node *diff, sr_event_t event, struct confd *confd);
 
 /* system-software.c */
diff --git a/src/confd/src/main.c b/src/confd/src/main.c
index fa390f770..067f323e9 100644
--- a/src/confd/src/main.c
+++ b/src/confd/src/main.c
@@ -35,6 +35,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /* Maximum number of sysrepo event pipe file descriptors across all plugins */
 #define MAX_EVENT_FDS 64
@@ -49,6 +50,12 @@
  */
 #define SENTINEL_PATH "/run/confd.boot"
 
+/* Migrated startup-config, loaded into running but never saved to /cfg */
+#define MIGRATED_PATH "/run/confd-migrated.cfg"
+
+/* Set when startup-config fails, next boot goes to fail-secure, issue #1637 */
+#define FAILED_PATH   "/mnt/aux/startup-config.failed"
+
 /*
  * Set a finit condition in the usr/ namespace, e.g.
  * "usr/startup-config-ok", used to signal IITO (and finit services) about
@@ -185,11 +192,15 @@ static void pump_sigterm(int sig)
 	pump_running = 0;
 }
 
-static void event_pump(struct plugin *plugins, int plugin_count)
+/* With a deadline, a hung callback resets us to fail-secure, issue #1637 */
+static void event_pump(struct plugin *plugins, int plugin_count, uint32_t deadline_ms)
 {
 	sr_subscription_ctx_t *subs[MAX_EVENT_FDS];
 	struct pollfd fds[MAX_EVENT_FDS];
+	time_t now, last = 0;
+	unsigned int ack;
 	int nfds = 0;
+	int id = -1;
 
 	for (int i = 0; i < plugin_count; i++) {
 		struct plugin *p = &plugins[i];
@@ -208,14 +219,30 @@ static void event_pump(struct plugin *plugins, int plugin_count)
 
 	signal(SIGTERM, pump_sigterm);
 
+	if (deadline_ms) {
+		id = wdog_subscribe("confd-bootstrap", deadline_ms, &ack);
+		if (id < 0)
+			WARN("Not supervised by watchdogd: %s", strerror(-id));
+	}
+
 	while (pump_running) {
 		if (poll(fds, nfds, 100) > 0) {
 			for (int i = 0; i < nfds; i++)
 				if (fds[i].revents & POLLIN)
 					sr_subscription_process_events(subs[i], NULL, NULL);
 		}
+
+		/* Each kick is a round trip to watchdogd, once a second is enough */
+		now = time(NULL);
+		if (id >= 0 && now != last) {
+			last = now;
+			wdog_kick2(id, &ack);
+		}
 	}
 
+	if (id >= 0)
+		wdog_unsubscribe(id, ack);
+
 	_exit(0);
 }
 
@@ -393,42 +420,63 @@ static void banner_append(const char *msg)
 }
 
 /*
- * Smart migration: only fork+exec the migrate script if the version
- * in the config file doesn't match the current confd version.
+ * Read the infix-meta version of a config file, "0.0" if unset.
  */
-static int maybe_migrate(const char *path)
+static int config_version(const char *path, char *buf, size_t len)
 {
-	const char *backup_dir = "/cfg/backup";
-	json_t *root, *meta, *ver;
-	const char *file_ver;
-	char backup[256];
-	int rc;
+	json_t *root, *ver;
 
 	root = json_load_file(path, 0, NULL);
 	if (!root)
 		return -1;
 
-	meta     = json_object_get(root, "infix-meta:meta");
-	ver      = meta ? json_object_get(meta, "version") : NULL;
-	file_ver = ver ? json_string_value(ver) : "0.0";
+	ver = json_object_get(json_object_get(root, "infix-meta:meta"), "version");
+	strlcpy(buf, json_string_value(ver) ?: "0.0", len);
+	json_decref(root);
 
-	if (!strcmp(file_ver, CONFD_VERSION)) {
-		json_decref(root);
-		return 0;
+	return 0;
+}
+
+/*
+ * Migrate a config file of an older version to a temporary file, leaving
+ * the original untouched until the user saves running-config, issue #1637.
+ * A newer file is loaded as-is.  Returns the file to load, NULL on error.
+ */
+static const char *maybe_migrate(const char *path, const char *ver)
+{
+	const char *backup_dir = "/cfg/backup";
+	char backup[256], newver[16];
+	int rc;
+
+	if (!strcmp(ver, CONFD_VERSION))
+		return path;
+
+	/*
+	 * After a downgrade the file is often usable, a newer version only
+	 * means new settings may be in it.  Try it, the strict parse rejects
+	 * any we don't know, instead of reverting to failure-config.
+	 */
+	if (strverscmp(ver, CONFD_VERSION) > 0) {
+		WARN("%s config version %s is newer than confd %s, trying as-is.",
+		     path, ver, CONFD_VERSION);
+		return path;
 	}
-	json_decref(root);
 
-	NOTE("%s config version %s vs confd %s, migrating ...", path, file_ver, CONFD_VERSION);
+	NOTE("%s config version %s vs confd %s, migrating ...", path, ver, CONFD_VERSION);
 
 	mkpath(backup_dir, 0770);
 	chown(backup_dir, 0, 10); /* root:wheel */
 
 	snprintf(backup, sizeof(backup), "%s/%s", backup_dir, basenm(path));
-	rc = systemf("migrate -i -b \"%s\" \"%s\"", backup, path);
-	if (rc)
-		ERROR("Migration of %s failed (rc=%d)", path, rc);
+	rc = systemf("migrate -b \"%s\" \"%s\" >%s", backup, path, MIGRATED_PATH);
+	if (rc || config_version(MIGRATED_PATH, newver, sizeof(newver)) ||
+	    strcmp(newver, CONFD_VERSION)) {
+		ERROR("Migration of %s to version %s failed (rc=%d)", path, CONFD_VERSION, rc);
+		unlink(MIGRATED_PATH);
+		return NULL;
+	}
 
-	return rc;
+	return MIGRATED_PATH;
 }
 
 /*
@@ -578,6 +626,14 @@ static int bootstrap_config(sr_conn_ctx_t *conn, sr_session_ctx_t *sess,
 	const char *config_path;
 	int r;
 
+	/* Cleared right away, or a failed fail-secure boot is a boot loop */
+	if (fexist(FAILED_PATH)) {
+		unlink(FAILED_PATH);
+		ERROR("Previous boot failed loading startup-config.");
+		handle_startup_failure(sess, failure_path, conn, timeout_ms);
+		return 1;
+	}
+
 	/* Test mode support */
 	if (fexist("/mnt/aux/test-mode")) {
 		if (fexist("/mnt/aux/test-override-startup")) {
@@ -592,23 +648,47 @@ static int bootstrap_config(sr_conn_ctx_t *conn, sr_session_ctx_t *sess,
 	}
 
 	if (fexist(config_path)) {
-		/* Run migration if needed */
-		maybe_migrate(config_path);
+		const char *load_path;
+		char ver[16];
+
+		if (config_version(config_path, ver, sizeof(ver))) {
+			ERROR("Parsing %s failed", config_path);
+			goto fail;
+		}
+
+		load_path = maybe_migrate(config_path, ver);
+		if (!load_path)
+			goto fail;
 
 		/* Load startup (or test) config */
 		NOTE("Loading %s ...", config_path);
-		if (load_config(conn, sess, config_path, timeout_ms)) {
-			handle_startup_failure(sess, failure_path, conn, timeout_ms);
-			return 1; /* fail-secure, keep running */
-		}
+		r = load_config(conn, sess, load_path, timeout_ms);
+		unlink(MIGRATED_PATH);
+		if (r)
+			goto fail;
 
 		NOTE("Loaded %s successfully, syncing startup datastore.", config_path);
 		sr_session_switch_ds(sess, SR_DS_STARTUP);
 		r = sr_copy_config(sess, NULL, SR_DS_RUNNING, timeout_ms);
-		sr_session_switch_ds(sess, SR_DS_RUNNING);
 		if (r != SR_ERR_OK)
 			WARN("Failed to sync startup datastore: %s", sr_strerror(r));
 
+		if (strcmp(ver, CONFD_VERSION)) {
+			char msg[160];
+
+			/* Differ from running, or a save is an empty diff and never hits disk */
+			if (r == SR_ERR_OK &&
+			    (sr_set_item_str(sess, "/infix-meta:meta/version", ver, NULL, 0) ||
+			     sr_apply_changes(sess, timeout_ms)))
+				WARN("Failed setting startup datastore version %s", ver);
+
+			snprintf(msg, sizeof(msg), "NOTE: %s version %s loaded as version %s, not saved.\n"
+				 "      Use 'copy running-config startup-config' to save.",
+				 basenm(config_path), ver, CONFD_VERSION);
+			banner_append(msg);
+		}
+		sr_session_switch_ds(sess, SR_DS_RUNNING);
+
 		set_finit_cond("startup-config-ok");
 		return 0;
 	}
@@ -630,6 +710,20 @@ static int bootstrap_config(sr_conn_ctx_t *conn, sr_session_ctx_t *sess,
 		set_finit_cond("startup-config-ok");
 
 	return 0;
+fail:
+	handle_startup_failure(sess, failure_path, conn, timeout_ms);
+
+	/* Finit ignores a regular reboot in runlevel S, so force it */
+	if (touch(FAILED_PATH)) {
+		ERRNO("Failed creating %s", FAILED_PATH);
+	} else {
+		sync();
+		systemf("reboot -f");
+		ERROR("Failed rebooting, staying in fail-secure mode.");
+		unlink(FAILED_PATH);
+	}
+
+	return 1; /* fail-secure, keep running */
 }
 
 int main(int argc, char **argv)
@@ -648,7 +742,7 @@ int main(int argc, char **argv)
 	uint32_t timeout_s = 60;
 	int plugin_count = 0;
 	int fatal_fail = 0;
-	uint32_t timeout_ms;
+	uint32_t timeout_ms, deadline_ms;
 	int status;
 
 	struct option options[] = {
@@ -841,13 +935,15 @@ int main(int argc, char **argv)
 		/* Phase 9: Fork event pump process for bootstrap.
 		 * With SR_SUBSCR_NO_THREAD, sr_replace_config() blocks waiting
 		 * for callbacks.  The pump process processes those events. */
+		/* Past the sysrepo timeout, not on a fail-secure boot (reset loop) */
+		deadline_ms = fexist(FAILED_PATH) ? 0 : timeout_ms + 10000;
 		pump_pid = fork();
 		if (pump_pid < 0) {
 			ERRNO("Failed to fork event pump");
 			goto cleanup;
 		}
 		if (pump_pid == 0)
-			event_pump(plugins, plugin_count);
+			event_pump(plugins, plugin_count, deadline_ms);
 
 		/* Phase 10: Load startup config -- plugins are now subscribed, so
 		 * sr_replace_config() will trigger their change callbacks.
diff --git a/src/confd/src/meta.c b/src/confd/src/meta.c
index abacb60a0..6bc6a2313 100644
--- a/src/confd/src/meta.c
+++ b/src/confd/src/meta.c
@@ -5,7 +5,7 @@
 #define META_XPATH  "/infix-meta:meta/version"
 
 
-static int set_version(sr_session_ctx_t *session)
+int meta_set_version(sr_session_ctx_t *session)
 {
 	int rc;
 
@@ -21,7 +21,7 @@ static int set_version(sr_session_ctx_t *session)
 int meta_change_cb(sr_session_ctx_t *session, struct lyd_node *config, struct lyd_node *diff, sr_event_t event, struct confd *confd)
 {
 	if (event == SR_EV_UPDATE)
-		return set_version(session);
+		return meta_set_version(session);
 
 	return SR_ERR_OK;
 }
diff --git a/src/statd/python/cli_pretty/cli_pretty.py b/src/statd/python/cli_pretty/cli_pretty.py
index d43a855d3..3ac5c9165 100755
--- a/src/statd/python/cli_pretty/cli_pretty.py
+++ b/src/statd/python/cli_pretty/cli_pretty.py
@@ -2207,10 +2207,16 @@ def show_software(json, name):
                f"{'VERSION':<{PadSoftware.version}}"
                f"{'DATE':<{PadSoftware.date}}")
         print(Decore.invert(hdr))
-        for _s in reversed(slots):
-            slot = Software(_s)
-            if slot.is_rootfs():
-                slot.print()
+        rootfs = [s for s in map(Software, reversed(slots)) if s.is_rootfs()]
+        for slot in rootfs:
+            slot.print()
+
+        booted = next((s for s in rootfs if s.state == "booted"), None)
+        others = [s for s in rootfs if s.version != booted.version] if booted else []
+        for slot in others:
+            print()
+            print(Decore.yellow(f"Note: the {slot.name} partition has {slot.version or 'unknown'},"
+                                f" this is {booted.version}.  Use 'upgrade' to update it."))
 
 
 def show_services(json):
diff --git a/src/webui/internal/handlers/software_test.go b/src/webui/internal/handlers/software_test.go
new file mode 100644
index 000000000..3b2c3915e
--- /dev/null
+++ b/src/webui/internal/handlers/software_test.go
@@ -0,0 +1,42 @@
+package handlers
+
+import "testing"
+
+func TestOtherSlots(t *testing.T) {
+	tests := []struct {
+		name  string
+		slots []slotEntry
+		want  []string
+	}{
+		{"same version", []slotEntry{
+			{Name: "primary", Version: "v1", Booted: true},
+			{Name: "secondary", Version: "v1"},
+		}, nil},
+		{"other is older", []slotEntry{
+			{Name: "primary", Version: "v2", Booted: true},
+			{Name: "secondary", Version: "v1"},
+		}, []string{"secondary"}},
+		{"other is empty", []slotEntry{
+			{Name: "primary", Version: "v2", Booted: true},
+			{Name: "secondary"},
+		}, []string{"secondary"}},
+		{"none booted", []slotEntry{
+			{Name: "primary", Version: "v2"},
+			{Name: "secondary", Version: "v1"},
+		}, nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := softwareData{Slots: tt.slots}.OtherSlots()
+			if len(got) != len(tt.want) {
+				t.Fatalf("got %v, want %v", got, tt.want)
+			}
+			for i := range got {
+				if got[i].Name != tt.want[i] {
+					t.Errorf("got %s, want %s", got[i].Name, tt.want[i])
+				}
+			}
+		})
+	}
+}
diff --git a/src/webui/internal/handlers/system.go b/src/webui/internal/handlers/system.go
index fa07ddd1a..6c1c5a528 100644
--- a/src/webui/internal/handlers/system.go
+++ b/src/webui/internal/handlers/system.go
@@ -520,6 +520,41 @@ type slotEntry struct {
 	Booted      bool
 }
 
+// bootedSlot returns the booted slot, or nil if unknown.
+func (d softwareData) bootedSlot() *slotEntry {
+	for i := range d.Slots {
+		if d.Slots[i].Booted {
+			return &d.Slots[i]
+		}
+	}
+	return nil
+}
+
+// BootedVersion returns the version of the booted slot, if known.
+func (d softwareData) BootedVersion() string {
+	if b := d.bootedSlot(); b != nil {
+		return b.Version
+	}
+	return ""
+}
+
+// OtherSlots returns the slots with a different version than the booted
+// one, i.e., those the user should also upgrade, issue #1637.
+func (d softwareData) OtherSlots() []slotEntry {
+	b := d.bootedSlot()
+	if b == nil {
+		return nil
+	}
+
+	var other []slotEntry
+	for _, s := range d.Slots {
+		if s.Version != b.Version {
+			other = append(other, s)
+		}
+	}
+	return other
+}
+
 type installerEntry struct {
 	Operation  string
 	Percentage int
diff --git a/src/webui/templates/pages/software.html b/src/webui/templates/pages/software.html
index 945006017..ccc7fb4cf 100644
--- a/src/webui/templates/pages/software.html
+++ b/src/webui/templates/pages/software.html
@@ -11,6 +11,10 @@
 
{{.Message}}
{{end}} +{{range .OtherSlots}} +
The {{.Name}} partition has {{if .Version}}{{.Version}}{{else}}an unknown version{{end}}, this is {{$.BootedVersion}}. Install the same software again to update it.
+{{end}} + {{$sseURL := "/software/progress"}}{{if .AutoReboot}}{{$sseURL = "/software/progress?auto-reboot=1"}}{{end}} {{if .Installing}}