Downgrade tool

.github/workflows/ci.yaml

@@ -235,6 +235,86 @@ jobs:
           ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang
           uv run make -j $(nproc) check-fuzz
 
+  check-downgrade:
+    name: Check we can downgrade the node
+    runs-on: ubuntu-22.04
+    needs:
+      - compile
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - CFG: compile-gcc
+            TEST_DB_PROVIDER: sqlite3
+            TEST_NETWORK: regtest
+            VALGRIND: 1
+          - CFG: compile-gcc
+            TEST_DB_PROVIDER: postgres
+            TEST_NETWORK: regtest
+          - CFG: compile-gcc
+            TEST_DB_PROVIDER: sqlite3
+            TEST_NETWORK: liquid-regtest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install dependencies
+        run: |
+          bash -x .github/scripts/setup.sh
+
+      - name: Install bitcoind
+        env:
+          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
+        run: .github/scripts/install-bitcoind.sh
+
+      - name: Download build
+        uses: actions/download-artifact@v4
+        with:
+          name: cln-${{ matrix.CFG }}.tar.bz2
+
+      - name: Unpack pre-built CLN
+        env:
+          CFG: ${{ matrix.CFG }}
+        run: |
+          tar -xaf cln-${CFG}.tar.bz2
+
+      - name: Fetch and unpack previous CLN
+        run: |
+          mkdir /tmp/old-cln
+          cd /tmp/old-cln
+          wget https://github.com/ElementsProject/lightning/releases/download/v25.09/clightning-v25.09-Ubuntu-22.04-amd64.tar.xz
+          tar -xaf clightning-v25.09-Ubuntu-22.04-amd64.tar.xz
+
+      - name: Switch network
+        if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
+        run: |
+          # Loading the network from config.vars rather than the envvar is a terrible idea...
+          sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
+          cat config.vars
+
+      - name: Test
+        env:
+          SLOW_MACHINE: 1
+          PYTEST_PAR: 10
+          TEST_DEBUG: 1
+          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
+          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
+          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
+          VALGRIND: ${{ matrix.VALGRIND }}
+          PREV_LIGHTNINGD: /tmp/old-cln/usr/bin/lightningd
+        run: |
+          env
+          cat config.vars
+          uv run eatmydata pytest tests/test_downgrade.py -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+
   integration:
     name: Test CLN ${{ matrix.name }}
     runs-on: ubuntu-22.04
@@ -627,6 +707,7 @@ jobs:
       - integration-valgrind
       - integration-sanitizers
       - min-btc-support
+      - check-downgrade
     if: ${{ always() }}
     steps:
       - name: Complete
@@ -638,6 +719,7 @@ jobs:
           SANITIZERS: ${{ needs['integration-sanitizers'].result }}
           DOCS: ${{ needs['update-docs-examples'].result }}
           BTC: ${{ needs['min-btc-support'].result }}
+          CHECK_DOWNGRADE: ${{ needs['check-downgrade'].result }}
         run: |
           failed=""
           for name in $JOB_NAMES; do

Makefile

@@ -9,6 +9,9 @@ $(info Building version $(VERSION))
 # Next release.
 CLN_NEXT_VERSION := v25.12
 
+# Previous release (for downgrade testing)
+CLN_PREV_VERSION := v25.09
+
 # --quiet / -s means quiet, dammit!
 ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s)
 ECHO := :
@@ -552,8 +555,8 @@ CHECK_BOLT_PREFIX=--prefix="BOLT-$(BOLTVERSION)"
 endif
 
 # Any mention of BOLT# must be followed by an exact quote, modulo whitespace.
-bolt-check/%: % bolt-precheck tools/check-bolt
-	@if [ -d .tmp.lightningrfc ]; then tools/check-bolt $(CHECK_BOLT_PREFIX) .tmp.lightningrfc $<; else echo "Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist" >&2; fi
+bolt-check/%: % bolt-precheck devtools/check-bolt
+	@if [ -d .tmp.lightningrfc ]; then devtools/check-bolt $(CHECK_BOLT_PREFIX) .tmp.lightningrfc $<; else echo "Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist" >&2; fi
 
 LOCAL_BOLTDIR=.tmp.lightningrfc
 
@@ -565,7 +568,7 @@ check-source-bolt: $(ALL_NONGEN_SRCFILES:%=bolt-check/%)
 check-whitespace/%: %
 	@if grep -Hn '[ 	]$$' $<; then echo Extraneous whitespace found >&2; exit 1; fi
 
-check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%)
+check-whitespace: check-whitespace/Makefile check-whitespace/devtools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%)
 
 check-spelling:
 	@tools/check-spelling.sh
@@ -934,8 +937,7 @@ TESTBINS = \
 	$(CLN_PLUGIN_EXAMPLES) \
 	tests/plugins/test_libplugin \
 	tests/plugins/channeld_fakenet \
-	tests/plugins/test_selfdisable_after_getmanifest \
-	tools/hsmtool
+	tests/plugins/test_selfdisable_after_getmanifest
 
 # The testpack is used in CI to transfer built artefacts between the
 # build and the test phase. This is necessary because the fixtures in
@@ -944,7 +946,7 @@ TESTBINS = \
 # version of `lightningd` leading to bogus results. We bundle up all
 # built artefacts here, and will unpack them on the tester (overlaying
 # on top of the checked out repo as if we had just built it in place).
-testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS)
+testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS) $(TOOLS)
 	tar -caf $@ $^
 
 uninstall:

README.md

@@ -198,7 +198,7 @@ If the two blockheights drift apart it might be necessary to intervene.
 
 ### HD wallet encryption
 
-You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key) by passing the `--encrypted-hsm` startup argument, or by using the `hsmtool` (which you can find in the `tool/` directory at the root of this repo) with the `encrypt` method. You can unencrypt an encrypted `hsm_secret` using the `hsmtool` with the `decrypt` method.
+You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key) by passing the `--encrypted-hsm` startup argument, or by using the `lightning-hsmtool` (which you can find in the `tool/` directory at the root of this repo) with the `encrypt` method. You can unencrypt an encrypted `hsm_secret` using the `lightning-hsmtool` with the `decrypt` method.
 
 If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` startup option to `lightningd`. Once your `hsm_secret` is encrypted, you __will not__ be able to access your funds without your password, so please beware with your password management. Also, beware of not feeling too safe with an encrypted `hsm_secret`: unlike for `bitcoind` where the wallet encryption can restrict the usage of some RPC command, `lightningd` always needs to access keys from the wallet which is thus __not locked__ (yet), even with an encrypted BIP32 master seed.
 

contrib/msggen/msggen/schema.json

@@ -7472,7 +7472,7 @@
               "Value to be decoded:",
               "    * a *bolt11* or *bolt12* string (optionally prefixed by `lightning:` or `LIGHTNING:`) as specified by the BOLT 11 and BOLT 12 specifications.",
               "    * a *rune* as created by lightning-commando-rune(7).",
-              "    * an *emergency_recover* string generated by hsmtool like `lightning-hsmtool getemergencyrecover <path/to/emergency.recover>`. It holds `emergency.recover` contents and starts with `clnemerg1`."
+              "    * an *emergency_recover* string generated by lightning-hsmtool like `lightning-hsmtool getemergencyrecover <path/to/emergency.recover>`. It holds `emergency.recover` contents and starts with `clnemerg1`."
             ]
           }
         }
@@ -30510,7 +30510,7 @@
       "description": [
         "The **recover** RPC command wipes your node and restarts it with the `--recover` option. This is only permitted if the node is unused: no channels, no bitcoin addresses issued (you can use `check` to see if recovery is possible).",
         "",
-        "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string.",
+        "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string.",
         "",
         "NOTE: this command only currently works with the `sqlite3` database backend."
       ],
@@ -30523,7 +30523,7 @@
           "hsmsecret": {
             "type": "string",
             "description": [
-              "Either a codex32 secret starting with `cl1` as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string."
+              "Either a codex32 secret starting with `cl1` as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string."
             ]
           }
         }

devtools/Makefile

@@ -1,4 +1,4 @@
-DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune devtools/gossmap-compress devtools/bip137-verifysignature devtools/convert-gossmap
+DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune devtools/gossmap-compress devtools/bip137-verifysignature devtools/convert-gossmap devtools/check-bolt
 ifeq ($(HAVE_SQLITE3),1)
 DEVTOOLS += devtools/checkchannels
 endif

doc/Makefile

@@ -161,6 +161,7 @@ NON_PREFIXED_MANPAGES := doc/lightning-cli.1 \
 	doc/lightningd-config.5 \
 	doc/lightningd-rpc.7 \
 	doc/lightning-hsmtool.8 \
+	doc/lightning-downgrade.8 \
 	doc/lightning-reckless.1
 
 MANPAGES := $(PREFIXED_MANPAGES) $(NON_PREFIXED_MANPAGES)

doc/beginners-guide/backup-and-recovery/hsm-secret.md

@@ -10,7 +10,7 @@ privacy:
 
 ## Generate HSM Secret
 
-If you are deploying a new node that has no funds and channels yet, you can generate BIP39 words using any process, and create the `hsm_secret` using the `hsmtool generatehsm` command. If you did `make install` then `hsmtool` is installed as [`lightning-hsmtool`](ref:lightning-hsmtool), else you can find it in the `tools/` directory of the build directory.
+If you are deploying a new node that has no funds and channels yet, you can generate BIP39 words using any process, and create the `hsm_secret` using the `lightning-hsmtool generatehsm` command. If you did `make install` then `hsmtool` is installed as [`lightning-hsmtool`](ref:lightning-hsmtool), else you can find it in the `tools/` directory of the build directory.
 
 ```shell
 lightning-hsmtool generatehsm hsm_secret
@@ -25,14 +25,14 @@ You can regenerate the same `hsm_secret` file using the same BIP39 words, which
 
 You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key):
 - either by passing the `--encrypted-hsm` startup argument
-- or by using the `encrypt` method from `/tools/hsmtool`. 
+- or by using the `encrypt` method from `/tools/lightning-hsmtool`. 
 
 If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` startup option to `lightningd`. Once your `hsm_secret` is encrypted, you **will not** be able to access your funds without your password, so please beware with your password management. Also, beware of not feeling too safe with an encrypted `hsm_secret`: unlike for `bitcoind` where the wallet encryption can restrict the usage of some RPC command, `lightningd` always needs to access keys from the wallet which is thus **not locked** (yet), even with an encrypted BIP32 master seed.
 
 
 ## Decrypt HSM Secret
 
-You can unencrypt an encrypted `hsm_secret` using the `hsmtool` with the `decrypt` method.
+You can unencrypt an encrypted `hsm_secret` using the `lightning-hsmtool` with the `decrypt` method.
 
 ```shell
 lightning-hsmtool decrypt ${LIGHTNINGDIR}/hsm_secret

doc/beginners-guide/backup.md

@@ -65,9 +65,9 @@ chmod 0400 hsm_secret
 
 #### Codex32 Format
 
-Run `tools/hsmtool getcodexsecret <hsm/secret/path> <id>` to get the `hsm_secret` in codex32 format.
+Run `tools/lightning-hsmtool getcodexsecret <hsm/secret/path> <id>` to get the `hsm_secret` in codex32 format.
 
-Example `tools/hsmtool getcodexsecret ~/.lightning/bitcoin/hsm_secret adt0`.
+Example `tools/lightning-hsmtool getcodexsecret ~/.lightning/bitcoin/hsm_secret adt0`.
 
 `hsm/secret/path` in the above command is `$LIGHTNINGDIR/hsm_secret`, and
 `id` is any 4 character string used to identify this secret. It **cannot** contain `i`, `o`, or `b`, but **can** contain all digits except `1`.

doc/index.rst

@@ -86,6 +86,7 @@ Core Lightning Documentation
    invoicerequest <invoicerequest.7.md>
    keysend <keysend.7.md>
    lightning-cli <lightning-cli.1.md>
+   lightning-downgrade <lightning-downgrade.8.md>
    lightning-hsmtool <lightning-hsmtool.8.md>
    lightning-reckless <lightning-reckless.1.md>
    lightningd <lightningd.8.md>

doc/lightning-downgrade.8.md

@@ -0,0 +1,102 @@
+lightning-downgrade -- Tool to revert core lightning to an older version
+========================================================================
+
+SYNOPSIS
+--------
+
+```bash
+lightning-downgrade method [ARGUMENTS]...
+```
+
+DESCRIPTION
+-----------
+
+**lightning-downgrade** reverts an upgrade by modifying the `lightningd`
+database back the prior version.  `lightningd` must **not** be running
+at the time.
+
+A downgrade may not be possible if a new feature has been used that would
+be incompatible with an older version.  In this case the downgrade will fail
+with a message and nothing will be changed.
+
+Use the latest `lightning-downgrade` to downgrade.  For example, the `v25.12` lightning-downgrade won't know how to downgrade `v26.06`.
+
+All minor versions are compatible, so a downgrade to v25.09 will work
+fine with v25.09.1 or v25.09.2, etc.
+
+VERSIONS
+--------
+
+* *v25.12*: downgrades to v25.09.
+
+  Downgrade is not possible if `withhold` `true` has been used with `fundchannel_complete`.
+
+* *v25.09*: downgrade is not supported.
+
+OPTIONS
+-------
+
+* **--lightning-dir**=*DIR*
+
+  Set the directory for the lightning daemon we're talking to; defaults to
+*$HOME/.lightning*.
+
+* **--conf**=*PATH*
+
+  Sets configuration file (default: **lightning-dir**/*config* ).
+
+* **--network**=*network*
+* **--mainnet**
+* **--testnet**
+* **--testnet4**
+* **--signet**
+* **--regtest**
+
+  Sets network explicitly.
+
+* **--rpc-file**=*FILE*
+
+  Named pipe to use to talk to lightning daemon: default is
+*lightning-rpc* in the lightning directory.
+
+* **wallet**=*DSN*
+
+  Identify the location of the wallet.  See lightningd-config(5) for details.
+
+* **--help**/**-h**
+
+  Pretty-print summary of options to standard output and exit.  The format can
+be changed using `-F`, `-R`, `-J`, `-H` etc.
+
+* **--version**/**-V**
+
+  Print version number to standard output and exit.
+
+
+BUGS
+----
+
+You should report bugs on our github issues page, and maybe submit a fix
+to gain our eternal gratitude!
+
+AUTHOR
+------
+
+Rusty Russell <<rusty@rustcorp.com.au>> wrote the initial version of **lightning-downgrade** and this man page.
+
+SEE ALSO
+--------
+
+lightningd(8), lightningd-config(5)
+
+RESOURCES
+---------
+
+Main web site: <https://github.com/ElementsProject/lightning>
+
+COPYING
+-------
+
+Note: the modules in the ccan/ directory have their own licenses, but
+the rest of the code is covered by the BSD-style MIT license.
+Main web site: <https://github.com/ElementsProject/lightning>

doc/lightningd-config.5.md

@@ -338,7 +338,7 @@ connections. Default is 9736.
 
 * **recover**=*hsmsecret*
 
-  Restore the node from a 32-byte secret encoded as either a codex32 secret string or a 64-character hex string: this will fail if the `hsm_secret` file exists.  Your node will start the node in offline mode, for manual recovery.  The secret can be extracted from the `hsm_secret` using hsmtool(8).
+  Restore the node from a 32-byte secret encoded as either a codex32 secret string or a 64-character hex string: this will fail if the `hsm_secret` file exists.  Your node will start the node in offline mode, for manual recovery.  The secret can be extracted from the `hsm_secret` using lightning-hsmtool(8).
 
 * **alias**=*NAME*
 

doc/node-operators-guide/faq.md

@@ -159,7 +159,7 @@ There are 3 types of 'rescans' you can make:
 
 ### Database corruption / channel state lost
 
-If you lose data (likely corrupted `lightningd.sqlite3`) about a channel **with `option_static_remotekey` enabled**, you can wait for your peer to unilateraly close the channel, then use `tools/hsmtool` with the `guesstoremote` command to attempt to recover your funds from the peer's published unilateral close transaction.
+If you lose data (likely corrupted `lightningd.sqlite3`) about a channel **with `option_static_remotekey` enabled**, you can wait for your peer to unilateraly close the channel, then use `tools/lightning-hsmtool` with the `guesstoremote` command to attempt to recover your funds from the peer's published unilateral close transaction.
 
 If `option_static_remotekey` was not enabled, you're probably out of luck. The keys for your funds in your peer's unilateral close transaction are derived from information you lost. Fortunately, since version `0.7.3` channels are created with `option_static_remotekey` by default if your peer supports it. Which is to say that channels created after block [598000](https://blockstream.info/block/0000000000000000000dd93b8fb5c622b9c903bf6f921ef48e266f0ead7faedb)  
 (short channel id starting with > 598000) have a high chance of supporting `option_static_remotekey`. You can verify it using the `features` field from the [`listpeers` command](ref:listpeers)'s result.

doc/schemas/decode.json

@@ -19,7 +19,7 @@
           "Value to be decoded:",
           "    * a *bolt11* or *bolt12* string (optionally prefixed by `lightning:` or `LIGHTNING:`) as specified by the BOLT 11 and BOLT 12 specifications.",
           "    * a *rune* as created by lightning-commando-rune(7).",
-          "    * an *emergency_recover* string generated by hsmtool like `lightning-hsmtool getemergencyrecover <path/to/emergency.recover>`. It holds `emergency.recover` contents and starts with `clnemerg1`."
+          "    * an *emergency_recover* string generated by lightning-hsmtool like `lightning-hsmtool getemergencyrecover <path/to/emergency.recover>`. It holds `emergency.recover` contents and starts with `clnemerg1`."
         ]
       }
     }

doc/schemas/recover.json

@@ -6,7 +6,7 @@
   "description": [
     "The **recover** RPC command wipes your node and restarts it with the `--recover` option. This is only permitted if the node is unused: no channels, no bitcoin addresses issued (you can use `check` to see if recovery is possible).",
     "",
-    "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string.",
+    "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string.",
     "",
     "NOTE: this command only currently works with the `sqlite3` database backend."
   ],
@@ -19,7 +19,7 @@
       "hsmsecret": {
         "type": "string",
         "description": [
-          "Either a codex32 secret starting with `cl1` as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string."
+          "Either a codex32 secret starting with `cl1` as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string."
         ]
       }
     }