diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 14ff313525a6..cd400b61eb26 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -235,6 +235,86 @@ jobs: ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang uv run make -j $(nproc) check-fuzz + check-downgrade: + name: Check we can downgrade the node + runs-on: ubuntu-22.04 + needs: + - compile + strategy: + fail-fast: false + matrix: + include: + - CFG: compile-gcc + TEST_DB_PROVIDER: sqlite3 + TEST_NETWORK: regtest + VALGRIND: 1 + - CFG: compile-gcc + TEST_DB_PROVIDER: postgres + TEST_NETWORK: regtest + - CFG: compile-gcc + TEST_DB_PROVIDER: sqlite3 + TEST_NETWORK: liquid-regtest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: | + bash -x .github/scripts/setup.sh + + - name: Install bitcoind + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} + run: .github/scripts/install-bitcoind.sh + + - name: Download build + uses: actions/download-artifact@v4 + with: + name: cln-${{ matrix.CFG }}.tar.bz2 + + - name: Unpack pre-built CLN + env: + CFG: ${{ matrix.CFG }} + run: | + tar -xaf cln-${CFG}.tar.bz2 + + - name: Fetch and unpack previous CLN + run: | + mkdir /tmp/old-cln + cd /tmp/old-cln + wget https://github.com/ElementsProject/lightning/releases/download/v25.09/clightning-v25.09-Ubuntu-22.04-amd64.tar.xz + tar -xaf clightning-v25.09-Ubuntu-22.04-amd64.tar.xz + + - name: Switch network + if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} + run: | + # Loading the network from config.vars rather than the envvar is a terrible idea... + sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars + cat config.vars + + - name: Test + env: + SLOW_MACHINE: 1 + PYTEST_PAR: 10 + TEST_DEBUG: 1 + TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} + LIGHTNINGD_POSTGRES_NO_VACUUM: 1 + VALGRIND: ${{ matrix.VALGRIND }} + PREV_LIGHTNINGD: /tmp/old-cln/usr/bin/lightningd + run: | + env + cat config.vars + uv run eatmydata pytest tests/test_downgrade.py -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} + integration: name: Test CLN ${{ matrix.name }} runs-on: ubuntu-22.04 @@ -627,6 +707,7 @@ jobs: - integration-valgrind - integration-sanitizers - min-btc-support + - check-downgrade if: ${{ always() }} steps: - name: Complete @@ -638,6 +719,7 @@ jobs: SANITIZERS: ${{ needs['integration-sanitizers'].result }} DOCS: ${{ needs['update-docs-examples'].result }} BTC: ${{ needs['min-btc-support'].result }} + CHECK_DOWNGRADE: ${{ needs['check-downgrade'].result }} run: | failed="" for name in $JOB_NAMES; do diff --git a/Makefile b/Makefile index f6cd7c1a0b71..d48b75b447ec 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ $(info Building version $(VERSION)) # Next release. CLN_NEXT_VERSION := v25.12 +# Previous release (for downgrade testing) +CLN_PREV_VERSION := v25.09 + # --quiet / -s means quiet, dammit! ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s) ECHO := : @@ -552,8 +555,8 @@ CHECK_BOLT_PREFIX=--prefix="BOLT-$(BOLTVERSION)" endif # Any mention of BOLT# must be followed by an exact quote, modulo whitespace. -bolt-check/%: % bolt-precheck tools/check-bolt - @if [ -d .tmp.lightningrfc ]; then tools/check-bolt $(CHECK_BOLT_PREFIX) .tmp.lightningrfc $<; else echo "Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist" >&2; fi +bolt-check/%: % bolt-precheck devtools/check-bolt + @if [ -d .tmp.lightningrfc ]; then devtools/check-bolt $(CHECK_BOLT_PREFIX) .tmp.lightningrfc $<; else echo "Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist" >&2; fi LOCAL_BOLTDIR=.tmp.lightningrfc @@ -565,7 +568,7 @@ check-source-bolt: $(ALL_NONGEN_SRCFILES:%=bolt-check/%) check-whitespace/%: % @if grep -Hn '[ ]$$' $<; then echo Extraneous whitespace found >&2; exit 1; fi -check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%) +check-whitespace: check-whitespace/Makefile check-whitespace/devtools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%) check-spelling: @tools/check-spelling.sh @@ -934,8 +937,7 @@ TESTBINS = \ $(CLN_PLUGIN_EXAMPLES) \ tests/plugins/test_libplugin \ tests/plugins/channeld_fakenet \ - tests/plugins/test_selfdisable_after_getmanifest \ - tools/hsmtool + tests/plugins/test_selfdisable_after_getmanifest # The testpack is used in CI to transfer built artefacts between the # build and the test phase. This is necessary because the fixtures in @@ -944,7 +946,7 @@ TESTBINS = \ # version of `lightningd` leading to bogus results. We bundle up all # built artefacts here, and will unpack them on the tester (overlaying # on top of the checked out repo as if we had just built it in place). -testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS) +testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS) $(TOOLS) tar -caf $@ $^ uninstall: diff --git a/README.md b/README.md index 46bd108bfe02..ca5969bb1785 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ If the two blockheights drift apart it might be necessary to intervene. ### HD wallet encryption -You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key) by passing the `--encrypted-hsm` startup argument, or by using the `hsmtool` (which you can find in the `tool/` directory at the root of this repo) with the `encrypt` method. You can unencrypt an encrypted `hsm_secret` using the `hsmtool` with the `decrypt` method. +You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key) by passing the `--encrypted-hsm` startup argument, or by using the `lightning-hsmtool` (which you can find in the `tool/` directory at the root of this repo) with the `encrypt` method. You can unencrypt an encrypted `hsm_secret` using the `lightning-hsmtool` with the `decrypt` method. If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` startup option to `lightningd`. Once your `hsm_secret` is encrypted, you __will not__ be able to access your funds without your password, so please beware with your password management. Also, beware of not feeling too safe with an encrypted `hsm_secret`: unlike for `bitcoind` where the wallet encryption can restrict the usage of some RPC command, `lightningd` always needs to access keys from the wallet which is thus __not locked__ (yet), even with an encrypted BIP32 master seed. diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index 6a373d1cc4b2..8b2e1abc6e7a 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -7472,7 +7472,7 @@ "Value to be decoded:", " * a *bolt11* or *bolt12* string (optionally prefixed by `lightning:` or `LIGHTNING:`) as specified by the BOLT 11 and BOLT 12 specifications.", " * a *rune* as created by lightning-commando-rune(7).", - " * an *emergency_recover* string generated by hsmtool like `lightning-hsmtool getemergencyrecover `. It holds `emergency.recover` contents and starts with `clnemerg1`." + " * an *emergency_recover* string generated by lightning-hsmtool like `lightning-hsmtool getemergencyrecover `. It holds `emergency.recover` contents and starts with `clnemerg1`." ] } } @@ -30511,7 +30511,7 @@ "description": [ "The **recover** RPC command wipes your node and restarts it with the `--recover` option. This is only permitted if the node is unused: no channels, no bitcoin addresses issued (you can use `check` to see if recovery is possible).", "", - "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string.", + "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string.", "", "NOTE: this command only currently works with the `sqlite3` database backend." ], @@ -30524,7 +30524,7 @@ "hsmsecret": { "type": "string", "description": [ - "Either a codex32 secret starting with `cl1` as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string." + "Either a codex32 secret starting with `cl1` as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string." ] } } diff --git a/devtools/Makefile b/devtools/Makefile index a61f3c3d0874..b7c4874032e3 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -1,4 +1,4 @@ -DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune devtools/gossmap-compress devtools/bip137-verifysignature devtools/convert-gossmap +DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune devtools/gossmap-compress devtools/bip137-verifysignature devtools/convert-gossmap devtools/check-bolt ifeq ($(HAVE_SQLITE3),1) DEVTOOLS += devtools/checkchannels endif diff --git a/tools/check-bolt.c b/devtools/check-bolt.c similarity index 100% rename from tools/check-bolt.c rename to devtools/check-bolt.c diff --git a/doc/Makefile b/doc/Makefile index 613036bb88c7..4603bd9c0f56 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -8,6 +8,7 @@ MARKDOWNPAGES := doc/addgossip.7 \ doc/addpsbtoutput.7 \ doc/askrene-age.7 \ doc/askrene-bias-channel.7 \ + doc/askrene-bias-node.7 \ doc/askrene-create-layer.7 \ doc/askrene-remove-layer.7 \ doc/askrene-create-channel.7 \ @@ -162,6 +163,7 @@ NON_PREFIXED_MANPAGES := doc/lightning-cli.1 \ doc/lightningd-config.5 \ doc/lightningd-rpc.7 \ doc/lightning-hsmtool.8 \ + doc/lightning-downgrade.8 \ doc/lightning-reckless.1 MANPAGES := $(PREFIXED_MANPAGES) $(NON_PREFIXED_MANPAGES) diff --git a/doc/beginners-guide/backup-and-recovery/hsm-secret.md b/doc/beginners-guide/backup-and-recovery/hsm-secret.md index e1fa3a02b569..d23ef5373f57 100644 --- a/doc/beginners-guide/backup-and-recovery/hsm-secret.md +++ b/doc/beginners-guide/backup-and-recovery/hsm-secret.md @@ -10,7 +10,7 @@ privacy: ## Generate HSM Secret -If you are deploying a new node that has no funds and channels yet, you can generate BIP39 words using any process, and create the `hsm_secret` using the `hsmtool generatehsm` command. If you did `make install` then `hsmtool` is installed as [`lightning-hsmtool`](ref:lightning-hsmtool), else you can find it in the `tools/` directory of the build directory. +If you are deploying a new node that has no funds and channels yet, you can generate BIP39 words using any process, and create the `hsm_secret` using the `lightning-hsmtool generatehsm` command. If you did `make install` then `hsmtool` is installed as [`lightning-hsmtool`](ref:lightning-hsmtool), else you can find it in the `tools/` directory of the build directory. ```shell lightning-hsmtool generatehsm hsm_secret @@ -25,14 +25,14 @@ You can regenerate the same `hsm_secret` file using the same BIP39 words, which You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key): - either by passing the `--encrypted-hsm` startup argument -- or by using the `encrypt` method from `/tools/hsmtool`. +- or by using the `encrypt` method from `/tools/lightning-hsmtool`. If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` startup option to `lightningd`. Once your `hsm_secret` is encrypted, you **will not** be able to access your funds without your password, so please beware with your password management. Also, beware of not feeling too safe with an encrypted `hsm_secret`: unlike for `bitcoind` where the wallet encryption can restrict the usage of some RPC command, `lightningd` always needs to access keys from the wallet which is thus **not locked** (yet), even with an encrypted BIP32 master seed. ## Decrypt HSM Secret -You can unencrypt an encrypted `hsm_secret` using the `hsmtool` with the `decrypt` method. +You can unencrypt an encrypted `hsm_secret` using the `lightning-hsmtool` with the `decrypt` method. ```shell lightning-hsmtool decrypt ${LIGHTNINGDIR}/hsm_secret diff --git a/doc/beginners-guide/backup.md b/doc/beginners-guide/backup.md index dc31df168c97..9811b9d26a2b 100644 --- a/doc/beginners-guide/backup.md +++ b/doc/beginners-guide/backup.md @@ -65,9 +65,9 @@ chmod 0400 hsm_secret #### Codex32 Format -Run `tools/hsmtool getcodexsecret ` to get the `hsm_secret` in codex32 format. +Run `tools/lightning-hsmtool getcodexsecret ` to get the `hsm_secret` in codex32 format. -Example `tools/hsmtool getcodexsecret ~/.lightning/bitcoin/hsm_secret adt0`. +Example `tools/lightning-hsmtool getcodexsecret ~/.lightning/bitcoin/hsm_secret adt0`. `hsm/secret/path` in the above command is `$LIGHTNINGDIR/hsm_secret`, and `id` is any 4 character string used to identify this secret. It **cannot** contain `i`, `o`, or `b`, but **can** contain all digits except `1`. diff --git a/doc/index.rst b/doc/index.rst index a7487d8718bd..8bb0ddbac0e5 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,6 +16,7 @@ Core Lightning Documentation addpsbtoutput askrene-age askrene-bias-channel + askrene-bias-node askrene-create-channel askrene-create-layer askrene-disable-node @@ -87,6 +88,7 @@ Core Lightning Documentation invoicerequest keysend lightning-cli + lightning-downgrade lightning-hsmtool lightning-reckless lightningd diff --git a/doc/lightning-downgrade.8.md b/doc/lightning-downgrade.8.md new file mode 100644 index 000000000000..6f154001d9b0 --- /dev/null +++ b/doc/lightning-downgrade.8.md @@ -0,0 +1,102 @@ +lightning-downgrade -- Tool to revert core lightning to an older version +======================================================================== + +SYNOPSIS +-------- + +```bash +lightning-downgrade [ARGUMENTS]... +``` + +DESCRIPTION +----------- + +**lightning-downgrade** reverts an upgrade by modifying the `lightningd` +database back the prior version. `lightningd` must **not** be running +at the time. + +A downgrade may not be possible if a new feature has been used that would +be incompatible with an older version. In this case the downgrade will fail +with a message and nothing will be changed. + +Use the latest `lightning-downgrade` to downgrade. For example, the `v25.12` lightning-downgrade won't know how to downgrade `v26.06`. + +All minor versions are compatible, so a downgrade to v25.09 will work +fine with v25.09.1 or v25.09.2, etc. + +VERSIONS +-------- + +* *v25.12*: downgrades to v25.09. + + Downgrade is not possible if `withhold` `true` has been used with `fundchannel_complete`, or if `askrene-bias-node` has been used. + +* *v25.09*: downgrade is not supported. + +OPTIONS +------- + +* **--lightning-dir**=*DIR* + + Set the directory for the lightning daemon we're talking to; defaults to +*$HOME/.lightning*. + +* **--conf**=*PATH* + + Sets configuration file (default: **lightning-dir**/*config* ). + +* **--network**=*network* +* **--mainnet** +* **--testnet** +* **--testnet4** +* **--signet** +* **--regtest** + + Sets network explicitly. + +* **--rpc-file**=*FILE* + + Named pipe to use to talk to lightning daemon: default is +*lightning-rpc* in the lightning directory. + +* **wallet**=*DSN* + + Identify the location of the wallet. See lightningd-config(5) for details. + +* **--help**/**-h** + + Pretty-print summary of options to standard output and exit. The format can +be changed using `-F`, `-R`, `-J`, `-H` etc. + +* **--version**/**-V** + + Print version number to standard output and exit. + + +BUGS +---- + +You should report bugs on our github issues page, and maybe submit a fix +to gain our eternal gratitude! + +AUTHOR +------ + +Rusty Russell <> wrote the initial version of **lightning-downgrade** and this man page. + +SEE ALSO +-------- + +lightningd(8), lightningd-config(5) + +RESOURCES +--------- + +Main web site: + +COPYING +------- + +Note: the modules in the ccan/ directory have their own licenses, but +the rest of the code is covered by the BSD-style MIT license. +Main web site: diff --git a/doc/lightningd-config.5.md b/doc/lightningd-config.5.md index a905d29e6208..e63d5b321b41 100644 --- a/doc/lightningd-config.5.md +++ b/doc/lightningd-config.5.md @@ -338,7 +338,7 @@ connections. Default is 9736. * **recover**=*hsmsecret* - Restore the node from a 32-byte secret encoded as either a codex32 secret string or a 64-character hex string: this will fail if the `hsm_secret` file exists. Your node will start the node in offline mode, for manual recovery. The secret can be extracted from the `hsm_secret` using hsmtool(8). + Restore the node from a 32-byte secret encoded as either a codex32 secret string or a 64-character hex string: this will fail if the `hsm_secret` file exists. Your node will start the node in offline mode, for manual recovery. The secret can be extracted from the `hsm_secret` using lightning-hsmtool(8). * **alias**=*NAME* diff --git a/doc/node-operators-guide/faq.md b/doc/node-operators-guide/faq.md index aaf8c23d4202..17ad832d7174 100644 --- a/doc/node-operators-guide/faq.md +++ b/doc/node-operators-guide/faq.md @@ -159,7 +159,7 @@ There are 3 types of 'rescans' you can make: ### Database corruption / channel state lost -If you lose data (likely corrupted `lightningd.sqlite3`) about a channel **with `option_static_remotekey` enabled**, you can wait for your peer to unilateraly close the channel, then use `tools/hsmtool` with the `guesstoremote` command to attempt to recover your funds from the peer's published unilateral close transaction. +If you lose data (likely corrupted `lightningd.sqlite3`) about a channel **with `option_static_remotekey` enabled**, you can wait for your peer to unilateraly close the channel, then use `tools/lightning-hsmtool` with the `guesstoremote` command to attempt to recover your funds from the peer's published unilateral close transaction. If `option_static_remotekey` was not enabled, you're probably out of luck. The keys for your funds in your peer's unilateral close transaction are derived from information you lost. Fortunately, since version `0.7.3` channels are created with `option_static_remotekey` by default if your peer supports it. Which is to say that channels created after block [598000](https://blockstream.info/block/0000000000000000000dd93b8fb5c622b9c903bf6f921ef48e266f0ead7faedb) (short channel id starting with > 598000) have a high chance of supporting `option_static_remotekey`. You can verify it using the `features` field from the [`listpeers` command](ref:listpeers)'s result. diff --git a/doc/schemas/decode.json b/doc/schemas/decode.json index 4a7ed03dc460..6635eff5c12e 100644 --- a/doc/schemas/decode.json +++ b/doc/schemas/decode.json @@ -19,7 +19,7 @@ "Value to be decoded:", " * a *bolt11* or *bolt12* string (optionally prefixed by `lightning:` or `LIGHTNING:`) as specified by the BOLT 11 and BOLT 12 specifications.", " * a *rune* as created by lightning-commando-rune(7).", - " * an *emergency_recover* string generated by hsmtool like `lightning-hsmtool getemergencyrecover `. It holds `emergency.recover` contents and starts with `clnemerg1`." + " * an *emergency_recover* string generated by lightning-hsmtool like `lightning-hsmtool getemergencyrecover `. It holds `emergency.recover` contents and starts with `clnemerg1`." ] } } diff --git a/doc/schemas/recover.json b/doc/schemas/recover.json index 0112a5e2ba5d..9732879cf10c 100644 --- a/doc/schemas/recover.json +++ b/doc/schemas/recover.json @@ -6,7 +6,7 @@ "description": [ "The **recover** RPC command wipes your node and restarts it with the `--recover` option. This is only permitted if the node is unused: no channels, no bitcoin addresses issued (you can use `check` to see if recovery is possible).", "", - "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string.", + "*hsmsecret* is either a codex32 secret starting with \"cl1\" as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string.", "", "NOTE: this command only currently works with the `sqlite3` database backend." ], @@ -19,7 +19,7 @@ "hsmsecret": { "type": "string", "description": [ - "Either a codex32 secret starting with `cl1` as returned by `hsmtool getcodexsecret`, or a raw 64 character hex string." + "Either a codex32 secret starting with `cl1` as returned by `lightning-hsmtool getcodexsecret`, or a raw 64 character hex string." ] } } diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index 97bee8f73a84..fc00a7c9c25e 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -237,7 +237,8 @@ struct anchor_details *create_anchor_details(const tal_t *ctx UNNEEDED, const struct bitcoin_tx *tx UNNEEDED) { fprintf(stderr, "create_anchor_details called!\n"); abort(); } /* Generated stub for delete_channel */ -void delete_channel(struct channel *channel STEALS UNNEEDED, bool completely_eliminate UNNEEDED) +void delete_channel(struct channel *channel STEALS UNNEEDED, + bool completely_eliminate UNNEEDED) { fprintf(stderr, "delete_channel called!\n"); abort(); } /* Generated stub for depthcb_update_scid */ bool depthcb_update_scid(struct channel *channel UNNEEDED, diff --git a/plugins/askrene/Makefile b/plugins/askrene/Makefile index 029234c53370..bd09f1b98847 100644 --- a/plugins/askrene/Makefile +++ b/plugins/askrene/Makefile @@ -1,5 +1,6 @@ PLUGIN_ASKRENE_SRC := \ plugins/askrene/askrene.c \ + plugins/askrene/datastore_wire.c \ plugins/askrene/layer.c \ plugins/askrene/reserve.c \ plugins/askrene/mcf.c \ @@ -13,6 +14,7 @@ PLUGIN_ASKRENE_SRC := \ PLUGIN_ASKRENE_HEADER := \ plugins/askrene/askrene.h \ + plugins/askrene/datastore_wire.h \ plugins/askrene/layer.h \ plugins/askrene/reserve.h \ plugins/askrene/mcf.h \ diff --git a/plugins/askrene/datastore_wire.c b/plugins/askrene/datastore_wire.c new file mode 100644 index 000000000000..022b7a60a762 --- /dev/null +++ b/plugins/askrene/datastore_wire.c @@ -0,0 +1,332 @@ +#include "config.h" +#include +#include + +/* FIXME: Generate this! */ + +/* Helper to append bool to data, and return value */ +static bool towire_bool_val(u8 **pptr, bool v) +{ + towire_bool(pptr, v); + return v; +} + +static void towire_short_channel_id_dir(u8 **pptr, const struct short_channel_id_dir *scidd) +{ + towire_short_channel_id(pptr, scidd->scid); + towire_u8(pptr, scidd->dir); +} + +static void fromwire_short_channel_id_dir(const u8 **cursor, size_t *max, + struct short_channel_id_dir *scidd) +{ + scidd->scid = fromwire_short_channel_id(cursor, max); + scidd->dir = fromwire_u8(cursor, max); +} + +static struct amount_msat *fromwire_opt_amount_msat(const tal_t *ctx, + const u8 **cursor, size_t *len) +{ + struct amount_msat *msat; + + if (!fromwire_bool(cursor, len)) + return NULL; + msat = tal(ctx, struct amount_msat); + *msat = fromwire_amount_msat(cursor, len); + return msat; +} + +static u32 *fromwire_opt_u32(const tal_t *ctx, const u8 **cursor, size_t *len) +{ + u32 *v; + + if (!fromwire_bool(cursor, len)) + return NULL; + v = tal(ctx, u32); + *v = fromwire_u32(cursor, len); + return v; +} + +static u16 *fromwire_opt_u16(const tal_t *ctx, const u8 **cursor, size_t *len) +{ + u16 *v; + + if (!fromwire_bool(cursor, len)) + return NULL; + v = tal(ctx, u16); + *v = fromwire_u16(cursor, len); + return v; +} + +static bool *fromwire_opt_bool(const tal_t *ctx, const u8 **cursor, size_t *len) +{ + bool *v; + + if (!fromwire_bool(cursor, len)) + return NULL; + v = tal(ctx, bool); + *v = fromwire_bool(cursor, len); + return v; +} + +static const char *fromwire_opt_wirestring(const tal_t *ctx, + const u8 **cursor, size_t *len) +{ + if (!fromwire_bool(cursor, len)) + return NULL; + return fromwire_wirestring(ctx, cursor, len); +} + +static void towire_opt_u32(u8 **p, const u32 *v) +{ + if (towire_bool_val(p, v != NULL)) + towire_u32(p, *v); +} + +static void towire_opt_u16(u8 **p, const u16 *v) +{ + if (towire_bool_val(p, v != NULL)) + towire_u16(p, *v); +} + +static void towire_opt_bool(u8 **p, const bool *v) +{ + if (towire_bool_val(p, v != NULL)) + towire_bool(p, *v); +} + +static void towire_opt_amount_msat(u8 **p, const struct amount_msat *v) +{ + if (towire_bool_val(p, v != NULL)) + towire_amount_msat(p, *v); +} + +static void towire_opt_wirestring(u8 **p, const char *v) +{ + if (towire_bool_val(p, v != NULL)) + towire_wirestring(p, v); +} + +bool fromwire_dstore_channel(const u8 **cursor, size_t *len, + struct node_id *n1, + struct node_id *n2, + struct short_channel_id *scid, + struct amount_msat *capacity) +{ + if (fromwire_u16(cursor, len) != DSTORE_CHANNEL) { + fromwire_fail(cursor, len); + return false; + } + fromwire_node_id(cursor, len, n1); + fromwire_node_id(cursor, len, n2); + *scid = fromwire_short_channel_id(cursor, len); + *capacity = fromwire_amount_msat(cursor, len); + + return *cursor != NULL; +} + +void towire_dstore_channel(u8 **data, + const struct node_id *n1, + const struct node_id *n2, + struct short_channel_id scid, + struct amount_msat capacity) +{ + towire_u16(data, DSTORE_CHANNEL); + towire_node_id(data, n1); + towire_node_id(data, n2); + towire_short_channel_id(data, scid); + towire_amount_msat(data, capacity); +} + +bool fromwire_dstore_channel_update(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + bool **enabled, + struct amount_msat **htlc_min, + struct amount_msat **htlc_max, + struct amount_msat **base_fee, + u32 **proportional_fee, + u16 **delay) +{ + if (fromwire_u16(cursor, len) != DSTORE_CHANNEL_UPDATE) { + fromwire_fail(cursor, len); + return false; + } + + fromwire_short_channel_id_dir(cursor, len, scidd); + *enabled = fromwire_opt_bool(ctx, cursor, len); + *htlc_min = fromwire_opt_amount_msat(ctx, cursor, len); + *htlc_max = fromwire_opt_amount_msat(ctx, cursor, len); + *base_fee = fromwire_opt_amount_msat(ctx, cursor, len); + *proportional_fee = fromwire_opt_u32(ctx, cursor, len); + *delay = fromwire_opt_u16(ctx, cursor, len); + + return *cursor != NULL; +} + +void towire_dstore_channel_update(u8 **data, + const struct short_channel_id_dir *scidd, + const bool *enabled, + const struct amount_msat *htlc_min, + const struct amount_msat *htlc_max, + const struct amount_msat *base_fee, + const u32 *proportional_fee, + const u16 *delay) +{ + towire_u16(data, DSTORE_CHANNEL_UPDATE); + towire_short_channel_id_dir(data, scidd); + towire_opt_bool(data, enabled); + towire_opt_amount_msat(data, htlc_min); + towire_opt_amount_msat(data, htlc_max); + towire_opt_amount_msat(data, base_fee); + towire_opt_u32(data, proportional_fee); + towire_opt_u16(data, delay); +} + +bool fromwire_dstore_channel_constraint(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + u64 *timestamp, + struct amount_msat **min, + struct amount_msat **max) +{ + if (fromwire_u16(cursor, len) != DSTORE_CHANNEL_CONSTRAINT) { + fromwire_fail(cursor, len); + return false; + } + + fromwire_short_channel_id_dir(cursor, len, scidd); + *timestamp = fromwire_u64(cursor, len); + *min = fromwire_opt_amount_msat(ctx, cursor, len); + *max = fromwire_opt_amount_msat(ctx, cursor, len); + + return *cursor != NULL; +} + +void towire_dstore_channel_constraint(u8 **data, + const struct short_channel_id_dir *scidd, + u64 timestamp, + const struct amount_msat *min, + const struct amount_msat *max) +{ + towire_u16(data, DSTORE_CHANNEL_CONSTRAINT); + towire_short_channel_id_dir(data, scidd); + towire_u64(data, timestamp); + towire_opt_amount_msat(data, min); + towire_opt_amount_msat(data, max); +} + +bool fromwire_dstore_channel_bias(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + s8 *bias_factor, + const char **description) +{ + if (fromwire_u16(cursor, len) != DSTORE_CHANNEL_BIAS) { + fromwire_fail(cursor, len); + return false; + } + + fromwire_short_channel_id_dir(cursor, len, scidd); + *bias_factor = fromwire_s8(cursor, len); + *description = fromwire_opt_wirestring(ctx, cursor, len); + + return *cursor != NULL; +} + +void towire_dstore_channel_bias(u8 **data, + const struct short_channel_id_dir *scidd, + s8 bias_factor, + const char *description) +{ + towire_u16(data, DSTORE_CHANNEL_BIAS); + towire_short_channel_id_dir(data, scidd); + towire_s8(data, bias_factor); + towire_opt_wirestring(data, description); +} + +bool fromwire_dstore_channel_bias_v2(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + s8 *bias_factor, + const char **description, + u64 *timestamp) +{ + if (fromwire_u16(cursor, len) != DSTORE_CHANNEL_BIAS_V2) { + fromwire_fail(cursor, len); + return false; + } + + fromwire_short_channel_id_dir(cursor, len, scidd); + *bias_factor = fromwire_s8(cursor, len); + *description = fromwire_opt_wirestring(ctx, cursor, len); + *timestamp = fromwire_u64(cursor, len); + + return *cursor != NULL; +} + +void towire_dstore_channel_bias_v2(u8 **data, + const struct short_channel_id_dir *scidd, + s8 bias_factor, + const char *description, + u64 timestamp) +{ + towire_u16(data, DSTORE_CHANNEL_BIAS_V2); + towire_short_channel_id_dir(data, scidd); + towire_s8(data, bias_factor); + towire_opt_wirestring(data, description); + towire_u64(data, timestamp); +} + +bool fromwire_dstore_node_bias(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct node_id *node, + const char **description, + s8 *in_bias, s8 *out_bias, + u64 *timestamp) +{ + if (fromwire_u16(cursor, len) != DSTORE_NODE_BIAS) { + fromwire_fail(cursor, len); + return false; + } + + fromwire_node_id(cursor, len, node); + *in_bias = fromwire_s8(cursor, len); + *out_bias = fromwire_s8(cursor, len); + *description = fromwire_opt_wirestring(ctx, cursor, len); + *timestamp = fromwire_u64(cursor, len); + + return *cursor != NULL; +} + +void towire_dstore_node_bias(u8 **data, + const struct node_id *node, + const char *description, + s8 in_bias, s8 out_bias, + u64 timestamp) +{ + towire_u16(data, DSTORE_NODE_BIAS); + towire_node_id(data, node); + towire_s8(data, in_bias); + towire_s8(data, out_bias); + towire_opt_wirestring(data, description); + towire_u64(data, timestamp); +} + +bool fromwire_dstore_disabled_node(const u8 **cursor, size_t *len, + struct node_id *node) +{ + if (fromwire_u16(cursor, len) != DSTORE_DISABLED_NODE) { + fromwire_fail(cursor, len); + return false; + } + + fromwire_node_id(cursor, len, node); + return *cursor != NULL; +} + +void towire_dstore_disabled_node(u8 **data, const struct node_id *node) +{ + towire_u16(data, DSTORE_DISABLED_NODE); + towire_node_id(data, node); +} diff --git a/plugins/askrene/datastore_wire.h b/plugins/askrene/datastore_wire.h new file mode 100644 index 000000000000..70282ad3f3b2 --- /dev/null +++ b/plugins/askrene/datastore_wire.h @@ -0,0 +1,101 @@ +#ifndef LIGHTNING_PLUGINS_ASKRENE_DATASTORE_WIRE_H +#define LIGHTNING_PLUGINS_ASKRENE_DATASTORE_WIRE_H +#include "config.h" +#include +#include +#include +#include +#include + +/* Different elements in the datastore */ +enum dstore_layer_type { + /* We don't use type 0, which fromwire_u16 returns on trunction */ + DSTORE_CHANNEL = 1, + DSTORE_CHANNEL_UPDATE = 2, + DSTORE_CHANNEL_CONSTRAINT = 3, + DSTORE_CHANNEL_BIAS = 4, + DSTORE_DISABLED_NODE = 5, + DSTORE_CHANNEL_BIAS_V2 = 6, + DSTORE_NODE_BIAS = 7, +}; + +bool fromwire_dstore_channel(const u8 **cursor, size_t *len, + struct node_id *n1, + struct node_id *n2, + struct short_channel_id *scid, + struct amount_msat *capacity); +void towire_dstore_channel(u8 **data, + const struct node_id *n1, + const struct node_id *n2, + struct short_channel_id scid, + struct amount_msat capacity); + +bool fromwire_dstore_channel_update(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + bool **enabled, + struct amount_msat **htlc_min, + struct amount_msat **htlc_max, + struct amount_msat **base_fee, + u32 **proportional_fee, + u16 **delay); +void towire_dstore_channel_update(u8 **data, + const struct short_channel_id_dir *scidd, + const bool *enabled, + const struct amount_msat *htlc_min, + const struct amount_msat *htlc_max, + const struct amount_msat *base_fee, + const u32 *proportional_fee, + const u16 *delay); + +bool fromwire_dstore_channel_constraint(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + u64 *timestamp, + struct amount_msat **min, + struct amount_msat **max); +void towire_dstore_channel_constraint(u8 **data, + const struct short_channel_id_dir *scidd, + u64 timestamp, + const struct amount_msat *min, + const struct amount_msat *max); + +bool fromwire_dstore_channel_bias(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + s8 *bias_factor, + const char **description); +void towire_dstore_channel_bias(u8 **data, + const struct short_channel_id_dir *scidd, + s8 bias_factor, + const char *description); + +bool fromwire_dstore_channel_bias_v2(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct short_channel_id_dir *scidd, + s8 *bias_factor, + const char **description, + u64 *timestamp); +void towire_dstore_channel_bias_v2(u8 **data, + const struct short_channel_id_dir *scidd, + s8 bias_factor, + const char *description, + u64 timestamp); + +bool fromwire_dstore_node_bias(const tal_t *ctx, + const u8 **cursor, size_t *len, + struct node_id *node, + const char **description, + s8 *in_bias, s8 *out_bias, + u64 *timestamp); +void towire_dstore_node_bias(u8 **data, + const struct node_id *node, + const char *description, + s8 in_bias, s8 out_bias, + u64 timestamp); + +void towire_dstore_disabled_node(u8 **data, const struct node_id *node); +bool fromwire_dstore_disabled_node(const u8 **cursor, size_t *len, + struct node_id *node); + +#endif /* LIGHTNING_PLUGINS_ASKRENE_DATASTORE_WIRE_H */ diff --git a/plugins/askrene/layer.c b/plugins/askrene/layer.c index 2ef27d8f17b5..8dccaaa28fde 100644 --- a/plugins/askrene/layer.c +++ b/plugins/askrene/layer.c @@ -8,24 +8,13 @@ #include #include #include +#include #include #include #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) -/* Different elements in the datastore */ -enum dstore_layer_type { - /* We don't use type 0, which fromwire_u16 returns on trunction */ - DSTORE_CHANNEL = 1, - DSTORE_CHANNEL_UPDATE = 2, - DSTORE_CHANNEL_CONSTRAINT = 3, - DSTORE_CHANNEL_BIAS = 4, - DSTORE_DISABLED_NODE = 5, - DSTORE_CHANNEL_BIAS_V2 = 6, - DSTORE_NODE_BIAS = 7, -}; - /* A channels which doesn't (necessarily) exist in the gossmap. */ struct local_channel { /* Canonical order, n1 < n2 */ @@ -447,39 +436,9 @@ static void append_layer_datastore(struct layer *layer, const u8 *data) send_outreq(req); } -/* We don't autogenerate our wire code here. Partially because the - * fromwire_ routines we generate are aimed towards single messages, - * not a continuous stream, but also because this needs to be - * consistent across updates, and the extensions to the wire format - * we use (for inter-daemon comms) do not have such a guarantee */ - -/* Helper to append bool to data, and return value */ -static bool towire_bool_val(u8 **pptr, bool v) -{ - towire_bool(pptr, v); - return v; -} - -static void towire_short_channel_id_dir(u8 **pptr, const struct short_channel_id_dir *scidd) -{ - towire_short_channel_id(pptr, scidd->scid); - towire_u8(pptr, scidd->dir); -} - -static void fromwire_short_channel_id_dir(const u8 **cursor, size_t *max, - struct short_channel_id_dir *scidd) -{ - scidd->scid = fromwire_short_channel_id(cursor, max); - scidd->dir = fromwire_u8(cursor, max); -} - static void towire_save_channel(u8 **data, const struct local_channel *lc) { - towire_u16(data, DSTORE_CHANNEL); - towire_node_id(data, &lc->n1); - towire_node_id(data, &lc->n2); - towire_short_channel_id(data, lc->scid); - towire_amount_msat(data, lc->capacity); + towire_dstore_channel(data, &lc->n1, &lc->n2, lc->scid, lc->capacity); } static void save_channel(struct layer *layer, const struct local_channel *lc) @@ -503,31 +462,20 @@ static void load_channel(struct plugin *plugin, struct short_channel_id scid; struct amount_msat capacity; - fromwire_node_id(cursor, len, &n1); - fromwire_node_id(cursor, len, &n2); - scid = fromwire_short_channel_id(cursor, len); - capacity = fromwire_amount_msat(cursor, len); - - if (*cursor) + if (fromwire_dstore_channel(cursor, len, &n1, &n2, &scid, &capacity)) add_local_channel(layer, &n1, &n2, scid, capacity); } static void towire_save_channel_update(u8 **data, const struct local_update *lu) { - towire_u16(data, DSTORE_CHANNEL_UPDATE); - towire_short_channel_id_dir(data, &lu->scidd); - if (towire_bool_val(data, lu->enabled != NULL)) - towire_bool(data, *lu->enabled); - if (towire_bool_val(data, lu->htlc_min != NULL)) - towire_amount_msat(data, *lu->htlc_min); - if (towire_bool_val(data, lu->htlc_max != NULL)) - towire_amount_msat(data, *lu->htlc_max); - if (towire_bool_val(data, lu->base_fee != NULL)) - towire_amount_msat(data, *lu->base_fee); - if (towire_bool_val(data, lu->proportional_fee != NULL)) - towire_u32(data, *lu->proportional_fee); - if (towire_bool_val(data, lu->delay != NULL)) - towire_u16(data, *lu->delay); + towire_dstore_channel_update(data, + &lu->scidd, + lu->enabled, + lu->htlc_min, + lu->htlc_max, + lu->base_fee, + lu->proportional_fee, + lu->delay); } static void save_channel_update(struct layer *layer, const struct local_update *lu) @@ -548,40 +496,19 @@ static void load_channel_update(struct plugin *plugin, size_t *len) { struct short_channel_id_dir scidd; - bool *enabled = NULL, enabled_val; - struct amount_msat *htlc_min = NULL, htlc_min_val; - struct amount_msat *htlc_max = NULL, htlc_max_val; - struct amount_msat *base_fee = NULL, base_fee_val; - u32 *proportional_fee = NULL, proportional_fee_val; - u16 *delay = NULL, delay_val; - - fromwire_short_channel_id_dir(cursor, len, &scidd); - if (fromwire_bool(cursor, len)) { - enabled_val = fromwire_bool(cursor, len); - enabled = &enabled_val; - } - if (fromwire_bool(cursor, len)) { - htlc_min_val = fromwire_amount_msat(cursor, len); - htlc_min = &htlc_min_val; - } - if (fromwire_bool(cursor, len)) { - htlc_max_val = fromwire_amount_msat(cursor, len); - htlc_max = &htlc_max_val; - } - if (fromwire_bool(cursor, len)) { - base_fee_val = fromwire_amount_msat(cursor, len); - base_fee = &base_fee_val; - } - if (fromwire_bool(cursor, len)) { - proportional_fee_val = fromwire_u32(cursor, len); - proportional_fee = &proportional_fee_val; - } - if (fromwire_bool(cursor, len)) { - delay_val = fromwire_u16(cursor, len); - delay = &delay_val; - } - - if (*cursor) + bool *enabled; + struct amount_msat *htlc_min, *htlc_max, *base_fee; + u32 *proportional_fee; + u16 *delay; + + if (fromwire_dstore_channel_update(tmpctx, cursor, len, + &scidd, + &enabled, + &htlc_min, + &htlc_max, + &base_fee, + &proportional_fee, + &delay)) add_update_channel(layer, &scidd, enabled, htlc_min, @@ -593,13 +520,9 @@ static void load_channel_update(struct plugin *plugin, static void towire_save_channel_constraint(u8 **data, const struct constraint *c) { - towire_u16(data, DSTORE_CHANNEL_CONSTRAINT); - towire_short_channel_id_dir(data, &c->scidd); - towire_u64(data, c->timestamp); - if (towire_bool_val(data, !amount_msat_is_zero(c->min))) - towire_amount_msat(data, c->min); - if (towire_bool_val(data, !amount_msat_eq(c->max, AMOUNT_MSAT(UINT64_MAX)))) - towire_amount_msat(data, c->max); + towire_dstore_channel_constraint(data, &c->scidd, c->timestamp, + amount_msat_is_zero(c->min) ? NULL : &c->min, + amount_msat_eq(c->max, AMOUNT_MSAT(UINT64_MAX)) ? NULL: &c->max); } static void save_channel_constraint(struct layer *layer, const struct constraint *c) @@ -620,47 +543,23 @@ static void load_channel_constraint(struct plugin *plugin, size_t *len) { struct short_channel_id_dir scidd; - struct amount_msat *min = NULL, min_val; - struct amount_msat *max = NULL, max_val; + struct amount_msat *min; + struct amount_msat *max; u64 timestamp; - fromwire_short_channel_id_dir(cursor, len, &scidd); - timestamp = fromwire_u64(cursor, len); - if (fromwire_bool(cursor, len)) { - min_val = fromwire_amount_msat(cursor, len); - min = &min_val; - } - if (fromwire_bool(cursor, len)) { - max_val = fromwire_amount_msat(cursor, len); - max = &max_val; - } - if (*cursor) + if (fromwire_dstore_channel_constraint(tmpctx, cursor, len, + &scidd, ×tamp, + &min, &max)) add_constraint(layer, &scidd, timestamp, min, max); } static void towire_save_channel_bias(u8 **data, const struct bias *bias) { - towire_u16(data, DSTORE_CHANNEL_BIAS_V2); - towire_short_channel_id_dir(data, &bias->scidd); - towire_s8(data, bias->bias); - // has description? - towire_bool_val(data, bias->description != NULL); - if (bias->description) - towire_wirestring(data, bias->description); - towire_u64(data, bias->timestamp); -} - -static void towire_save_node_bias(u8 **data, const struct node_bias *bias) -{ - towire_u16(data, DSTORE_NODE_BIAS); - towire_node_id(data, &bias->node); - towire_s8(data, bias->in_bias); - towire_s8(data, bias->out_bias); - // has description? - towire_bool_val(data, bias->description != NULL); - if (bias->description) - towire_wirestring(data, bias->description); - towire_u64(data, bias->timestamp); + towire_dstore_channel_bias_v2(data, + &bias->scidd, + bias->bias, + bias->description, + bias->timestamp); } static void save_channel_bias(struct layer *layer, const struct bias *bias) @@ -675,6 +574,16 @@ static void save_channel_bias(struct layer *layer, const struct bias *bias) append_layer_datastore(layer, data); } +static void towire_save_node_bias(u8 **data, const struct node_bias *bias) +{ + towire_dstore_node_bias(data, + &bias->node, + bias->description, + bias->in_bias, + bias->out_bias, + bias->timestamp); +} + static void save_node_bias(struct layer *layer, const struct node_bias *bias) { u8 *data; @@ -693,17 +602,16 @@ static void load_channel_bias(struct plugin *plugin, size_t *len) { struct short_channel_id_dir scidd; - char *description; + const char *description; s8 bias_factor; /* If we read an old version without timestamp, just put the current * time. */ u64 timestamp = clock_time().ts.tv_sec; - fromwire_short_channel_id_dir(cursor, len, &scidd); - bias_factor = fromwire_s8(cursor, len); - description = fromwire_wirestring(tmpctx, cursor, len); - - if (*cursor) + if (fromwire_dstore_channel_bias(tmpctx, cursor, len, + &scidd, + &bias_factor, + &description)) set_bias(layer, &scidd, take(description), bias_factor, false, timestamp); } @@ -714,18 +622,15 @@ static void load_channel_bias_v2(struct plugin *plugin, size_t *len) { struct short_channel_id_dir scidd; - char *description = NULL; + const char *description; s8 bias_factor; u64 timestamp; - fromwire_short_channel_id_dir(cursor, len, &scidd); - bias_factor = fromwire_s8(cursor, len); - if (fromwire_bool(cursor, len)) { - description = fromwire_wirestring(tmpctx, cursor, len); - } - timestamp = fromwire_u64(cursor, len); - - if (*cursor) + if (fromwire_dstore_channel_bias_v2(tmpctx, cursor, len, + &scidd, + &bias_factor, + &description, + ×tamp)) set_bias(layer, &scidd, take(description), bias_factor, false, timestamp); } @@ -736,19 +641,16 @@ static void load_node_bias(struct plugin *plugin, size_t *len) { struct node_id node; - char *description = NULL; + const char *description; s8 in_bias, out_bias; u64 timestamp; - fromwire_node_id(cursor, len, &node); - in_bias = fromwire_s8(cursor, len); - out_bias = fromwire_s8(cursor, len); - if (fromwire_bool(cursor, len)) { - description = fromwire_wirestring(tmpctx, cursor, len); - } - timestamp = fromwire_u64(cursor, len); - - if (*cursor){ + if (fromwire_dstore_node_bias(tmpctx, cursor, len, + &node, + &description, + &in_bias, + &out_bias, + ×tamp)) { set_node_bias(layer, &node, take(description), in_bias, /* relative = */ false, /* out dir = */ false, timestamp); @@ -758,12 +660,6 @@ static void load_node_bias(struct plugin *plugin, } } -static void towire_save_disabled_node(u8 **data, const struct node_id *node) -{ - towire_u16(data, DSTORE_DISABLED_NODE); - towire_node_id(data, node); -} - static void save_disabled_node(struct layer *layer, const struct node_id *node) { u8 *data; @@ -771,7 +667,7 @@ static void save_disabled_node(struct layer *layer, const struct node_id *node) if (!layer->persistent) return; data = tal_arr(tmpctx, u8, 0); - towire_save_disabled_node(&data, node); + towire_dstore_disabled_node(&data, node); append_layer_datastore(layer, data); } @@ -782,8 +678,7 @@ static void load_disabled_node(struct plugin *plugin, { struct node_id node; - fromwire_node_id(cursor, len, &node); - if (*cursor) + if (fromwire_dstore_disabled_node(cursor, len, &node)) add_disabled_node(layer, &node); } @@ -807,7 +702,7 @@ static void save_complete_layer(struct layer *layer) data = tal_arr(tmpctx, u8, 0); for (size_t i = 0; i < tal_count(layer->disabled_nodes); i++) - towire_save_disabled_node(&data, &layer->disabled_nodes[i]); + towire_dstore_disabled_node(&data, &layer->disabled_nodes[i]); for (lc = local_channel_hash_first(layer->local_channels, &lcit); lc; @@ -863,7 +758,7 @@ static void populate_layer(struct askrene *askrene, while (len != 0) { enum dstore_layer_type type; - type = fromwire_u16(&data, &len); + type = fromwire_peektypen(data, len); switch (type) { case DSTORE_CHANNEL: diff --git a/tests/autogenerate-rpc-examples.py b/tests/autogenerate-rpc-examples.py index 0a8f880df775..89c906609705 100644 --- a/tests/autogenerate-rpc-examples.py +++ b/tests/autogenerate-rpc-examples.py @@ -1827,7 +1827,7 @@ def get_hsm_secret(n): """Returns codex32 and hex""" try: hsmfile = os.path.join(n.daemon.lightning_dir, TEST_NETWORK, "hsm_secret") - codex32 = subprocess.check_output(["tools/hsmtool", "getcodexsecret", hsmfile, "leet"]).decode('utf-8').strip() + codex32 = subprocess.check_output(["tools/lightning-hsmtool", "getcodexsecret", hsmfile, "leet"]).decode('utf-8').strip() with open(hsmfile, "rb") as f: hexhsm = f.read().hex() return codex32, hexhsm diff --git a/tests/test_askrene.py b/tests/test_askrene.py index f5e986ade798..bf87287e9b05 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -1671,7 +1671,7 @@ def test_askrene_fake_channeld(node_factory, bitcoind): hsmfile = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, "hsm_secret") # Needs peer node id and channel dbid (1, it's the first channel), prints out: # "shaseed: xxxxxxx\n" - shaseed = subprocess.check_output(["tools/hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] + shaseed = subprocess.check_output(["tools/lightning-hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] l1.rpc.dev_peer_shachain(l2.info['id'], shaseed) TEMPORARY_CHANNEL_FAILURE = 0x1007 diff --git a/tests/test_downgrade.py b/tests/test_downgrade.py new file mode 100644 index 000000000000..e460f3db2249 --- /dev/null +++ b/tests/test_downgrade.py @@ -0,0 +1,112 @@ +from fixtures import * # noqa: F401,F403 +from utils import ( + TIMEOUT, # noqa: F401 + first_scid, only_one, +) + +import os +import subprocess + +# From the binary: +# ERROR_DBVERSION = 1 +ERROR_DBFAIL = 2 +ERROR_USAGE = 3 +# ERROR_INTERNAL = 99 + + +def downgrade_cmdline(node): + # lightning-downgrade understands a subset of the options + # to lightningd. + downgrade_opts = [] + for o in node.daemon.opts: + if o in ('network', 'lightning-dir', 'conf', 'rpc-file', 'wallet'): + if node.daemon.opts[o] is None: + downgrade_opts.append(f"--{o}") + else: + downgrade_opts.append(f"--{o}={node.daemon.opts[o]}") + + cmd_line = ["tools/lightning-downgrade"] + downgrade_opts + if os.getenv("VALGRIND") == "1": + cmd_line = ['valgrind', '-q', '--error-exitcode=7'] + cmd_line + return cmd_line + + +def test_downgrade(node_factory, executor): + l1, l2 = node_factory.line_graph(2, opts={'may_reconnect': True}, wait_for_announce=True) + + bias_scidd = f"{first_scid(l1, l2)}/0" + # Create a bias for this channel. + l1.rpc.askrene_bias_channel('xpay', bias_scidd, 1) + bias = only_one(only_one(l1.rpc.askrene_listlayers('xpay')['layers'])['biases']) + assert bias['short_channel_id_dir'] == bias_scidd + assert bias['bias'] == 1 + + # Make a payment, which means we update layer information. + old_inv = l2.rpc.invoice(1000, 'test_downgrade1', 'test_downgrade') + l1.rpc.xpay(old_inv['bolt11']) + + cmd_line = downgrade_cmdline(l1) + + # No downgrade on live nodes! + retcode = subprocess.call(cmd_line, timeout=TIMEOUT) + assert retcode == ERROR_USAGE + + l1.stop() + subprocess.check_call(cmd_line) + + # Test with old lightningd if it's available. + old_cln = os.getenv('PREV_LIGHTNINGD') + if old_cln: + current_executable = l1.daemon.executable + l1.daemon.executable = old_cln + + l1.start() + + # Disable schema checking here: the node is OLD! + l1.rpc.jsonschemas = {} + + # It should connect to l2 no problems, make payment. + l1.connect(l2) + inv = l2.rpc.invoice(1000, 'test_downgrade', 'test_downgrade') + l1.rpc.xpay(inv['bolt11']) + + # It should see the bias! + bias = only_one(only_one(l1.rpc.askrene_listlayers('xpay')['layers'])['biases']) + assert bias['short_channel_id_dir'] == bias_scidd + assert bias['bias'] == 1 + + l1.stop() + l1.daemon.executable = current_executable + + # Another downgrade is a noop. + assert subprocess.check_output(cmd_line).decode("utf8").startswith("Already compatible with ") + + # Should be able to upgrade without any trouble + l1.daemon.opts['database-upgrade'] = True + l1.start() + assert l1.daemon.is_in_log("Updating database from version") + + l1.connect(l2) + inv2 = l2.rpc.invoice(1000, 'test_downgrade2', 'test_downgrade2') + l1.rpc.xpay(inv2['bolt11']) + + # bias still present + bias = only_one(only_one(l1.rpc.askrene_listlayers('xpay')['layers'])['biases']) + assert bias['short_channel_id_dir'] == bias_scidd + assert bias['bias'] == 1 + + +def test_downgrade_fail(node_factory, executor): + """If we have created as node bias, we cannot downgrade""" + l1, l2 = node_factory.line_graph(2, opts={'may_reconnect': True}, wait_for_announce=True) + + l1.rpc.askrene_bias_node('xpay', l2.info['id'], 'in', 1) + cmd_line = downgrade_cmdline(l1) + + l1.stop() + + p = subprocess.Popen(cmd_line, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) + _, err = p.communicate(timeout=TIMEOUT) + assert p.returncode == ERROR_DBFAIL + assert 'Askrene has a node bias, which is not supported in v25.09' in err.decode('utf-8') diff --git a/tests/test_misc.py b/tests/test_misc.py index 31a9c3c9e600..00fc0d6a8cae 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -1510,7 +1510,7 @@ def test_decode(node_factory, bitcoind): """Test the decode option to decode the contents of emergency recovery. """ l1 = node_factory.get_node() - cmd_line = ["tools/hsmtool", "getemergencyrecover", os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "emergency.recover")] + cmd_line = ["tools/lightning-hsmtool", "getemergencyrecover", os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "emergency.recover")] out = subprocess.check_output(cmd_line).decode('utf-8') bech32_out = out.strip('\n') assert bech32_out.startswith('clnemerg1') @@ -1533,7 +1533,7 @@ def test_recover(node_factory, bitcoind): os.unlink(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "hsm_secret")) l1.daemon.start() - cmd_line = ["tools/hsmtool", "getcodexsecret", os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "hsm_secret")] + cmd_line = ["tools/lightning-hsmtool", "getcodexsecret", os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "hsm_secret")] out = subprocess.check_output(cmd_line + ["leet", "0"]).decode('utf-8') assert out == "cl10leetsllhdmn9m42vcsamx24zrxgs3qrl7ahwvhw4fnzrhve25gvezzyqqjdsjnzedu43ns\n" @@ -4593,7 +4593,7 @@ def test_recover_command(node_factory, bitcoind): def get_hsm_secret(n): """Returns codex32 and hex""" hsmfile = os.path.join(n.daemon.lightning_dir, TEST_NETWORK, "hsm_secret") - codex32 = subprocess.check_output(["tools/hsmtool", "getcodexsecret", hsmfile, "leet"]).decode('utf-8').strip() + codex32 = subprocess.check_output(["tools/lightning-hsmtool", "getcodexsecret", hsmfile, "leet"]).decode('utf-8').strip() with open(hsmfile, "rb") as f: hexhsm = f.read().hex() return codex32, hexhsm diff --git a/tests/test_plugin.py b/tests/test_plugin.py index a036ecc2c59a..3198fcd271b0 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -4427,12 +4427,12 @@ def test_exposesecret(node_factory): with pytest.raises(RpcError, match="must be valid bech32 string"): l1.rpc.exposesecret(passphrase='test_exposesecret', identifier=invalid) - # As given by hsmtool: - # $ ./tools/hsmtool getcodexsecret /tmp/ltests-10uyxcnw/test_exposesecret_1/lightning-1/regtest/hsm_secret junr + # As given by lightning-hsmtool: + # $ ./tools/lightning-hsmtool getcodexsecret /tmp/ltests-10uyxcnw/test_exposesecret_1/lightning-1/regtest/hsm_secret junr # cl10junrsd35kw6r5de5kueedxyqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqj00m675kxffh - # $ ./tools/hsmtool getcodexsecret /tmp/ltests-10uyxcnw/test_exposesecret_1/lightning-1/regtest/hsm_secret junx + # $ ./tools/lightning-hsmtool getcodexsecret /tmp/ltests-10uyxcnw/test_exposesecret_1/lightning-1/regtest/hsm_secret junx # cl10junxsd35kw6r5de5kueedxyqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq6mdtn5lql6p8m - # $ ./tools/hsmtool getcodexsecret /tmp/ltests-10uyxcnw/test_exposesecret_1/lightning-1/regtest/hsm_secret cln2 + # $ ./tools/lightning-hsmtool getcodexsecret /tmp/ltests-10uyxcnw/test_exposesecret_1/lightning-1/regtest/hsm_secret cln2 # cl10cln2sd35kw6r5de5kueedxyqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq2v3y60yxxn4mq assert l1.rpc.exposesecret(passphrase='test_exposesecret') == {'codex32': 'cl10junrsd35kw6r5de5kueedxyqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqj00m675kxffh', 'identifier': 'junr'} diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 0dfa66cd1243..8781161e2a60 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -1274,10 +1274,10 @@ def write_all(fd, bytestr): class HsmTool(TailableProc): """Helper for testing the hsmtool as a subprocess""" def __init__(self, directory, *args): - self.prefix = "hsmtool" - TailableProc.__init__(self, os.path.join(directory, "hsmtool")) + self.prefix = "lightning-hsmtool" + TailableProc.__init__(self, os.path.join(directory, "lightning-hsmtool")) assert hasattr(self, "env") - self.cmd_line = ["tools/hsmtool", *args] + self.cmd_line = ["tools/lightning-hsmtool", *args] @unittest.skipIf(VALGRIND, "It does not play well with prompt and key derivation.") @@ -1653,7 +1653,7 @@ def test_hsmtool_all_commands_work_with_mnemonic_formats(node_factory): ] for cmd_args, expected_output in test_commands: - cmd_line = ["tools/hsmtool"] + cmd_args + cmd_line = ["tools/lightning-hsmtool"] + cmd_args out = subprocess.check_output(cmd_line).decode("utf8") actual_output = out.strip() assert actual_output == expected_output, f"Command {cmd_args[0]} output mismatch" diff --git a/tests/test_xpay.py b/tests/test_xpay.py index ed24a4b76909..949584121fcd 100644 --- a/tests/test_xpay.py +++ b/tests/test_xpay.py @@ -40,7 +40,7 @@ def test_pay_fakenet(node_factory): hsmfile = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, "hsm_secret") # Needs peer node id and channel dbid (1, it's the first channel), prints out: # "shaseed: xxxxxxx\n" - shaseed = subprocess.check_output(["tools/hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] + shaseed = subprocess.check_output(["tools/lightning-hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] l1.rpc.dev_peer_shachain(l2.info['id'], shaseed) # Failure from final (unknown payment hash) @@ -247,7 +247,7 @@ def test_xpay_fake_channeld(node_factory, bitcoind, chainparams, slow_mode): hsmfile = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, "hsm_secret") # Needs peer node id and channel dbid (1, it's the first channel), prints out: # "shaseed: xxxxxxx\n" - shaseed = subprocess.check_output(["tools/hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] + shaseed = subprocess.check_output(["tools/lightning-hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] l1.rpc.dev_peer_shachain(l2.info['id'], shaseed) # Toggle whether we wait for all the parts to finish. @@ -564,7 +564,7 @@ def test_xpay_maxfee(node_factory, bitcoind, chainparams): hsmfile = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, "hsm_secret") # Needs peer node id and channel dbid (1, it's the first channel), prints out: # "shaseed: xxxxxxx\n" - shaseed = subprocess.check_output(["tools/hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] + shaseed = subprocess.check_output(["tools/lightning-hsmtool", "dumpcommitments", l1.info['id'], "1", "0", hsmfile]).decode('utf-8').strip().partition(": ")[2] l1.rpc.dev_peer_shachain(l2.info['id'], shaseed) # This one triggers the bug! diff --git a/tools/.gitignore b/tools/.gitignore index a17763bb919d..fa376c3387e2 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -1,4 +1,3 @@ headerversions check-bolt -hsmtool lightning-hsmtool diff --git a/tools/Makefile b/tools/Makefile index 932ed45ea838..2d86221df962 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,5 +1,5 @@ #! /usr/bin/make -TOOLS := tools/hsmtool tools/check-bolt +TOOLS := tools/lightning-hsmtool tools/lightning-downgrade TOOLS_SRC := $(TOOLS:=.c) TOOLS_OBJ := $(TOOLS_SRC:.c=.o) @@ -13,16 +13,29 @@ ALL_PROGRAMS += $(TOOLS) tools/headerversions: $(FORCE) tools/headerversions.o libccan.a @trap "rm -f $@.tmp.$$$$" EXIT; $(LINK.o) tools/headerversions.o libccan.a $(LOADLIBES) $(LDLIBS) -o $@.tmp.$$$$ && mv -f $@.tmp.$$$$ $@ +$(TOOLS): libcommon.a + tools/headerversions.o: ccan/config.h -tools/check-bolt: tools/check-bolt.o libcommon.a -tools/hsmtool: tools/hsmtool.o libcommon.a +tools/lightning-hsmtool: tools/lightning-hsmtool.o +tools/lightning-downgrade.o: CFLAGS:=$(CFLAGS) -DCLN_PREV_VERSION=$(CLN_PREV_VERSION) + +tools/lightning-downgrade: \ + db/exec.o \ + db/bindings.o \ + db/utils.o \ + wallet/datastore.o \ + wallet/migrations.o \ + plugins/askrene/datastore_wire.o \ + $(DB_OBJS) \ + $(WALLET_DB_QUERIES:.c=.o) \ + tools/lightning-downgrade.o -tools/lightning-hsmtool: tools/hsmtool - cp $< $@ +update-mocks: $(tools/lightning-downgrade.c:%=update-mocks/%.c) clean: tools-clean tools-clean: + $(RM) $(TOOLS) $(TOOLS_OBJ) $(RM) tools/headerversions $(RM) tools/headerversions.o diff --git a/tools/lightning-downgrade.c b/tools/lightning-downgrade.c new file mode 100644 index 000000000000..6b52dffb83c6 --- /dev/null +++ b/tools/lightning-downgrade.c @@ -0,0 +1,406 @@ +/* Tool for limited downgrade of an offline node */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ERROR_DBVERSION 1 +#define ERROR_DBFAIL 2 +#define ERROR_USAGE 3 +#define ERROR_INTERNAL 99 + +#define PREV_VERSION stringify(CLN_PREV_VERSION) + +struct db_version { + const char *name; + size_t db_height; + const char *(*downgrade_datastore)(const tal_t *ctx, struct db *db); + bool gossip_store_compatible; +}; + +struct layer { + const char **key; + const u8 *data; +}; + +static void copy_data(u8 **out, const u8 *in, size_t len) +{ + size_t oldlen = tal_bytelen(*out); + + tal_resize(out, oldlen + len); + memcpy(*out + oldlen, in, len); +} + +/* askrene added DSTORE_CHANNEL_BIAS_V2 (convertable) and + * DSTORE_NODE_BIAS (not convertable) */ +static const char *convert_layer_data(const tal_t *ctx, + const char *layername, + const u8 *data_in, + const u8 **data_out) +{ + size_t len = tal_bytelen(data_in); + struct node_id n; + struct short_channel_id scid; + struct amount_msat msat, *msat_ptr; + struct short_channel_id_dir scidd; + bool *bool_ptr; + u64 timestamp; + u32 *u32_ptr; + u16 *u16_ptr; + s8 bias; + const char *string; + u8 *out = tal_arr(ctx, u8, 0); + + /* Unfortunately, there are no explicit lengths, so we have + * to read all records even if we don't care about them. */ + while (len != 0) { + enum dstore_layer_type type; + const u8 *olddata = data_in; + type = fromwire_peektypen(data_in, len); + + switch (type) { + /* These are all simply digested and copied */ + case DSTORE_CHANNEL: + if (fromwire_dstore_channel(&data_in, &len, + &n, &n, &scid, &msat)) + copy_data(&out, data_in, olddata - data_in); + continue; + case DSTORE_CHANNEL_UPDATE: + if (fromwire_dstore_channel_update(tmpctx, &data_in, &len, + &scidd, &bool_ptr, + &msat_ptr, &msat_ptr, &msat_ptr, + &u32_ptr, &u16_ptr)) + copy_data(&out, data_in, olddata - data_in); + continue; + case DSTORE_CHANNEL_CONSTRAINT: + if (fromwire_dstore_channel_constraint(tmpctx, &data_in, &len, + &scidd, ×tamp, + &msat_ptr, &msat_ptr)) + copy_data(&out, data_in, olddata - data_in); + continue; + case DSTORE_CHANNEL_BIAS: + if (fromwire_dstore_channel_bias(tmpctx, &data_in, &len, + &scidd, &bias, + &string)) + copy_data(&out, data_in, olddata - data_in); + continue; + case DSTORE_DISABLED_NODE: + if (fromwire_dstore_disabled_node(&data_in, &len, &n)) + copy_data(&out, data_in, olddata - data_in); + continue; + + /* Convert back, lose timestamp */ + case DSTORE_CHANNEL_BIAS_V2: + if (fromwire_dstore_channel_bias_v2(tmpctx, &data_in, &len, + &scidd, &bias, + &string, ×tamp)) { + towire_dstore_channel_bias(&out, &scidd, bias, string); + } + continue; + + case DSTORE_NODE_BIAS: + return "Askrene has a node bias, which is not supported in v25.09"; + } + + return tal_fmt(ctx, "Unknown askrene layer record %u in %s", type, layername); + } + + if (!data_in) + return tal_fmt(ctx, "Corrupt askrene layer record for %s", layername); + + *data_out = out; + return NULL; +} + +static const char *downgrade_askrene_layers(const tal_t *ctx, struct db *db) +{ + const char **base, **k; + const u8 *data; + struct db_stmt *stmt; + struct layer **layers = tal_arr(tmpctx, struct layer *, 0); + + base = tal_arr(tmpctx, const char *, 2); + base[0] = "askrene"; + base[1] = "layers"; + + /* Gather and convert */ + for (stmt = db_datastore_first(tmpctx, db, base, + &k, &data, NULL); + stmt; + stmt = db_datastore_next(tmpctx, stmt, base, + &k, &data, NULL)) { + struct layer *layer; + const char *err; + + if (!data) + continue; + layer = tal(layers, struct layer); + layer->key = tal_steal(layer, k); + err = convert_layer_data(layer, k[2], data, &layer->data); + if (err) { + tal_free(stmt); + return err; + } + tal_arr_expand(&layers, layer); + } + + /* Write back */ + for (size_t i = 0; i < tal_count(layers); i++) + db_datastore_update(db, layers[i]->key, layers[i]->data); + return NULL; +} + +static const struct db_version db_versions[] = { + { "v25.09", 276, downgrade_askrene_layers, false }, + /* When we implement v25.12 downgrade: { "v25.12", 280, ???}, */ +}; + +static const struct db_version *version_db(const char *version) +{ + for (size_t i = 0; i < ARRAY_SIZE(db_versions); i++) { + if (streq(db_versions[i].name, version)) + return &db_versions[i]; + } + errx(ERROR_INTERNAL, "Unknown version %s", version); +} + +static void db_error(void *unused, bool fatal, const char *fmt, va_list ap) +{ + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + if (fatal) + exit(ERROR_DBFAIL); +} + +/* The standard opt_log_stderr_exit exits with status 1 */ +static void opt_log_stderr_exit_usage(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + va_end(ap); + exit(ERROR_USAGE); +} + +int main(int argc, char *argv[]) +{ + char *config_filename, *base_dir, *net_dir, *rpc_filename, *wallet_dsn = NULL; + const struct db_version *prev_version; + size_t current, num_migrations; + struct db *db; + const struct db_migration *migrations; + struct db_stmt *stmt; + + setup_locale(); + err_set_progname(argv[0]); + + minimal_config_opts(tmpctx, argc, argv, &config_filename, &base_dir, + &net_dir, &rpc_filename); + opt_register_early_arg("--wallet", opt_set_talstr, NULL, + &wallet_dsn, + "Location of the wallet database."); + opt_register_noarg("--help|-h", opt_usage_and_exit, + "A tool to downgrade an offline Core Lightning Node to " PREV_VERSION, + "Print this message."); + opt_early_parse(argc, argv, opt_log_stderr_exit_usage); + opt_parse(&argc, argv, opt_log_stderr_exit_usage); + + if (argc != 1) + opt_usage_exit_fail("No arguments expected"); + + if (!wallet_dsn) + wallet_dsn = tal_fmt(tmpctx, "sqlite3://%s/lightningd.sqlite3", net_dir); + + if (path_is_file(path_join(tmpctx, base_dir, + tal_fmt(tmpctx, "lightningd-%s.pid", + chainparams->network_name)))) { + errx(ERROR_USAGE, + "Lightningd PID file exists, aborting: lightningd must not be running"); + } + + migrations = get_db_migrations(&num_migrations); + prev_version = version_db(PREV_VERSION); + + /* Open db, check it's the expected version */ + db = db_open(tmpctx, wallet_dsn, false, false, db_error, NULL); + if (!db) + err(1, "Could not open database %s", wallet_dsn); + db->report_changes_fn = NULL; + + db_begin_transaction(db); + db->data_version = db_data_version_get(db); + current = db_get_version(db); + + if (current < prev_version->db_height) + errx(ERROR_DBVERSION, "Database version %zu already less than %zu expected for %s", + current, prev_version->db_height, PREV_VERSION); + if (current == prev_version->db_height) { + printf("Already compatible with %s\n", PREV_VERSION); + exit(0); + } + if (current >= num_migrations) + errx(ERROR_DBVERSION, "Unknown database version %zu: I only know up to %zu (%s)", + current, num_migrations, stringify(CLN_NEXT_VERSION)); + + /* current version is the last migration we did. */ + while (current > prev_version->db_height) { + if (migrations[current].revertsql) { + stmt = db_prepare_v2(db, migrations[current].revertsql); + db_exec_prepared_v2(stmt); + tal_free(stmt); + } + if (migrations[current].revertfn) { + const char *error = migrations[current].revertfn(tmpctx, db); + if (error) + errx(ERROR_DBFAIL, "Downgrade failed: %s", error); + } + current--; + } + + if (prev_version->downgrade_datastore) { + const char *error = prev_version->downgrade_datastore(tmpctx, db); + if (error) + errx(ERROR_DBFAIL, "Downgrade failed: %s", error); + } + + /* Finally update the version number in the version table */ + stmt = db_prepare_v2(db, SQL("UPDATE version SET version=?;")); + db_bind_int(stmt, current); + db_exec_prepared_v2(stmt); + tal_free(stmt); + + printf("Downgrade to %s succeeded. Committing.\n", PREV_VERSION); + db_commit_transaction(db); + tal_free(db); + + if (!version_db(PREV_VERSION)->gossip_store_compatible) { + printf("Deleting incompatible gossip_store\n"); + unlink(path_join(tmpctx, net_dir, "gossip_store")); + } +} + +/*** We don't actually perform migrations, so these are stubs which abort. ***/ +/* Remake with `make update-mocks` or `make update-mocks/tools/lightning-downgrade.c` */ + +/* AUTOGENERATED MOCKS START */ +/* Generated stub for fillin_missing_channel_blockheights */ +void fillin_missing_channel_blockheights(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "fillin_missing_channel_blockheights called!\n"); abort(); } +/* Generated stub for fillin_missing_channel_id */ +void fillin_missing_channel_id(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "fillin_missing_channel_id called!\n"); abort(); } +/* Generated stub for fillin_missing_lease_satoshi */ +void fillin_missing_lease_satoshi(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "fillin_missing_lease_satoshi called!\n"); abort(); } +/* Generated stub for fillin_missing_local_basepoints */ +void fillin_missing_local_basepoints(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "fillin_missing_local_basepoints called!\n"); abort(); } +/* Generated stub for fillin_missing_scriptpubkeys */ +void fillin_missing_scriptpubkeys(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "fillin_missing_scriptpubkeys called!\n"); abort(); } +/* Generated stub for insert_addrtype_to_addresses */ +void insert_addrtype_to_addresses(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "insert_addrtype_to_addresses called!\n"); abort(); } +/* Generated stub for migrate_channels_scids_as_integers */ +void migrate_channels_scids_as_integers(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_channels_scids_as_integers called!\n"); abort(); } +/* Generated stub for migrate_convert_old_channel_keyidx */ +void migrate_convert_old_channel_keyidx(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_convert_old_channel_keyidx called!\n"); abort(); } +/* Generated stub for migrate_datastore_commando_runes */ +void migrate_datastore_commando_runes(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_datastore_commando_runes called!\n"); abort(); } +/* Generated stub for migrate_fail_pending_payments_without_htlcs */ +void migrate_fail_pending_payments_without_htlcs(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_fail_pending_payments_without_htlcs called!\n"); abort(); } +/* Generated stub for migrate_fill_in_channel_type */ +void migrate_fill_in_channel_type(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_fill_in_channel_type called!\n"); abort(); } +/* Generated stub for migrate_forwards_add_rowid */ +void migrate_forwards_add_rowid(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_forwards_add_rowid called!\n"); abort(); } +/* Generated stub for migrate_from_account_db */ +void migrate_from_account_db(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_from_account_db called!\n"); abort(); } +/* Generated stub for migrate_inflight_last_tx_to_psbt */ +void migrate_inflight_last_tx_to_psbt(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_inflight_last_tx_to_psbt called!\n"); abort(); } +/* Generated stub for migrate_initialize_alias_local */ +void migrate_initialize_alias_local(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_initialize_alias_local called!\n"); abort(); } +/* Generated stub for migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards */ +void migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards called!\n"); abort(); } +/* Generated stub for migrate_initialize_forwards_wait_indexes */ +void migrate_initialize_forwards_wait_indexes(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_initialize_forwards_wait_indexes called!\n"); abort(); } +/* Generated stub for migrate_initialize_invoice_wait_indexes */ +void migrate_initialize_invoice_wait_indexes(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_initialize_invoice_wait_indexes called!\n"); abort(); } +/* Generated stub for migrate_initialize_payment_wait_indexes */ +void migrate_initialize_payment_wait_indexes(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_initialize_payment_wait_indexes called!\n"); abort(); } +/* Generated stub for migrate_invalid_last_tx_psbts */ +void migrate_invalid_last_tx_psbts(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_invalid_last_tx_psbts called!\n"); abort(); } +/* Generated stub for migrate_invoice_created_index_var */ +void migrate_invoice_created_index_var(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_invoice_created_index_var called!\n"); abort(); } +/* Generated stub for migrate_last_tx_to_psbt */ +void migrate_last_tx_to_psbt(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_last_tx_to_psbt called!\n"); abort(); } +/* Generated stub for migrate_normalize_invstr */ +void migrate_normalize_invstr(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_normalize_invstr called!\n"); abort(); } +/* Generated stub for migrate_our_funding */ +void migrate_our_funding(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_our_funding called!\n"); abort(); } +/* Generated stub for migrate_payments_scids_as_integers */ +void migrate_payments_scids_as_integers(struct lightningd *ld UNNEEDED, + struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_payments_scids_as_integers called!\n"); abort(); } +/* Generated stub for migrate_pr2342_feerate_per_channel */ +void migrate_pr2342_feerate_per_channel(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_pr2342_feerate_per_channel called!\n"); abort(); } +/* Generated stub for migrate_remove_chain_moves_duplicates */ +void migrate_remove_chain_moves_duplicates(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_remove_chain_moves_duplicates called!\n"); abort(); } +/* Generated stub for migrate_runes_idfix */ +void migrate_runes_idfix(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) +{ fprintf(stderr, "migrate_runes_idfix called!\n"); abort(); } +/* AUTOGENERATED MOCKS END */ diff --git a/tools/hsmtool.c b/tools/lightning-hsmtool.c similarity index 100% rename from tools/hsmtool.c rename to tools/lightning-hsmtool.c diff --git a/wallet/Makefile b/wallet/Makefile index b0a503dc9ee0..34ea6d4cd3f8 100644 --- a/wallet/Makefile +++ b/wallet/Makefile @@ -2,8 +2,10 @@ WALLET_LIB_SRC := \ wallet/account_migration.c \ + wallet/datastore.c \ wallet/db.c \ wallet/invoices.c \ + wallet/migrations.c \ wallet/psbt_fixup.c \ wallet/txfilter.c \ wallet/wallet.c \ @@ -33,12 +35,16 @@ WALLET_SQL_FILES := \ $(DB_SQL_FILES) \ wallet/account_migration.c \ wallet/db.c \ + wallet/datastore.c \ wallet/invoices.c \ + wallet/migrations.c \ wallet/wallet.c \ wallet/test/run-db.c \ wallet/test/run-wallet.c \ wallet/test/run-chain_moves_duplicate-detect.c \ wallet/test/run-migrate_remove_chain_moves_duplicates.c \ + tools/lightning-downgrade.c \ + wallet/statements_gettextgen.po: $(WALLET_SQL_FILES) $(FORCE) @if $(call SHA256STAMP_CHANGED); then \ diff --git a/wallet/account_migration.c b/wallet/account_migration.c index b1de096bb8fa..812b4d6aefb4 100644 --- a/wallet/account_migration.c +++ b/wallet/account_migration.c @@ -11,6 +11,7 @@ #include #include #include +#include /* These functions and definitions copied almost exactly from old * plugins/bkpr/{recorder.c,chain_event.h,channel_event.h} diff --git a/wallet/datastore.c b/wallet/datastore.c new file mode 100644 index 000000000000..27112a2dd0af --- /dev/null +++ b/wallet/datastore.c @@ -0,0 +1,177 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +/* Does k1 match k2 as far as k2 goes? */ +bool datastore_key_startswith(const char **k1, const char **k2) +{ + size_t k1len = tal_count(k1), k2len = tal_count(k2); + + if (k2len > k1len) + return false; + + for (size_t i = 0; i < k2len; i++) { + if (!streq(k1[i], k2[i])) + return false; + } + return true; +} + +bool datastore_key_eq(const char **k1, const char **k2) +{ + return tal_count(k1) == tal_count(k2) + && datastore_key_startswith(k1, k2); +} + +/* We join key parts with nuls for now. */ +void db_bind_datastore_key(struct db_stmt *stmt, const char **key) +{ + u8 *joined; + size_t len; + + if (tal_count(key) == 1) { + db_bind_blob(stmt, (u8 *)key[0], strlen(key[0])); + return; + } + + len = strlen(key[0]); + joined = (u8 *)tal_strdup(tmpctx, key[0]); + for (size_t i = 1; i < tal_count(key); i++) { + tal_resize(&joined, len + 1 + strlen(key[i])); + joined[len] = '\0'; + memcpy(joined + len + 1, key[i], strlen(key[i])); + len += 1 + strlen(key[i]); + } + db_bind_blob(stmt, joined, len); +} + +u8 *db_datastore_get(const tal_t *ctx, + struct db *db, + const char **key, + u64 *generation) +{ + struct db_stmt *stmt; + u8 *ret; + + stmt = db_prepare_v2(db, + SQL("SELECT data, generation" + " FROM datastore" + " WHERE key = ?")); + db_bind_datastore_key(stmt, key); + db_query_prepared(stmt); + + if (!db_step(stmt)) { + tal_free(stmt); + return NULL; + } + + ret = db_col_arr(ctx, stmt, "data", u8); + if (generation) + *generation = db_col_u64(stmt, "generation"); + else + db_col_ignore(stmt, "generation"); + tal_free(stmt); + return ret; +} + +static const char **db_col_datastore_key(const tal_t *ctx, + struct db_stmt *stmt, + const char *colname) +{ + char **key; + const u8 *joined = db_col_blob(stmt, colname); + size_t len = db_col_bytes(stmt, colname); + + key = tal_arr(ctx, char *, 0); + do { + size_t partlen; + for (partlen = 0; partlen < len; partlen++) { + if (joined[partlen] == '\0') { + partlen++; + break; + } + } + tal_arr_expand(&key, tal_strndup(key, (char *)joined, partlen)); + len -= partlen; + joined += partlen; + } while (len != 0); + + return cast_const2(const char **, key); +} + +struct db_stmt *db_datastore_next(const tal_t *ctx, + struct db_stmt *stmt, + const char **startkey, + const char ***key, + const u8 **data, + u64 *generation) +{ + if (!db_step(stmt)) + return tal_free(stmt); + + *key = db_col_datastore_key(ctx, stmt, "key"); + + /* We select from startkey onwards, so once we're past it, stop */ + if (startkey && !datastore_key_startswith(*key, startkey)) { + db_col_ignore(stmt, "data"); + db_col_ignore(stmt, "generation"); + return tal_free(stmt); + } + + if (data) + *data = db_col_arr(ctx, stmt, "data", u8); + else + db_col_ignore(stmt, "data"); + + if (generation) + *generation = db_col_u64(stmt, "generation"); + else + db_col_ignore(stmt, "generation"); + + return stmt; +} + +struct db_stmt *db_datastore_first(const tal_t *ctx, + struct db *db, + const char **startkey, + const char ***key, + const u8 **data, + u64 *generation) +{ + struct db_stmt *stmt; + + if (startkey) { + stmt = db_prepare_v2(db, + SQL("SELECT key, data, generation" + " FROM datastore" + " WHERE key >= ?" + " ORDER BY key;")); + db_bind_datastore_key(stmt, startkey); + } else { + stmt = db_prepare_v2(db, + SQL("SELECT key, data, generation" + " FROM datastore" + " ORDER BY key;")); + } + db_query_prepared(stmt); + + return db_datastore_next(ctx, stmt, startkey, key, data, generation); +} + +void db_datastore_update(struct db *db, const char **key, const u8 *data) +{ + struct db_stmt *stmt; + + stmt = db_prepare_v2(db, + SQL("UPDATE datastore SET data=?, generation=generation+1 WHERE key=?;")); + db_bind_talarr(stmt, data); + db_bind_datastore_key(stmt, key); + db_exec_prepared_v2(take(stmt)); +} + diff --git a/wallet/datastore.h b/wallet/datastore.h new file mode 100644 index 000000000000..3b96f35db6b4 --- /dev/null +++ b/wallet/datastore.h @@ -0,0 +1,37 @@ +#ifndef LIGHTNING_WALLET_DATASTORE_H +#define LIGHTNING_WALLET_DATASTORE_H +/* Access routines for the datastore: here so that tools/lightningd-downgrade + * can use them */ +#include "config.h" +#include +#include +#include + +struct db; +struct db_stmt; + +/* Does k1 match k2 as far as k2 goes? */ +bool datastore_key_startswith(const char **k1, const char **k2); +bool datastore_key_eq(const char **k1, const char **k2); +void db_bind_datastore_key(struct db_stmt *stmt, const char **key); +u8 *db_datastore_get(const tal_t *ctx, + struct db *db, + const char **key, + u64 *generation); +struct db_stmt *db_datastore_next(const tal_t *ctx, + struct db_stmt *stmt, + const char **startkey, + const char ***key, + const u8 **data, + u64 *generation); + +struct db_stmt *db_datastore_first(const tal_t *ctx, + struct db *db, + const char **startkey, + const char ***key, + const u8 **data, + u64 *generation); + +/* Update existing record */ +void db_datastore_update(struct db *db, const char **key, const u8 *data); +#endif /* LIGHTNING_WALLET_DATASTORE_H */ diff --git a/wallet/db.c b/wallet/db.c index a0387220b119..dc85b4f2a315 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -14,1096 +14,10 @@ #include #include #include +#include #include #include -struct migration { - const char *sql; - void (*func)(struct lightningd *ld, struct db *db); -}; - -static void migrate_pr2342_feerate_per_channel(struct lightningd *ld, struct db *db); - -static void migrate_our_funding(struct lightningd *ld, struct db *db); - -static void migrate_last_tx_to_psbt(struct lightningd *ld, struct db *db); - -static void -migrate_inflight_last_tx_to_psbt(struct lightningd *ld, struct db *db); - -static void fillin_missing_scriptpubkeys(struct lightningd *ld, struct db *db); - -static void fillin_missing_channel_id(struct lightningd *ld, struct db *db); - -static void fillin_missing_local_basepoints(struct lightningd *ld, - struct db *db); - -static void fillin_missing_channel_blockheights(struct lightningd *ld, - struct db *db); - -static void migrate_channels_scids_as_integers(struct lightningd *ld, - struct db *db); - -static void migrate_payments_scids_as_integers(struct lightningd *ld, - struct db *db); - -static void fillin_missing_lease_satoshi(struct lightningd *ld, - struct db *db); - -static void fillin_missing_lease_satoshi(struct lightningd *ld, - struct db *db); - -static void migrate_invalid_last_tx_psbts(struct lightningd *ld, - struct db *db); - -static void migrate_fill_in_channel_type(struct lightningd *ld, - struct db *db); - -static void migrate_normalize_invstr(struct lightningd *ld, - struct db *db); - -static void migrate_initialize_invoice_wait_indexes(struct lightningd *ld, - struct db *db); - -static void migrate_invoice_created_index_var(struct lightningd *ld, - struct db *db); - -static void migrate_initialize_payment_wait_indexes(struct lightningd *ld, - struct db *db); - -static void migrate_forwards_add_rowid(struct lightningd *ld, - struct db *db); - -static void migrate_initialize_forwards_wait_indexes(struct lightningd *ld, - struct db *db); -static void migrate_initialize_alias_local(struct lightningd *ld, - struct db *db); -static void insert_addrtype_to_addresses(struct lightningd *ld, - struct db *db); -static void migrate_convert_old_channel_keyidx(struct lightningd *ld, - struct db *db); -static void migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards(struct lightningd *ld, - struct db *db); -static void migrate_fail_pending_payments_without_htlcs(struct lightningd *ld, - struct db *db); - -/* Do not reorder or remove elements from this array, it is used to - * migrate existing databases from a previous state, based on the - * string indices */ -static struct migration dbmigrations[] = { - {SQL("CREATE TABLE version (version INTEGER)"), NULL}, - {SQL("INSERT INTO version VALUES (1)"), NULL}, - {SQL("CREATE TABLE outputs (" - " prev_out_tx BLOB" - ", prev_out_index INTEGER" - ", value BIGINT" - ", type INTEGER" - ", status INTEGER" - ", keyindex INTEGER" - ", PRIMARY KEY (prev_out_tx, prev_out_index));"), - NULL}, - {SQL("CREATE TABLE vars (" - " name VARCHAR(32)" - ", val VARCHAR(255)" - ", PRIMARY KEY (name)" - ");"), - NULL}, - {SQL("CREATE TABLE shachains (" - " id BIGSERIAL" - ", min_index BIGINT" - ", num_valid BIGINT" - ", PRIMARY KEY (id)" - ");"), - NULL}, - {SQL("CREATE TABLE shachain_known (" - " shachain_id BIGINT REFERENCES shachains(id) ON DELETE CASCADE" - ", pos INTEGER" - ", idx BIGINT" - ", hash BLOB" - ", PRIMARY KEY (shachain_id, pos)" - ");"), - NULL}, - {SQL("CREATE TABLE peers (" - " id BIGSERIAL" - ", node_id BLOB UNIQUE" /* pubkey */ - ", address TEXT" - ", PRIMARY KEY (id)" - ");"), - NULL}, - {SQL("CREATE TABLE channels (" - " id BIGSERIAL," /* chan->id */ - /* FIXME: We deliberately never delete a peer with channels, so this constraint is - * unnecessary! */ - " peer_id BIGINT REFERENCES peers(id) ON DELETE CASCADE," - " short_channel_id TEXT," - " channel_config_local BIGINT," - " channel_config_remote BIGINT," - " state INTEGER," - " funder INTEGER," - " channel_flags INTEGER," - " minimum_depth INTEGER," - " next_index_local BIGINT," - " next_index_remote BIGINT," - " next_htlc_id BIGINT," - " funding_tx_id BLOB," - " funding_tx_outnum INTEGER," - " funding_satoshi BIGINT," - " funding_locked_remote INTEGER," - " push_msatoshi BIGINT," - " msatoshi_local BIGINT," /* our_msatoshi */ - /* START channel_info */ - " fundingkey_remote BLOB," - " revocation_basepoint_remote BLOB," - " payment_basepoint_remote BLOB," - " htlc_basepoint_remote BLOB," - " delayed_payment_basepoint_remote BLOB," - " per_commit_remote BLOB," - " old_per_commit_remote BLOB," - " local_feerate_per_kw INTEGER," - " remote_feerate_per_kw INTEGER," - /* END channel_info */ - " shachain_remote_id BIGINT," - " shutdown_scriptpubkey_remote BLOB," - " shutdown_keyidx_local BIGINT," - " last_sent_commit_state BIGINT," - " last_sent_commit_id INTEGER," - " last_tx BLOB," - " last_sig BLOB," - " closing_fee_received INTEGER," - " closing_sig_received BLOB," - " PRIMARY KEY (id)" - ");"), - NULL}, - {SQL("CREATE TABLE channel_configs (" - " id BIGSERIAL," - " dust_limit_satoshis BIGINT," - " max_htlc_value_in_flight_msat BIGINT," - " channel_reserve_satoshis BIGINT," - " htlc_minimum_msat BIGINT," - " to_self_delay INTEGER," - " max_accepted_htlcs INTEGER," - " PRIMARY KEY (id)" - ");"), - NULL}, - {SQL("CREATE TABLE channel_htlcs (" - " id BIGSERIAL," - " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," - " channel_htlc_id BIGINT," - " direction INTEGER," - " origin_htlc BIGINT," - " msatoshi BIGINT," - " cltv_expiry INTEGER," - " payment_hash BLOB," - " payment_key BLOB," - " routing_onion BLOB," - " failuremsg BLOB," /* Note: This is in fact the failure onionreply, - * but renaming columns is hard! */ - " malformed_onion INTEGER," - " hstate INTEGER," - " shared_secret BLOB," - " PRIMARY KEY (id)," - " UNIQUE (channel_id, channel_htlc_id, direction)" - ");"), - NULL}, - {SQL("CREATE TABLE invoices (" - " id BIGSERIAL," - " state INTEGER," - " msatoshi BIGINT," - " payment_hash BLOB," - " payment_key BLOB," - " label TEXT," - " PRIMARY KEY (id)," - " UNIQUE (label)," - " UNIQUE (payment_hash)" - ");"), - NULL}, - {SQL("CREATE TABLE payments (" - " id BIGSERIAL," - " timestamp INTEGER," - " status INTEGER," - " payment_hash BLOB," - " direction INTEGER," - " destination BLOB," - " msatoshi BIGINT," - " PRIMARY KEY (id)," - " UNIQUE (payment_hash)" - ");"), - NULL}, - /* Add expiry field to invoices (effectively infinite). */ - {SQL("ALTER TABLE invoices ADD expiry_time BIGINT;"), NULL}, - {SQL("UPDATE invoices SET expiry_time=9223372036854775807;"), NULL}, - /* Add pay_index field to paid invoices (initially, same order as id). */ - {SQL("ALTER TABLE invoices ADD pay_index BIGINT;"), NULL}, - {SQL("CREATE UNIQUE INDEX invoices_pay_index ON invoices(pay_index);"), - NULL}, - {SQL("UPDATE invoices SET pay_index=id WHERE state=1;"), - NULL}, /* only paid invoice */ - /* Create next_pay_index variable (highest pay_index). */ - {SQL("INSERT INTO vars(name, val)" - " VALUES('next_pay_index', " - " COALESCE((SELECT MAX(pay_index) FROM invoices WHERE state=1), 0) " - "+ 1" - " );"), - NULL}, - /* Create first_block field; initialize from channel id if any. - * This fails for channels still awaiting lockin, but that only applies to - * pre-release software, so it's forgivable. */ - {SQL("ALTER TABLE channels ADD first_blocknum BIGINT;"), NULL}, - {SQL("UPDATE channels SET first_blocknum=1 WHERE short_channel_id IS NOT NULL;"), - NULL}, - {SQL("ALTER TABLE outputs ADD COLUMN channel_id BIGINT;"), NULL}, - {SQL("ALTER TABLE outputs ADD COLUMN peer_id BLOB;"), NULL}, - {SQL("ALTER TABLE outputs ADD COLUMN commitment_point BLOB;"), NULL}, - {SQL("ALTER TABLE invoices ADD COLUMN msatoshi_received BIGINT;"), NULL}, - /* Normally impossible, so at least we'll know if databases are ancient. */ - {SQL("UPDATE invoices SET msatoshi_received=0 WHERE state=1;"), NULL}, - {SQL("ALTER TABLE channels ADD COLUMN last_was_revoke INTEGER;"), NULL}, - /* We no longer record incoming payments: invoices cover that. - * Without ALTER_TABLE DROP COLUMN support we need to do this by - * rename & copy, which works because there are no triggers etc. */ - {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, - {SQL("CREATE TABLE payments (" - " id BIGSERIAL," - " timestamp INTEGER," - " status INTEGER," - " payment_hash BLOB," - " destination BLOB," - " msatoshi BIGINT," - " PRIMARY KEY (id)," - " UNIQUE (payment_hash)" - ");"), - NULL}, - {SQL("INSERT INTO payments SELECT id, timestamp, status, payment_hash, " - "destination, msatoshi FROM temp_payments WHERE direction=1;"), - NULL}, - {SQL("DROP TABLE temp_payments;"), NULL}, - /* We need to keep the preimage in case they ask to pay again. */ - {SQL("ALTER TABLE payments ADD COLUMN payment_preimage BLOB;"), NULL}, - /* We need to keep the shared secrets to decode error returns. */ - {SQL("ALTER TABLE payments ADD COLUMN path_secrets BLOB;"), NULL}, - /* Create time-of-payment of invoice, default already-paid - * invoices to current time. */ - {SQL("ALTER TABLE invoices ADD paid_timestamp BIGINT;"), NULL}, - {SQL("UPDATE invoices" - " SET paid_timestamp = CURRENT_TIMESTAMP()" - " WHERE state = 1;"), - NULL}, - /* We need to keep the route node pubkeys and short channel ids to - * correctly mark routing failures. We separate short channel ids - * because we cannot safely save them as blobs due to byteorder - * concerns. */ - {SQL("ALTER TABLE payments ADD COLUMN route_nodes BLOB;"), NULL}, - {SQL("ALTER TABLE payments ADD COLUMN route_channels BLOB;"), NULL}, - {SQL("CREATE TABLE htlc_sigs (channelid INTEGER REFERENCES channels(id) ON " - "DELETE CASCADE, signature BLOB);"), - NULL}, - {SQL("CREATE INDEX channel_idx ON htlc_sigs (channelid)"), NULL}, - /* Get rid of OPENINGD entries; we don't put them in db any more */ - {SQL("DELETE FROM channels WHERE state=1"), NULL}, - /* Keep track of db upgrades, for debugging */ - {SQL("CREATE TABLE db_upgrades (upgrade_from INTEGER, lightning_version " - "TEXT);"), - NULL}, - /* We used not to clean up peers when their channels were gone. */ - {SQL("DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);"), - NULL}, - /* The ONCHAIND_CHEATED/THEIR_UNILATERAL/OUR_UNILATERAL/MUTUAL are now one - */ - {SQL("UPDATE channels SET STATE = 8 WHERE state > 8;"), NULL}, - /* Add bolt11 to invoices table*/ - {SQL("ALTER TABLE invoices ADD bolt11 TEXT;"), NULL}, - /* What do we think the head of the blockchain looks like? Used - * primarily to track confirmations across restarts and making - * sure we handle reorgs correctly. */ - {SQL("CREATE TABLE blocks (height INT, hash BLOB, prev_hash BLOB, " - "UNIQUE(height));"), - NULL}, - /* ON DELETE CASCADE would have been nice for confirmation_height, - * so that we automatically delete outputs that fall off the - * blockchain and then we rediscover them if they are included - * again. However, we have the their_unilateral/to_us which we - * can't simply recognize from the chain without additional - * hints. So we just mark them as unconfirmed should the block - * die. */ - {SQL("ALTER TABLE outputs ADD COLUMN confirmation_height INTEGER " - "REFERENCES blocks(height) ON DELETE SET NULL;"), - NULL}, - {SQL("ALTER TABLE outputs ADD COLUMN spend_height INTEGER REFERENCES " - "blocks(height) ON DELETE SET NULL;"), - NULL}, - /* Create a covering index that covers both fields */ - {SQL("CREATE INDEX output_height_idx ON outputs (confirmation_height, " - "spend_height);"), - NULL}, - {SQL("CREATE TABLE utxoset (" - " txid BLOB," - " outnum INT," - " blockheight INT REFERENCES blocks(height) ON DELETE CASCADE," - " spendheight INT REFERENCES blocks(height) ON DELETE SET NULL," - " txindex INT," - " scriptpubkey BLOB," - " satoshis BIGINT," - " PRIMARY KEY(txid, outnum));"), - NULL}, - {SQL("CREATE INDEX short_channel_id ON utxoset (blockheight, txindex, " - "outnum)"), - NULL}, - /* Necessary index for long rollbacks of the blockchain, otherwise we're - * doing table scans for every block removed. */ - {SQL("CREATE INDEX utxoset_spend ON utxoset (spendheight)"), NULL}, - /* Assign key 0 to unassigned shutdown_keyidx_local. */ - {SQL("UPDATE channels SET shutdown_keyidx_local=0 WHERE " - "shutdown_keyidx_local = -1;"), - NULL}, - /* FIXME: We should rename shutdown_keyidx_local to final_key_index */ - /* -- Payment routing failure information -- */ - /* BLOB if failure was due to unparseable onion, NULL otherwise */ - {SQL("ALTER TABLE payments ADD failonionreply BLOB;"), NULL}, - /* 0 if we could theoretically retry, 1 if PERM fail at payee */ - {SQL("ALTER TABLE payments ADD faildestperm INTEGER;"), NULL}, - /* Contents of routing_failure (only if not unparseable onion) */ - {SQL("ALTER TABLE payments ADD failindex INTEGER;"), - NULL}, /* erring_index */ - {SQL("ALTER TABLE payments ADD failcode INTEGER;"), NULL}, /* failcode */ - {SQL("ALTER TABLE payments ADD failnode BLOB;"), NULL}, /* erring_node */ - {SQL("ALTER TABLE payments ADD failchannel TEXT;"), - NULL}, /* erring_channel */ - {SQL("ALTER TABLE payments ADD failupdate BLOB;"), - NULL}, /* channel_update - can be NULL*/ - /* -- Payment routing failure information ends -- */ - /* Delete route data for already succeeded or failed payments */ - {SQL("UPDATE payments" - " SET path_secrets = NULL" - " , route_nodes = NULL" - " , route_channels = NULL" - " WHERE status <> 0;"), - NULL}, /* PAYMENT_PENDING */ - /* -- Routing statistics -- */ - {SQL("ALTER TABLE channels ADD in_payments_offered INTEGER DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD in_payments_fulfilled INTEGER DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD in_msatoshi_offered BIGINT DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD in_msatoshi_fulfilled BIGINT DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD out_payments_offered INTEGER DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD out_payments_fulfilled INTEGER DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD out_msatoshi_offered BIGINT DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD out_msatoshi_fulfilled BIGINT DEFAULT 0;"), NULL}, - {SQL("UPDATE channels" - " SET in_payments_offered = 0, in_payments_fulfilled = 0" - " , in_msatoshi_offered = 0, in_msatoshi_fulfilled = 0" - " , out_payments_offered = 0, out_payments_fulfilled = 0" - " , out_msatoshi_offered = 0, out_msatoshi_fulfilled = 0" - " ;"), - NULL}, - /* -- Routing statistics ends --*/ - /* Record the msatoshi actually sent in a payment. */ - {SQL("ALTER TABLE payments ADD msatoshi_sent BIGINT;"), NULL}, - {SQL("UPDATE payments SET msatoshi_sent = msatoshi;"), NULL}, - /* Delete dangling utxoset entries due to Issue #1280 */ - {SQL("DELETE FROM utxoset WHERE blockheight IN (" - " SELECT DISTINCT(blockheight)" - " FROM utxoset LEFT OUTER JOIN blocks on (blockheight = " - "blocks.height) " - " WHERE blocks.hash IS NULL" - ");"), - NULL}, - /* Record feerate range, to optimize onchaind grinding actual fees. */ - {SQL("ALTER TABLE channels ADD min_possible_feerate INTEGER;"), NULL}, - {SQL("ALTER TABLE channels ADD max_possible_feerate INTEGER;"), NULL}, - /* https://bitcoinfees.github.io/#1d says Dec 17 peak was ~1M sat/kb - * which is 250,000 sat/Sipa */ - {SQL("UPDATE channels SET min_possible_feerate=0, " - "max_possible_feerate=250000;"), - NULL}, - /* -- Min and max msatoshi_to_us -- */ - {SQL("ALTER TABLE channels ADD msatoshi_to_us_min BIGINT;"), NULL}, - {SQL("ALTER TABLE channels ADD msatoshi_to_us_max BIGINT;"), NULL}, - {SQL("UPDATE channels" - " SET msatoshi_to_us_min = msatoshi_local" - " , msatoshi_to_us_max = msatoshi_local" - " ;"), - NULL}, - /* -- Min and max msatoshi_to_us ends -- */ - /* Transactions we are interested in. Either we sent them ourselves or we - * are watching them. We don't cascade block height deletes so we don't - * forget any of them by accident.*/ - {SQL("CREATE TABLE transactions (" - " id BLOB" - ", blockheight INTEGER REFERENCES blocks(height) ON DELETE SET NULL" - ", txindex INTEGER" - ", rawtx BLOB" - ", PRIMARY KEY (id)" - ");"), - NULL}, - /* -- Detailed payment failure -- */ - {SQL("ALTER TABLE payments ADD faildetail TEXT;"), NULL}, - {SQL("UPDATE payments" - " SET faildetail = 'unspecified payment failure reason'" - " WHERE status = 2;"), - NULL}, /* PAYMENT_FAILED */ - /* -- Detailed payment faiure ends -- */ - {SQL("CREATE TABLE channeltxs (" - /* The id serves as insertion order and short ID */ - " id BIGSERIAL" - ", channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE" - ", type INTEGER" - ", transaction_id BLOB REFERENCES transactions(id) ON DELETE CASCADE" - /* The input_num is only used by the txo_watch, 0 if txwatch */ - ", input_num INTEGER" - /* The height at which we sent the depth notice */ - ", blockheight INTEGER REFERENCES blocks(height) ON DELETE CASCADE" - ", PRIMARY KEY(id)" - ");"), - NULL}, - /* -- Set the correct rescan height for PR #1398 -- */ - /* Delete blocks that are higher than our initial scan point, this is a - * no-op if we don't have a channel. */ - {SQL("DELETE FROM blocks WHERE height > (SELECT MIN(first_blocknum) FROM " - "channels);"), - NULL}, - /* Now make sure we have the lower bound block with the first_blocknum - * height. This may introduce a block with NULL height if we didn't have any - * blocks, remove that in the next. */ - {SQL("INSERT INTO blocks (height) VALUES ((SELECT " - "MIN(first_blocknum) FROM channels)) " - "ON CONFLICT(height) DO NOTHING;"), - NULL}, - {SQL("DELETE FROM blocks WHERE height IS NULL;"), NULL}, - /* -- End of PR #1398 -- */ - {SQL("ALTER TABLE invoices ADD description TEXT;"), NULL}, - /* FIXME: payments table 'description' is really a 'label' */ - {SQL("ALTER TABLE payments ADD description TEXT;"), NULL}, - /* future_per_commitment_point if other side proves we're out of date -- */ - {SQL("ALTER TABLE channels ADD future_per_commitment_point BLOB;"), NULL}, - /* last_sent_commit array fix */ - {SQL("ALTER TABLE channels ADD last_sent_commit BLOB;"), NULL}, - /* Stats table to track forwarded HTLCs. The values in the HTLCs - * and their states are replicated here and the entries are not - * deleted when the HTLC entries or the channel entries are - * deleted to avoid unexpected drops in statistics. */ - {SQL("CREATE TABLE forwarded_payments (" - " in_htlc_id BIGINT REFERENCES channel_htlcs(id) ON DELETE SET NULL" - ", out_htlc_id BIGINT REFERENCES channel_htlcs(id) ON DELETE SET NULL" - ", in_channel_scid BIGINT" - ", out_channel_scid BIGINT" - ", in_msatoshi BIGINT" - ", out_msatoshi BIGINT" - ", state INTEGER" - ", UNIQUE(in_htlc_id, out_htlc_id)" - ");"), - NULL}, - /* Add a direction for failed payments. */ - {SQL("ALTER TABLE payments ADD faildirection INTEGER;"), - NULL}, /* erring_direction */ - /* Fix dangling peers with no channels. */ - {SQL("DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);"), - NULL}, - {SQL("ALTER TABLE outputs ADD scriptpubkey BLOB;"), NULL}, - /* Keep bolt11 string for payments. */ - {SQL("ALTER TABLE payments ADD bolt11 TEXT;"), NULL}, - /* PR #2342 feerate per channel */ - {SQL("ALTER TABLE channels ADD feerate_base INTEGER;"), NULL}, - {SQL("ALTER TABLE channels ADD feerate_ppm INTEGER;"), NULL}, - {NULL, migrate_pr2342_feerate_per_channel}, - {SQL("ALTER TABLE channel_htlcs ADD received_time BIGINT"), NULL}, - {SQL("ALTER TABLE forwarded_payments ADD received_time BIGINT"), NULL}, - {SQL("ALTER TABLE forwarded_payments ADD resolved_time BIGINT"), NULL}, - {SQL("ALTER TABLE channels ADD remote_upfront_shutdown_script BLOB;"), - NULL}, - /* PR #2524: Add failcode into forward_payment */ - {SQL("ALTER TABLE forwarded_payments ADD failcode INTEGER;"), NULL}, - /* remote signatures for channel announcement */ - {SQL("ALTER TABLE channels ADD remote_ann_node_sig BLOB;"), NULL}, - {SQL("ALTER TABLE channels ADD remote_ann_bitcoin_sig BLOB;"), NULL}, - /* FIXME: We now use the transaction_annotations table to type each - * input and output instead of type and channel_id! */ - /* Additional information for transaction tracking and listing */ - {SQL("ALTER TABLE transactions ADD type BIGINT;"), NULL}, - /* Not a foreign key on purpose since we still delete channels from - * the DB which would remove this. It is mainly used to group payments - * in the list view anyway, e.g., show all close and htlc transactions - * as a single bundle. */ - {SQL("ALTER TABLE transactions ADD channel_id BIGINT;"), NULL}, - /* Convert pre-Adelaide short_channel_ids */ - {SQL("UPDATE channels" - " SET short_channel_id = REPLACE(short_channel_id, ':', 'x')" - " WHERE short_channel_id IS NOT NULL;"), NULL }, - {SQL("UPDATE payments SET failchannel = REPLACE(failchannel, ':', 'x')" - " WHERE failchannel IS NOT NULL;"), NULL }, - /* option_static_remotekey is nailed at creation time. */ - {SQL("ALTER TABLE channels ADD COLUMN option_static_remotekey INTEGER" - " DEFAULT 0;"), NULL }, - {SQL("ALTER TABLE vars ADD COLUMN intval INTEGER"), NULL}, - {SQL("ALTER TABLE vars ADD COLUMN blobval BLOB"), NULL}, - {SQL("UPDATE vars SET intval = CAST(val AS INTEGER) WHERE name IN ('bip32_max_index', 'last_processed_block', 'next_pay_index')"), NULL}, - {SQL("UPDATE vars SET blobval = CAST(val AS BLOB) WHERE name = 'genesis_hash'"), NULL}, - {SQL("CREATE TABLE transaction_annotations (" - /* Not making this a reference since we usually filter the TX by - * walking its inputs and outputs, and only afterwards storing it in - * the DB. Having a reference here would point into the void until we - * add the matching TX. */ - " txid BLOB" - ", idx INTEGER" /* 0 when location is the tx, the index of the output or input otherwise */ - ", location INTEGER" /* The transaction itself, the output at idx, or the input at idx */ - ", type INTEGER" - ", channel BIGINT REFERENCES channels(id)" - ", UNIQUE(txid, idx)" - ");"), NULL}, - {SQL("ALTER TABLE channels ADD shutdown_scriptpubkey_local BLOB;"), - NULL}, - /* See https://github.com/ElementsProject/lightning/issues/3189 */ - {SQL("UPDATE forwarded_payments SET received_time=0 WHERE received_time IS NULL;"), - NULL}, - {SQL("ALTER TABLE invoices ADD COLUMN features BLOB DEFAULT '';"), NULL}, - /* We can now have multiple payments in progress for a single hash, so - * add two fields; combination of payment_hash & partid is unique. */ - {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, - {SQL("CREATE TABLE payments (" - " id BIGSERIAL" - ", timestamp INTEGER" - ", status INTEGER" - ", payment_hash BLOB" - ", destination BLOB" - ", msatoshi BIGINT" - ", payment_preimage BLOB" - ", path_secrets BLOB" - ", route_nodes BLOB" - ", route_channels BLOB" - ", failonionreply BLOB" - ", faildestperm INTEGER" - ", failindex INTEGER" - ", failcode INTEGER" - ", failnode BLOB" - ", failchannel TEXT" - ", failupdate BLOB" - ", msatoshi_sent BIGINT" - ", faildetail TEXT" - ", description TEXT" - ", faildirection INTEGER" - ", bolt11 TEXT" - ", total_msat BIGINT" - ", partid BIGINT" - ", PRIMARY KEY (id)" - ", UNIQUE (payment_hash, partid))"), NULL}, - {SQL("INSERT INTO payments (" - "id" - ", timestamp" - ", status" - ", payment_hash" - ", destination" - ", msatoshi" - ", payment_preimage" - ", path_secrets" - ", route_nodes" - ", route_channels" - ", failonionreply" - ", faildestperm" - ", failindex" - ", failcode" - ", failnode" - ", failchannel" - ", failupdate" - ", msatoshi_sent" - ", faildetail" - ", description" - ", faildirection" - ", bolt11)" - "SELECT id" - ", timestamp" - ", status" - ", payment_hash" - ", destination" - ", msatoshi" - ", payment_preimage" - ", path_secrets" - ", route_nodes" - ", route_channels" - ", failonionreply" - ", faildestperm" - ", failindex" - ", failcode" - ", failnode" - ", failchannel" - ", failupdate" - ", msatoshi_sent" - ", faildetail" - ", description" - ", faildirection" - ", bolt11 FROM temp_payments;"), NULL}, - {SQL("UPDATE payments SET total_msat = msatoshi;"), NULL}, - {SQL("UPDATE payments SET partid = 0;"), NULL}, - {SQL("DROP TABLE temp_payments;"), NULL}, - {SQL("ALTER TABLE channel_htlcs ADD partid BIGINT;"), NULL}, - {SQL("UPDATE channel_htlcs SET partid = 0;"), NULL}, - {SQL("CREATE TABLE channel_feerates (" - " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," - " hstate INTEGER," - " feerate_per_kw INTEGER," - " UNIQUE (channel_id, hstate)" - ");"), - NULL}, - /* Cast old-style per-side feerates into most likely layout for statewise - * feerates. */ - /* If we're funder (LOCAL=0): - * Then our feerate is set last (SENT_ADD_ACK_REVOCATION = 4) */ - {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" - " SELECT id, 4, local_feerate_per_kw FROM channels WHERE funder = 0;"), - NULL}, - /* If different, assume their feerate is in state SENT_ADD_COMMIT = 1 */ - {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" - " SELECT id, 1, remote_feerate_per_kw FROM channels WHERE funder = 0 and local_feerate_per_kw != remote_feerate_per_kw;"), - NULL}, - /* If they're funder (REMOTE=1): - * Then their feerate is set last (RCVD_ADD_ACK_REVOCATION = 14) */ - {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" - " SELECT id, 14, remote_feerate_per_kw FROM channels WHERE funder = 1;"), - NULL}, - /* If different, assume their feerate is in state RCVD_ADD_COMMIT = 11 */ - {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" - " SELECT id, 11, local_feerate_per_kw FROM channels WHERE funder = 1 and local_feerate_per_kw != remote_feerate_per_kw;"), - NULL}, - /* FIXME: Remove now-unused local_feerate_per_kw and remote_feerate_per_kw from channels */ - {SQL("INSERT INTO vars (name, intval) VALUES ('data_version', 0);"), NULL}, - /* For outgoing HTLCs, we now keep a localmsg instead of a failcode. - * Turn anything in transition into a WIRE_TEMPORARY_NODE_FAILURE. */ - {SQL("ALTER TABLE channel_htlcs ADD localfailmsg BLOB;"), NULL}, - {SQL("UPDATE channel_htlcs SET localfailmsg=decode('2002', 'hex') WHERE malformed_onion != 0 AND direction = 1;"), NULL}, - {SQL("ALTER TABLE channels ADD our_funding_satoshi BIGINT DEFAULT 0;"), migrate_our_funding}, - {SQL("CREATE TABLE penalty_bases (" - " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE" - ", commitnum BIGINT" - ", txid BLOB" - ", outnum INTEGER" - ", amount BIGINT" - ", PRIMARY KEY (channel_id, commitnum)" - ");"), NULL}, - /* For incoming HTLCs, we now keep track of whether or not we provided - * the preimage for it, or not. */ - {SQL("ALTER TABLE channel_htlcs ADD we_filled INTEGER;"), NULL}, - /* We track the counter for coin_moves, as a convenience for notification consumers */ - {SQL("INSERT INTO vars (name, intval) VALUES ('coin_moves_count', 0);"), NULL}, - {NULL, migrate_last_tx_to_psbt}, - {SQL("ALTER TABLE outputs ADD reserved_til INTEGER DEFAULT NULL;"), NULL}, - {NULL, fillin_missing_scriptpubkeys}, - /* option_anchor_outputs is nailed at creation time. */ - {SQL("ALTER TABLE channels ADD COLUMN option_anchor_outputs INTEGER" - " DEFAULT 0;"), NULL }, - /* We need to know if it was option_anchor_outputs to spend to_remote */ - {SQL("ALTER TABLE outputs ADD option_anchor_outputs INTEGER" - " DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD full_channel_id BLOB DEFAULT NULL;"), fillin_missing_channel_id}, - {SQL("ALTER TABLE channels ADD funding_psbt BLOB DEFAULT NULL;"), NULL}, - /* Channel closure reason */ - {SQL("ALTER TABLE channels ADD closer INTEGER DEFAULT 2;"), NULL}, - {SQL("ALTER TABLE channels ADD state_change_reason INTEGER DEFAULT 0;"), NULL}, - {SQL("CREATE TABLE channel_state_changes (" - " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," - " timestamp BIGINT," - " old_state INTEGER," - " new_state INTEGER," - " cause INTEGER," - " message TEXT" - ");"), NULL}, - {SQL("CREATE TABLE offers (" - " offer_id BLOB" - ", bolt12 TEXT" - ", label TEXT" - ", status INTEGER" - ", PRIMARY KEY (offer_id)" - ");"), NULL}, - /* A reference into our own offers table, if it was made from one */ - {SQL("ALTER TABLE invoices ADD COLUMN local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id);"), NULL}, - /* A reference into our own offers table, if it was made from one */ - {SQL("ALTER TABLE payments ADD COLUMN local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id);"), NULL}, - {SQL("ALTER TABLE channels ADD funding_tx_remote_sigs_received INTEGER DEFAULT 0;"), NULL}, - /* Speeds up deletion of one peer from the database, measurements suggest - * it cuts down the time by 80%. */ - {SQL("CREATE INDEX forwarded_payments_out_htlc_id" - " ON forwarded_payments (out_htlc_id);"), NULL}, - {SQL("UPDATE channel_htlcs SET malformed_onion = 0 WHERE malformed_onion IS NULL"), NULL}, - /* Speed up forwarded_payments lookup based on state */ - {SQL("CREATE INDEX forwarded_payments_state ON forwarded_payments (state)"), NULL}, - {SQL("CREATE TABLE channel_funding_inflights (" - " channel_id BIGSERIAL REFERENCES channels(id) ON DELETE CASCADE" - ", funding_tx_id BLOB" - ", funding_tx_outnum INTEGER" - ", funding_feerate INTEGER" - ", funding_satoshi BIGINT" - ", our_funding_satoshi BIGINT" - ", funding_psbt BLOB" - ", last_tx BLOB" - ", last_sig BLOB" - ", funding_tx_remote_sigs_received INTEGER" - ", PRIMARY KEY (channel_id, funding_tx_id)" - ");"), - NULL}, - {SQL("ALTER TABLE channels ADD revocation_basepoint_local BLOB"), NULL}, - {SQL("ALTER TABLE channels ADD payment_basepoint_local BLOB"), NULL}, - {SQL("ALTER TABLE channels ADD htlc_basepoint_local BLOB"), NULL}, - {SQL("ALTER TABLE channels ADD delayed_payment_basepoint_local BLOB"), NULL}, - {SQL("ALTER TABLE channels ADD funding_pubkey_local BLOB"), NULL}, - {NULL, fillin_missing_local_basepoints}, - /* Oops, can I haz money back plz? */ - {SQL("ALTER TABLE channels ADD shutdown_wrong_txid BLOB DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channels ADD shutdown_wrong_outnum INTEGER DEFAULT NULL"), NULL}, - {NULL, migrate_inflight_last_tx_to_psbt}, - /* Channels can now change their type at specific commit indexes. */ - {SQL("ALTER TABLE channels ADD local_static_remotekey_start BIGINT DEFAULT 0"), - NULL}, - {SQL("ALTER TABLE channels ADD remote_static_remotekey_start BIGINT DEFAULT 0"), - NULL}, - /* Set counter past 2^48 if they don't have option */ - {SQL("UPDATE channels SET" - " remote_static_remotekey_start = 9223372036854775807," - " local_static_remotekey_start = 9223372036854775807" - " WHERE option_static_remotekey = 0"), - NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD lease_commit_sig BLOB DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD lease_chan_max_msat BIGINT DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD lease_chan_max_ppt INTEGER DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD lease_expiry INTEGER DEFAULT 0"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD lease_blockheight_start INTEGER DEFAULT 0"), NULL}, - {SQL("ALTER TABLE channels ADD lease_commit_sig BLOB DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channels ADD lease_chan_max_msat INTEGER DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channels ADD lease_chan_max_ppt INTEGER DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE channels ADD lease_expiry INTEGER DEFAULT 0"), NULL}, - {SQL("CREATE TABLE channel_blockheights (" - " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," - " hstate INTEGER," - " blockheight INTEGER," - " UNIQUE (channel_id, hstate)" - ");"), - fillin_missing_channel_blockheights}, - {SQL("ALTER TABLE outputs ADD csv_lock INTEGER DEFAULT 1;"), NULL}, - {SQL("CREATE TABLE datastore (" - " key BLOB," - " data BLOB," - " generation BIGINT," - " PRIMARY KEY (key)" - ");"), - NULL}, - {SQL("CREATE INDEX channel_state_changes_channel_id" - " ON channel_state_changes (channel_id);"), NULL}, - /* We need to switch the unique key to cover the groupid as well, - * so we can attempt payments multiple times. */ - {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, - {SQL("CREATE TABLE payments (" - " id BIGSERIAL" - ", timestamp INTEGER" - ", status INTEGER" - ", payment_hash BLOB" - ", destination BLOB" - ", msatoshi BIGINT" - ", payment_preimage BLOB" - ", path_secrets BLOB" - ", route_nodes BLOB" - ", route_channels BLOB" - ", failonionreply BLOB" - ", faildestperm INTEGER" - ", failindex INTEGER" - ", failcode INTEGER" - ", failnode BLOB" - ", failchannel TEXT" - ", failupdate BLOB" - ", msatoshi_sent BIGINT" - ", faildetail TEXT" - ", description TEXT" - ", faildirection INTEGER" - ", bolt11 TEXT" - ", total_msat BIGINT" - ", partid BIGINT" - ", groupid BIGINT NOT NULL DEFAULT 0" - ", local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id)" - ", PRIMARY KEY (id)" - ", UNIQUE (payment_hash, partid, groupid))"), NULL}, - {SQL("INSERT INTO payments (" - "id" - ", timestamp" - ", status" - ", payment_hash" - ", destination" - ", msatoshi" - ", payment_preimage" - ", path_secrets" - ", route_nodes" - ", route_channels" - ", failonionreply" - ", faildestperm" - ", failindex" - ", failcode" - ", failnode" - ", failchannel" - ", failupdate" - ", msatoshi_sent" - ", faildetail" - ", description" - ", faildirection" - ", bolt11" - ", groupid" - ", local_offer_id)" - "SELECT id" - ", timestamp" - ", status" - ", payment_hash" - ", destination" - ", msatoshi" - ", payment_preimage" - ", path_secrets" - ", route_nodes" - ", route_channels" - ", failonionreply" - ", faildestperm" - ", failindex" - ", failcode" - ", failnode" - ", failchannel" - ", failupdate" - ", msatoshi_sent" - ", faildetail" - ", description" - ", faildirection" - ", bolt11" - ", 0" - ", local_offer_id FROM temp_payments;"), NULL}, - {SQL("DROP TABLE temp_payments;"), NULL}, - /* HTLCs also need to carry the groupid around so we can - * selectively update them. */ - {SQL("ALTER TABLE channel_htlcs ADD groupid BIGINT;"), NULL}, - {SQL("ALTER TABLE channel_htlcs ADD COLUMN" - " min_commit_num BIGINT default 0;"), NULL}, - {SQL("ALTER TABLE channel_htlcs ADD COLUMN" - " max_commit_num BIGINT default NULL;"), NULL}, - /* Set max_commit_num for dead (RCVD_REMOVE_ACK_REVOCATION or SENT_REMOVE_ACK_REVOCATION) HTLCs based on latest indexes */ - {SQL("UPDATE channel_htlcs SET max_commit_num =" - " (SELECT GREATEST(next_index_local, next_index_remote)" - " FROM channels WHERE id=channel_id)" - " WHERE (hstate=9 OR hstate=19);"), NULL}, - /* Remove unused fields which take much room in db. */ - {SQL("UPDATE channel_htlcs SET" - " payment_key=NULL," - " routing_onion=NULL," - " failuremsg=NULL," - " shared_secret=NULL," - " localfailmsg=NULL" - " WHERE (hstate=9 OR hstate=19);"), NULL}, - /* We default to 50k sats */ - {SQL("ALTER TABLE channel_configs ADD max_dust_htlc_exposure_msat BIGINT DEFAULT 50000000"), NULL}, - {SQL("ALTER TABLE channel_htlcs ADD fail_immediate INTEGER DEFAULT 0"), NULL}, - - /* Issue #4887: reset the payments.id sequence after the migration above. Since this is a SELECT statement that would otherwise fail, make it an INSERT into the `vars` table.*/ - {SQL("/*PSQL*/INSERT INTO vars (name, intval) VALUES ('payment_id_reset', setval(pg_get_serial_sequence('payments', 'id'), COALESCE((SELECT MAX(id)+1 FROM payments), 1)))"), NULL}, - - /* Issue #4901: Partial index speeds up startup on nodes with ~1000 channels. */ - {&SQL("CREATE INDEX channel_htlcs_speedup_unresolved_idx" - " ON channel_htlcs(channel_id, direction)" - " WHERE hstate NOT IN (9, 19);") - [BUILD_ASSERT_OR_ZERO( 9 == RCVD_REMOVE_ACK_REVOCATION) + - BUILD_ASSERT_OR_ZERO(19 == SENT_REMOVE_ACK_REVOCATION)], - NULL}, - {SQL("ALTER TABLE channel_htlcs ADD fees_msat BIGINT DEFAULT 0"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD lease_fee BIGINT DEFAULT 0"), NULL}, - /* Default is too big; we set to max after loading */ - {SQL("ALTER TABLE channels ADD htlc_maximum_msat BIGINT DEFAULT 2100000000000000"), NULL}, - {SQL("ALTER TABLE channels ADD htlc_minimum_msat BIGINT DEFAULT 0"), NULL}, - {SQL("ALTER TABLE forwarded_payments ADD forward_style INTEGER DEFAULT NULL"), NULL}, - /* "description" is used for label, so we use "paydescription" here */ - {SQL("ALTER TABLE payments ADD paydescription TEXT;"), NULL}, - /* Alias we sent to the remote side, for zeroconf and - * option_scid_alias, can be a list of short_channel_ids if - * required, but keeping it a single SCID for now. */ - {SQL("ALTER TABLE channels ADD alias_local BIGINT DEFAULT NULL"), NULL}, - /* Alias we received from the peer, and which we should be using - * in routehints in invoices. The peer will remember all the - * aliases, but we only ever need one. */ - {SQL("ALTER TABLE channels ADD alias_remote BIGINT DEFAULT NULL"), NULL}, - /* Cheeky immediate completion as best effort approximation of real completion time */ - {SQL("ALTER TABLE payments ADD completed_at INTEGER DEFAULT NULL;"), NULL}, - {SQL("UPDATE payments SET completed_at = timestamp WHERE status != 0;"), NULL}, - {SQL("CREATE INDEX payments_idx ON payments (payment_hash)"), NULL}, - /* forwards table outlives the channels, so we move there from old forwarded_payments table; - * but here the ids are the HTLC numbers, not the internal db ids. */ - {SQL("CREATE TABLE forwards (" - "in_channel_scid BIGINT" - ", in_htlc_id BIGINT" - ", out_channel_scid BIGINT" - ", out_htlc_id BIGINT" - ", in_msatoshi BIGINT" - ", out_msatoshi BIGINT" - ", state INTEGER" - ", received_time BIGINT" - ", resolved_time BIGINT" - ", failcode INTEGER" - ", forward_style INTEGER" - ", PRIMARY KEY(in_channel_scid, in_htlc_id))"), NULL}, - {SQL("INSERT INTO forwards SELECT" - " in_channel_scid" - ", COALESCE(" - " (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.in_htlc_id)," - " -_ROWID_" - " )" - ", out_channel_scid" - ", (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.out_htlc_id)" - ", in_msatoshi" - ", out_msatoshi" - ", state" - ", received_time" - ", resolved_time" - ", failcode" - ", forward_style" - " FROM forwarded_payments"), NULL}, - {SQL("DROP INDEX forwarded_payments_state;"), NULL}, - {SQL("DROP INDEX forwarded_payments_out_htlc_id;"), NULL}, - {SQL("DROP TABLE forwarded_payments;"), NULL}, - /* Adds scid column, then moves short_channel_id across to it */ - {SQL("ALTER TABLE channels ADD scid BIGINT;"), migrate_channels_scids_as_integers}, - {SQL("ALTER TABLE payments ADD failscid BIGINT;"), migrate_payments_scids_as_integers}, - {SQL("ALTER TABLE outputs ADD is_in_coinbase INTEGER DEFAULT 0;"), NULL}, - {SQL("CREATE TABLE invoicerequests (" - " invreq_id BLOB" - ", bolt12 TEXT" - ", label TEXT" - ", status INTEGER" - ", PRIMARY KEY (invreq_id)" - ");"), NULL}, - /* A reference into our own invoicerequests table, if it was made from one */ - {SQL("ALTER TABLE payments ADD COLUMN local_invreq_id BLOB DEFAULT NULL REFERENCES invoicerequests(invreq_id);"), NULL}, - /* FIXME: Remove payments local_offer_id column! */ - {SQL("ALTER TABLE channel_funding_inflights ADD COLUMN lease_satoshi BIGINT;"), NULL}, - {SQL("ALTER TABLE channels ADD require_confirm_inputs_remote INTEGER DEFAULT 0;"), NULL}, - {SQL("ALTER TABLE channels ADD require_confirm_inputs_local INTEGER DEFAULT 0;"), NULL}, - {NULL, fillin_missing_lease_satoshi}, - {NULL, migrate_invalid_last_tx_psbts}, - {SQL("ALTER TABLE channels ADD channel_type BLOB DEFAULT NULL;"), NULL}, - {NULL, migrate_fill_in_channel_type}, - {SQL("ALTER TABLE peers ADD feature_bits BLOB DEFAULT NULL;"), NULL}, - {NULL, migrate_normalize_invstr}, - {SQL("CREATE TABLE runes (id BIGSERIAL, rune TEXT, PRIMARY KEY (id));"), NULL}, - {SQL("CREATE TABLE runes_blacklist (start_index BIGINT, end_index BIGINT);"), NULL}, - {SQL("ALTER TABLE channels ADD ignore_fee_limits INTEGER DEFAULT 0;"), NULL}, - {NULL, migrate_initialize_invoice_wait_indexes}, - {SQL("ALTER TABLE invoices ADD updated_index BIGINT DEFAULT 0"), NULL}, - {SQL("CREATE INDEX invoice_update_idx ON invoices (updated_index)"), NULL}, - {NULL, migrate_datastore_commando_runes}, - {NULL, migrate_invoice_created_index_var}, - /* Splicing requires us to store HTLC sigs for inflight splices and allows us to discard old sigs after splice confirmation. */ - {SQL("ALTER TABLE htlc_sigs ADD inflight_tx_id BLOB"), NULL}, - {SQL("ALTER TABLE htlc_sigs ADD inflight_tx_outnum INTEGER"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD splice_amnt BIGINT DEFAULT 0"), NULL}, - {SQL("ALTER TABLE channel_funding_inflights ADD i_am_initiator INTEGER DEFAULT 0"), NULL}, - {NULL, migrate_runes_idfix}, - {SQL("ALTER TABLE runes ADD last_used_nsec BIGINT DEFAULT NULL"), NULL}, - {SQL("DELETE FROM vars WHERE name = 'runes_uniqueid'"), NULL}, - {SQL("CREATE TABLE invoice_fallbacks (" - " scriptpubkey BLOB," - " invoice_id BIGINT REFERENCES invoices(id) ON DELETE CASCADE," - " PRIMARY KEY (scriptpubkey)" - ");"), - NULL}, - {SQL("ALTER TABLE invoices ADD paid_txid BLOB DEFAULT NULL"), NULL}, - {SQL("ALTER TABLE invoices ADD paid_outnum INTEGER DEFAULT NULL"), NULL}, - {SQL("CREATE TABLE local_anchors (" - " channel_id BIGSERIAL REFERENCES channels(id)," - " commitment_index BIGINT," - " commitment_txid BLOB," - " commitment_anchor_outnum INTEGER," - " commitment_fee BIGINT," - " commitment_weight INTEGER)"), NULL}, - {SQL("CREATE INDEX local_anchors_idx ON local_anchors (channel_id)"), NULL}, - {SQL("ALTER TABLE payments ADD updated_index BIGINT DEFAULT 0"), NULL}, - {SQL("CREATE INDEX payments_update_idx ON payments (updated_index)"), NULL}, - {NULL, migrate_initialize_payment_wait_indexes}, - {NULL, migrate_forwards_add_rowid}, - {SQL("ALTER TABLE forwards ADD updated_index BIGINT DEFAULT 0"), NULL}, - {SQL("CREATE INDEX forwards_updated_idx ON forwards (updated_index)"), NULL}, - {NULL, migrate_initialize_forwards_wait_indexes}, - {SQL("ALTER TABLE channel_funding_inflights ADD force_sign_first INTEGER DEFAULT 0"), NULL}, - {SQL("ALTER TABLE channels ADD remote_feerate_base INTEGER DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE channels ADD remote_feerate_ppm INTEGER DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE channels ADD remote_cltv_expiry_delta INTEGER DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE channels ADD remote_htlc_maximum_msat BIGINT DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE channels ADD remote_htlc_minimum_msat BIGINT DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE channels ADD last_stable_connection BIGINT DEFAULT 0;"), NULL}, - {NULL, NULL}, /* old migrate_initialize_alias_local */ - {SQL("CREATE TABLE addresses (" - " keyidx BIGINT," - " addrtype INTEGER)"), NULL}, - {NULL, insert_addrtype_to_addresses}, - {SQL("ALTER TABLE channel_funding_inflights ADD remote_funding BLOB DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE peers ADD last_known_address BLOB DEFAULT NULL;"), NULL}, - {SQL("ALTER TABLE channels ADD close_attempt_height INTEGER DEFAULT 0;"), NULL}, - {NULL, migrate_convert_old_channel_keyidx}, - {SQL("INSERT INTO vars(name, intval)" - " VALUES('needs_p2wpkh_close_rescan', 1)"), NULL}, - {SQL("ALTER TABLE channel_htlcs ADD updated_index BIGINT DEFAULT 0"), NULL}, - {SQL("CREATE INDEX channel_htlcs_updated_idx ON channel_htlcs (updated_index)"), NULL}, - {NULL, NULL}, /* Old, incorrect channel_htlcs_wait_indexes migration */ - {SQL("ALTER TABLE channel_funding_inflights ADD locked_scid BIGINT DEFAULT 0;"), NULL}, - {NULL, migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards}, - {SQL("ALTER TABLE channel_funding_inflights ADD i_sent_sigs INTEGER DEFAULT 0"), NULL}, - {SQL("ALTER TABLE channels ADD old_scids BLOB DEFAULT NULL;"), NULL}, - {NULL, migrate_initialize_alias_local}, - /* Avoids duplication in chain_moves and coin_moves tables */ - {SQL("CREATE TABLE move_accounts (" - " id BIGSERIAL," - " name TEXT," - " PRIMARY KEY (id)," - " UNIQUE (name)" - ")"), NULL}, - {SQL("CREATE TABLE chain_moves (" - " id BIGSERIAL," - /* One of these is null */ - " account_channel_id BIGINT references channels(id)," - " account_nonchannel_id BIGINT references move_accounts(id)," - " tag_bitmap BIGINT NOT NULL," - " credit_or_debit BIGINT NOT NULL," - " timestamp BIGINT NOT NULL," - " utxo BLOB NOT NULL," - " spending_txid BLOB," - /* This does NOT reference peers(node_id), since we can have - * MVT_CHANNEL_PROPOSED events on zeroconf channels where we end up - * forgetting the channel, thus the peer */ - " peer_id BLOB," - " payment_hash BLOB," - " block_height INTEGER NOT NULL," - " output_sat BIGINT NOT NULL," - /* One of these is null */ - " originating_channel_id BIGINT references channels(id)," - " originating_nonchannel_id BIGINT references move_accounts(id)," - " output_count INTEGER," - " PRIMARY KEY (id)" - ")"), NULL}, - {SQL("CREATE TABLE channel_moves (" - " id BIGSERIAL," - /* One of these is null */ - " account_channel_id BIGINT references channels(id)," - " account_nonchannel_id BIGINT references move_accounts(id)," - " tag_bitmap BIGINT NOT NULL," - " credit_or_debit BIGINT NOT NULL," - " timestamp BIGINT NOT NULL," - " payment_hash BLOB," - " payment_part_id BIGINT," - " payment_group_id BIGINT," - " fees BIGINT NOT NULL," - " PRIMARY KEY (id)" - ")"), NULL}, - /* We do a lookup before each append, to avoid duplicates */ - {SQL("CREATE INDEX chain_moves_utxo_idx ON chain_moves (utxo)"), NULL}, - {NULL, migrate_from_account_db}, - /* We accidentally allowed duplicate entries */ - {NULL, migrate_remove_chain_moves_duplicates}, - {SQL("CREATE TABLE network_events (" - " id BIGSERIAL," - " peer_id BLOB NOT NULL," - " type INTEGER NOT NULL," - " timestamp BIGINT," - " reason TEXT," - " duration_nsec BIGINT," - " connect_attempted INTEGER NOT NULL," - " PRIMARY KEY (id)" - ")"), NULL}, - {NULL, migrate_fail_pending_payments_without_htlcs}, - {SQL("ALTER TABLE channels ADD withheld INTEGER DEFAULT 0;"), NULL}, -}; - /** * db_migrate - Apply all remaining migrations from the current version */ @@ -1112,11 +26,14 @@ static bool db_migrate(struct lightningd *ld, struct db *db, { /* Attempt to read the version from the database */ int current, orig, available; + size_t num_migrations; char *err_msg; struct db_stmt *stmt; + const struct db_migration *dbmigrations = get_db_migrations(&num_migrations); + /* This is the final number, not the count! */ + available = num_migrations - 1; orig = current = db_get_version(db); - available = ARRAY_SIZE(dbmigrations) - 1; if (current == -1) log_info(ld->log, "Creating database"); @@ -1205,7 +122,7 @@ struct db *db_setup(const tal_t *ctx, struct lightningd *ld, } /* Will apply the current config fee settings to all channels */ -static void migrate_pr2342_feerate_per_channel(struct lightningd *ld, struct db *db) +void migrate_pr2342_feerate_per_channel(struct lightningd *ld, struct db *db) { struct db_stmt *stmt = db_prepare_v2( db, SQL("UPDATE channels SET feerate_base = ?, feerate_ppm = ?;")); @@ -1223,7 +140,7 @@ static void migrate_pr2342_feerate_per_channel(struct lightningd *ld, struct db * is the same as the funding_satoshi for every channel where we are * the `funder` */ -static void migrate_our_funding(struct lightningd *ld, struct db *db) +void migrate_our_funding(struct lightningd *ld, struct db *db) { struct db_stmt *stmt; @@ -1322,7 +239,7 @@ void fillin_missing_scriptpubkeys(struct lightningd *ld, struct db *db) * could simply derive the channel_id whenever it was required, but since there * are now two ways to do it, we save the derived channel id. */ -static void fillin_missing_channel_id(struct lightningd *ld, struct db *db) +void fillin_missing_channel_id(struct lightningd *ld, struct db *db) { struct db_stmt *stmt; @@ -1358,8 +275,8 @@ static void fillin_missing_channel_id(struct lightningd *ld, struct db *db) tal_free(stmt); } -static void fillin_missing_local_basepoints(struct lightningd *ld, - struct db *db) +void fillin_missing_local_basepoints(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; @@ -1421,8 +338,8 @@ static void fillin_missing_local_basepoints(struct lightningd *ld, /* New 'channel_blockheights' table, every existing channel gets a * 'initial blockheight' of 0 */ -static void fillin_missing_channel_blockheights(struct lightningd *ld, - struct db *db) +void fillin_missing_channel_blockheights(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; @@ -1643,8 +560,8 @@ void migrate_last_tx_to_psbt(struct lightningd *ld, struct db *db) } /* We used to store scids as strings... */ -static void migrate_channels_scids_as_integers(struct lightningd *ld, - struct db *db) +void migrate_channels_scids_as_integers(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; char **scids = tal_arr(tmpctx, char *, 0); @@ -1700,8 +617,8 @@ static void migrate_channels_scids_as_integers(struct lightningd *ld, db_exec_prepared_v2(take(stmt)); } -static void migrate_payments_scids_as_integers(struct lightningd *ld, - struct db *db) +void migrate_payments_scids_as_integers(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; const char *colnames[] = {"failchannel"}; @@ -1736,8 +653,8 @@ static void migrate_payments_scids_as_integers(struct lightningd *ld, db_fatal(db, "Could not delete payments.failchannel"); } -static void fillin_missing_lease_satoshi(struct lightningd *ld, - struct db *db) +void fillin_missing_lease_satoshi(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; @@ -1748,8 +665,8 @@ static void fillin_missing_lease_satoshi(struct lightningd *ld, tal_free(stmt); } -static void migrate_fill_in_channel_type(struct lightningd *ld, - struct db *db) +void migrate_fill_in_channel_type(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; @@ -1791,8 +708,8 @@ static void migrate_fill_in_channel_type(struct lightningd *ld, tal_free(stmt); } -static void migrate_initialize_invoice_wait_indexes(struct lightningd *ld, - struct db *db) +void migrate_initialize_invoice_wait_indexes(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; bool res; @@ -1809,7 +726,7 @@ static void migrate_initialize_invoice_wait_indexes(struct lightningd *ld, tal_free(stmt); } -static void migrate_invoice_created_index_var(struct lightningd *ld, struct db *db) +void migrate_invoice_created_index_var(struct lightningd *ld, struct db *db) { struct db_stmt *stmt; s64 badindex, realindex; @@ -1870,8 +787,8 @@ static void migrate_initialize_wait_indexes(struct db *db, tal_free(stmt); } -static void migrate_initialize_payment_wait_indexes(struct lightningd *ld, - struct db *db) +void migrate_initialize_payment_wait_indexes(struct lightningd *ld, + struct db *db) { migrate_initialize_wait_indexes(db, WAIT_SUBSYSTEM_SENDPAY, @@ -1880,8 +797,8 @@ static void migrate_initialize_payment_wait_indexes(struct lightningd *ld, "MAX(id)"); } -static void migrate_forwards_add_rowid(struct lightningd *ld, - struct db *db) +void migrate_forwards_add_rowid(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; @@ -1907,8 +824,8 @@ static void migrate_forwards_add_rowid(struct lightningd *ld, db_exec_prepared_v2(take(stmt)); } -static void migrate_initialize_forwards_wait_indexes(struct lightningd *ld, - struct db *db) +void migrate_initialize_forwards_wait_indexes(struct lightningd *ld, + struct db *db) { migrate_initialize_wait_indexes(db, WAIT_SUBSYSTEM_FORWARD, @@ -1917,8 +834,8 @@ static void migrate_initialize_forwards_wait_indexes(struct lightningd *ld, "MAX(rowid)"); } -static void migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards(struct lightningd *ld, - struct db *db) +void migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards(struct lightningd *ld, + struct db *db) { /* A previous badly-written migration (now NULL-ed out) set * the forwards, not htlc index! Set the htlcs migration, and fixup forwards. */ @@ -1948,8 +865,7 @@ static void complain_unfixed(struct lightningd *ld, } } -static void migrate_invalid_last_tx_psbts(struct lightningd *ld, - struct db *db) +void migrate_invalid_last_tx_psbts(struct lightningd *ld, struct db *db) { struct db_stmt *stmt; @@ -2009,7 +925,7 @@ static void migrate_invalid_last_tx_psbts(struct lightningd *ld, * See also `to_canonical_invstr` in `common/bolt11.c` the definition of * canonical invoice. */ -static void migrate_normalize_invstr(struct lightningd *ld, struct db *db) +void migrate_normalize_invstr(struct lightningd *ld, struct db *db) { struct db_stmt *stmt; @@ -2064,8 +980,8 @@ static void migrate_normalize_invstr(struct lightningd *ld, struct db *db) /* We required local aliases to be set on established channels, * but we forgot about already-existing ones in the db! */ -static void migrate_initialize_alias_local(struct lightningd *ld, - struct db *db) +void migrate_initialize_alias_local(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; u64 *ids = tal_arr(tmpctx, u64, 0); @@ -2090,8 +1006,8 @@ static void migrate_initialize_alias_local(struct lightningd *ld, } /* Insert address type as `ADDR_ALL` for issued addresses */ -static void insert_addrtype_to_addresses(struct lightningd *ld, - struct db *db) +void insert_addrtype_to_addresses(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; u64 bip32_max_index = db_get_intvar(db, "bip32_max_index", 0); @@ -2113,8 +1029,8 @@ static void insert_addrtype_to_addresses(struct lightningd *ld, * don't have access to the peers' features in the db, so instead convert all * the keys to ADDR_ALL. Users with closed channels may still need to * rescan! */ -static void migrate_convert_old_channel_keyidx(struct lightningd *ld, - struct db *db) +void migrate_convert_old_channel_keyidx(struct lightningd *ld, + struct db *db) { struct db_stmt *stmt; @@ -2132,8 +1048,8 @@ static void migrate_convert_old_channel_keyidx(struct lightningd *ld, db_exec_prepared_v2(take(stmt)); } -static void migrate_fail_pending_payments_without_htlcs(struct lightningd *ld, - struct db *db) +void migrate_fail_pending_payments_without_htlcs(struct lightningd *ld, + struct db *db) { /* If channeld died or was offline at the right moment, we * could register a payment as pending, but then not create an diff --git a/wallet/db.h b/wallet/db.h index 29c35fcf8670..f891d18a6fa2 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -26,8 +26,4 @@ struct db *db_setup(const tal_t *ctx, struct lightningd *ld, /* We store last wait indices in our var table. */ void load_indexes(struct db *db, struct indexes *indexes); -/* Migration function for old commando datastore runes. */ -void migrate_datastore_commando_runes(struct lightningd *ld, struct db *db); -/* Migrate old runes with incorrect id fields */ -void migrate_runes_idfix(struct lightningd *ld, struct db *db); #endif /* LIGHTNING_WALLET_DB_H */ diff --git a/wallet/migrations.c b/wallet/migrations.c new file mode 100644 index 000000000000..b58c3e750c79 --- /dev/null +++ b/wallet/migrations.c @@ -0,0 +1,1088 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *revert_too_early(const tal_t *ctx, struct db *db) +{ + return tal_strdup(ctx, "Downgrade to before v25.09 not supported"); +} + +/* Don't allow downgrade if they've *used* the withheld column. */ +static const char *revert_withheld_column(const tal_t *ctx, struct db *db) +{ + struct db_stmt *stmt; + struct channel_id *cid; + struct short_channel_id *scid; + + stmt = db_prepare_v2(db, SQL("SELECT full_channel_id, scid FROM channels WHERE withheld != 0")); + db_query_prepared(stmt); + if (db_step(stmt)) { + cid = tal(tmpctx, struct channel_id); + db_col_channel_id(stmt, "full_channel_id", cid); + if (db_col_is_null(stmt, "scid")) + scid = NULL; + else { + scid = tal(tmpctx, struct short_channel_id); + *scid = db_col_short_channel_id(stmt, "scid"); + } + } else { + cid = NULL; + scid = NULL; + } + tal_free(stmt); + + if (cid) { + return tal_fmt(tmpctx, "Channel %s (%s) used v25.12's withheld flag", + fmt_channel_id(tmpctx, cid), + scid ? fmt_short_channel_id(tmpctx, *scid) : "no short channel id"); + } + + /* For sqlite3 needs "2021-03-12 (3.35.0)" or above */ + stmt = db_prepare_v2(db, SQL("ALTER TABLE channels DROP COLUMN withheld")); + db_exec_prepared_v2(take(stmt)); + return NULL; +} + +/* Do not reorder or remove elements from this array, it is used to + * migrate existing databases from a previous state, based on the + * string indices */ +static const struct db_migration dbmigrations[] = { + {SQL("CREATE TABLE version (version INTEGER)"), NULL}, + {SQL("INSERT INTO version VALUES (1)"), NULL}, + {SQL("CREATE TABLE outputs (" + " prev_out_tx BLOB" + ", prev_out_index INTEGER" + ", value BIGINT" + ", type INTEGER" + ", status INTEGER" + ", keyindex INTEGER" + ", PRIMARY KEY (prev_out_tx, prev_out_index));"), + NULL}, + {SQL("CREATE TABLE vars (" + " name VARCHAR(32)" + ", val VARCHAR(255)" + ", PRIMARY KEY (name)" + ");"), + NULL}, + {SQL("CREATE TABLE shachains (" + " id BIGSERIAL" + ", min_index BIGINT" + ", num_valid BIGINT" + ", PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE shachain_known (" + " shachain_id BIGINT REFERENCES shachains(id) ON DELETE CASCADE" + ", pos INTEGER" + ", idx BIGINT" + ", hash BLOB" + ", PRIMARY KEY (shachain_id, pos)" + ");"), + NULL}, + {SQL("CREATE TABLE peers (" + " id BIGSERIAL" + ", node_id BLOB UNIQUE" /* pubkey */ + ", address TEXT" + ", PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE channels (" + " id BIGSERIAL," /* chan->id */ + /* FIXME: We deliberately never delete a peer with channels, so this constraint is + * unnecessary! */ + " peer_id BIGINT REFERENCES peers(id) ON DELETE CASCADE," + " short_channel_id TEXT," + " channel_config_local BIGINT," + " channel_config_remote BIGINT," + " state INTEGER," + " funder INTEGER," + " channel_flags INTEGER," + " minimum_depth INTEGER," + " next_index_local BIGINT," + " next_index_remote BIGINT," + " next_htlc_id BIGINT," + " funding_tx_id BLOB," + " funding_tx_outnum INTEGER," + " funding_satoshi BIGINT," + " funding_locked_remote INTEGER," + " push_msatoshi BIGINT," + " msatoshi_local BIGINT," /* our_msatoshi */ + /* START channel_info */ + " fundingkey_remote BLOB," + " revocation_basepoint_remote BLOB," + " payment_basepoint_remote BLOB," + " htlc_basepoint_remote BLOB," + " delayed_payment_basepoint_remote BLOB," + " per_commit_remote BLOB," + " old_per_commit_remote BLOB," + " local_feerate_per_kw INTEGER," + " remote_feerate_per_kw INTEGER," + /* END channel_info */ + " shachain_remote_id BIGINT," + " shutdown_scriptpubkey_remote BLOB," + " shutdown_keyidx_local BIGINT," + " last_sent_commit_state BIGINT," + " last_sent_commit_id INTEGER," + " last_tx BLOB," + " last_sig BLOB," + " closing_fee_received INTEGER," + " closing_sig_received BLOB," + " PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE channel_configs (" + " id BIGSERIAL," + " dust_limit_satoshis BIGINT," + " max_htlc_value_in_flight_msat BIGINT," + " channel_reserve_satoshis BIGINT," + " htlc_minimum_msat BIGINT," + " to_self_delay INTEGER," + " max_accepted_htlcs INTEGER," + " PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE channel_htlcs (" + " id BIGSERIAL," + " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," + " channel_htlc_id BIGINT," + " direction INTEGER," + " origin_htlc BIGINT," + " msatoshi BIGINT," + " cltv_expiry INTEGER," + " payment_hash BLOB," + " payment_key BLOB," + " routing_onion BLOB," + " failuremsg BLOB," /* Note: This is in fact the failure onionreply, + * but renaming columns is hard! */ + " malformed_onion INTEGER," + " hstate INTEGER," + " shared_secret BLOB," + " PRIMARY KEY (id)," + " UNIQUE (channel_id, channel_htlc_id, direction)" + ");"), + NULL}, + {SQL("CREATE TABLE invoices (" + " id BIGSERIAL," + " state INTEGER," + " msatoshi BIGINT," + " payment_hash BLOB," + " payment_key BLOB," + " label TEXT," + " PRIMARY KEY (id)," + " UNIQUE (label)," + " UNIQUE (payment_hash)" + ");"), + NULL}, + {SQL("CREATE TABLE payments (" + " id BIGSERIAL," + " timestamp INTEGER," + " status INTEGER," + " payment_hash BLOB," + " direction INTEGER," + " destination BLOB," + " msatoshi BIGINT," + " PRIMARY KEY (id)," + " UNIQUE (payment_hash)" + ");"), + NULL}, + /* Add expiry field to invoices (effectively infinite). */ + {SQL("ALTER TABLE invoices ADD expiry_time BIGINT;"), NULL}, + {SQL("UPDATE invoices SET expiry_time=9223372036854775807;"), NULL}, + /* Add pay_index field to paid invoices (initially, same order as id). */ + {SQL("ALTER TABLE invoices ADD pay_index BIGINT;"), NULL}, + {SQL("CREATE UNIQUE INDEX invoices_pay_index ON invoices(pay_index);"), + NULL}, + {SQL("UPDATE invoices SET pay_index=id WHERE state=1;"), + NULL}, /* only paid invoice */ + /* Create next_pay_index variable (highest pay_index). */ + {SQL("INSERT INTO vars(name, val)" + " VALUES('next_pay_index', " + " COALESCE((SELECT MAX(pay_index) FROM invoices WHERE state=1), 0) " + "+ 1" + " );"), + NULL}, + /* Create first_block field; initialize from channel id if any. + * This fails for channels still awaiting lockin, but that only applies to + * pre-release software, so it's forgivable. */ + {SQL("ALTER TABLE channels ADD first_blocknum BIGINT;"), NULL}, + {SQL("UPDATE channels SET first_blocknum=1 WHERE short_channel_id IS NOT NULL;"), + NULL}, + {SQL("ALTER TABLE outputs ADD COLUMN channel_id BIGINT;"), NULL}, + {SQL("ALTER TABLE outputs ADD COLUMN peer_id BLOB;"), NULL}, + {SQL("ALTER TABLE outputs ADD COLUMN commitment_point BLOB;"), NULL}, + {SQL("ALTER TABLE invoices ADD COLUMN msatoshi_received BIGINT;"), NULL}, + /* Normally impossible, so at least we'll know if databases are ancient. */ + {SQL("UPDATE invoices SET msatoshi_received=0 WHERE state=1;"), NULL}, + {SQL("ALTER TABLE channels ADD COLUMN last_was_revoke INTEGER;"), NULL}, + /* We no longer record incoming payments: invoices cover that. + * Without ALTER_TABLE DROP COLUMN support we need to do this by + * rename & copy, which works because there are no triggers etc. */ + {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, + {SQL("CREATE TABLE payments (" + " id BIGSERIAL," + " timestamp INTEGER," + " status INTEGER," + " payment_hash BLOB," + " destination BLOB," + " msatoshi BIGINT," + " PRIMARY KEY (id)," + " UNIQUE (payment_hash)" + ");"), + NULL}, + {SQL("INSERT INTO payments SELECT id, timestamp, status, payment_hash, " + "destination, msatoshi FROM temp_payments WHERE direction=1;"), + NULL}, + {SQL("DROP TABLE temp_payments;"), NULL}, + /* We need to keep the preimage in case they ask to pay again. */ + {SQL("ALTER TABLE payments ADD COLUMN payment_preimage BLOB;"), NULL}, + /* We need to keep the shared secrets to decode error returns. */ + {SQL("ALTER TABLE payments ADD COLUMN path_secrets BLOB;"), NULL}, + /* Create time-of-payment of invoice, default already-paid + * invoices to current time. */ + {SQL("ALTER TABLE invoices ADD paid_timestamp BIGINT;"), NULL}, + {SQL("UPDATE invoices" + " SET paid_timestamp = CURRENT_TIMESTAMP()" + " WHERE state = 1;"), + NULL}, + /* We need to keep the route node pubkeys and short channel ids to + * correctly mark routing failures. We separate short channel ids + * because we cannot safely save them as blobs due to byteorder + * concerns. */ + {SQL("ALTER TABLE payments ADD COLUMN route_nodes BLOB;"), NULL}, + {SQL("ALTER TABLE payments ADD COLUMN route_channels BLOB;"), NULL}, + {SQL("CREATE TABLE htlc_sigs (channelid INTEGER REFERENCES channels(id) ON " + "DELETE CASCADE, signature BLOB);"), + NULL}, + {SQL("CREATE INDEX channel_idx ON htlc_sigs (channelid)"), NULL}, + /* Get rid of OPENINGD entries; we don't put them in db any more */ + {SQL("DELETE FROM channels WHERE state=1"), NULL}, + /* Keep track of db upgrades, for debugging */ + {SQL("CREATE TABLE db_upgrades (upgrade_from INTEGER, lightning_version " + "TEXT);"), + NULL}, + /* We used not to clean up peers when their channels were gone. */ + {SQL("DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);"), + NULL}, + /* The ONCHAIND_CHEATED/THEIR_UNILATERAL/OUR_UNILATERAL/MUTUAL are now one + */ + {SQL("UPDATE channels SET STATE = 8 WHERE state > 8;"), NULL}, + /* Add bolt11 to invoices table*/ + {SQL("ALTER TABLE invoices ADD bolt11 TEXT;"), NULL}, + /* What do we think the head of the blockchain looks like? Used + * primarily to track confirmations across restarts and making + * sure we handle reorgs correctly. */ + {SQL("CREATE TABLE blocks (height INT, hash BLOB, prev_hash BLOB, " + "UNIQUE(height));"), + NULL}, + /* ON DELETE CASCADE would have been nice for confirmation_height, + * so that we automatically delete outputs that fall off the + * blockchain and then we rediscover them if they are included + * again. However, we have the their_unilateral/to_us which we + * can't simply recognize from the chain without additional + * hints. So we just mark them as unconfirmed should the block + * die. */ + {SQL("ALTER TABLE outputs ADD COLUMN confirmation_height INTEGER " + "REFERENCES blocks(height) ON DELETE SET NULL;"), + NULL}, + {SQL("ALTER TABLE outputs ADD COLUMN spend_height INTEGER REFERENCES " + "blocks(height) ON DELETE SET NULL;"), + NULL}, + /* Create a covering index that covers both fields */ + {SQL("CREATE INDEX output_height_idx ON outputs (confirmation_height, " + "spend_height);"), + NULL}, + {SQL("CREATE TABLE utxoset (" + " txid BLOB," + " outnum INT," + " blockheight INT REFERENCES blocks(height) ON DELETE CASCADE," + " spendheight INT REFERENCES blocks(height) ON DELETE SET NULL," + " txindex INT," + " scriptpubkey BLOB," + " satoshis BIGINT," + " PRIMARY KEY(txid, outnum));"), + NULL}, + {SQL("CREATE INDEX short_channel_id ON utxoset (blockheight, txindex, " + "outnum)"), + NULL}, + /* Necessary index for long rollbacks of the blockchain, otherwise we're + * doing table scans for every block removed. */ + {SQL("CREATE INDEX utxoset_spend ON utxoset (spendheight)"), NULL}, + /* Assign key 0 to unassigned shutdown_keyidx_local. */ + {SQL("UPDATE channels SET shutdown_keyidx_local=0 WHERE " + "shutdown_keyidx_local = -1;"), + NULL}, + /* FIXME: We should rename shutdown_keyidx_local to final_key_index */ + /* -- Payment routing failure information -- */ + /* BLOB if failure was due to unparseable onion, NULL otherwise */ + {SQL("ALTER TABLE payments ADD failonionreply BLOB;"), NULL}, + /* 0 if we could theoretically retry, 1 if PERM fail at payee */ + {SQL("ALTER TABLE payments ADD faildestperm INTEGER;"), NULL}, + /* Contents of routing_failure (only if not unparseable onion) */ + {SQL("ALTER TABLE payments ADD failindex INTEGER;"), + NULL}, /* erring_index */ + {SQL("ALTER TABLE payments ADD failcode INTEGER;"), NULL}, /* failcode */ + {SQL("ALTER TABLE payments ADD failnode BLOB;"), NULL}, /* erring_node */ + {SQL("ALTER TABLE payments ADD failchannel TEXT;"), + NULL}, /* erring_channel */ + {SQL("ALTER TABLE payments ADD failupdate BLOB;"), + NULL}, /* channel_update - can be NULL*/ + /* -- Payment routing failure information ends -- */ + /* Delete route data for already succeeded or failed payments */ + {SQL("UPDATE payments" + " SET path_secrets = NULL" + " , route_nodes = NULL" + " , route_channels = NULL" + " WHERE status <> 0;"), + NULL}, /* PAYMENT_PENDING */ + /* -- Routing statistics -- */ + {SQL("ALTER TABLE channels ADD in_payments_offered INTEGER DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD in_payments_fulfilled INTEGER DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD in_msatoshi_offered BIGINT DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD in_msatoshi_fulfilled BIGINT DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD out_payments_offered INTEGER DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD out_payments_fulfilled INTEGER DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD out_msatoshi_offered BIGINT DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD out_msatoshi_fulfilled BIGINT DEFAULT 0;"), NULL}, + {SQL("UPDATE channels" + " SET in_payments_offered = 0, in_payments_fulfilled = 0" + " , in_msatoshi_offered = 0, in_msatoshi_fulfilled = 0" + " , out_payments_offered = 0, out_payments_fulfilled = 0" + " , out_msatoshi_offered = 0, out_msatoshi_fulfilled = 0" + " ;"), + NULL}, + /* -- Routing statistics ends --*/ + /* Record the msatoshi actually sent in a payment. */ + {SQL("ALTER TABLE payments ADD msatoshi_sent BIGINT;"), NULL}, + {SQL("UPDATE payments SET msatoshi_sent = msatoshi;"), NULL}, + /* Delete dangling utxoset entries due to Issue #1280 */ + {SQL("DELETE FROM utxoset WHERE blockheight IN (" + " SELECT DISTINCT(blockheight)" + " FROM utxoset LEFT OUTER JOIN blocks on (blockheight = " + "blocks.height) " + " WHERE blocks.hash IS NULL" + ");"), + NULL}, + /* Record feerate range, to optimize onchaind grinding actual fees. */ + {SQL("ALTER TABLE channels ADD min_possible_feerate INTEGER;"), NULL}, + {SQL("ALTER TABLE channels ADD max_possible_feerate INTEGER;"), NULL}, + /* https://bitcoinfees.github.io/#1d says Dec 17 peak was ~1M sat/kb + * which is 250,000 sat/Sipa */ + {SQL("UPDATE channels SET min_possible_feerate=0, " + "max_possible_feerate=250000;"), + NULL}, + /* -- Min and max msatoshi_to_us -- */ + {SQL("ALTER TABLE channels ADD msatoshi_to_us_min BIGINT;"), NULL}, + {SQL("ALTER TABLE channels ADD msatoshi_to_us_max BIGINT;"), NULL}, + {SQL("UPDATE channels" + " SET msatoshi_to_us_min = msatoshi_local" + " , msatoshi_to_us_max = msatoshi_local" + " ;"), + NULL}, + /* -- Min and max msatoshi_to_us ends -- */ + /* Transactions we are interested in. Either we sent them ourselves or we + * are watching them. We don't cascade block height deletes so we don't + * forget any of them by accident.*/ + {SQL("CREATE TABLE transactions (" + " id BLOB" + ", blockheight INTEGER REFERENCES blocks(height) ON DELETE SET NULL" + ", txindex INTEGER" + ", rawtx BLOB" + ", PRIMARY KEY (id)" + ");"), + NULL}, + /* -- Detailed payment failure -- */ + {SQL("ALTER TABLE payments ADD faildetail TEXT;"), NULL}, + {SQL("UPDATE payments" + " SET faildetail = 'unspecified payment failure reason'" + " WHERE status = 2;"), + NULL}, /* PAYMENT_FAILED */ + /* -- Detailed payment faiure ends -- */ + {SQL("CREATE TABLE channeltxs (" + /* The id serves as insertion order and short ID */ + " id BIGSERIAL" + ", channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE" + ", type INTEGER" + ", transaction_id BLOB REFERENCES transactions(id) ON DELETE CASCADE" + /* The input_num is only used by the txo_watch, 0 if txwatch */ + ", input_num INTEGER" + /* The height at which we sent the depth notice */ + ", blockheight INTEGER REFERENCES blocks(height) ON DELETE CASCADE" + ", PRIMARY KEY(id)" + ");"), + NULL}, + /* -- Set the correct rescan height for PR #1398 -- */ + /* Delete blocks that are higher than our initial scan point, this is a + * no-op if we don't have a channel. */ + {SQL("DELETE FROM blocks WHERE height > (SELECT MIN(first_blocknum) FROM " + "channels);"), + NULL}, + /* Now make sure we have the lower bound block with the first_blocknum + * height. This may introduce a block with NULL height if we didn't have any + * blocks, remove that in the next. */ + {SQL("INSERT INTO blocks (height) VALUES ((SELECT " + "MIN(first_blocknum) FROM channels)) " + "ON CONFLICT(height) DO NOTHING;"), + NULL}, + {SQL("DELETE FROM blocks WHERE height IS NULL;"), NULL}, + /* -- End of PR #1398 -- */ + {SQL("ALTER TABLE invoices ADD description TEXT;"), NULL}, + /* FIXME: payments table 'description' is really a 'label' */ + {SQL("ALTER TABLE payments ADD description TEXT;"), NULL}, + /* future_per_commitment_point if other side proves we're out of date -- */ + {SQL("ALTER TABLE channels ADD future_per_commitment_point BLOB;"), NULL}, + /* last_sent_commit array fix */ + {SQL("ALTER TABLE channels ADD last_sent_commit BLOB;"), NULL}, + /* Stats table to track forwarded HTLCs. The values in the HTLCs + * and their states are replicated here and the entries are not + * deleted when the HTLC entries or the channel entries are + * deleted to avoid unexpected drops in statistics. */ + {SQL("CREATE TABLE forwarded_payments (" + " in_htlc_id BIGINT REFERENCES channel_htlcs(id) ON DELETE SET NULL" + ", out_htlc_id BIGINT REFERENCES channel_htlcs(id) ON DELETE SET NULL" + ", in_channel_scid BIGINT" + ", out_channel_scid BIGINT" + ", in_msatoshi BIGINT" + ", out_msatoshi BIGINT" + ", state INTEGER" + ", UNIQUE(in_htlc_id, out_htlc_id)" + ");"), + NULL}, + /* Add a direction for failed payments. */ + {SQL("ALTER TABLE payments ADD faildirection INTEGER;"), + NULL}, /* erring_direction */ + /* Fix dangling peers with no channels. */ + {SQL("DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);"), + NULL}, + {SQL("ALTER TABLE outputs ADD scriptpubkey BLOB;"), NULL}, + /* Keep bolt11 string for payments. */ + {SQL("ALTER TABLE payments ADD bolt11 TEXT;"), NULL}, + /* PR #2342 feerate per channel */ + {SQL("ALTER TABLE channels ADD feerate_base INTEGER;"), NULL}, + {SQL("ALTER TABLE channels ADD feerate_ppm INTEGER;"), NULL}, + {NULL, migrate_pr2342_feerate_per_channel}, + {SQL("ALTER TABLE channel_htlcs ADD received_time BIGINT"), NULL}, + {SQL("ALTER TABLE forwarded_payments ADD received_time BIGINT"), NULL}, + {SQL("ALTER TABLE forwarded_payments ADD resolved_time BIGINT"), NULL}, + {SQL("ALTER TABLE channels ADD remote_upfront_shutdown_script BLOB;"), + NULL}, + /* PR #2524: Add failcode into forward_payment */ + {SQL("ALTER TABLE forwarded_payments ADD failcode INTEGER;"), NULL}, + /* remote signatures for channel announcement */ + {SQL("ALTER TABLE channels ADD remote_ann_node_sig BLOB;"), NULL}, + {SQL("ALTER TABLE channels ADD remote_ann_bitcoin_sig BLOB;"), NULL}, + /* FIXME: We now use the transaction_annotations table to type each + * input and output instead of type and channel_id! */ + /* Additional information for transaction tracking and listing */ + {SQL("ALTER TABLE transactions ADD type BIGINT;"), NULL}, + /* Not a foreign key on purpose since we still delete channels from + * the DB which would remove this. It is mainly used to group payments + * in the list view anyway, e.g., show all close and htlc transactions + * as a single bundle. */ + {SQL("ALTER TABLE transactions ADD channel_id BIGINT;"), NULL}, + /* Convert pre-Adelaide short_channel_ids */ + {SQL("UPDATE channels" + " SET short_channel_id = REPLACE(short_channel_id, ':', 'x')" + " WHERE short_channel_id IS NOT NULL;"), NULL }, + {SQL("UPDATE payments SET failchannel = REPLACE(failchannel, ':', 'x')" + " WHERE failchannel IS NOT NULL;"), NULL }, + /* option_static_remotekey is nailed at creation time. */ + {SQL("ALTER TABLE channels ADD COLUMN option_static_remotekey INTEGER" + " DEFAULT 0;"), NULL }, + {SQL("ALTER TABLE vars ADD COLUMN intval INTEGER"), NULL}, + {SQL("ALTER TABLE vars ADD COLUMN blobval BLOB"), NULL}, + {SQL("UPDATE vars SET intval = CAST(val AS INTEGER) WHERE name IN ('bip32_max_index', 'last_processed_block', 'next_pay_index')"), NULL}, + {SQL("UPDATE vars SET blobval = CAST(val AS BLOB) WHERE name = 'genesis_hash'"), NULL}, + {SQL("CREATE TABLE transaction_annotations (" + /* Not making this a reference since we usually filter the TX by + * walking its inputs and outputs, and only afterwards storing it in + * the DB. Having a reference here would point into the void until we + * add the matching TX. */ + " txid BLOB" + ", idx INTEGER" /* 0 when location is the tx, the index of the output or input otherwise */ + ", location INTEGER" /* The transaction itself, the output at idx, or the input at idx */ + ", type INTEGER" + ", channel BIGINT REFERENCES channels(id)" + ", UNIQUE(txid, idx)" + ");"), NULL}, + {SQL("ALTER TABLE channels ADD shutdown_scriptpubkey_local BLOB;"), + NULL}, + /* See https://github.com/ElementsProject/lightning/issues/3189 */ + {SQL("UPDATE forwarded_payments SET received_time=0 WHERE received_time IS NULL;"), + NULL}, + {SQL("ALTER TABLE invoices ADD COLUMN features BLOB DEFAULT '';"), NULL}, + /* We can now have multiple payments in progress for a single hash, so + * add two fields; combination of payment_hash & partid is unique. */ + {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, + {SQL("CREATE TABLE payments (" + " id BIGSERIAL" + ", timestamp INTEGER" + ", status INTEGER" + ", payment_hash BLOB" + ", destination BLOB" + ", msatoshi BIGINT" + ", payment_preimage BLOB" + ", path_secrets BLOB" + ", route_nodes BLOB" + ", route_channels BLOB" + ", failonionreply BLOB" + ", faildestperm INTEGER" + ", failindex INTEGER" + ", failcode INTEGER" + ", failnode BLOB" + ", failchannel TEXT" + ", failupdate BLOB" + ", msatoshi_sent BIGINT" + ", faildetail TEXT" + ", description TEXT" + ", faildirection INTEGER" + ", bolt11 TEXT" + ", total_msat BIGINT" + ", partid BIGINT" + ", PRIMARY KEY (id)" + ", UNIQUE (payment_hash, partid))"), NULL}, + {SQL("INSERT INTO payments (" + "id" + ", timestamp" + ", status" + ", payment_hash" + ", destination" + ", msatoshi" + ", payment_preimage" + ", path_secrets" + ", route_nodes" + ", route_channels" + ", failonionreply" + ", faildestperm" + ", failindex" + ", failcode" + ", failnode" + ", failchannel" + ", failupdate" + ", msatoshi_sent" + ", faildetail" + ", description" + ", faildirection" + ", bolt11)" + "SELECT id" + ", timestamp" + ", status" + ", payment_hash" + ", destination" + ", msatoshi" + ", payment_preimage" + ", path_secrets" + ", route_nodes" + ", route_channels" + ", failonionreply" + ", faildestperm" + ", failindex" + ", failcode" + ", failnode" + ", failchannel" + ", failupdate" + ", msatoshi_sent" + ", faildetail" + ", description" + ", faildirection" + ", bolt11 FROM temp_payments;"), NULL}, + {SQL("UPDATE payments SET total_msat = msatoshi;"), NULL}, + {SQL("UPDATE payments SET partid = 0;"), NULL}, + {SQL("DROP TABLE temp_payments;"), NULL}, + {SQL("ALTER TABLE channel_htlcs ADD partid BIGINT;"), NULL}, + {SQL("UPDATE channel_htlcs SET partid = 0;"), NULL}, + {SQL("CREATE TABLE channel_feerates (" + " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," + " hstate INTEGER," + " feerate_per_kw INTEGER," + " UNIQUE (channel_id, hstate)" + ");"), + NULL}, + /* Cast old-style per-side feerates into most likely layout for statewise + * feerates. */ + /* If we're funder (LOCAL=0): + * Then our feerate is set last (SENT_ADD_ACK_REVOCATION = 4) */ + {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" + " SELECT id, 4, local_feerate_per_kw FROM channels WHERE funder = 0;"), + NULL}, + /* If different, assume their feerate is in state SENT_ADD_COMMIT = 1 */ + {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" + " SELECT id, 1, remote_feerate_per_kw FROM channels WHERE funder = 0 and local_feerate_per_kw != remote_feerate_per_kw;"), + NULL}, + /* If they're funder (REMOTE=1): + * Then their feerate is set last (RCVD_ADD_ACK_REVOCATION = 14) */ + {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" + " SELECT id, 14, remote_feerate_per_kw FROM channels WHERE funder = 1;"), + NULL}, + /* If different, assume their feerate is in state RCVD_ADD_COMMIT = 11 */ + {SQL("INSERT INTO channel_feerates(channel_id, hstate, feerate_per_kw)" + " SELECT id, 11, local_feerate_per_kw FROM channels WHERE funder = 1 and local_feerate_per_kw != remote_feerate_per_kw;"), + NULL}, + /* FIXME: Remove now-unused local_feerate_per_kw and remote_feerate_per_kw from channels */ + {SQL("INSERT INTO vars (name, intval) VALUES ('data_version', 0);"), NULL}, + /* For outgoing HTLCs, we now keep a localmsg instead of a failcode. + * Turn anything in transition into a WIRE_TEMPORARY_NODE_FAILURE. */ + {SQL("ALTER TABLE channel_htlcs ADD localfailmsg BLOB;"), NULL}, + {SQL("UPDATE channel_htlcs SET localfailmsg=decode('2002', 'hex') WHERE malformed_onion != 0 AND direction = 1;"), NULL}, + {SQL("ALTER TABLE channels ADD our_funding_satoshi BIGINT DEFAULT 0;"), migrate_our_funding}, + {SQL("CREATE TABLE penalty_bases (" + " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE" + ", commitnum BIGINT" + ", txid BLOB" + ", outnum INTEGER" + ", amount BIGINT" + ", PRIMARY KEY (channel_id, commitnum)" + ");"), NULL}, + /* For incoming HTLCs, we now keep track of whether or not we provided + * the preimage for it, or not. */ + {SQL("ALTER TABLE channel_htlcs ADD we_filled INTEGER;"), NULL}, + /* We track the counter for coin_moves, as a convenience for notification consumers */ + {SQL("INSERT INTO vars (name, intval) VALUES ('coin_moves_count', 0);"), NULL}, + {NULL, migrate_last_tx_to_psbt}, + {SQL("ALTER TABLE outputs ADD reserved_til INTEGER DEFAULT NULL;"), NULL}, + {NULL, fillin_missing_scriptpubkeys}, + /* option_anchor_outputs is nailed at creation time. */ + {SQL("ALTER TABLE channels ADD COLUMN option_anchor_outputs INTEGER" + " DEFAULT 0;"), NULL }, + /* We need to know if it was option_anchor_outputs to spend to_remote */ + {SQL("ALTER TABLE outputs ADD option_anchor_outputs INTEGER" + " DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD full_channel_id BLOB DEFAULT NULL;"), fillin_missing_channel_id}, + {SQL("ALTER TABLE channels ADD funding_psbt BLOB DEFAULT NULL;"), NULL}, + /* Channel closure reason */ + {SQL("ALTER TABLE channels ADD closer INTEGER DEFAULT 2;"), NULL}, + {SQL("ALTER TABLE channels ADD state_change_reason INTEGER DEFAULT 0;"), NULL}, + {SQL("CREATE TABLE channel_state_changes (" + " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," + " timestamp BIGINT," + " old_state INTEGER," + " new_state INTEGER," + " cause INTEGER," + " message TEXT" + ");"), NULL}, + {SQL("CREATE TABLE offers (" + " offer_id BLOB" + ", bolt12 TEXT" + ", label TEXT" + ", status INTEGER" + ", PRIMARY KEY (offer_id)" + ");"), NULL}, + /* A reference into our own offers table, if it was made from one */ + {SQL("ALTER TABLE invoices ADD COLUMN local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id);"), NULL}, + /* A reference into our own offers table, if it was made from one */ + {SQL("ALTER TABLE payments ADD COLUMN local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id);"), NULL}, + {SQL("ALTER TABLE channels ADD funding_tx_remote_sigs_received INTEGER DEFAULT 0;"), NULL}, + /* Speeds up deletion of one peer from the database, measurements suggest + * it cuts down the time by 80%. */ + {SQL("CREATE INDEX forwarded_payments_out_htlc_id" + " ON forwarded_payments (out_htlc_id);"), NULL}, + {SQL("UPDATE channel_htlcs SET malformed_onion = 0 WHERE malformed_onion IS NULL"), NULL}, + /* Speed up forwarded_payments lookup based on state */ + {SQL("CREATE INDEX forwarded_payments_state ON forwarded_payments (state)"), NULL}, + {SQL("CREATE TABLE channel_funding_inflights (" + " channel_id BIGSERIAL REFERENCES channels(id) ON DELETE CASCADE" + ", funding_tx_id BLOB" + ", funding_tx_outnum INTEGER" + ", funding_feerate INTEGER" + ", funding_satoshi BIGINT" + ", our_funding_satoshi BIGINT" + ", funding_psbt BLOB" + ", last_tx BLOB" + ", last_sig BLOB" + ", funding_tx_remote_sigs_received INTEGER" + ", PRIMARY KEY (channel_id, funding_tx_id)" + ");"), + NULL}, + {SQL("ALTER TABLE channels ADD revocation_basepoint_local BLOB"), NULL}, + {SQL("ALTER TABLE channels ADD payment_basepoint_local BLOB"), NULL}, + {SQL("ALTER TABLE channels ADD htlc_basepoint_local BLOB"), NULL}, + {SQL("ALTER TABLE channels ADD delayed_payment_basepoint_local BLOB"), NULL}, + {SQL("ALTER TABLE channels ADD funding_pubkey_local BLOB"), NULL}, + {NULL, fillin_missing_local_basepoints}, + /* Oops, can I haz money back plz? */ + {SQL("ALTER TABLE channels ADD shutdown_wrong_txid BLOB DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channels ADD shutdown_wrong_outnum INTEGER DEFAULT NULL"), NULL}, + {NULL, migrate_inflight_last_tx_to_psbt}, + /* Channels can now change their type at specific commit indexes. */ + {SQL("ALTER TABLE channels ADD local_static_remotekey_start BIGINT DEFAULT 0"), + NULL}, + {SQL("ALTER TABLE channels ADD remote_static_remotekey_start BIGINT DEFAULT 0"), + NULL}, + /* Set counter past 2^48 if they don't have option */ + {SQL("UPDATE channels SET" + " remote_static_remotekey_start = 9223372036854775807," + " local_static_remotekey_start = 9223372036854775807" + " WHERE option_static_remotekey = 0"), + NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD lease_commit_sig BLOB DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD lease_chan_max_msat BIGINT DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD lease_chan_max_ppt INTEGER DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD lease_expiry INTEGER DEFAULT 0"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD lease_blockheight_start INTEGER DEFAULT 0"), NULL}, + {SQL("ALTER TABLE channels ADD lease_commit_sig BLOB DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channels ADD lease_chan_max_msat INTEGER DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channels ADD lease_chan_max_ppt INTEGER DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE channels ADD lease_expiry INTEGER DEFAULT 0"), NULL}, + {SQL("CREATE TABLE channel_blockheights (" + " channel_id BIGINT REFERENCES channels(id) ON DELETE CASCADE," + " hstate INTEGER," + " blockheight INTEGER," + " UNIQUE (channel_id, hstate)" + ");"), + fillin_missing_channel_blockheights}, + {SQL("ALTER TABLE outputs ADD csv_lock INTEGER DEFAULT 1;"), NULL}, + {SQL("CREATE TABLE datastore (" + " key BLOB," + " data BLOB," + " generation BIGINT," + " PRIMARY KEY (key)" + ");"), + NULL}, + {SQL("CREATE INDEX channel_state_changes_channel_id" + " ON channel_state_changes (channel_id);"), NULL}, + /* We need to switch the unique key to cover the groupid as well, + * so we can attempt payments multiple times. */ + {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, + {SQL("CREATE TABLE payments (" + " id BIGSERIAL" + ", timestamp INTEGER" + ", status INTEGER" + ", payment_hash BLOB" + ", destination BLOB" + ", msatoshi BIGINT" + ", payment_preimage BLOB" + ", path_secrets BLOB" + ", route_nodes BLOB" + ", route_channels BLOB" + ", failonionreply BLOB" + ", faildestperm INTEGER" + ", failindex INTEGER" + ", failcode INTEGER" + ", failnode BLOB" + ", failchannel TEXT" + ", failupdate BLOB" + ", msatoshi_sent BIGINT" + ", faildetail TEXT" + ", description TEXT" + ", faildirection INTEGER" + ", bolt11 TEXT" + ", total_msat BIGINT" + ", partid BIGINT" + ", groupid BIGINT NOT NULL DEFAULT 0" + ", local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id)" + ", PRIMARY KEY (id)" + ", UNIQUE (payment_hash, partid, groupid))"), NULL}, + {SQL("INSERT INTO payments (" + "id" + ", timestamp" + ", status" + ", payment_hash" + ", destination" + ", msatoshi" + ", payment_preimage" + ", path_secrets" + ", route_nodes" + ", route_channels" + ", failonionreply" + ", faildestperm" + ", failindex" + ", failcode" + ", failnode" + ", failchannel" + ", failupdate" + ", msatoshi_sent" + ", faildetail" + ", description" + ", faildirection" + ", bolt11" + ", groupid" + ", local_offer_id)" + "SELECT id" + ", timestamp" + ", status" + ", payment_hash" + ", destination" + ", msatoshi" + ", payment_preimage" + ", path_secrets" + ", route_nodes" + ", route_channels" + ", failonionreply" + ", faildestperm" + ", failindex" + ", failcode" + ", failnode" + ", failchannel" + ", failupdate" + ", msatoshi_sent" + ", faildetail" + ", description" + ", faildirection" + ", bolt11" + ", 0" + ", local_offer_id FROM temp_payments;"), NULL}, + {SQL("DROP TABLE temp_payments;"), NULL}, + /* HTLCs also need to carry the groupid around so we can + * selectively update them. */ + {SQL("ALTER TABLE channel_htlcs ADD groupid BIGINT;"), NULL}, + {SQL("ALTER TABLE channel_htlcs ADD COLUMN" + " min_commit_num BIGINT default 0;"), NULL}, + {SQL("ALTER TABLE channel_htlcs ADD COLUMN" + " max_commit_num BIGINT default NULL;"), NULL}, + /* Set max_commit_num for dead (RCVD_REMOVE_ACK_REVOCATION or SENT_REMOVE_ACK_REVOCATION) HTLCs based on latest indexes */ + {SQL("UPDATE channel_htlcs SET max_commit_num =" + " (SELECT GREATEST(next_index_local, next_index_remote)" + " FROM channels WHERE id=channel_id)" + " WHERE (hstate=9 OR hstate=19);"), NULL}, + /* Remove unused fields which take much room in db. */ + {SQL("UPDATE channel_htlcs SET" + " payment_key=NULL," + " routing_onion=NULL," + " failuremsg=NULL," + " shared_secret=NULL," + " localfailmsg=NULL" + " WHERE (hstate=9 OR hstate=19);"), NULL}, + /* We default to 50k sats */ + {SQL("ALTER TABLE channel_configs ADD max_dust_htlc_exposure_msat BIGINT DEFAULT 50000000"), NULL}, + {SQL("ALTER TABLE channel_htlcs ADD fail_immediate INTEGER DEFAULT 0"), NULL}, + + /* Issue #4887: reset the payments.id sequence after the migration above. Since this is a SELECT statement that would otherwise fail, make it an INSERT into the `vars` table.*/ + {SQL("/*PSQL*/INSERT INTO vars (name, intval) VALUES ('payment_id_reset', setval(pg_get_serial_sequence('payments', 'id'), COALESCE((SELECT MAX(id)+1 FROM payments), 1)))"), NULL}, + + /* Issue #4901: Partial index speeds up startup on nodes with ~1000 channels. */ + {&SQL("CREATE INDEX channel_htlcs_speedup_unresolved_idx" + " ON channel_htlcs(channel_id, direction)" + " WHERE hstate NOT IN (9, 19);") + [BUILD_ASSERT_OR_ZERO( 9 == RCVD_REMOVE_ACK_REVOCATION) + + BUILD_ASSERT_OR_ZERO(19 == SENT_REMOVE_ACK_REVOCATION)], + NULL}, + {SQL("ALTER TABLE channel_htlcs ADD fees_msat BIGINT DEFAULT 0"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD lease_fee BIGINT DEFAULT 0"), NULL}, + /* Default is too big; we set to max after loading */ + {SQL("ALTER TABLE channels ADD htlc_maximum_msat BIGINT DEFAULT 2100000000000000"), NULL}, + {SQL("ALTER TABLE channels ADD htlc_minimum_msat BIGINT DEFAULT 0"), NULL}, + {SQL("ALTER TABLE forwarded_payments ADD forward_style INTEGER DEFAULT NULL"), NULL}, + /* "description" is used for label, so we use "paydescription" here */ + {SQL("ALTER TABLE payments ADD paydescription TEXT;"), NULL}, + /* Alias we sent to the remote side, for zeroconf and + * option_scid_alias, can be a list of short_channel_ids if + * required, but keeping it a single SCID for now. */ + {SQL("ALTER TABLE channels ADD alias_local BIGINT DEFAULT NULL"), NULL}, + /* Alias we received from the peer, and which we should be using + * in routehints in invoices. The peer will remember all the + * aliases, but we only ever need one. */ + {SQL("ALTER TABLE channels ADD alias_remote BIGINT DEFAULT NULL"), NULL}, + /* Cheeky immediate completion as best effort approximation of real completion time */ + {SQL("ALTER TABLE payments ADD completed_at INTEGER DEFAULT NULL;"), NULL}, + {SQL("UPDATE payments SET completed_at = timestamp WHERE status != 0;"), NULL}, + {SQL("CREATE INDEX payments_idx ON payments (payment_hash)"), NULL}, + /* forwards table outlives the channels, so we move there from old forwarded_payments table; + * but here the ids are the HTLC numbers, not the internal db ids. */ + {SQL("CREATE TABLE forwards (" + "in_channel_scid BIGINT" + ", in_htlc_id BIGINT" + ", out_channel_scid BIGINT" + ", out_htlc_id BIGINT" + ", in_msatoshi BIGINT" + ", out_msatoshi BIGINT" + ", state INTEGER" + ", received_time BIGINT" + ", resolved_time BIGINT" + ", failcode INTEGER" + ", forward_style INTEGER" + ", PRIMARY KEY(in_channel_scid, in_htlc_id))"), NULL}, + {SQL("INSERT INTO forwards SELECT" + " in_channel_scid" + ", COALESCE(" + " (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.in_htlc_id)," + " -_ROWID_" + " )" + ", out_channel_scid" + ", (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.out_htlc_id)" + ", in_msatoshi" + ", out_msatoshi" + ", state" + ", received_time" + ", resolved_time" + ", failcode" + ", forward_style" + " FROM forwarded_payments"), NULL}, + {SQL("DROP INDEX forwarded_payments_state;"), NULL}, + {SQL("DROP INDEX forwarded_payments_out_htlc_id;"), NULL}, + {SQL("DROP TABLE forwarded_payments;"), NULL}, + /* Adds scid column, then moves short_channel_id across to it */ + {SQL("ALTER TABLE channels ADD scid BIGINT;"), migrate_channels_scids_as_integers}, + {SQL("ALTER TABLE payments ADD failscid BIGINT;"), migrate_payments_scids_as_integers}, + {SQL("ALTER TABLE outputs ADD is_in_coinbase INTEGER DEFAULT 0;"), NULL}, + {SQL("CREATE TABLE invoicerequests (" + " invreq_id BLOB" + ", bolt12 TEXT" + ", label TEXT" + ", status INTEGER" + ", PRIMARY KEY (invreq_id)" + ");"), NULL}, + /* A reference into our own invoicerequests table, if it was made from one */ + {SQL("ALTER TABLE payments ADD COLUMN local_invreq_id BLOB DEFAULT NULL REFERENCES invoicerequests(invreq_id);"), NULL}, + /* FIXME: Remove payments local_offer_id column! */ + {SQL("ALTER TABLE channel_funding_inflights ADD COLUMN lease_satoshi BIGINT;"), NULL}, + {SQL("ALTER TABLE channels ADD require_confirm_inputs_remote INTEGER DEFAULT 0;"), NULL}, + {SQL("ALTER TABLE channels ADD require_confirm_inputs_local INTEGER DEFAULT 0;"), NULL}, + {NULL, fillin_missing_lease_satoshi}, + {NULL, migrate_invalid_last_tx_psbts}, + {SQL("ALTER TABLE channels ADD channel_type BLOB DEFAULT NULL;"), NULL}, + {NULL, migrate_fill_in_channel_type}, + {SQL("ALTER TABLE peers ADD feature_bits BLOB DEFAULT NULL;"), NULL}, + {NULL, migrate_normalize_invstr}, + {SQL("CREATE TABLE runes (id BIGSERIAL, rune TEXT, PRIMARY KEY (id));"), NULL}, + {SQL("CREATE TABLE runes_blacklist (start_index BIGINT, end_index BIGINT);"), NULL}, + {SQL("ALTER TABLE channels ADD ignore_fee_limits INTEGER DEFAULT 0;"), NULL}, + {NULL, migrate_initialize_invoice_wait_indexes}, + {SQL("ALTER TABLE invoices ADD updated_index BIGINT DEFAULT 0"), NULL}, + {SQL("CREATE INDEX invoice_update_idx ON invoices (updated_index)"), NULL}, + {NULL, migrate_datastore_commando_runes}, + {NULL, migrate_invoice_created_index_var}, + /* Splicing requires us to store HTLC sigs for inflight splices and allows us to discard old sigs after splice confirmation. */ + {SQL("ALTER TABLE htlc_sigs ADD inflight_tx_id BLOB"), NULL}, + {SQL("ALTER TABLE htlc_sigs ADD inflight_tx_outnum INTEGER"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD splice_amnt BIGINT DEFAULT 0"), NULL}, + {SQL("ALTER TABLE channel_funding_inflights ADD i_am_initiator INTEGER DEFAULT 0"), NULL}, + {NULL, migrate_runes_idfix}, + {SQL("ALTER TABLE runes ADD last_used_nsec BIGINT DEFAULT NULL"), NULL}, + {SQL("DELETE FROM vars WHERE name = 'runes_uniqueid'"), NULL}, + {SQL("CREATE TABLE invoice_fallbacks (" + " scriptpubkey BLOB," + " invoice_id BIGINT REFERENCES invoices(id) ON DELETE CASCADE," + " PRIMARY KEY (scriptpubkey)" + ");"), + NULL}, + {SQL("ALTER TABLE invoices ADD paid_txid BLOB DEFAULT NULL"), NULL}, + {SQL("ALTER TABLE invoices ADD paid_outnum INTEGER DEFAULT NULL"), NULL}, + {SQL("CREATE TABLE local_anchors (" + " channel_id BIGSERIAL REFERENCES channels(id)," + " commitment_index BIGINT," + " commitment_txid BLOB," + " commitment_anchor_outnum INTEGER," + " commitment_fee BIGINT," + " commitment_weight INTEGER)"), NULL}, + {SQL("CREATE INDEX local_anchors_idx ON local_anchors (channel_id)"), NULL}, + {SQL("ALTER TABLE payments ADD updated_index BIGINT DEFAULT 0"), NULL}, + {SQL("CREATE INDEX payments_update_idx ON payments (updated_index)"), NULL}, + {NULL, migrate_initialize_payment_wait_indexes}, + {NULL, migrate_forwards_add_rowid}, + {SQL("ALTER TABLE forwards ADD updated_index BIGINT DEFAULT 0"), NULL}, + {SQL("CREATE INDEX forwards_updated_idx ON forwards (updated_index)"), NULL}, + {NULL, migrate_initialize_forwards_wait_indexes}, + {SQL("ALTER TABLE channel_funding_inflights ADD force_sign_first INTEGER DEFAULT 0"), NULL}, + {SQL("ALTER TABLE channels ADD remote_feerate_base INTEGER DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channels ADD remote_feerate_ppm INTEGER DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channels ADD remote_cltv_expiry_delta INTEGER DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channels ADD remote_htlc_maximum_msat BIGINT DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channels ADD remote_htlc_minimum_msat BIGINT DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channels ADD last_stable_connection BIGINT DEFAULT 0;"), NULL}, + {NULL, NULL}, /* old migrate_initialize_alias_local */ + {SQL("CREATE TABLE addresses (" + " keyidx BIGINT," + " addrtype INTEGER)"), NULL}, + {NULL, insert_addrtype_to_addresses}, + {SQL("ALTER TABLE channel_funding_inflights ADD remote_funding BLOB DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE peers ADD last_known_address BLOB DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channels ADD close_attempt_height INTEGER DEFAULT 0;"), NULL}, + {NULL, migrate_convert_old_channel_keyidx}, + {SQL("INSERT INTO vars(name, intval)" + " VALUES('needs_p2wpkh_close_rescan', 1)"), NULL}, + {SQL("ALTER TABLE channel_htlcs ADD updated_index BIGINT DEFAULT 0"), NULL}, + {SQL("CREATE INDEX channel_htlcs_updated_idx ON channel_htlcs (updated_index)"), NULL}, + {NULL, NULL}, /* Old, incorrect channel_htlcs_wait_indexes migration */ + {SQL("ALTER TABLE channel_funding_inflights ADD locked_scid BIGINT DEFAULT 0;"), NULL}, + {NULL, migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards}, + {SQL("ALTER TABLE channel_funding_inflights ADD i_sent_sigs INTEGER DEFAULT 0"), NULL}, + {SQL("ALTER TABLE channels ADD old_scids BLOB DEFAULT NULL;"), NULL}, + {NULL, migrate_initialize_alias_local}, + /* Avoids duplication in chain_moves and coin_moves tables */ + {SQL("CREATE TABLE move_accounts (" + " id BIGSERIAL," + " name TEXT," + " PRIMARY KEY (id)," + " UNIQUE (name)" + ")"), NULL}, + {SQL("CREATE TABLE chain_moves (" + " id BIGSERIAL," + /* One of these is null */ + " account_channel_id BIGINT references channels(id)," + " account_nonchannel_id BIGINT references move_accounts(id)," + " tag_bitmap BIGINT NOT NULL," + " credit_or_debit BIGINT NOT NULL," + " timestamp BIGINT NOT NULL," + " utxo BLOB NOT NULL," + " spending_txid BLOB," + /* This does NOT reference peers(node_id), since we can have + * MVT_CHANNEL_PROPOSED events on zeroconf channels where we end up + * forgetting the channel, thus the peer */ + " peer_id BLOB," + " payment_hash BLOB," + " block_height INTEGER NOT NULL," + " output_sat BIGINT NOT NULL," + /* One of these is null */ + " originating_channel_id BIGINT references channels(id)," + " originating_nonchannel_id BIGINT references move_accounts(id)," + " output_count INTEGER," + " PRIMARY KEY (id)" + ")"), NULL}, + {SQL("CREATE TABLE channel_moves (" + " id BIGSERIAL," + /* One of these is null */ + " account_channel_id BIGINT references channels(id)," + " account_nonchannel_id BIGINT references move_accounts(id)," + " tag_bitmap BIGINT NOT NULL," + " credit_or_debit BIGINT NOT NULL," + " timestamp BIGINT NOT NULL," + " payment_hash BLOB," + " payment_part_id BIGINT," + " payment_group_id BIGINT," + " fees BIGINT NOT NULL," + " PRIMARY KEY (id)" + ")"), NULL}, + /* We do a lookup before each append, to avoid duplicates */ + {SQL("CREATE INDEX chain_moves_utxo_idx ON chain_moves (utxo)"), NULL}, + {NULL, migrate_from_account_db, NULL, revert_too_early}, + /* ^v25.09 */ + + /* We accidentally allowed duplicate entries */ + {NULL, migrate_remove_chain_moves_duplicates, + /* Removing duplicates is idempotent, so no revert needed */ + NULL, NULL}, + {SQL("CREATE TABLE network_events (" + " id BIGSERIAL," + " peer_id BLOB NOT NULL," + " type INTEGER NOT NULL," + " timestamp BIGINT," + " reason TEXT," + " duration_nsec BIGINT," + " connect_attempted INTEGER NOT NULL," + " PRIMARY KEY (id)" + ")"), NULL, + /* Simply drop table. */ + SQL("DROP TABLE network_events")}, + {NULL, migrate_fail_pending_payments_without_htlcs, + /* Failing pending payments is idempotent, so no revert needed */ + NULL, NULL}, + {SQL("ALTER TABLE channels ADD withheld INTEGER DEFAULT 0;"), NULL, + /* Need to make sure that withheld isn't used. */ + NULL, revert_withheld_column}, + /* ^v25.12 */ + +}; + +const struct db_migration *get_db_migrations(size_t *num) +{ + *num = ARRAY_SIZE(dbmigrations); + return dbmigrations; +} diff --git a/wallet/migrations.h b/wallet/migrations.h new file mode 100644 index 000000000000..39d0b9b0c7bb --- /dev/null +++ b/wallet/migrations.h @@ -0,0 +1,65 @@ +#ifndef LIGHTNING_WALLET_MIGRATIONS_H +#define LIGHTNING_WALLET_MIGRATIONS_H + +#include "config.h" + +struct lightningd; + +struct db_migration { + const char *sql; + void (*func)(struct lightningd *ld, struct db *db); + const char *revertsql; + /* If non-NULL, returns string explaining why downgrade is impossible */ + const char *(*revertfn)(const tal_t *ctx, struct db *db); +}; + +const struct db_migration *get_db_migrations(size_t *num); + +/* All the functions provided by migrations.c */ +void migrate_pr2342_feerate_per_channel(struct lightningd *ld, struct db *db); +void migrate_our_funding(struct lightningd *ld, struct db *db); +void migrate_last_tx_to_psbt(struct lightningd *ld, struct db *db); +void migrate_inflight_last_tx_to_psbt(struct lightningd *ld, struct db *db); +void fillin_missing_scriptpubkeys(struct lightningd *ld, struct db *db); +void fillin_missing_channel_id(struct lightningd *ld, struct db *db); +void fillin_missing_local_basepoints(struct lightningd *ld, + struct db *db); +void fillin_missing_channel_blockheights(struct lightningd *ld, + struct db *db); +void migrate_channels_scids_as_integers(struct lightningd *ld, + struct db *db); +void migrate_payments_scids_as_integers(struct lightningd *ld, + struct db *db); +void fillin_missing_lease_satoshi(struct lightningd *ld, + struct db *db); +void migrate_invalid_last_tx_psbts(struct lightningd *ld, + struct db *db); +void migrate_fill_in_channel_type(struct lightningd *ld, + struct db *db); +void migrate_normalize_invstr(struct lightningd *ld, + struct db *db); +void migrate_initialize_invoice_wait_indexes(struct lightningd *ld, + struct db *db); +void migrate_invoice_created_index_var(struct lightningd *ld, + struct db *db); +void migrate_initialize_payment_wait_indexes(struct lightningd *ld, + struct db *db); +void migrate_forwards_add_rowid(struct lightningd *ld, + struct db *db); +void migrate_initialize_forwards_wait_indexes(struct lightningd *ld, + struct db *db); +void migrate_initialize_alias_local(struct lightningd *ld, + struct db *db); +void insert_addrtype_to_addresses(struct lightningd *ld, + struct db *db); +void migrate_convert_old_channel_keyidx(struct lightningd *ld, + struct db *db); +void migrate_initialize_channel_htlcs_wait_indexes_and_fixup_forwards(struct lightningd *ld, + struct db *db); +void migrate_fail_pending_payments_without_htlcs(struct lightningd *ld, + struct db *db); +void migrate_remove_chain_moves_duplicates(struct lightningd *ld, struct db *db); +void migrate_from_account_db(struct lightningd *ld, struct db *db); +void migrate_datastore_commando_runes(struct lightningd *ld, struct db *db); +void migrate_runes_idfix(struct lightningd *ld, struct db *db); +#endif /* LIGHTNING_WALLET_MIGRATIONS_H */ diff --git a/wallet/test/run-chain_moves_duplicate-detect.c b/wallet/test/run-chain_moves_duplicate-detect.c index 63a68ad3e815..428768d27e38 100644 --- a/wallet/test/run-chain_moves_duplicate-detect.c +++ b/wallet/test/run-chain_moves_duplicate-detect.c @@ -21,7 +21,9 @@ static void db_log_(struct logger *log UNUSED, enum log_level level UNUSED, cons #include "db/db_sqlite3.c" #include "db/exec.c" #include "db/utils.c" +#include "wallet/datastore.c" #include "wallet/db.c" +#include "wallet/migrations.c" #include "common/coin_mvt.c" /* AUTOGENERATED MOCKS START */ diff --git a/wallet/test/run-db.c b/wallet/test/run-db.c index 6ab8e6e59ae7..c1b4cb1bdaf5 100644 --- a/wallet/test/run-db.c +++ b/wallet/test/run-db.c @@ -10,8 +10,10 @@ static void db_log_(struct logger *log UNUSED, enum log_level level UNUSED, cons #include "db/db_sqlite3.c" #include "db/exec.c" #include "db/utils.c" +#include "wallet/datastore.c" #include "wallet/db.c" #include "wallet/wallet.c" +#include "wallet/migrations.c" #include "test_utils.h" diff --git a/wallet/test/run-migrate_remove_chain_moves_duplicates.c b/wallet/test/run-migrate_remove_chain_moves_duplicates.c index 480e2837eac8..2bb792e2ade5 100644 --- a/wallet/test/run-migrate_remove_chain_moves_duplicates.c +++ b/wallet/test/run-migrate_remove_chain_moves_duplicates.c @@ -78,6 +78,34 @@ void connect_htlc_in(struct htlc_in_map *map UNNEEDED, struct htlc_in *hin UNNEE /* Generated stub for connect_htlc_out */ void connect_htlc_out(struct htlc_out_map *map UNNEEDED, struct htlc_out *hout UNNEEDED) { fprintf(stderr, "connect_htlc_out called!\n"); abort(); } +/* Generated stub for db_bind_datastore_key */ +void db_bind_datastore_key(struct db_stmt *stmt UNNEEDED, const char **key UNNEEDED) +{ fprintf(stderr, "db_bind_datastore_key called!\n"); abort(); } +/* Generated stub for db_datastore_first */ +struct db_stmt *db_datastore_first(const tal_t *ctx UNNEEDED, + struct db *db UNNEEDED, + const char **startkey UNNEEDED, + const char ***key UNNEEDED, + const u8 **data UNNEEDED, + u64 *generation UNNEEDED) +{ fprintf(stderr, "db_datastore_first called!\n"); abort(); } +/* Generated stub for db_datastore_get */ +u8 *db_datastore_get(const tal_t *ctx UNNEEDED, + struct db *db UNNEEDED, + const char **key UNNEEDED, + u64 *generation UNNEEDED) +{ fprintf(stderr, "db_datastore_get called!\n"); abort(); } +/* Generated stub for db_datastore_next */ +struct db_stmt *db_datastore_next(const tal_t *ctx UNNEEDED, + struct db_stmt *stmt UNNEEDED, + const char **startkey UNNEEDED, + const char ***key UNNEEDED, + const u8 **data UNNEEDED, + u64 *generation UNNEEDED) +{ fprintf(stderr, "db_datastore_next called!\n"); abort(); } +/* Generated stub for db_datastore_update */ +void db_datastore_update(struct db *db UNNEEDED, const char **key UNNEEDED, const u8 *data UNNEEDED) +{ fprintf(stderr, "db_datastore_update called!\n"); abort(); } /* Generated stub for fatal */ void fatal(const char *fmt UNNEEDED, ...) { fprintf(stderr, "fatal called!\n"); abort(); } @@ -124,6 +152,9 @@ void get_channel_basepoints(struct lightningd *ld UNNEEDED, struct basepoints *local_basepoints UNNEEDED, struct pubkey *local_funding_pubkey UNNEEDED) { fprintf(stderr, "get_channel_basepoints called!\n"); abort(); } +/* Generated stub for get_db_migrations */ +const struct db_migration *get_db_migrations(size_t *num UNNEEDED) +{ fprintf(stderr, "get_db_migrations called!\n"); abort(); } /* Generated stub for hash_cid */ size_t hash_cid(const struct channel_id *cid UNNEEDED) { fprintf(stderr, "hash_cid called!\n"); abort(); } @@ -172,9 +203,6 @@ struct invoices *invoices_new(const tal_t *ctx UNNEEDED, void logv(struct logger *logger UNNEEDED, enum log_level level UNNEEDED, const struct node_id *node_id UNNEEDED, bool call_notifier UNNEEDED, const char *fmt UNNEEDED, va_list ap UNNEEDED) { fprintf(stderr, "logv called!\n"); abort(); } -/* Generated stub for migrate_from_account_db */ -void migrate_from_account_db(struct lightningd *ld UNNEEDED, struct db *db UNNEEDED) -{ fprintf(stderr, "migrate_from_account_db called!\n"); abort(); } /* Generated stub for new_channel */ struct channel *new_channel(struct peer *peer UNNEEDED, u64 dbid UNNEEDED, /* NULL or stolen */ diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 74e02307fe54..69b141472b76 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -35,7 +35,9 @@ static void test_error(struct lightningd *ld, bool fatal, const char *fmt, va_li #include "db/db_sqlite3.c" #include "db/exec.c" #include "db/utils.c" +#include "wallet/datastore.c" #include "wallet/db.c" +#include "wallet/migrations.c" #include #include diff --git a/wallet/wallet.c b/wallet/wallet.c index a9791584729d..c6a272a211a7 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -24,7 +24,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -6332,63 +6334,9 @@ void wallet_invoice_request_mark_used(struct db *db, const struct sha256 *invreq } } -/* We join key parts with nuls for now. */ -static void db_bind_datastore_key(struct db_stmt *stmt, - const char **key) -{ - u8 *joined; - size_t len; - - if (tal_count(key) == 1) { - db_bind_blob(stmt, (u8 *)key[0], strlen(key[0])); - return; - } - - len = strlen(key[0]); - joined = (u8 *)tal_strdup(tmpctx, key[0]); - for (size_t i = 1; i < tal_count(key); i++) { - tal_resize(&joined, len + 1 + strlen(key[i])); - joined[len] = '\0'; - memcpy(joined + len + 1, key[i], strlen(key[i])); - len += 1 + strlen(key[i]); - } - db_bind_blob(stmt, joined, len); -} - -static const char **db_col_datastore_key(const tal_t *ctx, - struct db_stmt *stmt, - const char *colname) -{ - char **key; - const u8 *joined = db_col_blob(stmt, colname); - size_t len = db_col_bytes(stmt, colname); - - key = tal_arr(ctx, char *, 0); - do { - size_t partlen; - for (partlen = 0; partlen < len; partlen++) { - if (joined[partlen] == '\0') { - partlen++; - break; - } - } - tal_arr_expand(&key, tal_strndup(key, (char *)joined, partlen)); - len -= partlen; - joined += partlen; - } while (len != 0); - - return cast_const2(const char **, key); -} - void wallet_datastore_update(struct wallet *w, const char **key, const u8 *data) { - struct db_stmt *stmt; - - stmt = db_prepare_v2(w->db, - SQL("UPDATE datastore SET data=?, generation=generation+1 WHERE key=?;")); - db_bind_talarr(stmt, data); - db_bind_datastore_key(stmt, key); - db_exec_prepared_v2(take(stmt)); + db_datastore_update(w->db, key, data); } static void db_datastore_create(struct db *db, const char **key, const u8 *data) @@ -6457,56 +6405,6 @@ void wallet_datastore_remove(struct wallet *w, const char **key) db_datastore_remove(w->db, key); } -/* Does k1 match k2 as far as k2 goes? */ -bool datastore_key_startswith(const char **k1, const char **k2) -{ - size_t k1len = tal_count(k1), k2len = tal_count(k2); - - if (k2len > k1len) - return false; - - for (size_t i = 0; i < k2len; i++) { - if (!streq(k1[i], k2[i])) - return false; - } - return true; -} - -bool datastore_key_eq(const char **k1, const char **k2) -{ - return tal_count(k1) == tal_count(k2) - && datastore_key_startswith(k1, k2); -} - -static u8 *db_datastore_get(const tal_t *ctx, - struct db *db, - const char **key, - u64 *generation) -{ - struct db_stmt *stmt; - u8 *ret; - - stmt = db_prepare_v2(db, - SQL("SELECT data, generation" - " FROM datastore" - " WHERE key = ?")); - db_bind_datastore_key(stmt, key); - db_query_prepared(stmt); - - if (!db_step(stmt)) { - tal_free(stmt); - return NULL; - } - - ret = db_col_arr(ctx, stmt, "data", u8); - if (generation) - *generation = db_col_u64(stmt, "generation"); - else - db_col_ignore(stmt, "generation"); - tal_free(stmt); - return ret; -} - u8 *wallet_datastore_get(const tal_t *ctx, struct wallet *w, const char **key, @@ -6515,65 +6413,6 @@ u8 *wallet_datastore_get(const tal_t *ctx, return db_datastore_get(ctx, w->db, key, generation); } -static struct db_stmt *db_datastore_next(const tal_t *ctx, - struct db_stmt *stmt, - const char **startkey, - const char ***key, - const u8 **data, - u64 *generation) -{ - if (!db_step(stmt)) - return tal_free(stmt); - - *key = db_col_datastore_key(ctx, stmt, "key"); - - /* We select from startkey onwards, so once we're past it, stop */ - if (startkey && !datastore_key_startswith(*key, startkey)) { - db_col_ignore(stmt, "data"); - db_col_ignore(stmt, "generation"); - return tal_free(stmt); - } - - if (data) - *data = db_col_arr(ctx, stmt, "data", u8); - else - db_col_ignore(stmt, "data"); - - if (generation) - *generation = db_col_u64(stmt, "generation"); - else - db_col_ignore(stmt, "generation"); - - return stmt; -} - -static struct db_stmt *db_datastore_first(const tal_t *ctx, - struct db *db, - const char **startkey, - const char ***key, - const u8 **data, - u64 *generation) -{ - struct db_stmt *stmt; - - if (startkey) { - stmt = db_prepare_v2(db, - SQL("SELECT key, data, generation" - " FROM datastore" - " WHERE key >= ?" - " ORDER BY key;")); - db_bind_datastore_key(stmt, startkey); - } else { - stmt = db_prepare_v2(db, - SQL("SELECT key, data, generation" - " FROM datastore" - " ORDER BY key;")); - } - db_query_prepared(stmt); - - return db_datastore_next(ctx, stmt, startkey, key, data, generation); -} - struct db_stmt *wallet_datastore_first(const tal_t *ctx, struct wallet *w, const char **startkey, diff --git a/wallet/wallet.h b/wallet/wallet.h index 5b33dfce3ca7..1f1b81da6aed 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -2006,6 +2006,5 @@ void wallet_datastore_save_payment_description(struct db *db, const struct sha256 *payment_hash, const char *desc); void migrate_setup_coinmoves(struct lightningd *ld, struct db *db); -void migrate_remove_chain_moves_duplicates(struct lightningd *ld, struct db *db); #endif /* LIGHTNING_WALLET_WALLET_H */ diff --git a/wire/fromwire.c b/wire/fromwire.c index 8e81c81571f4..9fded589e2d0 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -41,10 +41,9 @@ const u8 *fromwire(const u8 **cursor, size_t *max, void *copy, size_t n) return memcheck(p, n); } -int fromwire_peektype(const u8 *cursor) +int fromwire_peektypen(const u8 *cursor, size_t max) { be16 be_type; - size_t max = tal_count(cursor); fromwire(&cursor, &max, &be_type, sizeof(be_type)); if (!cursor) @@ -52,6 +51,11 @@ int fromwire_peektype(const u8 *cursor) return be16_to_cpu(be_type); } +int fromwire_peektype(const u8 *cursor) +{ + return fromwire_peektypen(cursor, tal_count(cursor)); +} + u8 fromwire_u8(const u8 **cursor, size_t *max) { u8 ret; diff --git a/wire/wire.h b/wire/wire.h index 8e8f12ac6164..da3858cc4979 100644 --- a/wire/wire.h +++ b/wire/wire.h @@ -15,6 +15,8 @@ typedef char utf8; /* Read the type; returns -1 if not long enough. cursor is a tal ptr. */ int fromwire_peektype(const u8 *cursor); +/* Same, but doesn't need to be a tal ptr */ +int fromwire_peektypen(const u8 *cursor, size_t len); void *fromwire_fail(const u8 **cursor, size_t *max); void towire(u8 **pptr, const void *data, size_t len);