From e5d0a1041e8eaa39253a5a729dc3329fb1d76982 Mon Sep 17 00:00:00 2001 From: masih Date: Thu, 19 Nov 2026 15:03:22 +0000 Subject: [PATCH 1/4] Re-sign a refused transaction at the same nonce with current fees When the node refuses a claim's stored transaction, for example because its fee cap is below the base fee, the sender re-signs the claim at the same nonce with the configured fees, persists it, and broadcasts it. Every hash signed for a claim is kept in a new claim_txs table, and a receipt for any of them settles the claim. --- AGENTS.md | 5 +- RUNBOOK.md | 5 +- internal/sender/sender.go | 106 +++++++++++++++++++++++++-------- internal/sender/sender_test.go | 66 +++++++++++++++++++- internal/store/store.go | 100 ++++++++++++++++++++++++++++--- internal/store/store_test.go | 65 ++++++++++++++++++++ 6 files changed, 306 insertions(+), 41 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 801026e..545ebed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,8 +11,9 @@ Testnet faucet for Sei EVM chains. Run `make verify` before pushing; it mirrors - **Persist before broadcast.** A signed transaction is written to the store before it is sent, so a restart can find and reconcile it. - **A claim keeps its nonce.** Once signed, a claim's transaction is only ever - rebroadcast at the same nonce until its receipt settles it. Re-signing it at a - new nonce while the first transaction can still land pays it twice. + rebroadcast, or re-signed with new fees, at the same nonce until a receipt for + any of its transactions settles it. Re-signing it at a new nonce while the + first transaction can still land pays it twice. - **The receipt decides.** Giga's EVM-only `CheckTx` is stateless, so a broadcast that succeeds can still be rejected in the block. A claim is confirmed only by a successful receipt. diff --git a/RUNBOOK.md b/RUNBOOK.md index 88214b2..ab5aff1 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -49,8 +49,9 @@ nonces: - **Pending is above latest and not moving:** the node holds the faucet's transactions but is not including them. Check that the chain is producing - blocks. The sender has no same-nonce fee bump yet, so a transaction priced - below what the chain now accepts stays stuck until the chain accepts it. + blocks. If the node refuses them as priced too low, raise + `config.gas-fee-cap-wei`: on its next resync the sender re-signs each refused + transfer at the same nonce with the new fees. - **Pending equals latest:** the node keeps dropping the transactions, and the rebroadcast errors in the logs say why. See WellspringSendErrors. - **The log says "faucet nonce spent by a transaction it did not send":** diff --git a/internal/sender/sender.go b/internal/sender/sender.go index 24f0cea..0ee3d77 100644 --- a/internal/sender/sender.go +++ b/internal/sender/sender.go @@ -6,6 +6,8 @@ // was signed at until its receipt settles it: after any doubt the sender // rebroadcasts the same transactions rather than re-signing claims at new // nonces, which could pay a claim twice if its first transaction still lands. +// A transaction the node refuses is re-signed at the same nonce with the +// current fees; only one of them can land, and a receipt for any settles it. package sender import ( @@ -175,19 +177,7 @@ func (s *Sender) dispatch(ctx context.Context) error { return err } for _, c := range claims { - tx, err := types.SignNewTx(s.cfg.Key, s.signer, &types.DynamicFeeTx{ - ChainID: s.cfg.ChainID, - Nonce: s.nonce, - GasTipCap: s.cfg.GasTipCap, - GasFeeCap: s.cfg.GasFeeCap, - Gas: transferGas, - To: &c.Address, - Value: c.Amount, - }) - if err != nil { - return err - } - raw, err := tx.MarshalBinary() + tx, raw, err := s.sign(s.nonce, c.Address, c.Amount) if err != nil { return err } @@ -217,7 +207,7 @@ func (s *Sender) confirm(ctx context.Context) error { return err } for _, c := range claims { - receipt, err := s.rpc.TransactionReceipt(ctx, common.HexToHash(c.TxHash)) + receipt, err := s.receipt(ctx, c, false) if errors.Is(err, ethereum.NotFound) { if time.Since(c.SentAt.Time) > s.cfg.ConfirmTimeout && time.Since(s.resyncedAt) > s.cfg.ConfirmTimeout { s.needResync = true @@ -225,7 +215,6 @@ func (s *Sender) confirm(ctx context.Context) error { continue } if err != nil { - s.metrics.rpcErrors.Inc() return err } if err := s.settle(ctx, c, receipt); err != nil { @@ -273,11 +262,7 @@ func (s *Sender) resync(ctx context.Context) error { return err } } - var tx types.Transaction - if err := tx.UnmarshalBinary(c.RawTx); err != nil { - return err - } - if err := s.rpc.SendTransaction(ctx, &tx); err != nil && !alreadyHave(err) { + if err := s.rebroadcast(ctx, c); err != nil { s.metrics.sendErrors.Inc() stuck = true slog.Warn("rebroadcast refused", "claim", c.ID, "nonce", nonce, "err", err) @@ -300,9 +285,68 @@ func alreadyHave(err error) bool { return strings.Contains(msg, "already known") || strings.Contains(msg, "already imported") || strings.Contains(msg, "nonce too low") } +// rebroadcast resends a claim's stored transaction. If the node refuses it, +// the claim is re-signed at the same nonce with the current fees, persisted, +// and the new transaction sent instead. +func (s *Sender) rebroadcast(ctx context.Context, c *store.Claim) error { + var old types.Transaction + if err := old.UnmarshalBinary(c.RawTx); err != nil { + return err + } + refused := s.rpc.SendTransaction(ctx, &old) + if refused == nil || alreadyHave(refused) { + return nil + } + tx, raw, err := s.sign(old.Nonce(), c.Address, c.Amount) + if err != nil { + return err + } + if tx.Hash() == old.Hash() { + return refused + } + if err := s.store.Resign(ctx, c.ID, tx.Hash(), raw, time.Now()); err != nil { + return err + } + slog.Info("re-signed refused transaction", "claim", c.ID, "nonce", tx.Nonce(), "old", old.Hash(), "new", tx.Hash(), "refused", refused) + if err := s.rpc.SendTransaction(ctx, tx); err != nil && !alreadyHave(err) { + return err + } + return nil +} + +// receipt returns the receipt that settles a claim, or ethereum.NotFound. A +// successful receipt for any transaction signed for the claim settles it. A +// failed one settles it only for the latest transaction or once the nonce is +// spent: on Giga a failed receipt leaves the nonce unspent, so a later +// re-signed transaction can still land. +func (s *Sender) receipt(ctx context.Context, c *store.Claim, spent bool) (*types.Receipt, error) { + hashes, err := s.store.TxHashes(ctx, c.ID) + if err != nil { + return nil, err + } + var failed *types.Receipt + for _, hash := range hashes { + receipt, err := s.rpc.TransactionReceipt(ctx, hash) + switch { + case errors.Is(err, ethereum.NotFound): + case err != nil: + s.metrics.rpcErrors.Inc() + return nil, err + case receipt.Status == types.ReceiptStatusSuccessful: + return receipt, nil + case spent || hash == common.HexToHash(c.TxHash): + failed = receipt + } + } + if failed == nil { + return nil, ethereum.NotFound + } + return failed, nil +} + // reconcile settles a sent claim whose nonce the chain has already spent. func (s *Sender) reconcile(ctx context.Context, c *store.Claim) error { - receipt, err := s.rpc.TransactionReceipt(ctx, common.HexToHash(c.TxHash)) + receipt, err := s.receipt(ctx, c, true) switch { case errors.Is(err, ethereum.NotFound): if time.Since(c.SentAt.Time) <= s.cfg.ConfirmTimeout { @@ -317,18 +361,28 @@ func (s *Sender) reconcile(ctx context.Context, c *store.Claim) error { } } -// fill spends nonce with a zero-value transfer to the faucet. Filling the same -// nonce again signs the identical transaction. -func (s *Sender) fill(ctx context.Context, nonce uint64) error { +// sign signs a transfer at nonce with the current fees. +func (s *Sender) sign(nonce uint64, to common.Address, value *big.Int) (*types.Transaction, []byte, error) { tx, err := types.SignNewTx(s.cfg.Key, s.signer, &types.DynamicFeeTx{ ChainID: s.cfg.ChainID, Nonce: nonce, GasTipCap: s.cfg.GasTipCap, GasFeeCap: s.cfg.GasFeeCap, Gas: transferGas, - To: &s.from, - Value: new(big.Int), + To: &to, + Value: value, }) + if err != nil { + return nil, nil, err + } + raw, err := tx.MarshalBinary() + return tx, raw, err +} + +// fill spends nonce with a zero-value transfer to the faucet. Filling the same +// nonce again signs the identical transaction. +func (s *Sender) fill(ctx context.Context, nonce uint64) error { + tx, _, err := s.sign(nonce, s.from, new(big.Int)) if err != nil { return err } diff --git a/internal/sender/sender_test.go b/internal/sender/sender_test.go index 7f995ef..2860b25 100644 --- a/internal/sender/sender_test.go +++ b/internal/sender/sender_test.go @@ -22,8 +22,9 @@ import ( var amount = big.NewInt(1e18) // chain mimics Giga: it admits only the sender's exact next nonce, and a -// transfer to a rejected address lands with a failed receipt without spending -// its nonce, so every transfer queued behind it is dropped. +// transfer to a rejected address, or a failing transaction, lands with a failed +// receipt without spending its nonce, so every transfer queued behind it is +// dropped. A transaction whose fee cap is below minFee is refused. type chain struct { signer types.Signer @@ -35,6 +36,8 @@ type chain struct { refuse int hang bool rejected map[common.Address]bool + failing map[common.Hash]bool + minFee *big.Int } func newChain() *chain { @@ -43,6 +46,8 @@ func newChain() *chain { receipts: map[common.Hash]*types.Receipt{}, paid: map[common.Address]int{}, rejected: map[common.Address]bool{}, + failing: map[common.Hash]bool{}, + minFee: new(big.Int), } } @@ -67,6 +72,9 @@ func (c *chain) SendTransaction(_ context.Context, tx *types.Transaction) error c.refuse-- return errors.New("node unavailable") } + if tx.GasFeeCap().Cmp(c.minFee) < 0 { + return errors.New("max fee per gas less than block base fee") + } for _, p := range c.pending { if p.Hash() == tx.Hash() { return errors.New("already known") @@ -92,7 +100,7 @@ func (c *chain) mine() { c.mu.Lock() defer c.mu.Unlock() for _, tx := range c.pending { - if c.rejected[*tx.To()] { + if c.rejected[*tx.To()] || c.failing[tx.Hash()] { c.receipts[tx.Hash()] = &types.Receipt{Status: types.ReceiptStatusFailed} break } @@ -277,6 +285,58 @@ func TestRefusedRebroadcastKeepsResyncing(t *testing.T) { h.paidOnce(addrs...) } +// TestResignedClaimSettlesOnce restarts the faucet with a raised fee cap after +// the node refuses a claim's transaction, and checks the claim is re-signed at +// its nonce and paid once, whichever of its transactions lands. +func TestResignedClaimSettlesOnce(t *testing.T) { + for name, refuse := range map[string]func(h *harness, old common.Hash){ + "fee cap below base fee": func(h *harness, _ common.Hash) { + h.chain.drop() + h.chain.minFee = big.NewInt(5) + }, + "old transaction lands after the re-sign": func(h *harness, _ common.Hash) { + h.chain.refuse = 1 + }, + "old transaction fails in its block": func(h *harness, old common.Hash) { + h.chain.failing[old] = true + h.chain.mine() + h.chain.refuse = 1 + }, + } { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + h := newHarness(t) + addrs := h.claim(1) + h.sender().step(ctx) + sent, err := h.store.InFlight(ctx) + if err != nil || len(sent) != 1 { + t.Fatalf("in flight: %v %v", sent, err) + } + refuse(h, common.HexToHash(sent[0].TxHash)) + + h.cfg.GasFeeCap = big.NewInt(5) + s := h.sender() + s.step(ctx) + hashes, err := h.store.TxHashes(ctx, sent[0].ID) + if err != nil || len(hashes) != 2 || hashes[0] != common.HexToHash(sent[0].TxHash) { + t.Fatalf("hashes after re-sign: %v %v", hashes, err) + } + if resigned, err := h.store.Get(ctx, sent[0].ID); err != nil || resigned.Nonce != sent[0].Nonce || common.HexToHash(resigned.TxHash) != hashes[1] { + t.Fatalf("re-signed claim: %+v %v", resigned, err) + } + + h.drain(s) + h.paidOnce(addrs...) + if c, err := h.store.Get(ctx, sent[0].ID); err != nil || c.Status != store.StatusConfirmed { + t.Fatalf("claim: %+v %v", c, err) + } + if h.chain.nonce != 1 { + t.Fatalf("chain nonce %d, want 1", h.chain.nonce) + } + }) + } +} + func TestGaugesTrackUnsettledClaimsAndBalance(t *testing.T) { h := newHarness(t) ctx := context.Background() diff --git a/internal/store/store.go b/internal/store/store.go index 8712a49..a387d2c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -7,6 +7,7 @@ import ( "database/sql" "encoding/hex" "errors" + "fmt" "math/big" "net/url" "time" @@ -63,7 +64,9 @@ type Store struct { db *sql.DB } -const schema = ` +// migrations upgrade the schema in order. The database's user_version counts +// those already applied. Append to the list; never edit an applied migration. +var migrations = []string{` CREATE TABLE IF NOT EXISTS claims ( id TEXT PRIMARY KEY, address TEXT NOT NULL, @@ -81,7 +84,16 @@ CREATE TABLE IF NOT EXISTS claims ( CREATE INDEX IF NOT EXISTS claims_address ON claims(address, created_at); CREATE INDEX IF NOT EXISTS claims_ip ON claims(ip, created_at); CREATE INDEX IF NOT EXISTS claims_status ON claims(status, created_at); -` +`, ` +CREATE TABLE claim_txs ( + hash TEXT PRIMARY KEY, + claim_id TEXT NOT NULL REFERENCES claims(id), + signed_at INTEGER NOT NULL +); +CREATE INDEX claim_txs_claim ON claim_txs(claim_id); +INSERT INTO claim_txs (hash, claim_id, signed_at) + SELECT tx_hash, id, COALESCE(sent_at, created_at) FROM claims WHERE tx_hash != ''; +`} // Open opens or creates the store at path. func Open(ctx context.Context, path string) (*Store, error) { @@ -94,13 +106,41 @@ func Open(ctx context.Context, path string) (*Store, error) { } // One connection serialises every write, which makes the limit checks atomic. db.SetMaxOpenConns(1) - if _, err := db.ExecContext(ctx, schema); err != nil { + if err := migrate(ctx, db); err != nil { _ = db.Close() return nil, err } return &Store{db: db}, nil } +func migrate(ctx context.Context, db *sql.DB) error { + var version int + if err := db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil { + return err + } + for ; version < len(migrations); version++ { + if err := applyMigration(ctx, db, version); err != nil { + return fmt.Errorf("migration %d: %w", version+1, err) + } + } + return nil +} + +func applyMigration(ctx context.Context, db *sql.DB, version int) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, migrations[version]); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version+1)); err != nil { + return err + } + return tx.Commit() +} + // Close closes the store. func (s *Store) Close() error { return s.db.Close() } @@ -184,8 +224,48 @@ func (s *Store) OldestUnsettled(ctx context.Context) (time.Time, bool, error) { // MarkSent records a queued claim's signed transaction. Call it before // broadcasting, so a restart can reconcile the transaction. func (s *Store) MarkSent(ctx context.Context, id string, nonce uint64, hash common.Hash, raw []byte, now time.Time) error { - return s.update(ctx, StatusQueued, `status = ?, nonce = ?, tx_hash = ?, raw_tx = ?, sent_at = ?`, - id, StatusSent, int64(nonce), hash.Hex(), raw, now.UnixMilli()) //nolint:gosec // nonces stay far below 2^63 + return s.sign(ctx, StatusQueued, id, hash, now, `status = ?, nonce = ?, tx_hash = ?, raw_tx = ?, sent_at = ?`, + StatusSent, int64(nonce), hash.Hex(), raw, now.UnixMilli()) //nolint:gosec // nonces stay far below 2^63 +} + +// Resign replaces a sent claim's transaction with one re-signed at the same +// nonce, keeping the earlier hashes in TxHashes. Call it before broadcasting. +func (s *Store) Resign(ctx context.Context, id string, hash common.Hash, raw []byte, now time.Time) error { + return s.sign(ctx, StatusSent, id, hash, now, `tx_hash = ?, raw_tx = ?`, hash.Hex(), raw) +} + +// TxHashes returns every transaction hash signed for a claim, oldest first. +func (s *Store) TxHashes(ctx context.Context, id string) ([]common.Hash, error) { + rows, err := s.db.QueryContext(ctx, `SELECT hash FROM claim_txs WHERE claim_id = ? ORDER BY rowid`, id) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var out []common.Hash + for rows.Next() { + var hash string + if err := rows.Scan(&hash); err != nil { + return nil, err + } + out = append(out, common.HexToHash(hash)) + } + return out, rows.Err() +} + +func (s *Store) sign(ctx context.Context, from Status, id string, hash common.Hash, now time.Time, set string, args ...any) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if err := update(ctx, tx, from, set, id, args...); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO claim_txs (hash, claim_id, signed_at) VALUES (?, ?, ?)`, + hash.Hex(), id, now.UnixMilli()); err != nil { + return err + } + return tx.Commit() } // Finish records a sent claim's final status. @@ -193,14 +273,18 @@ func (s *Store) Finish(ctx context.Context, id string, status Status, errMsg str if status != StatusConfirmed && status != StatusFailed { return ErrWrongStatus } - return s.update(ctx, StatusSent, `status = ?, error = ?, done_at = ?`, id, status, errMsg, now.UnixMilli()) + return update(ctx, s.db, StatusSent, `status = ?, error = ?, done_at = ?`, id, status, errMsg, now.UnixMilli()) } // ErrWrongStatus is returned for a transition the claim's status does not allow. var ErrWrongStatus = errors.New("claim is not in the expected status") -func (s *Store) update(ctx context.Context, from Status, set, id string, args ...any) error { - res, err := s.db.ExecContext(ctx, `UPDATE claims SET `+set+` WHERE id = ? AND status = ?`, append(args, id, from)...) //nolint:gosec // set is a constant from this file +type execer interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) +} + +func update(ctx context.Context, db execer, from Status, set, id string, args ...any) error { + res, err := db.ExecContext(ctx, `UPDATE claims SET `+set+` WHERE id = ? AND status = ?`, append(args, id, from)...) //nolint:gosec // set is a constant from this file if err != nil { return err } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 204b97b..41411af 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "database/sql" "errors" "math/big" "path/filepath" @@ -134,6 +135,70 @@ func TestLifecyclePersists(t *testing.T) { } } +func TestResignKeepsEveryHash(t *testing.T) { + ctx := context.Background() + s, _ := open(t) + c := claim(t, s, alice, "1.1.1.1", limits, t0) + if err := s.Resign(ctx, c.ID, common.Hash{1}, []byte{1}, t0); !errors.Is(err, ErrWrongStatus) { + t.Fatalf("Resign of a queued claim: %v", err) + } + if err := s.MarkSent(ctx, c.ID, 7, common.Hash{1}, []byte{1}, t0); err != nil { + t.Fatal(err) + } + if err := s.Resign(ctx, c.ID, common.Hash{2}, []byte{2}, t0); err != nil { + t.Fatal(err) + } + if got, err := s.Get(ctx, c.ID); err != nil || got.Nonce.Int64 != 7 || got.TxHash != (common.Hash{2}).Hex() || string(got.RawTx) != "\x02" { + t.Fatalf("re-signed claim: %+v %v", got, err) + } + if hashes, err := s.TxHashes(ctx, c.ID); err != nil || len(hashes) != 2 || hashes[0] != (common.Hash{1}) || hashes[1] != (common.Hash{2}) { + t.Fatalf("hashes: %v %v", hashes, err) + } + if err := s.Finish(ctx, c.ID, StatusConfirmed, "", t0); err != nil { + t.Fatal(err) + } + if err := s.Resign(ctx, c.ID, common.Hash{3}, []byte{3}, t0); !errors.Is(err, ErrWrongStatus) { + t.Fatalf("Resign of a confirmed claim: %v", err) + } +} + +func TestMigrationKeepsSentHashes(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "wellspring.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, migrations[0]); err != nil { + t.Fatal(err) + } + hash := common.HexToHash("0xabc") + if _, err := db.ExecContext(ctx, `INSERT INTO claims (id, address, ip, amount, status, nonce, tx_hash, created_at, sent_at) + VALUES ('sent', ?, '1.1.1.1', '1', ?, 0, ?, 1, 2), ('queued', ?, '2.2.2.2', '1', ?, NULL, '', 1, NULL)`, + alice.Hex(), StatusSent, hash.Hex(), bob.Hex(), StatusQueued); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + for range 2 { + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if hashes, err := s.TxHashes(ctx, "sent"); err != nil || len(hashes) != 1 || hashes[0] != hash { + t.Fatalf("sent claim hashes: %v %v", hashes, err) + } + if hashes, err := s.TxHashes(ctx, "queued"); err != nil || len(hashes) != 0 { + t.Fatalf("queued claim hashes: %v %v", hashes, err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + } +} + func TestConcurrentClaimsForOneAddress(t *testing.T) { s, _ := open(t) var wg sync.WaitGroup From 61feac0749ad5cba045a56a64bacbf4cbb7da498 Mon Sep 17 00:00:00 2001 From: masih Date: Thu, 19 Nov 2026 22:03:48 +0000 Subject: [PATCH 2/4] Settle a failed transfer only once its nonce is spent A failed receipt for the latest transaction no longer fails the claim while an earlier one can still land: resync spends the nonce and reconcile settles it. Re-signing back to an earlier hash is idempotent, Resign restarts the sent time, and receipts also check the claim's current hash. --- internal/sender/sender.go | 41 ++++++++++++++------ internal/sender/sender_test.go | 69 +++++++++++++++++++++++++--------- internal/store/store.go | 24 +++++++++--- internal/store/store_test.go | 12 +++++- 4 files changed, 110 insertions(+), 36 deletions(-) diff --git a/internal/sender/sender.go b/internal/sender/sender.go index 0ee3d77..a759992 100644 --- a/internal/sender/sender.go +++ b/internal/sender/sender.go @@ -16,6 +16,7 @@ import ( "errors" "log/slog" "math/big" + "slices" "strings" "sync" "time" @@ -217,6 +218,13 @@ func (s *Sender) confirm(ctx context.Context) error { if err != nil { return err } + if receipt.Status != types.ReceiptStatusSuccessful { + // On Giga a failed receipt leaves the nonce unspent, and an earlier + // transaction for the claim can still land at it. Resync spends the + // nonce and reconcile settles the claim. + s.needResync = true + continue + } if err := s.settle(ctx, c, receipt); err != nil { return err } @@ -224,20 +232,19 @@ func (s *Sender) confirm(ctx context.Context) error { return nil } -// settle finishes a claim from its receipt. On Giga a failed receipt leaves -// its nonce unspent, so every transfer signed after it needs a resync. +// settle finishes a claim from its receipt. A failed receipt settles a claim +// only once its nonce is spent. func (s *Sender) settle(ctx context.Context, c *store.Claim, receipt *types.Receipt) error { if receipt.Status == types.ReceiptStatusSuccessful { s.metrics.claimTime.Observe(time.Since(c.CreatedAt).Seconds()) return s.finish(ctx, c, store.StatusConfirmed, "") } - s.needResync = true return s.finish(ctx, c, store.StatusFailed, "transfer rejected by the chain") } // resync settles the sent claims below the committed nonce and rebroadcasts -// the rest at the nonces they were signed at. A nonce no sent claim holds, -// left by a claim whose transfer was rejected, is filled with a zero-value +// the rest at the nonces they were signed at. A nonce no sent claim holds, or +// one whose claim's latest transfer was rejected, is filled with a zero-value // transfer to the faucet so the transfers after it can land. func (s *Sender) resync(ctx context.Context) error { committed, err := s.rpc.NonceAt(ctx, s.from, nil) @@ -262,7 +269,15 @@ func (s *Sender) resync(ctx context.Context) error { return err } } - if err := s.rebroadcast(ctx, c); err != nil { + receipt, err := s.receipt(ctx, c, false) + if err != nil && !errors.Is(err, ethereum.NotFound) { + return err + } + if receipt != nil && receipt.Status != types.ReceiptStatusSuccessful { + if err := s.fill(ctx, nonce); err != nil { + return err + } + } else if err := s.rebroadcast(ctx, c); err != nil { s.metrics.sendErrors.Inc() stuck = true slog.Warn("rebroadcast refused", "claim", c.ID, "nonce", nonce, "err", err) @@ -314,16 +329,18 @@ func (s *Sender) rebroadcast(ctx context.Context, c *store.Claim) error { return nil } -// receipt returns the receipt that settles a claim, or ethereum.NotFound. A -// successful receipt for any transaction signed for the claim settles it. A -// failed one settles it only for the latest transaction or once the nonce is -// spent: on Giga a failed receipt leaves the nonce unspent, so a later -// re-signed transaction can still land. +// receipt returns a successful receipt for any transaction signed for a +// claim, else a failed receipt for its latest transaction, or for any of them +// once the nonce is spent, else ethereum.NotFound. func (s *Sender) receipt(ctx context.Context, c *store.Claim, spent bool) (*types.Receipt, error) { hashes, err := s.store.TxHashes(ctx, c.ID) if err != nil { return nil, err } + latest := common.HexToHash(c.TxHash) + if !slices.Contains(hashes, latest) { + hashes = append(hashes, latest) + } var failed *types.Receipt for _, hash := range hashes { receipt, err := s.rpc.TransactionReceipt(ctx, hash) @@ -334,7 +351,7 @@ func (s *Sender) receipt(ctx context.Context, c *store.Claim, spent bool) (*type return nil, err case receipt.Status == types.ReceiptStatusSuccessful: return receipt, nil - case spent || hash == common.HexToHash(c.TxHash): + case spent || hash == latest: failed = receipt } } diff --git a/internal/sender/sender_test.go b/internal/sender/sender_test.go index 2860b25..d54bbb2 100644 --- a/internal/sender/sender_test.go +++ b/internal/sender/sender_test.go @@ -2,6 +2,7 @@ package sender import ( "context" + "database/sql" "errors" "math/big" "path/filepath" @@ -22,9 +23,9 @@ import ( var amount = big.NewInt(1e18) // chain mimics Giga: it admits only the sender's exact next nonce, and a -// transfer to a rejected address, or a failing transaction, lands with a failed -// receipt without spending its nonce, so every transfer queued behind it is -// dropped. A transaction whose fee cap is below minFee is refused. +// transfer to a rejected address lands with a failed receipt without spending +// its nonce, so every transfer queued behind it is dropped. A transaction whose +// fee cap is below minFee is refused. type chain struct { signer types.Signer @@ -36,7 +37,6 @@ type chain struct { refuse int hang bool rejected map[common.Address]bool - failing map[common.Hash]bool minFee *big.Int } @@ -46,7 +46,6 @@ func newChain() *chain { receipts: map[common.Hash]*types.Receipt{}, paid: map[common.Address]int{}, rejected: map[common.Address]bool{}, - failing: map[common.Hash]bool{}, minFee: new(big.Int), } } @@ -100,7 +99,7 @@ func (c *chain) mine() { c.mu.Lock() defer c.mu.Unlock() for _, tx := range c.pending { - if c.rejected[*tx.To()] || c.failing[tx.Hash()] { + if c.rejected[*tx.To()] { c.receipts[tx.Hash()] = &types.Receipt{Status: types.ReceiptStatusFailed} break } @@ -123,18 +122,20 @@ type harness struct { t *testing.T chain *chain store *store.Store + path string cfg Config } func newHarness(t *testing.T) *harness { t.Helper() - st, err := store.Open(context.Background(), filepath.Join(t.TempDir(), "wellspring.db")) + path := filepath.Join(t.TempDir(), "wellspring.db") + st, err := store.Open(context.Background(), path) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) key, _ := crypto.GenerateKey() - return &harness{t: t, chain: newChain(), store: st, cfg: Config{ + return &harness{t: t, chain: newChain(), store: st, path: path, cfg: Config{ ChainID: big.NewInt(1330), Key: key, GasTipCap: big.NewInt(1), @@ -289,18 +290,31 @@ func TestRefusedRebroadcastKeepsResyncing(t *testing.T) { // the node refuses a claim's transaction, and checks the claim is re-signed at // its nonce and paid once, whichever of its transactions lands. func TestResignedClaimSettlesOnce(t *testing.T) { - for name, refuse := range map[string]func(h *harness, old common.Hash){ - "fee cap below base fee": func(h *harness, _ common.Hash) { + for name, tc := range map[string]struct { + refuse func(h *harness, old common.Hash) + afterResign func(h *harness, hashes []common.Hash) + }{ + "fee cap below base fee": {refuse: func(h *harness, _ common.Hash) { h.chain.drop() h.chain.minFee = big.NewInt(5) - }, - "old transaction lands after the re-sign": func(h *harness, _ common.Hash) { + }}, + "old transaction lands after the re-sign": {refuse: func(h *harness, _ common.Hash) { h.chain.refuse = 1 + }}, + "old transaction fails after the re-sign": { + refuse: func(h *harness, _ common.Hash) { + h.chain.drop() + h.chain.refuse = 1 + }, + afterResign: func(h *harness, hashes []common.Hash) { + h.chain.receipts[hashes[0]] = &types.Receipt{Status: types.ReceiptStatusFailed} + }, }, - "old transaction fails in its block": func(h *harness, old common.Hash) { - h.chain.failing[old] = true - h.chain.mine() - h.chain.refuse = 1 + "re-signed transaction fails before the old one lands": { + refuse: func(h *harness, _ common.Hash) { h.chain.refuse = 1 }, + afterResign: func(h *harness, hashes []common.Hash) { + h.chain.receipts[hashes[1]] = &types.Receipt{Status: types.ReceiptStatusFailed} + }, }, } { t.Run(name, func(t *testing.T) { @@ -312,7 +326,7 @@ func TestResignedClaimSettlesOnce(t *testing.T) { if err != nil || len(sent) != 1 { t.Fatalf("in flight: %v %v", sent, err) } - refuse(h, common.HexToHash(sent[0].TxHash)) + tc.refuse(h, common.HexToHash(sent[0].TxHash)) h.cfg.GasFeeCap = big.NewInt(5) s := h.sender() @@ -324,6 +338,9 @@ func TestResignedClaimSettlesOnce(t *testing.T) { if resigned, err := h.store.Get(ctx, sent[0].ID); err != nil || resigned.Nonce != sent[0].Nonce || common.HexToHash(resigned.TxHash) != hashes[1] { t.Fatalf("re-signed claim: %+v %v", resigned, err) } + if tc.afterResign != nil { + tc.afterResign(h, hashes) + } h.drain(s) h.paidOnce(addrs...) @@ -337,6 +354,24 @@ func TestResignedClaimSettlesOnce(t *testing.T) { } } +// TestSettlesClaimWithoutHashHistory covers a claim sent by a binary that +// predates claim_txs, after a rollback past the migration. +func TestSettlesClaimWithoutHashHistory(t *testing.T) { + h := newHarness(t) + addrs := h.claim(1) + h.sender().step(context.Background()) + db, err := sql.Open("sqlite", h.path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + if _, err := db.ExecContext(context.Background(), `DELETE FROM claim_txs`); err != nil { + t.Fatal(err) + } + h.drain(h.sender()) + h.paidOnce(addrs...) +} + func TestGaugesTrackUnsettledClaimsAndBalance(t *testing.T) { h := newHarness(t) ctx := context.Background() diff --git a/internal/store/store.go b/internal/store/store.go index a387d2c..27cbe1a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -229,12 +229,17 @@ func (s *Store) MarkSent(ctx context.Context, id string, nonce uint64, hash comm } // Resign replaces a sent claim's transaction with one re-signed at the same -// nonce, keeping the earlier hashes in TxHashes. Call it before broadcasting. +// nonce, keeping the earlier hashes in TxHashes, and restarts its sent time. +// Call it before broadcasting. func (s *Store) Resign(ctx context.Context, id string, hash common.Hash, raw []byte, now time.Time) error { - return s.sign(ctx, StatusSent, id, hash, now, `tx_hash = ?, raw_tx = ?`, hash.Hex(), raw) + return s.sign(ctx, StatusSent, id, hash, now, `tx_hash = ?, raw_tx = ?, sent_at = ?`, hash.Hex(), raw, now.UnixMilli()) } -// TxHashes returns every transaction hash signed for a claim, oldest first. +// ErrHashTaken is returned for a transaction hash already signed for another claim. +var ErrHashTaken = errors.New("transaction hash belongs to another claim") + +// TxHashes returns every transaction hash signed for a claim, in the order +// they were first signed. func (s *Store) TxHashes(ctx context.Context, id string) ([]common.Hash, error) { rows, err := s.db.QueryContext(ctx, `SELECT hash FROM claim_txs WHERE claim_id = ? ORDER BY rowid`, id) if err != nil { @@ -261,9 +266,18 @@ func (s *Store) sign(ctx context.Context, from Status, id string, hash common.Ha if err := update(ctx, tx, from, set, id, args...); err != nil { return err } - if _, err := tx.ExecContext(ctx, `INSERT INTO claim_txs (hash, claim_id, signed_at) VALUES (?, ?, ?)`, - hash.Hex(), id, now.UnixMilli()); err != nil { + // Signing is deterministic, so fees changed back to earlier values sign a + // hash the claim already has. + res, err := tx.ExecContext(ctx, `INSERT INTO claim_txs (hash, claim_id, signed_at) VALUES (?, ?, ?) + ON CONFLICT (hash) DO UPDATE SET signed_at = excluded.signed_at WHERE claim_id = excluded.claim_id`, + hash.Hex(), id, now.UnixMilli()) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil { return err + } else if n != 1 { + return ErrHashTaken } return tx.Commit() } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 41411af..9288f1d 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -145,15 +145,23 @@ func TestResignKeepsEveryHash(t *testing.T) { if err := s.MarkSent(ctx, c.ID, 7, common.Hash{1}, []byte{1}, t0); err != nil { t.Fatal(err) } - if err := s.Resign(ctx, c.ID, common.Hash{2}, []byte{2}, t0); err != nil { + t1 := t0.Add(time.Hour) + if err := s.Resign(ctx, c.ID, common.Hash{2}, []byte{2}, t1); err != nil { t.Fatal(err) } - if got, err := s.Get(ctx, c.ID); err != nil || got.Nonce.Int64 != 7 || got.TxHash != (common.Hash{2}).Hex() || string(got.RawTx) != "\x02" { + if got, err := s.Get(ctx, c.ID); err != nil || got.Nonce.Int64 != 7 || got.TxHash != (common.Hash{2}).Hex() || string(got.RawTx) != "\x02" || !got.SentAt.Time.Equal(t1) { t.Fatalf("re-signed claim: %+v %v", got, err) } + if err := s.Resign(ctx, c.ID, common.Hash{1}, []byte{1}, t1); err != nil { + t.Fatalf("re-sign back to an earlier hash: %v", err) + } if hashes, err := s.TxHashes(ctx, c.ID); err != nil || len(hashes) != 2 || hashes[0] != (common.Hash{1}) || hashes[1] != (common.Hash{2}) { t.Fatalf("hashes: %v %v", hashes, err) } + other := claim(t, s, bob, "2.2.2.2", limits, t0) + if err := s.MarkSent(ctx, other.ID, 8, common.Hash{1}, []byte{1}, t0); !errors.Is(err, ErrHashTaken) { + t.Fatalf("MarkSent with another claim's hash: %v", err) + } if err := s.Finish(ctx, c.ID, StatusConfirmed, "", t0); err != nil { t.Fatal(err) } From 3654a2921e2f1edd36ecdd870a898970c841e0f8 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 20 Nov 2026 16:27:07 +0000 Subject: [PATCH 3/4] Keep the hash a re-sign replaces when it has no history --- internal/store/store.go | 7 +++++++ internal/store/store_test.go | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/internal/store/store.go b/internal/store/store.go index 27cbe1a..6a1b6a0 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -263,6 +263,13 @@ func (s *Store) sign(ctx context.Context, from Status, id string, hash common.Ha return err } defer func() { _ = tx.Rollback() }() + // A binary from before claim_txs may have sent the claim without recording + // its hash, so keep the hash being replaced. + if _, err := tx.ExecContext(ctx, `INSERT INTO claim_txs (hash, claim_id, signed_at) + SELECT tx_hash, id, COALESCE(sent_at, created_at) FROM claims WHERE id = ? AND tx_hash != '' + ON CONFLICT (hash) DO NOTHING`, id); err != nil { + return err + } if err := update(ctx, tx, from, set, id, args...); err != nil { return err } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 9288f1d..0ae3870 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -170,6 +170,25 @@ func TestResignKeepsEveryHash(t *testing.T) { } } +func TestResignKeepsHashSentWithoutHistory(t *testing.T) { + ctx := context.Background() + s, _ := open(t) + c := claim(t, s, alice, "1.1.1.1", limits, t0) + if err := s.MarkSent(ctx, c.ID, 7, common.Hash{1}, []byte{1}, t0); err != nil { + t.Fatal(err) + } + // As if sent by a binary from before claim_txs. + if _, err := s.db.ExecContext(ctx, `DELETE FROM claim_txs`); err != nil { + t.Fatal(err) + } + if err := s.Resign(ctx, c.ID, common.Hash{2}, []byte{2}, t0.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if hashes, err := s.TxHashes(ctx, c.ID); err != nil || len(hashes) != 2 || hashes[0] != (common.Hash{1}) || hashes[1] != (common.Hash{2}) { + t.Fatalf("hashes: %v %v", hashes, err) + } +} + func TestMigrationKeepsSentHashes(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "wellspring.db") From 37ee4491bc944b8cbb9474e279c3952c747d3151 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 20 Nov 2026 16:36:17 +0000 Subject: [PATCH 4/4] Treat a fill the node already has as sent --- internal/sender/sender.go | 5 ++++- internal/sender/sender_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/internal/sender/sender.go b/internal/sender/sender.go index a759992..1f98792 100644 --- a/internal/sender/sender.go +++ b/internal/sender/sender.go @@ -404,7 +404,10 @@ func (s *Sender) fill(ctx context.Context, nonce uint64) error { return err } slog.Warn("filling unspent nonce", "nonce", nonce) - return s.rpc.SendTransaction(ctx, tx) + if err := s.rpc.SendTransaction(ctx, tx); err != nil && !alreadyHave(err) { + return err + } + return nil } func (s *Sender) finish(ctx context.Context, c *store.Claim, status store.Status, msg string) error { diff --git a/internal/sender/sender_test.go b/internal/sender/sender_test.go index d54bbb2..ba10771 100644 --- a/internal/sender/sender_test.go +++ b/internal/sender/sender_test.go @@ -223,6 +223,21 @@ func TestFailedReceiptFillsItsNonce(t *testing.T) { } } +func TestFillAlreadyPendingKeepsResyncing(t *testing.T) { + h := newHarness(t) + addrs := h.claim(1) + h.chain.rejected[addrs[0]] = true + s := h.sender() + ctx := context.Background() + s.step(ctx) + h.chain.mine() + for range 2 { + if err := s.resync(ctx); err != nil { + t.Fatal(err) + } + } +} + func TestDroppedTransactionsResend(t *testing.T) { h := newHarness(t) h.cfg.ConfirmTimeout = 0