wallet: make sure we re-watch outpoints after blocks are rolled back.

At startup, we load the outpoints to watch, *then* roll back 15
blocks.  If there were things in those blocks we wanted to watch, we
no longer do!

1. We load the utxoset into memory: everything in the utxoset table
   which has spendheight null.
2. We roll back 15 blocks to re-read.  Deleting a block from the
   database causes the utxo spentheights referring to it to be set
   to null.
3. We roll forward, but we didn't update the in-memory utxoset,
   so we're not watching those utxos which are spent.

The main symptom of this is that we spam peers with obsolete gossip
(if we get sent a channel announcement for a closed channel, we can
think it isn't spent yet).  But it could *also* mean we don't notice
onchain txs, if we restart at the wrong time!

Changelog-Fixed: lightningd: we could miss tx spends which happened in the past blocks when we restarted.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

tests/test_wallet.py

@@ -2538,7 +2538,6 @@ def test_hsm_wrong_passphrase_crash(node_factory):
     os.close(slave_fd2)
 
 
-@pytest.mark.xfail(strict=True)
 def test_unspend_during_reorg(node_factory, bitcoind):
     l1, l2 = node_factory.line_graph(2)
     scid = first_scid(l1, l2)

wallet/wallet.c

@@ -180,32 +180,39 @@ static void our_addresses_init(struct wallet *w)
 	w->our_addresses_maxindex = w->keyscan_gap;
 }
 
-static void outpointfilters_init(struct wallet *w)
+/* Idempotent: outpointfilter_add is a noop if it already exists. */
+static void refill_outpointfilters(struct wallet *w)
 {
 	struct db_stmt *stmt;
-	struct utxo **utxos = wallet_get_all_utxos(NULL, w);
-	struct bitcoin_outpoint outpoint;
-
-	w->owned_outpoints = outpointfilter_new(w);
-	for (size_t i = 0; i < tal_count(utxos); i++)
-		outpointfilter_add(w->owned_outpoints, &utxos[i]->outpoint);
 
-	tal_free(utxos);
-
-	w->utxoset_outpoints = outpointfilter_new(w);
 	stmt = db_prepare_v2(
 	    w->db,
 	    SQL("SELECT txid, outnum FROM utxoset WHERE spendheight is NULL"));
 	db_query_prepared(stmt);
 
 	while (db_step(stmt)) {
+		struct bitcoin_outpoint outpoint;
 		db_col_txid(stmt, "txid", &outpoint.txid);
 		outpoint.n = db_col_int(stmt, "outnum");
 		outpointfilter_add(w->utxoset_outpoints, &outpoint);
 	}
 	tal_free(stmt);
 }
 
+static void outpointfilters_init(struct wallet *w)
+{
+	struct utxo **utxos = wallet_get_all_utxos(NULL, w);
+
+	w->owned_outpoints = outpointfilter_new(w);
+	for (size_t i = 0; i < tal_count(utxos); i++)
+		outpointfilter_add(w->owned_outpoints, &utxos[i]->outpoint);
+
+	tal_free(utxos);
+
+	w->utxoset_outpoints = outpointfilter_new(w);
+	refill_outpointfilters(w);
+}
+
 struct wallet *wallet_new(struct lightningd *ld, struct timers *timers)
 {
 	struct wallet *wallet = tal(ld, struct wallet);
@@ -4867,6 +4874,9 @@ void wallet_block_remove(struct wallet *w, struct block *b)
 	db_query_prepared(stmt);
 	assert(!db_step(stmt));
 	tal_free(stmt);
+
+	/* We might need to watch more now-unspent UTXOs */
+	refill_outpointfilters(w);
 }
 
 void wallet_blocks_rollback(struct wallet *w, u32 height)
@@ -4875,6 +4885,7 @@ void wallet_blocks_rollback(struct wallet *w, u32 height)
 							"WHERE height > ?"));
 	db_bind_int(stmt, height);
 	db_exec_prepared_v2(take(stmt));
+	refill_outpointfilters(w);
 }
 
 bool wallet_outpoint_spend(const tal_t *ctx, struct wallet *w, const u32 blockheight,