autogenerate-rpc-examples.py: use fixed port numbers.

This makes many of our information outputs consistent.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

contrib/pyln-testing/pyln/testing/utils.py

@@ -398,14 +398,10 @@ class BitcoinD(TailableProc):
     def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
         TailableProc.__init__(self, bitcoin_dir, verbose=False)
 
-        if rpcport is None:
-            self.reserved_rpcport = reserve_unused_port()
-            rpcport = self.reserved_rpcport
-        else:
-            self.reserved_rpcport = None
-
         self.bitcoin_dir = bitcoin_dir
         self.rpcport = rpcport
+        self.reserved_rpcport = None
+        self.port_setup = False
         self.prefix = 'bitcoind'
         self.canned_blocks = None
 
@@ -429,21 +425,29 @@ def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
             '-debug=validation',
             '-rpcthreads=20',
         ]
-        # For up to and including 0.16.1, this needs to be in main section.
-        BITCOIND_CONFIG['rpcport'] = rpcport
-        # For after 0.16.1 (eg. 3f398d7a17f136cd4a67998406ca41a124ae2966), this
-        # needs its own [regtest] section.
-        BITCOIND_REGTEST = {'rpcport': rpcport}
         self.conf_file = os.path.join(bitcoin_dir, 'bitcoin.conf')
+
+    def set_port(self, rpcport):
+        assert self.port_setup is False
+
+        BITCOIND_REGTEST = {'rpcport': rpcport}
         write_config(self.conf_file, BITCOIND_CONFIG, BITCOIND_REGTEST)
-        self.rpc = SimpleBitcoinProxy(btc_conf_file=self.conf_file)
-        self.proxies = []
+        self.port_setup = True
 
     def __del__(self):
         if self.reserved_rpcport is not None:
             drop_unused_port(self.reserved_rpcport)
 
     def start(self, wallet_file=None):
+        if not self.port_setup:
+            if self.rpcport is None:
+                self.reserved_rpcport = reserve_unused_port()
+                self.rpcport = self.reserved_rpcport
+            self.set_port(self.rpcport)
+
+        self.rpc = SimpleBitcoinProxy(btc_conf_file=self.conf_file)
+        self.proxies = []
+
         TailableProc.start(self)
         self.wait_for_log("Done loading", timeout=TIMEOUT)
 
@@ -1666,6 +1670,7 @@ def split_options(self, opts):
             'start',
             'gossip_store_file',
             'no_entropy',
+            'base_port',
         ]
         node_opts = {k: v for k, v in opts.items() if k in node_opt_keys}
         cli_opts = {k: v for k, v in opts.items() if k not in node_opt_keys}
@@ -1712,10 +1717,14 @@ def get_nodes(self, num_nodes, opts=None):
     def get_node(self, node_id=None, options=None, dbfile=None,
                  bkpr_dbfile=None, feerates=(15000, 11000, 7500, 3750),
                  start=True, wait_for_bitcoind_sync=True, may_fail=False,
-                 expect_fail=False, cleandir=True, gossip_store_file=None, unused_grpc_port=True, **kwargs):
+                 expect_fail=False, cleandir=True, gossip_store_file=None, unused_grpc_port=True, base_port=None, **kwargs):
         node_id = self.get_node_id() if not node_id else node_id
-        port = reserve_unused_port()
-        grpc_port = self.get_unused_port() if unused_grpc_port else None
+        if base_port:
+            port = base_port + node_id * 2 - 1
+            grpc_port = base_port + node_id * 2
+        else:
+            port = reserve_unused_port()
+            grpc_port = self.get_unused_port() if unused_grpc_port else None
 
         lightning_dir = os.path.join(
             self.directory, "lightning-{}/".format(node_id))
@@ -1739,7 +1748,8 @@ def get_node(self, node_id=None, options=None, dbfile=None,
             node.set_feerates(feerates, False)
 
         self.nodes.append(node)
-        self.reserved_ports.append(port)
+        if not base_port:
+            self.reserved_ports.append(port)
         if dbfile:
             out = open(os.path.join(node.daemon.lightning_dir, TEST_NETWORK,
                                     'lightningd.sqlite3'), 'xb')

tests/autogenerate-rpc-examples.py

@@ -10,6 +10,7 @@
 from pyln.client import RpcError, Millisatoshi  # type: ignore
 from pyln.testing.utils import GENERATE_EXAMPLES
 from utils import only_one, mine_funding_to_announce, sync_blockheight, wait_for, first_scid, serialize_payload_tlv, serialize_payload_final_tlv
+import socket
 import sys
 import os
 import time
@@ -32,6 +33,7 @@
 EXAMPLES_JSON = {}
 LOG_FILE = './tests/autogenerate-examples-status.log'
 IGNORE_RPCS_LIST = ['dev-splice', 'reckless', 'sql-template']
+BASE_PORTNUM = 30000
 
 if os.path.exists(LOG_FILE):
     open(LOG_FILE, 'w').close()
@@ -42,6 +44,16 @@ class MissingExampleError(Exception):
     pass
 
 
+def check_ports(portrange):
+    for port in portrange:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.bind(("127.0.0.1", port))
+            except OSError:
+                logger.error(f'Port {port} in use!')
+                raise
+
+
 def update_examples_in_schema_files():
     """Update examples in JSON schema files"""
     try:
@@ -129,6 +141,7 @@ def setup_test_nodes(node_factory, bitcoind, regenerate_blockchain):
                 'broken_log': '.*',
                 'dev-bitcoind-poll': 3,    # Default 1; increased to avoid rpc failures
                 'no_entropy': True,
+                'base_port': BASE_PORTNUM,
             }.copy()
             for i in range(6)
         ]
@@ -688,6 +701,7 @@ def generate_splice_examples(node_factory, bitcoind, regenerate_blockchain):
                 'broken_log': '.*',
                 'dev-bitcoind-poll': 3,
                 'no_entropy': True,
+                'base_port': BASE_PORTNUM,
             }.copy()
             for i in range(2)
         ]
@@ -745,6 +759,7 @@ def generate_channels_examples(node_factory, bitcoind, l1, l3, l4, l5, regenerat
                 'broken_log': '.*',
                 'dev-bitcoind-poll': 3,
                 'no_entropy': True,
+                'base_port': BASE_PORTNUM,
             }.copy()
             for i in range(2)
         ]
@@ -796,6 +811,7 @@ def generate_channels_examples(node_factory, bitcoind, l1, l3, l4, l5, regenerat
                 'broken_log': '.*',
                 'dev-bitcoind-poll': 3,
                 'no_entropy': True,
+                'base_port': BASE_PORTNUM,
             }.copy()
             for i in range(2)
         ]
@@ -1009,7 +1025,7 @@ def generate_backup_recovery_examples(node_factory, l4, l5, l6, regenerate_block
         logger.info('Backup and Recovery Start...')
 
         # New node l13 used for recover and exposesecret examples
-        l13 = node_factory.get_node(options={'exposesecret-passphrase': "test_exposesecret"}, no_entropy=True)
+        l13 = node_factory.get_node(options={'exposesecret-passphrase': "test_exposesecret"}, no_entropy=True, base_portnum=BASE_PORTNUM)
         update_example(node=l13, method='exposesecret', params={'passphrase': 'test_exposesecret'})
         update_example(node=l13, method='exposesecret', params=['test_exposesecret', 'cln2'])
 
@@ -1169,6 +1185,13 @@ def test_generate_examples(node_factory, bitcoind, executor):
     # Change this to True to regenerate bitcoin block & wallet.
     regenerate_blockchain = (os.environ.get("REGENERATE_BLOCKCHAIN") == "1")
     wallet_exists = os.access("tests/data/autogenerate-bitcoind-wallet.dat", os.F_OK)
+
+    # Make sure we can get the ports we expect.
+    check_ports(range(BASE_PORTNUM + 1, BASE_PORTNUM + 40))
+
+    # Make sure bitcoind doesn't steal our ports!
+    bitcoind.set_port(BASE_PORTNUM)
+
     try:
         global ALL_RPC_EXAMPLES, REGENERATING_RPCS