From 4566178528aff7c1fed113545bec6b48da43e2e4 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Fri, 24 Jul 2026 13:03:34 -0700 Subject: [PATCH] pyln-testing: wait for listen port release before starting a node A node's connectd is a separate process holding the listen socket, and on shutdown it exits on its own schedule after lightningd itself is gone -- under valgrind, its teardown can lag by tens of seconds. If a test restarts the node in that window, the new connectd fails with 'Address already in use', lightningd exits, and the test times out waiting for 'Server started with public key'. Seen in CI in test_emergencyrecoverpenaltytxn, where the port was still held 37 seconds after the old connectd began shutting down. The filesystem port locks don't cover this: they keep other workers from reserving the port, but the restarting node owns its reservation both times; nothing waits for the old connectd to actually release the socket. Before launching lightningd, bind-probe the node's port (with SO_REUSEADDR, matching connectd, so TIME_WAIT sockets don't count as in use) and only proceed once the bind succeeds. Fixes: #9354 Changelog-None --- contrib/pyln-testing/pyln/testing/utils.py | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index 16c63ed92669..a2ab613dbea2 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -15,6 +15,7 @@ import ephemeral_port_reserve # type: ignore import tempfile +import errno import json import logging import lzma @@ -215,6 +216,40 @@ def cleanup_stale_port_locks(): pass # best-effort, never crash the test run over cleanup +def wait_for_port_released(port, timeout=TIMEOUT): + """Wait until 127.0.0.1:port can be bound again. + + A stopped node's connectd holds the listen socket until it exits, + which can be several seconds after lightningd itself is gone + (subdaemons are separate processes, and die slowly under valgrind). + Restarting the node before the port is released makes the new + connectd fail with 'Address already in use'. + """ + start_time = time.time() + while True: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Match connectd's SO_REUSEADDR, so sockets lingering in + # TIME_WAIT don't count as "still in use". + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + s.bind(('127.0.0.1', port)) + break + except OSError as e: + if e.errno != errno.EADDRINUSE: + raise + if time.time() - start_time > timeout: + raise TimeoutError( + "Port {} was not released within {} seconds" + .format(port, timeout)) + finally: + s.close() + time.sleep(0.1) + + waited = time.time() - start_time + if waited >= 1: + logging.info("Port %d took %.1fs to be released", port, waited) + + class TailableProc(object): """A monitorable process that we can start, stop and tail. @@ -848,6 +883,9 @@ def cmd_line(self): def start(self, stdin=None, wait_for_initialized=True, stderr_redir=False): self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport + # On restart, the previous incarnation's connectd may still be + # dying and holding our listen port: don't launch until it's free. + wait_for_port_released(self.port) TailableProc.start(self, stdin, stdout_redir=False, stderr_redir=stderr_redir) if wait_for_initialized: self.wait_for_log("Server started with public key")