diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index 16c63ed92669..a2ab613dbea2 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -15,6 +15,7 @@ import ephemeral_port_reserve # type: ignore import tempfile +import errno import json import logging import lzma @@ -215,6 +216,40 @@ def cleanup_stale_port_locks(): pass # best-effort, never crash the test run over cleanup +def wait_for_port_released(port, timeout=TIMEOUT): + """Wait until 127.0.0.1:port can be bound again. + + A stopped node's connectd holds the listen socket until it exits, + which can be several seconds after lightningd itself is gone + (subdaemons are separate processes, and die slowly under valgrind). + Restarting the node before the port is released makes the new + connectd fail with 'Address already in use'. + """ + start_time = time.time() + while True: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Match connectd's SO_REUSEADDR, so sockets lingering in + # TIME_WAIT don't count as "still in use". + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + s.bind(('127.0.0.1', port)) + break + except OSError as e: + if e.errno != errno.EADDRINUSE: + raise + if time.time() - start_time > timeout: + raise TimeoutError( + "Port {} was not released within {} seconds" + .format(port, timeout)) + finally: + s.close() + time.sleep(0.1) + + waited = time.time() - start_time + if waited >= 1: + logging.info("Port %d took %.1fs to be released", port, waited) + + class TailableProc(object): """A monitorable process that we can start, stop and tail. @@ -848,6 +883,9 @@ def cmd_line(self): def start(self, stdin=None, wait_for_initialized=True, stderr_redir=False): self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport + # On restart, the previous incarnation's connectd may still be + # dying and holding our listen port: don't launch until it's free. + wait_for_port_released(self.port) TailableProc.start(self, stdin, stdout_redir=False, stderr_redir=stderr_redir) if wait_for_initialized: self.wait_for_log("Server started with public key")