From f723d4075e4beffc57cced0bc183863a9a16839d Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Fri, 12 Sep 2025 07:35:31 +0000 Subject: [PATCH 01/13] mod_systemd: add missing space Submitted by: Christian Hesse Github: closes #557 Merge r1928356 from trunk. (cherry picked from commit 9cd6c92c95a8049f418123f4759df1ae106a8f6e) --- modules/arch/unix/mod_systemd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/arch/unix/mod_systemd.c b/modules/arch/unix/mod_systemd.c index 22482fd6bbc..395da965a2d 100644 --- a/modules/arch/unix/mod_systemd.c +++ b/modules/arch/unix/mod_systemd.c @@ -114,7 +114,7 @@ static int systemd_monitor(apr_pool_t *p, server_rec *s) / (float) up_time), bps); sd_notifyf(0, "READY=1\n" - "STATUS=Total requests: %lu; Idle/Busy workers %d/%d;" + "STATUS=Total requests: %lu; Idle/Busy workers %d/%d; " "Requests/sec: %.3g; Bytes served/sec: %sB/sec\n", sload.access_count, sload.idle, sload.busy, ((float) sload.access_count) / (float) up_time, bps); From 122322cc51a8af3b855224932b122a254a173274 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sat, 29 Aug 2026 09:11:02 +0000 Subject: [PATCH 02/13] * test/modules/arch/linux/: New test suite. * test/pyhttpd/env.py (HttpdTestEnv.set_httpd_env, _clean_path_env): Allow a suite to add variables to the environment httpd is started with. * test/pyhttpd/conf/stop.conf.template: Set DefaultRuntimeDir and PidFile to match httpd.conf, so that "apachectl -k stop" finds the running server. Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #723 Merge r1937596 from trunk. (cherry picked from commit 4abdd206a7a8d590e0e09adec22287f8b1a5e8be) --- test/modules/arch/linux/README | 79 +++ test/modules/arch/linux/conftest.py | 40 ++ test/modules/arch/linux/env.py | 526 ++++++++++++++++++ test/modules/arch/linux/test_001_notify.py | 141 +++++ test/modules/arch/linux/test_002_monitor.py | 103 ++++ .../arch/linux/test_003_extended_status.py | 66 +++ .../arch/linux/test_004_socket_activation.py | 90 +++ test/modules/arch/linux/test_005_service.py | 89 +++ test/pyhttpd/conf/stop.conf.template | 5 + test/pyhttpd/env.py | 15 + 10 files changed, 1154 insertions(+) create mode 100644 test/modules/arch/linux/README create mode 100644 test/modules/arch/linux/conftest.py create mode 100644 test/modules/arch/linux/env.py create mode 100644 test/modules/arch/linux/test_001_notify.py create mode 100644 test/modules/arch/linux/test_002_monitor.py create mode 100644 test/modules/arch/linux/test_003_extended_status.py create mode 100644 test/modules/arch/linux/test_004_socket_activation.py create mode 100644 test/modules/arch/linux/test_005_service.py diff --git a/test/modules/arch/linux/README b/test/modules/arch/linux/README new file mode 100644 index 00000000000..3894c6158fa --- /dev/null +++ b/test/modules/arch/linux/README @@ -0,0 +1,79 @@ +mod_systemd tests +================= + +What is different about this module +----------------------------------- +mod_systemd has no directives. Everything it does is driven by the +environment httpd was started with, and almost everything it produces goes +to the service manager rather than to a client: + + - sd_notify(3) datagrams sent to $NOTIFY_SOCKET at four points: reading + the configuration (pre_config), configuration loaded (post_config), + the MPM starting (pre_mpm, the only notification carrying MAINPID), + and a status line refreshed by the monitor hook. + - listening sockets inherited from the service manager, found through + $LISTEN_FDS. mod_systemd registers the two optional functions + server/listen.c calls, so loading the module is what enables socket + activation and not loading it is what disables it. + - ap_extended_status forced on in pre_config, so that the monitor hook + has request counts to report. + +None of that is observable over HTTP, so the tests observe it directly. + +How the tests run without systemd, and without privileges +--------------------------------------------------------- +There is no need for a service manager to exercise the protocol. Only +test_005 involves systemd at all; the rest run anywhere, and need nothing +from the systemd package beyond the libsystemd httpd itself is linked +against. + + test_001_notify.py $NOTIFY_SOCKET is an ordinary AF_UNIX datagram + socket the test binds itself (env.NotifyListener). + sd_notify writes to whatever that variable names, so + the test reads httpd's notifications straight off the + socket. HttpdTestEnv.set_httpd_env() puts the + variable into the environment apachectl passes on. + + test_002_monitor.py The periodic status line. ap_run_monitor() is + called once every ten turns of the parent's ~1s + loop, so these tests wait up to 25 seconds. + + test_003_extended_status.py + The ExtendedStatus side effect, observed through + mod_status. + + test_004_socket_activation.py + The test opens the listening socket itself and + hands it to httpd as descriptor 3 with LISTEN_FDS + and LISTEN_PID set, which is the whole protocol. + systemd-socket-activate would do the same, but its + --now option is newer than the systemd on some + distributions, and doing it directly needs no + systemd tooling at all. + + test_005_service.py The real thing: a transient Type=notify unit run + with "systemd-run --user". This is what checks that + systemd holds the unit in "activating" until READY=1 + arrives, tracks the right MainPID, and shows the + reported STATUS= as the unit's status text. + Skipped when the user has no systemd manager. + +The whole package is skipped unless mod_systemd was built, which needs +configure --enable-systemd. A static module is enough for everything +except the test which has to leave mod_systemd out of the configuration; +that one needs --enable-systemd=shared. + +Running the last suite where there is no user session +----------------------------------------------------- +"systemctl --user" needs a per-user manager, which a login session has but +a CI container does not; enabling lingering needs privileges. Where that +is a problem, run the tests inside a container with systemd as pid 1, +which rootless podman supports on a cgroup v2 host: + + podman run --rm -it --systemd=always \ + -v $PWD:/src:z -w /src registry.fedoraproject.org/fedora:latest \ + /usr/sbin/init + +then, in another terminal, "podman exec" into it and run pytest as a +non-root user with XDG_RUNTIME_DIR set. The other four suites need none of +this and run anywhere. diff --git a/test/modules/arch/linux/conftest.py b/test/modules/arch/linux/conftest.py new file mode 100644 index 00000000000..7b1a0d66cd7 --- /dev/null +++ b/test/modules/arch/linux/conftest.py @@ -0,0 +1,40 @@ +import logging +import os +import sys + +import pytest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../..')) + +from .env import SystemdTestEnv + + +def pytest_report_header(config, start_path): + env = SystemdTestEnv() + return f"mod_systemd [apache: {env.get_httpd_version()}, " \ + f"mpm: {env.mpm_module}, {env.prefix}]" + + +@pytest.fixture(scope="package") +def env(pytestconfig) -> SystemdTestEnv: + level = logging.INFO + console = logging.StreamHandler() + console.setLevel(level) + console.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) + logging.getLogger('').addHandler(console) + logging.getLogger('').setLevel(level=level) + env = SystemdTestEnv(pytestconfig=pytestconfig) + if not env.has_systemd_module: + pytest.skip("mod_systemd is not built, configure with --enable-systemd") + env.setup_httpd() + env.apache_access_log_clear() + env.httpd_error_log.clear_log() + env.start_notify_listener() + yield env + env.stop_notify_listener() + + +@pytest.fixture(autouse=True, scope="package") +def _stop_package_scope(env): + yield + assert env.apache_stop() == 0 diff --git a/test/modules/arch/linux/env.py b/test/modules/arch/linux/env.py new file mode 100644 index 00000000000..a09fb5f7d68 --- /dev/null +++ b/test/modules/arch/linux/env.py @@ -0,0 +1,526 @@ +import inspect +import logging +import os +import re +import shutil +import signal +import socket +import subprocess +import threading +import time +from datetime import timedelta +from typing import Callable, Dict, List, Optional + +from pyhttpd.env import HttpdTestEnv, HttpdTestSetup + +log = logging.getLogger(__name__) + + +class SystemdTestSetup(HttpdTestSetup): + + def __init__(self, env: 'HttpdTestEnv'): + super().__init__(env=env) + self.add_source_dir(os.path.dirname(inspect.getfile(SystemdTestSetup))) + # mod_systemd is only built with --enable-systemd, so it must not be + # a hard requirement; the tests skip when it is absent. + self.add_optional_modules(["systemd"]) + + +class SystemdTestEnv(HttpdTestEnv): + + def __init__(self, pytestconfig=None): + super().__init__(pytestconfig=pytestconfig) + self.add_httpd_log_modules(["core"]) + self._notify = None + + def setup_httpd(self, setup: HttpdTestSetup = None): + super().setup_httpd(setup=SystemdTestSetup(env=self)) + + @property + def systemd_is_dso(self) -> bool: + return os.path.isfile(os.path.join(self.libexec_dir, 'mod_systemd.so')) + + @property + def has_systemd_module(self) -> bool: + """Whether mod_systemd is available, however it was built: + --enable-systemd links it statically, --enable-systemd=shared + builds the DSO.""" + if self.systemd_is_dso: + return True + p = subprocess.run([self.httpd_bin, '-l'], capture_output=True, + text=True) + return re.search(r'^\s+mod_systemd\.c$', p.stdout, re.M) is not None + + @property + def notify(self) -> 'NotifyListener': + """The stand-in notification socket httpd reports to.""" + return self._notify + + def start_notify_listener(self) -> 'NotifyListener': + """Bind the notification socket and point httpd's $NOTIFY_SOCKET at it. + + This must happen before the server is first started, since libsystemd + reads $NOTIFY_SOCKET from the environment of the httpd process. + """ + assert self._notify is None + self._notify = NotifyListener( + os.path.join(self.server_dir, 'systemd-notify.sock')) + self.set_httpd_env('NOTIFY_SOCKET', self._notify.path) + return self._notify + + def stop_notify_listener(self): + if self._notify is not None: + self._notify.close() + self._notify = None + + def server_env(self) -> Dict[str, str]: + """The environment httpd is started with, as apachectl gets it.""" + return self._clean_path_env() + + @property + def httpd_bin(self) -> str: + return os.path.join(self.bin_dir, 'httpd') + + def apache_hard_restart(self) -> int: + """Restart without the "graceful" flag, so the MPM starts over.""" + r = self._run_apachectl("restart") + if r.exit_code == 0: + return 0 if self.is_live(self._http_base, timeout=timedelta(seconds=10)) else -1 + return r.exit_code + + def read_pid_file(self, name: str = 'httpd.pid') -> Optional[int]: + # Where PidFile lands depends on how the httpd under test resolves + # a relative path against DefaultRuntimeDir, which has differed + # between versions; look in both places rather than assume. + for d in (self.server_logs_dir, self.server_dir): + try: + with open(os.path.join(d, name)) as fd: + return int(fd.read().strip()) + except (OSError, ValueError): + continue + return None + + +class NotifyListener: + """A stand-in for the systemd notification socket. + + sd_notify(3) does nothing more than send a datagram to the AF_UNIX + socket named by $NOTIFY_SOCKET, so an unconnected datagram socket is + enough to observe everything mod_systemd reports, with no systemd + instance and no privileges involved. + + Datagrams are drained by a background thread so that the periodic + notifications from the monitor hook cannot fill the socket buffer + while a test is doing something else. + """ + + def __init__(self, path: str): + # sockaddr_un is limited to 108 bytes; well within reach for a + # source tree in a home directory, but check rather than fail + # obscurely inside bind(). + assert len(path) < 100, f"notification socket path too long: {path}" + if os.path.exists(path): + os.unlink(path) + self.path = path + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._sock.bind(path) + self._sock.settimeout(0.1) + self._lock = threading.Lock() + self._messages: List[Dict[str, str]] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._drain, daemon=True) + self._thread.start() + + def _drain(self): + while not self._stop.is_set(): + try: + data = self._sock.recv(8192) + except socket.timeout: + continue + except OSError: + break + try: + msg = self.parse(data) + except Exception as ex: + # Never let one unreadable datagram stop the listener: the + # tests would then see silence rather than a failure. + log.warning(f"undecodable notification {data!r}: {ex}") + continue + log.debug(f"notify: {msg}") + with self._lock: + self._messages.append(msg) + + def close(self): + self._stop.set() + self._thread.join(timeout=2) + self._sock.close() + if os.path.exists(self.path): + os.unlink(self.path) + + @staticmethod + def parse(data: bytes) -> Dict[str, str]: + """Split one notification datagram into its NAME=VALUE assignments. + + The last line carries no trailing newline in the MAINPID + notification, and a value may itself contain '='. + """ + msg = {} + for line in data.decode(errors='replace').split('\n'): + name, sep, value = line.partition('=') + if sep: + msg[name] = value + return msg + + @property + def messages(self) -> List[Dict[str, str]]: + with self._lock: + return list(self._messages) + + def clear(self): + with self._lock: + self._messages.clear() + + def wait_for(self, match: Callable[[Dict[str, str]], bool], + timeout: float = 5.0) -> Optional[Dict[str, str]]: + """Return the first message satisfying `match`, waiting for it to + arrive if it has not already. Returns None on timeout.""" + end = time.time() + timeout + seen = 0 + while True: + with self._lock: + pending = self._messages[seen:] + seen = len(self._messages) + for msg in pending: + if match(msg): + return msg + if time.time() >= end: + return None + time.sleep(0.05) + + def wait_for_key(self, key: str, timeout: float = 5.0) \ + -> Optional[Dict[str, str]]: + return self.wait_for(lambda m: key in m, timeout=timeout) + + def wait_for_status(self, pattern: str, timeout: float = 5.0) \ + -> Optional[Dict[str, str]]: + rx = re.compile(pattern) + return self.wait_for(lambda m: 'STATUS' in m and rx.search(m['STATUS']), + timeout=timeout) + + def statuses(self) -> List[str]: + return [m['STATUS'] for m in self.messages if 'STATUS' in m] + + +# The STATUS= line the monitor hook reports, from systemd_monitor() in +# modules/arch/unix/mod_systemd.c. "Idle/Busy workers" are in fact the +# percentages computed by ap_get_sload(), not worker counts. +MONITOR_STATUS = re.compile( + r'^Total requests: (?P\d+);\s*' + r'Idle/Busy workers (?P-?\d+)/(?P-?\d+);\s*' + r'Requests/sec: (?P\S+);\s*' + r'Bytes served/sec: (?P.*)B/sec$') + +# ap_run_monitor() is called every INTERVAL_OF_WRITABLE_PROBES (10) turns of +# the ~1s parent loop in ap_wait_or_timeout(), so a status update is up to +# roughly 10 seconds away. Waiting for one costs that; waiting to be sure +# none is coming costs the whole timeout, so keep it to a small multiple. +MONITOR_TIMEOUT = 25.0 + +# Showing that no report is coming costs the whole wait, so it only has to +# comfortably outlast one turn of that cycle. +NO_MONITOR_TIMEOUT = 15.0 + + +# A configuration for a server run directly rather than through apachectl, +# sharing the server root, module list and error log with the rest of the +# suite but with its own pid file and port. +STANDALONE_CONF = """ +ServerRoot "${server_dir}" +DefaultRuntimeDir logs +PidFile "${pidfile}" +Include "conf/${modules_conf}" +ServerName standalone.test +ErrorLog "logs/error_log" +LogLevel ${loglevel} +DocumentRoot "${server_dir}/htdocs" + + Require all granted + + + SSLSessionCache "shmcb:ssl_gcache_data(32000)" + +${extra} +Listen ${port} +""" + + +def write_server_conf(env: SystemdTestEnv, name: str, port: int, + modules_conf: str = 'modules.conf', + extra: str = '') -> str: + """Write a standalone configuration and return its path.""" + path = os.path.join(env.server_conf_dir, f'{name}.conf') + with open(path, 'w') as fd: + fd.write(STANDALONE_CONF + .replace('${server_dir}', env.server_dir) + .replace('${pidfile}', + os.path.join(env.server_logs_dir, f'{name}.pid')) + .replace('${loglevel}', 'debug' if env.verbosity else 'warn') + .replace('${modules_conf}', modules_conf) + .replace('${extra}', extra) + .replace('${port}', str(port))) + return path + + +def http_responds(port: int, timeout: float = 2.0) -> bool: + """One HTTP request, without curl, so that a listening socket which + nothing is serving cannot be mistaken for a running server: under socket + activation the listener exists before httpd does.""" + try: + with socket.create_connection(('127.0.0.1', port), 1.0) as c: + c.settimeout(timeout) + c.sendall(b'GET / HTTP/1.0\r\nHost: standalone.test\r\n\r\n') + return c.recv(64).startswith(b'HTTP/1.') + except OSError: + return False + + +class ActivatedServer: + """An httpd handed a listening socket the way a service manager does. + + The protocol is only $LISTEN_FDS descriptors starting at 3, and + $LISTEN_PID naming the process they were meant for, so the test opens + the socket and speaks it directly. systemd-socket-activate would do + the same, but its --now option is too recent to rely on, and this + needs no systemd tooling at all. + + apachectl cannot pass descriptors, so httpd is run directly, in the + foreground: LISTEN_PID has to be the process which calls + sd_listen_fds(), and a daemonised parent would not be it. + """ + + def __init__(self, env: SystemdTestEnv, port: int, name: str = 'activate', + extra: str = '', listen_port: int = None, + modules_conf: str = 'modules.conf'): + self.env = env + self.port = port + # The port systemd-socket-activate binds, which is the same as the + # configured one unless a test wants them to disagree. + self.listen_port = port if listen_port is None else listen_port + self.name = name + self.pid_file = os.path.join(env.server_logs_dir, f'{name}.pid') + self.proc = None + self.stdout = None + self.stderr = None + self.conf_file = write_server_conf(env, name, port, + modules_conf=modules_conf, + extra=extra) + + @staticmethod + def modules_conf_without(env: SystemdTestEnv, module: str) -> str: + """Write a copy of the generated module list with one module left + out, to check what happens when it is not loaded.""" + name = f'modules-no-{module}.conf' + src = os.path.join(env.server_conf_dir, 'modules.conf') + rx = re.compile(rf'^\s*LoadModule\s+{module}_module\b') + with open(src) as fd: + lines = [l for l in fd if not rx.match(l)] + with open(os.path.join(env.server_conf_dir, name), 'w') as fd: + fd.writelines(lines) + return name + + def args(self, fd: int) -> List[str]: + # The shell moves the inherited socket to descriptor 3 and names + # itself in LISTEN_PID before exec'ing httpd in its place, which is + # the one thing this cannot do from the parent: the pid has to be + # the one which will call sd_listen_fds(). bash rather than sh + # because dash parses only one digit in a redirection, and the + # socket lands well above descriptor 9. + return [ + 'bash', '-c', + f'exec 3<&{fd}; export LISTEN_FDS=1 LISTEN_FDNAMES=activate ' + 'LISTEN_PID=$$; exec "$0" "$@"', + self.env.httpd_bin, '-DFOREGROUND', + '-d', self.env.server_dir, '-f', self.conf_file, + ] + + def start(self) -> 'ActivatedServer': + lsock = socket.create_server(('', self.listen_port), + family=socket.AF_INET6, + dualstack_ipv6=True, backlog=128) + try: + # A new session so that the whole group can be signalled on the + # way out. + self.proc = subprocess.Popen( + self.args(lsock.fileno()), env=self.env.server_env(), + start_new_session=True, pass_fds=(lsock.fileno(),), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + finally: + # The child holds it now; keeping a copy here would leave the + # port bound after the server is gone. + lsock.close() + return self + + def _reap(self): + """Collect the output of a server which has exited.""" + if self.stderr is None and self.proc.poll() is not None: + self.stdout, self.stderr = self.proc.communicate() + + def wait_exit(self, timeout: float = 10.0) -> int: + """Wait for a server which is expected to fail to start.""" + self.stdout, self.stderr = self.proc.communicate(timeout=timeout) + return self.proc.returncode + + def is_live(self, timeout: float = 10.0) -> bool: + end = time.time() + timeout + while True: + if self.proc.poll() is not None: + # Collect its diagnostics, so that a test reporting the + # server did not come up can say why. + self._reap() + return False + if http_responds(self.port): + return True + if time.time() >= end: + return False + time.sleep(0.2) + + def is_running(self, settle: float = 2.0) -> bool: + """Whether the server is still up once it has had time to fail. + + A restart which cannot find its sockets takes a few milliseconds to + bring the parent down, and its old children go on serving after it, + so neither an immediate check nor a request proves anything. + """ + end = time.time() + settle + while time.time() < end: + if self.proc.poll() is not None: + return False + time.sleep(0.1) + return True + + def reload(self) -> int: + """Ask the running server to restart gracefully.""" + r = self.env.run([self.env.httpd_bin, '-d', self.env.server_dir, + '-f', self.conf_file, '-k', 'graceful'], + env=self.env.server_env()) + return r.exit_code + + def _signal_group(self, sig: int) -> bool: + """Signal every process still in the server's group, reporting + whether any remained. start_new_session() made the process we + launched the group leader, so its pid is the group id whether or + not it is still alive.""" + try: + os.killpg(self.proc.pid, sig) + return True + except OSError: + return False + + def stop(self): + if self.proc is None: + return + self._signal_group(signal.SIGTERM) + if self.proc.poll() is None: + try: + self.stdout, self.stderr = self.proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + self._signal_group(signal.SIGKILL) + self.stdout, self.stderr = self.proc.communicate() + # A parent which died during a failed restart leaves its children + # behind, still holding the listening socket and still answering. + # They have to go too, or the next test finds the port taken. + end = time.time() + 5 + while self._signal_group(0): + if time.time() >= end: + self._signal_group(signal.SIGKILL) + break + time.sleep(0.1) + + def __enter__(self) -> 'ActivatedServer': + return self.start() + + def __exit__(self, *args): + self.stop() + + +class TransientService: + """httpd run as a real transient systemd unit, with systemd-run. + + This is the only harness here which exercises the notification protocol + against systemd itself rather than a stand-in socket: systemd provides + NOTIFY_SOCKET, holds the service in "activating" until READY=1 arrives, + tracks MAINPID, and shows the reported STATUS= as the unit's status + text. It needs a per-user service manager, which a login session has + but a bare CI container does not. + """ + + def __init__(self, env: SystemdTestEnv, port: int, + name: str = None, extra: str = ''): + self.env = env + self.port = port + self.unit = name or f'httpd-test-{os.getpid()}' + self.conf_file = write_server_conf(env, 'transient', port, extra=extra) + self.pid_file = os.path.join(env.server_logs_dir, 'transient.pid') + + def read_pid(self) -> Optional[int]: + try: + with open(self.pid_file) as fd: + return int(fd.read().strip()) + except (OSError, ValueError): + return None + + @staticmethod + def is_available() -> bool: + """Whether this user has a systemd manager to run services under.""" + if shutil.which('systemd-run') is None: + return False + if not os.environ.get('XDG_RUNTIME_DIR'): + return False + # Talking to the manager at all is the test: without a login + # session, or with lingering off, there is none to talk to. + try: + return subprocess.run(['systemctl', '--user', 'show', '-p', + 'Version'], capture_output=True, + timeout=15).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + def systemctl(self, *args) -> subprocess.CompletedProcess: + return subprocess.run(['systemctl', '--user', *args], + capture_output=True, text=True) + + def show(self, prop: str) -> str: + r = self.systemctl('show', '-p', prop, '--value', f'{self.unit}.service') + return r.stdout.strip() + + def start(self, timeout: float = 20.0) -> subprocess.CompletedProcess: + httpd = self.env.httpd_bin + r = subprocess.run([ + 'systemd-run', '--user', '--collect', '--quiet', + '--unit', self.unit, + '--service-type=notify', + '--property=KillMode=mixed', + f'--property=ExecReload={httpd} -d {self.env.server_dir} ' + f'-f {self.conf_file} -k graceful', + httpd, '-DFOREGROUND', + '-d', self.env.server_dir, '-f', self.conf_file, + ], capture_output=True, text=True, timeout=timeout) + return r + + def wait_active(self, timeout: float = 20.0) -> bool: + end = time.time() + timeout + while time.time() < end: + if self.show('ActiveState') == 'active': + return True + time.sleep(0.2) + return False + + def stop(self): + self.systemctl('stop', f'{self.unit}.service') + self.systemctl('reset-failed', f'{self.unit}.service') + + def __enter__(self) -> 'TransientService': + return self + + def __exit__(self, *args): + self.stop() diff --git a/test/modules/arch/linux/test_001_notify.py b/test/modules/arch/linux/test_001_notify.py new file mode 100644 index 00000000000..a6fb1788492 --- /dev/null +++ b/test/modules/arch/linux/test_001_notify.py @@ -0,0 +1,141 @@ +import pytest + +from pyhttpd.conf import HttpdConf + + +class TestSystemdNotify: + """The service notifications mod_systemd sends over $NOTIFY_SOCKET + across the server lifecycle.""" + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + conf = HttpdConf(env) + conf.add_vhost_test1() + conf.install() + # Each test drives startup itself, so leave the server down. + assert env.apache_stop() == 0 + + @staticmethod + def index_of(messages, match): + for i, msg in enumerate(messages): + if match(msg): + return i + return -1 + + def test_systemd_001_01_startup(self, env): + """Startup reports configuration reading, then readiness, then that + the MPM is serving.""" + env.notify.clear() + assert env.apache_restart() == 0 + assert env.notify.wait_for_status(r'^Reading configuration\.\.\.$'), \ + f"no reload notification, got {env.notify.statuses()}" + assert env.notify.wait_for_status(r'^Configuration loaded\.$'), \ + f"no ready notification, got {env.notify.statuses()}" + assert env.notify.wait_for_status(r'^Processing requests\.\.\.$'), \ + f"no pre_mpm notification, got {env.notify.statuses()}" + + def test_systemd_001_02_reloading_before_ready(self, env): + """RELOADING=1 is sent while the configuration is read, and READY=1 + only once it has been.""" + env.notify.clear() + assert env.apache_restart() == 0 + assert env.notify.wait_for_status(r'^Configuration loaded\.$') + msgs = env.notify.messages + reloading = self.index_of( + msgs, lambda m: m.get('RELOADING') == '1' + and m.get('STATUS') == 'Reading configuration...') + ready = self.index_of( + msgs, lambda m: m.get('READY') == '1' + and m.get('STATUS') == 'Configuration loaded.') + assert reloading >= 0 and ready >= 0 + assert reloading < ready, \ + "READY=1 was reported before the configuration was read" + + def test_systemd_001_03_mainpid(self, env): + """MAINPID is the pid of the parent process, the one httpd records + in its pid file.""" + env.notify.clear() + assert env.apache_restart() == 0 + msg = env.notify.wait_for_key('MAINPID') + assert msg, f"no MAINPID reported, got {env.notify.messages}" + assert msg.get('READY') == '1' + assert msg.get('STATUS') == 'Processing requests...' + assert int(msg['MAINPID']) == env.read_pid_file() + + def test_systemd_001_04_reload(self, env): + """A graceful restart reports reading the configuration and then + being ready again.""" + assert env.apache_restart() == 0 + pid = env.read_pid_file() + env.notify.clear() + assert env.apache_reload() == 0 + assert env.notify.wait_for_status(r'^Reading configuration\.\.\.$'), \ + f"no reload notification, got {env.notify.statuses()}" + assert env.notify.wait_for_status(r'^Configuration loaded\.$'), \ + f"no ready notification, got {env.notify.statuses()}" + # The parent survives a graceful restart, and the MPM is not started + # over, so the pid systemd tracks neither changes nor is re-reported. + assert env.read_pid_file() == pid + for msg in env.notify.messages: + assert 'MAINPID' not in msg or int(msg['MAINPID']) == pid + + def test_systemd_001_05_hard_restart(self, env): + """An ungraceful restart starts the MPM over, and re-reports the + main pid, which is still that of the surviving parent.""" + assert env.apache_restart() == 0 + pid = env.read_pid_file() + env.notify.clear() + assert env.apache_hard_restart() == 0 + assert env.notify.wait_for_status(r'^Configuration loaded\.$'), \ + f"no ready notification, got {env.notify.statuses()}" + msg = env.notify.wait_for_key('MAINPID') + assert msg, f"no MAINPID after restart, got {env.notify.messages}" + assert int(msg['MAINPID']) == pid + assert env.read_pid_file() == pid + + def test_systemd_001_06_notify_socket_kept(self, env): + """mod_systemd leaves NOTIFY_SOCKET in the environment, so a second + start after a stop is reported just like the first.""" + assert env.apache_restart() == 0 + assert env.apache_stop() == 0 + env.notify.clear() + assert env.apache_restart() == 0 + assert env.notify.wait_for_status(r'^Configuration loaded\.$') + + @pytest.mark.xfail(reason="mod_systemd sends no STOPPING=1; systemd " + "learns of the shutdown only from the process " + "exiting") + def test_systemd_001_07_stopping(self, env): + """Shutdown ought to be announced before the process goes away.""" + assert env.apache_restart() == 0 + env.notify.clear() + assert env.apache_stop() == 0 + # The server is already gone, so anything it sent has arrived. + assert env.notify.wait_for_key('STOPPING', timeout=1), \ + f"no STOPPING=1 on shutdown, got {env.notify.messages}" + + @pytest.mark.xfail(reason="RELOADING=1 is sent without MONOTONIC_USEC, " + "which Type=notify-reload requires since " + "systemd 253") + def test_systemd_001_08_reloading_monotonic(self, env): + assert env.apache_restart() == 0 + env.notify.clear() + assert env.apache_reload() == 0 + msg = env.notify.wait_for(lambda m: m.get('RELOADING') == '1') + assert msg, "no RELOADING=1 on graceful restart" + assert 'MONOTONIC_USEC' in msg, \ + f"RELOADING=1 sent without MONOTONIC_USEC: {msg}" + + @pytest.mark.xfail(reason="mod_systemd implements no watchdog keepalive, " + "so a unit using WatchdogSec= would be killed") + def test_systemd_001_09_watchdog(self, env): + assert env.apache_stop() == 0 + env.set_httpd_env('WATCHDOG_USEC', '2000000') # ping every 1s + try: + env.notify.clear() + assert env.apache_restart() == 0 + assert env.notify.wait_for_key('WATCHDOG', timeout=4), \ + "no watchdog keepalive was sent" + finally: + env.set_httpd_env('WATCHDOG_USEC', None) + assert env.apache_restart() == 0 diff --git a/test/modules/arch/linux/test_002_monitor.py b/test/modules/arch/linux/test_002_monitor.py new file mode 100644 index 00000000000..659ae4e1038 --- /dev/null +++ b/test/modules/arch/linux/test_002_monitor.py @@ -0,0 +1,103 @@ +import re + +import pytest + +from pyhttpd.conf import HttpdConf + +from .env import MONITOR_STATUS, MONITOR_TIMEOUT + + +class TestSystemdMonitor: + """The periodic STATUS= line the monitor hook reports, which is what + "systemctl status httpd" shows below the unit description.""" + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + conf = HttpdConf(env, extras={ + 'base': """ + + SetHandler server-status + + """ + }) + conf.add_vhost_test1() + conf.install() + assert env.apache_restart() == 0 + + @staticmethod + def wait_report(env, above=-1): + """Wait for a report from the monitor hook accounting for more than + `above` requests, and return its fields.""" + msg = env.notify.wait_for( + lambda m: 'STATUS' in m + and (g := MONITOR_STATUS.match(m['STATUS'])) is not None + and int(g.group('requests')) > above, + timeout=MONITOR_TIMEOUT) + assert msg, f"no monitor report within {MONITOR_TIMEOUT}s, " \ + f"server {'up' if env.is_live() else 'down'}, " \ + f"got {env.notify.messages}" + return MONITOR_STATUS.match(msg['STATUS']) + + @pytest.fixture(scope='class') + def reports(self, env, _class_scope): + """Two consecutive reports with requests served between them, and + the mod_status report as of the second. + + The hook runs once every ten turns of the parent's one second + loop, so waiting for a report costs up to ten seconds. The tests + below share one pair rather than each waiting for its own. + """ + env.notify.clear() + first = self.wait_report(env) + for _ in range(10): + r = env.curl_get(env.mkurl("http", "test1", "/")) + assert r.response['status'] == 200 + second = self.wait_report(env, above=int(first.group('requests'))) + r = env.curl_get(env.mkurl("http", "test1", "/server-status?auto")) + assert r.response['status'] == 200 + auto = {} + for line in r.response['body'].decode().splitlines(): + key, sep, value = line.partition(':') + if sep and key not in auto: + auto[key] = value.strip() + return {'first': first, 'second': second, 'auto': auto} + + def test_systemd_002_01_report_format(self, reports): + m = reports['first'] + assert int(m.group('requests')) >= 0 + assert m.group('bps') + + def test_systemd_002_02_requests_counted(self, reports): + """The request count reported to systemd tracks requests served.""" + before = int(reports['first'].group('requests')) + after = int(reports['second'].group('requests')) + assert after >= before + 10, \ + f"{after - before} requests reported, at least 10 were served" + + def test_systemd_002_03_rates_are_finite(self, reports): + """Neither rate is inf or nan. + + systemd_monitor() divides by an uptime in whole seconds, which is + zero for a report arriving in the first second after the scoreboard + records a restart. + """ + for which in ('first', 'second'): + m = reports[which] + rate = m.group('rate') + assert re.match(r'^-?\d', rate), f"{which} request rate is {rate!r}" + assert float(rate) >= 0 + assert 'inf' not in m.group('bps') and 'nan' not in m.group('bps'), \ + f"{which} byte rate is {m.group('bps')!r}" + + def test_systemd_002_04_report_repeats(self, env, reports): + """The status line keeps being refreshed while the server runs.""" + assert reports['first'].group(0) != reports['second'].group(0) + + @pytest.mark.xfail(reason="ap_get_sload() reports idle and busy as " + "percentages, but mod_systemd labels them " + "'Idle/Busy workers'") + def test_systemd_002_05_worker_counts(self, reports): + """The numbers presented as worker counts are worker counts.""" + m, auto = reports['second'], reports['auto'] + assert int(m.group('idle')) == int(auto['IdleWorkers']) + assert int(m.group('busy')) == int(auto['BusyWorkers']) diff --git a/test/modules/arch/linux/test_003_extended_status.py b/test/modules/arch/linux/test_003_extended_status.py new file mode 100644 index 00000000000..badc1483944 --- /dev/null +++ b/test/modules/arch/linux/test_003_extended_status.py @@ -0,0 +1,66 @@ +import pytest + +from pyhttpd.conf import HttpdConf + +from .env import NO_MONITOR_TIMEOUT + + +class TestSystemdExtendedStatus: + """mod_systemd turns ExtendedStatus on so that it has request counts to + report, which changes what the rest of the server records too.""" + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + yield + conf = HttpdConf(env) + conf.add_vhost_test1() + conf.install() + assert env.apache_restart() == 0 + + def auto_report(self, env): + r = env.curl_get(env.mkurl("http", "test1", "/server-status?auto")) + assert r.response['status'] == 200 + return r.response['body'].decode() + + def install(self, env, extra=''): + conf = HttpdConf(env, extras={ + 'base': f""" + {extra} + + SetHandler server-status + + """ + }) + conf.add_vhost_test1() + conf.install() + assert env.apache_restart() == 0 + + def test_systemd_003_01_enabled_by_default(self, env): + """Loading mod_systemd is enough to get extended status; no + ExtendedStatus directive is present in this configuration.""" + self.install(env) + body = self.auto_report(env) + assert 'Total Accesses:' in body, \ + "ExtendedStatus was not enabled by mod_systemd" + assert 'Total kBytes:' in body + + def test_systemd_003_02_directive_wins(self, env): + """An explicit ExtendedStatus off still takes effect: mod_systemd + sets the default in pre_config, before the configuration is read.""" + self.install(env, extra='ExtendedStatus off') + body = self.auto_report(env) + assert 'Total Accesses:' not in body, \ + "ExtendedStatus off was overridden by mod_systemd" + + def test_systemd_003_03_no_report_without_extended_status(self, env): + """With extended status off the monitor hook declines, so no status + line is reported to systemd.""" + env.notify.clear() + self.install(env, extra='ExtendedStatus off') + # Startup notifications are still sent... + assert env.notify.wait_for_status(r'^Processing requests\.\.\.$', + timeout=10) is not None + # ...but the periodic status line is not. + assert env.notify.wait_for_status(r'^Total requests: ', + timeout=NO_MONITOR_TIMEOUT) is None, \ + "a monitor report was sent with ExtendedStatus off" diff --git a/test/modules/arch/linux/test_004_socket_activation.py b/test/modules/arch/linux/test_004_socket_activation.py new file mode 100644 index 00000000000..bd1339a9cf1 --- /dev/null +++ b/test/modules/arch/linux/test_004_socket_activation.py @@ -0,0 +1,90 @@ +import pytest + +from .env import ActivatedServer + + +class TestSystemdSocketActivation: + """Listening sockets passed in by the service manager. + + mod_systemd exports the two optional functions server/listen.c uses to + find them, so socket activation is enabled by loading the module and + disabled by not loading it, whatever the environment says. + + apachectl cannot pass file descriptors, so these tests open the + listening socket themselves and run httpd directly, in the foreground, + with the descriptor and the environment a service manager would give + it. No systemd process is involved. + """ + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + # The activated servers use the same server root; keep the one + # started by other tests out of the way. + assert env.apache_stop() == 0 + yield + assert env.apache_stop() == 0 + + def test_systemd_004_01_activated_listener(self, env): + """httpd serves on a socket it never opened itself.""" + env.notify.clear() + with ActivatedServer(env, port=env.http_port2) as server: + assert server.is_live(), \ + f"server did not come up: {server.stderr}" + r = env.curl_get(f"http://{env.http_addr}:{env.http_port2}/") + assert r.response['status'] == 200 + # The notification handshake works the same way as when httpd opens + # its own sockets. + assert env.notify.wait_for_status(r'^Configuration loaded\.$', timeout=0) + msg = env.notify.wait_for_key('MAINPID', timeout=0) + assert msg, f"httpd sent no MAINPID, got {env.notify.messages}" + assert msg['STATUS'] == 'Processing requests...' + + def test_systemd_004_02_no_socket_for_port(self, env): + """A Listen port the service manager did not pass is an error, not + a port httpd quietly opens for itself.""" + server = ActivatedServer(env, port=env.http_port2, + listen_port=env.proxy_port, + name='activate-wrongport') + with server: + assert server.wait_exit() != 0, "httpd started without a socket" + assert b'not configured in systemd' in server.stderr, \ + f"unexpected startup diagnostic: {server.stderr}" + + def test_systemd_004_03_disabled_without_module(self, env): + """Without mod_systemd the passed sockets are ignored and httpd + opens the configured port itself, the same arrangement that fails + in the test above.""" + if not env.systemd_is_dso: + pytest.skip("mod_systemd is linked statically and cannot be " + "left out of the configuration") + modules_conf = ActivatedServer.modules_conf_without(env, 'systemd') + server = ActivatedServer(env, port=env.http_port2, + listen_port=env.proxy_port, + name='activate-nomodule', + modules_conf=modules_conf) + with server: + assert server.is_live(), \ + f"server did not come up: {server.stderr}" + r = env.curl_get(f"http://{env.http_addr}:{env.http_port2}/") + assert r.response['status'] == 200 + + @pytest.mark.xfail(reason="ap_setup_listeners() clears LISTEN_FDS from " + "the environment after startup, so re-reading " + "the configuration finds no sockets") + def test_systemd_004_04_graceful_restart(self, env): + """An activated server survives a graceful restart.""" + with ActivatedServer(env, port=env.http_port2, + name='activate-reload') as server: + assert server.is_live(), \ + f"server did not come up: {server.stderr}" + server.reload() + env.httpd_error_log.ignore_recent(lognos=['AH02487']) + # Check the parent first: when it dies here its children carry + # on holding the listening socket and answering, so a request + # succeeding proves nothing on its own. + assert server.is_running(), \ + "the parent exited on graceful restart" + assert server.is_live(timeout=5), \ + "the server did not survive a graceful restart" + r = env.curl_get(f"http://{env.http_addr}:{env.http_port2}/") + assert r.response['status'] == 200 diff --git a/test/modules/arch/linux/test_005_service.py b/test/modules/arch/linux/test_005_service.py new file mode 100644 index 00000000000..ef220426302 --- /dev/null +++ b/test/modules/arch/linux/test_005_service.py @@ -0,0 +1,89 @@ +import time + +import pytest + +from .env import MONITOR_STATUS, MONITOR_TIMEOUT, TransientService, http_responds + +pytestmark = pytest.mark.skipif( + not TransientService.is_available(), + reason="no per-user systemd manager to run a transient service under") + + +class TestSystemdService: + """httpd as a real systemd service, end to end. + + Everything else here checks what mod_systemd sends. These check what + systemd does with it: hold the unit in "activating" until httpd is + ready, track the right process, and show the reported status text. + """ + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + assert env.apache_stop() == 0 + yield + assert env.apache_stop() == 0 + + @pytest.fixture + def service(self, env) -> TransientService: + svc = TransientService(env, port=env.http_port2) + yield svc + svc.stop() + + def test_systemd_005_01_type_notify(self, env, service): + """systemd-run returns once the unit is active, which for a + Type=notify service means READY=1 has been received, which + mod_systemd sends only after the configuration is loaded.""" + r = service.start() + assert r.returncode == 0, f"systemd-run failed: {r.stderr}" + assert service.show('ActiveState') == 'active' + assert http_responds(env.http_port2), \ + "the unit was active before the server would answer" + + def test_systemd_005_02_main_pid(self, env, service): + """The process systemd tracks is the httpd parent.""" + assert service.start().returncode == 0 + assert service.wait_active() + main_pid = int(service.show('MainPID')) + assert main_pid > 0 + assert main_pid == service.read_pid() + + def test_systemd_005_03_status_text(self, env, service): + """The status line "systemctl status" shows comes from the monitor + hook, and is refreshed while the server runs.""" + assert service.start().returncode == 0 + assert service.wait_active() + # First the post_config report, then the periodic one. + assert service.show('StatusText') in ('Configuration loaded.', + 'Processing requests...') + end = time.time() + MONITOR_TIMEOUT + text = None + while time.time() < end: + text = service.show('StatusText') + if MONITOR_STATUS.match(text): + break + time.sleep(0.5) + assert MONITOR_STATUS.match(text), \ + f"status text was never refreshed by the monitor hook: {text!r}" + + def test_systemd_005_04_reload(self, env, service): + """systemctl reload runs httpd -k graceful and the unit stays + active throughout.""" + assert service.start().returncode == 0 + assert service.wait_active() + pid = int(service.show('MainPID')) + r = service.systemctl('reload', f'{service.unit}.service') + assert r.returncode == 0, f"reload failed: {r.stderr}" + assert service.show('ActiveState') == 'active' + assert int(service.show('MainPID')) == pid + assert http_responds(env.http_port2) + + def test_systemd_005_05_stop(self, env, service): + """The unit stops cleanly, without systemd having to time out and + kill it.""" + assert service.start().returncode == 0 + assert service.wait_active() + r = service.systemctl('stop', f'{service.unit}.service') + assert r.returncode == 0, f"stop failed: {r.stderr}" + assert service.show('ActiveState') == 'inactive' + assert service.show('Result') == 'success' + assert not http_responds(env.http_port2) diff --git a/test/pyhttpd/conf/stop.conf.template b/test/pyhttpd/conf/stop.conf.template index 21bae845f8d..5e76b2d6e24 100644 --- a/test/pyhttpd/conf/stop.conf.template +++ b/test/pyhttpd/conf/stop.conf.template @@ -5,6 +5,11 @@ ServerName localhost ServerRoot "${server_dir}" +# Must agree with httpd.conf, or the running server cannot be found: +# the built-in default varies between httpd versions. +DefaultRuntimeDir logs +PidFile httpd.pid + Include "conf/modules.conf" DocumentRoot "${server_dir}/htdocs" diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index a30a027fb93..e9fdc5d3170 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -303,6 +303,7 @@ def __init__(self, pytestconfig=None): self._verbosity = pytestconfig.option.verbose if pytestconfig is not None else 0 self._test_conf = os.path.join(self._server_conf_dir, "test.conf") self._httpd_base_conf = [] + self._httpd_env = {} self._httpd_log_modules = ['aptest'] self._log_interesting = None self._setup = None @@ -327,6 +328,19 @@ def add_httpd_conf(self, lines: List[str]): def add_httpd_log_modules(self, modules: List[str]): self._httpd_log_modules.extend(modules) + def set_httpd_env(self, name: str, value: Optional[str]): + """Add a variable to the environment httpd is started with, or + remove it again when passed None. + + Used by tests for modules which take their input from the + environment rather than from the configuration, such as + mod_systemd reading $NOTIFY_SOCKET. + """ + if value is None: + self._httpd_env.pop(name, None) + else: + self._httpd_env[name] = value + def issue_certs(self): if self._ca is None: self._ca = HttpdTestCA.create_root(name=self.http_tld, @@ -690,6 +704,7 @@ def _clean_path_env(self) -> dict: parts.insert(0, venv_bin) env = os.environ.copy() env['PATH'] = os.pathsep.join(parts) + env.update(self._httpd_env) return env def _run_apachectl(self, cmd) -> ExecResult: From 75691afe5c149108db2e891fc7574c03511f36f1 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sat, 29 Aug 2026 09:11:16 +0000 Subject: [PATCH 03/13] * modules/arch/unix/mod_systemd.c (systemd_monitor): Avoid dividing by a zero uptime, and report the idle and busy figures as percentages. Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #723 Merge r1937597 from trunk. (cherry picked from commit f9ebab5e548c63b8e55b6bd85927c074b1064ac3) --- modules/arch/unix/mod_systemd.c | 17 +++++++++++------ test/modules/arch/linux/env.py | 6 +++--- test/modules/arch/linux/test_002_monitor.py | 18 +++++++++++------- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/modules/arch/unix/mod_systemd.c b/modules/arch/unix/mod_systemd.c index 395da965a2d..084949522b7 100644 --- a/modules/arch/unix/mod_systemd.c +++ b/modules/arch/unix/mod_systemd.c @@ -106,15 +106,20 @@ static int systemd_monitor(apr_pool_t *p, server_rec *s) } ap_get_sload(&sload); - /* up_time in seconds */ - up_time = (apr_uint32_t) apr_time_sec(apr_time_now() - - ap_scoreboard_image->global->restart_time); + /* up_time in seconds, and never zero: a restart resets restart_time, + * so this hook can run in the same second it was set. */ + up_time = apr_time_sec(apr_time_now() - + ap_scoreboard_image->global->restart_time); + if (up_time < 1) { + up_time = 1; + } - apr_strfsize((unsigned long)((float) (sload.bytes_served) - / (float) up_time), bps); + apr_strfsize(sload.bytes_served / up_time, bps); + /* ap_get_sload() gives idle and busy as percentages of the workers + * available, not as counts. */ sd_notifyf(0, "READY=1\n" - "STATUS=Total requests: %lu; Idle/Busy workers %d/%d; " + "STATUS=Total requests: %lu; Idle/Busy workers %d%%/%d%%; " "Requests/sec: %.3g; Bytes served/sec: %sB/sec\n", sload.access_count, sload.idle, sload.busy, ((float) sload.access_count) / (float) up_time, bps); diff --git a/test/modules/arch/linux/env.py b/test/modules/arch/linux/env.py index a09fb5f7d68..96c2bbb2cbd 100644 --- a/test/modules/arch/linux/env.py +++ b/test/modules/arch/linux/env.py @@ -212,11 +212,11 @@ def statuses(self) -> List[str]: # The STATUS= line the monitor hook reports, from systemd_monitor() in -# modules/arch/unix/mod_systemd.c. "Idle/Busy workers" are in fact the -# percentages computed by ap_get_sload(), not worker counts. +# modules/arch/unix/mod_systemd.c. Idle and busy are the percentages +# ap_get_sload() computes, and are -1 when there are no workers at all. MONITOR_STATUS = re.compile( r'^Total requests: (?P\d+);\s*' - r'Idle/Busy workers (?P-?\d+)/(?P-?\d+);\s*' + r'Idle/Busy workers (?P-?\d+)%/(?P-?\d+)%;\s*' r'Requests/sec: (?P\S+);\s*' r'Bytes served/sec: (?P.*)B/sec$') diff --git a/test/modules/arch/linux/test_002_monitor.py b/test/modules/arch/linux/test_002_monitor.py index 659ae4e1038..368d83ae74f 100644 --- a/test/modules/arch/linux/test_002_monitor.py +++ b/test/modules/arch/linux/test_002_monitor.py @@ -93,11 +93,15 @@ def test_systemd_002_04_report_repeats(self, env, reports): """The status line keeps being refreshed while the server runs.""" assert reports['first'].group(0) != reports['second'].group(0) - @pytest.mark.xfail(reason="ap_get_sload() reports idle and busy as " - "percentages, but mod_systemd labels them " - "'Idle/Busy workers'") - def test_systemd_002_05_worker_counts(self, reports): - """The numbers presented as worker counts are worker counts.""" + def test_systemd_002_05_worker_percentages(self, reports): + """Idle and busy are percentages of the workers available, and are + reported as such.""" m, auto = reports['second'], reports['auto'] - assert int(m.group('idle')) == int(auto['IdleWorkers']) - assert int(m.group('busy')) == int(auto['BusyWorkers']) + idle, busy = int(m.group('idle')), int(m.group('busy')) + assert 0 <= idle <= 100 and 0 <= busy <= 100 + # Integer division loses at most one point between the two. + assert 99 <= idle + busy <= 100 + # Busy workers are a minority of a mostly idle test server, which + # is what mod_status reports over the same scoreboard. + assert int(auto['IdleWorkers']) > int(auto['BusyWorkers']) + assert idle > busy From 90fde14d9e4a830ec55e46e90d57b8a2e1f4f704 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sat, 29 Aug 2026 09:11:29 +0000 Subject: [PATCH 04/13] * modules/arch/unix/mod_systemd.c (ap_systemd_listen_fds, ap_find_systemd_socket): Remember the number of sockets passed, so a graceful restart of a socket-activated server works. Drop the unchecked getenv("LISTEN_FDS"). Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #723 Merge r1937598 from trunk. (cherry picked from commit 3e26a675719a32a5f37dcc5ba878dad5df683be5) --- modules/arch/unix/mod_systemd.c | 33 ++++++++++++++----- .../arch/linux/test_004_socket_activation.py | 7 ++-- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/modules/arch/unix/mod_systemd.c b/modules/arch/unix/mod_systemd.c index 084949522b7..ca5523aa05a 100644 --- a/modules/arch/unix/mod_systemd.c +++ b/modules/arch/unix/mod_systemd.c @@ -127,9 +127,31 @@ static int systemd_monitor(apr_pool_t *p, server_rec *s) return DECLINED; } +/* The number of sockets passed by the service manager has to be + * remembered: the configuration is read again on restart, by which time + * the environment sd_listen_fds() reads has been cleared, and the module + * itself has been unloaded and loaded again. Hence retained data rather + * than a static. */ +static const char *const retained_key = "mod_systemd_listen_fds"; + +static int ap_systemd_listen_fds(int unset_environment) +{ + int *fds = ap_retained_data_get(retained_key); + + if (fds == NULL) { + fds = ap_retained_data_create(retained_key, sizeof(*fds)); + *fds = sd_listen_fds(0); + } + if (unset_environment) { + /* Take the variables out of the environment, keeping the count. */ + sd_listen_fds(1); + } + return *fds; +} + static int ap_find_systemd_socket(process_rec * process, apr_port_t port) { - int fdcount, fd; - int sdc = sd_listen_fds(0); + int fd; + int sdc = ap_systemd_listen_fds(0); if (sdc < 0) { ap_log_perror(APLOG_MARK, APLOG_CRIT, sdc, process->pool, APLOGNO(02486) @@ -144,8 +166,7 @@ static int ap_find_systemd_socket(process_rec * process, apr_port_t port) { return -1; } - fdcount = atoi(getenv("LISTEN_FDS")); - for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + fdcount; fd++) { + for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + sdc; fd++) { if (sd_is_socket_inet(fd, 0, 0, -1, port) > 0) { return fd; } @@ -154,10 +175,6 @@ static int ap_find_systemd_socket(process_rec * process, apr_port_t port) { return -1; } -static int ap_systemd_listen_fds(int unset_environment){ - return sd_listen_fds(unset_environment); -} - static void systemd_register_hooks(apr_pool_t *p) { APR_REGISTER_OPTIONAL_FN(ap_systemd_listen_fds); diff --git a/test/modules/arch/linux/test_004_socket_activation.py b/test/modules/arch/linux/test_004_socket_activation.py index bd1339a9cf1..b92fadd36c7 100644 --- a/test/modules/arch/linux/test_004_socket_activation.py +++ b/test/modules/arch/linux/test_004_socket_activation.py @@ -68,11 +68,10 @@ def test_systemd_004_03_disabled_without_module(self, env): r = env.curl_get(f"http://{env.http_addr}:{env.http_port2}/") assert r.response['status'] == 200 - @pytest.mark.xfail(reason="ap_setup_listeners() clears LISTEN_FDS from " - "the environment after startup, so re-reading " - "the configuration finds no sockets") def test_systemd_004_04_graceful_restart(self, env): - """An activated server survives a graceful restart.""" + """An activated server survives a graceful restart, which means + finding the passed sockets again after the environment naming them + has been cleared.""" with ActivatedServer(env, port=env.http_port2, name='activate-reload') as server: assert server.is_live(), \ From e33edfcca92150c8de077a6b1d48f51a61a92dd9 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sat, 29 Aug 2026 09:11:39 +0000 Subject: [PATCH 05/13] * modules/arch/unix/mod_systemd.c (monotonic_usec, systemd_pre_config): Send MONOTONIC_USEC with RELOADING=1. Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #723 Merge r1937599 from trunk. (cherry picked from commit 651b6a9a7a6c3286b84d4aa40dfe02f32918489f) --- modules/arch/unix/mod_systemd.c | 31 +++++++++++++++++++--- test/modules/arch/linux/test_001_notify.py | 8 +++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/modules/arch/unix/mod_systemd.c b/modules/arch/unix/mod_systemd.c index ca5523aa05a..2aa981c4907 100644 --- a/modules/arch/unix/mod_systemd.c +++ b/modules/arch/unix/mod_systemd.c @@ -16,6 +16,7 @@ */ #include +#include #include #include "ap_mpm.h" #include "ap_listen.h" @@ -39,12 +40,36 @@ #include #endif +/* Microseconds on the clock systemd compares RELOADING=1 against, or + * zero if it cannot be read. */ +static apr_uint64_t monotonic_usec(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + return (apr_uint64_t)ts.tv_sec * APR_USEC_PER_SEC + ts.tv_nsec / 1000; +} + static int systemd_pre_config(apr_pool_t *pconf, apr_pool_t *plog, apr_pool_t *ptemp) { - sd_notify(0, - "RELOADING=1\n" - "STATUS=Reading configuration...\n"); + apr_uint64_t usec = monotonic_usec(); + + /* A Type=notify-reload service ignores a reload notification which + * does not say when it was sent. */ + if (usec) { + sd_notifyf(0, + "RELOADING=1\n" + "MONOTONIC_USEC=%" APR_UINT64_T_FMT "\n" + "STATUS=Reading configuration...\n", usec); + } + else { + sd_notify(0, + "RELOADING=1\n" + "STATUS=Reading configuration...\n"); + } ap_extended_status = 1; return OK; } diff --git a/test/modules/arch/linux/test_001_notify.py b/test/modules/arch/linux/test_001_notify.py index a6fb1788492..38745bb5c7b 100644 --- a/test/modules/arch/linux/test_001_notify.py +++ b/test/modules/arch/linux/test_001_notify.py @@ -1,3 +1,5 @@ +import time + import pytest from pyhttpd.conf import HttpdConf @@ -114,9 +116,6 @@ def test_systemd_001_07_stopping(self, env): assert env.notify.wait_for_key('STOPPING', timeout=1), \ f"no STOPPING=1 on shutdown, got {env.notify.messages}" - @pytest.mark.xfail(reason="RELOADING=1 is sent without MONOTONIC_USEC, " - "which Type=notify-reload requires since " - "systemd 253") def test_systemd_001_08_reloading_monotonic(self, env): assert env.apache_restart() == 0 env.notify.clear() @@ -125,6 +124,9 @@ def test_systemd_001_08_reloading_monotonic(self, env): assert msg, "no RELOADING=1 on graceful restart" assert 'MONOTONIC_USEC' in msg, \ f"RELOADING=1 sent without MONOTONIC_USEC: {msg}" + # systemd compares this against its own reading of the same clock. + assert 0 < int(msg['MONOTONIC_USEC']) <= time.clock_gettime_ns( + time.CLOCK_MONOTONIC) // 1000 @pytest.mark.xfail(reason="mod_systemd implements no watchdog keepalive, " "so a unit using WatchdogSec= would be killed") From 6162e49b36db63b7c11776b8f1d6ac5ce11bb152 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sat, 29 Aug 2026 09:12:09 +0000 Subject: [PATCH 06/13] * modules/arch/unix/mod_systemd.c (systemd_stopping, systemd_post_config): Notify STOPPING=1 on shutdown. Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #723 Merge r1937600 from trunk. (cherry picked from commit 75d0e8c0c02243c9bd559045c9ae05d0057e1062) --- modules/arch/unix/mod_systemd.c | 16 ++++++++++++++++ test/modules/arch/linux/test_001_notify.py | 11 +++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/modules/arch/unix/mod_systemd.c b/modules/arch/unix/mod_systemd.c index 2aa981c4907..43b24ef2d0b 100644 --- a/modules/arch/unix/mod_systemd.c +++ b/modules/arch/unix/mod_systemd.c @@ -88,6 +88,17 @@ static void log_selinux_context(void) } #endif +/* pconf is also cleared on a restart, where the service is not stopping + * at all, so distinguish the two by the state of the process. */ +static apr_status_t systemd_stopping(void *unused) +{ + if (ap_state_query(AP_SQ_MAIN_STATE) == AP_SQ_MS_EXITING) { + sd_notify(0, "STOPPING=1\n" + "STATUS=Shutting down.\n"); + } + return APR_SUCCESS; +} + /* Report the service is ready in post_config, which could be during * startup or after a reload. The server could still hit a fatal * startup error after this point during ap_run_mpm(), so this is @@ -105,6 +116,11 @@ static int systemd_post_config(apr_pool_t *pconf, apr_pool_t *plog, log_selinux_context(); #endif + /* Not reached by "httpd -k stop" and friends, which signal the + * running server and exit before post_config. */ + apr_pool_cleanup_register(pconf, NULL, systemd_stopping, + apr_pool_cleanup_null); + sd_notify(0, "READY=1\n" "STATUS=Configuration loaded.\n"); return OK; diff --git a/test/modules/arch/linux/test_001_notify.py b/test/modules/arch/linux/test_001_notify.py index 38745bb5c7b..76b999fcf26 100644 --- a/test/modules/arch/linux/test_001_notify.py +++ b/test/modules/arch/linux/test_001_notify.py @@ -104,17 +104,16 @@ def test_systemd_001_06_notify_socket_kept(self, env): assert env.apache_restart() == 0 assert env.notify.wait_for_status(r'^Configuration loaded\.$') - @pytest.mark.xfail(reason="mod_systemd sends no STOPPING=1; systemd " - "learns of the shutdown only from the process " - "exiting") def test_systemd_001_07_stopping(self, env): - """Shutdown ought to be announced before the process goes away.""" + """Shutdown is announced before the process goes away.""" assert env.apache_restart() == 0 env.notify.clear() assert env.apache_stop() == 0 # The server is already gone, so anything it sent has arrived. - assert env.notify.wait_for_key('STOPPING', timeout=1), \ - f"no STOPPING=1 on shutdown, got {env.notify.messages}" + msg = env.notify.wait_for_key('STOPPING', timeout=1) + assert msg, f"no STOPPING=1 on shutdown, got {env.notify.messages}" + assert msg['STOPPING'] == '1' + assert msg.get('STATUS') == 'Shutting down.' def test_systemd_001_08_reloading_monotonic(self, env): assert env.apache_restart() == 0 From 342a47d0255910989ff7fc69812ea25e3eb0ebe0 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sat, 29 Aug 2026 09:12:17 +0000 Subject: [PATCH 07/13] * docs/manual/mod/mod_systemd.xml: Socket activation is supported. Generated HTML not rebuilt. Co-Authored-By: Claude Opus 5 (1M context) GitHub: closes #723 Merge r1937601 from trunk. (cherry picked from commit 7bbc2b3c7ba19ff9c41379a63ca0ce473b8b250a) --- docs/manual/mod/mod_systemd.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/manual/mod/mod_systemd.xml b/docs/manual/mod/mod_systemd.xml index 089d6296609..043ec0bd674 100644 --- a/docs/manual/mod/mod_systemd.xml +++ b/docs/manual/mod/mod_systemd.xml @@ -69,7 +69,12 @@ WantedBy=multi-user.target href="https://www.freedesktop.org/software/systemd/man/systemd.kill.html">systemd.kill(5) for more information.

-

This module does not provide support for Systemd socket activation.

+

Systemd socket activation is supported if httpd was built with + it. Each Listen port + must then be one passed in by systemd; a port which was not is a + fatal configuration error rather than one httpd opens for itself. + Socket activation is used only if this module is loaded, so it can + be built in and left unused.

ExtendedStatus is enabled by default if the module is loaded. If Date: Sat, 29 Aug 2026 09:19:37 +0000 Subject: [PATCH 08/13] * test/modules/arch/: Add missing __init__.py files lost in translation from the PR. Fixes: 4abdd206a7a8d590e0e09adec22287f8b1a5e8be (r1937596) Merge r1937602 from trunk. (cherry picked from commit 4fe1b8aafbe77bb5988bbfae008cd03db76a17b9) --- test/modules/arch/__init__.py | 0 test/modules/arch/linux/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/modules/arch/__init__.py create mode 100644 test/modules/arch/linux/__init__.py diff --git a/test/modules/arch/__init__.py b/test/modules/arch/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/modules/arch/linux/__init__.py b/test/modules/arch/linux/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 0d1a17288a221e08e382dd50acc92cae063690d4 Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sun, 30 Aug 2026 06:09:01 +0000 Subject: [PATCH 09/13] * test/pyhttpd/env.py (HttpdTestEnv.apache_hard_restart, read_pid_file): Moved here from the mod_systemd tests, for use by other suites. [skip ci] Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #727 Merge r1937635 from trunk. (cherry picked from commit 31d91bb768e7883f909ff1cb621811ccc608d2c7) --- test/modules/arch/linux/env.py | 20 -------------------- test/pyhttpd/env.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/test/modules/arch/linux/env.py b/test/modules/arch/linux/env.py index 96c2bbb2cbd..0e3b4f6dd9d 100644 --- a/test/modules/arch/linux/env.py +++ b/test/modules/arch/linux/env.py @@ -8,7 +8,6 @@ import subprocess import threading import time -from datetime import timedelta from typing import Callable, Dict, List, Optional from pyhttpd.env import HttpdTestEnv, HttpdTestSetup @@ -81,25 +80,6 @@ def server_env(self) -> Dict[str, str]: def httpd_bin(self) -> str: return os.path.join(self.bin_dir, 'httpd') - def apache_hard_restart(self) -> int: - """Restart without the "graceful" flag, so the MPM starts over.""" - r = self._run_apachectl("restart") - if r.exit_code == 0: - return 0 if self.is_live(self._http_base, timeout=timedelta(seconds=10)) else -1 - return r.exit_code - - def read_pid_file(self, name: str = 'httpd.pid') -> Optional[int]: - # Where PidFile lands depends on how the httpd under test resolves - # a relative path against DefaultRuntimeDir, which has differed - # between versions; look in both places rather than assume. - for d in (self.server_logs_dir, self.server_dir): - try: - with open(os.path.join(d, name)) as fd: - return int(fd.read().strip()) - except (OSError, ValueError): - continue - return None - class NotifyListener: """A stand-in for the systemd notification socket. diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index e9fdc5d3170..695176d54c7 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -756,6 +756,26 @@ def apache_fail(self): rv = 0 return rv + def apache_hard_restart(self) -> int: + """Restart without the "graceful" flag, so the MPM starts over.""" + r = self._run_apachectl("restart") + if r.exit_code == 0: + return 0 if self.is_live(self._http_base, + timeout=timedelta(seconds=10)) else -1 + return r.exit_code + + def read_pid_file(self, name: str = 'httpd.pid') -> Optional[int]: + # Where PidFile lands depends on how the httpd under test resolves + # a relative path against DefaultRuntimeDir, which has differed + # between versions; look in both places rather than assume. + for d in (self._server_logs_dir, self._server_dir): + try: + with open(os.path.join(d, name)) as fd: + return int(fd.read().strip()) + except (OSError, ValueError): + continue + return None + def apache_access_log_clear(self): if os.path.isfile(self._server_access_log): os.remove(self._server_access_log) From a857e78a2c61fb6ca6f93a500cc6be1adab75ced Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sun, 30 Aug 2026 08:57:27 +0000 Subject: [PATCH 10/13] mod_systemd: Implement watchdog support: * modules/arch/unix/mod_systemd.c (watchdog_ping): New. (systemd_pre_config, systemd_post_config, systemd_monitor): Send the watchdog keep-alive, and warn (AH10621) if WatchdogSec is too short to be met. * docs/manual/mod/mod_systemd.xml: Document watchdog support. * test/modules/arch/linux/test_006_watchdog.py: New test suite. * test/modules/arch/linux/env.py (ForegroundServer): New, running httpd in the foreground so $WATCHDOG_PID can name it. (TransientService): Take extra unit properties. * test/modules/arch/linux/test_001_notify.py (test_systemd_001_09_watchdog): No longer xfail; use a WatchdogSec which does not draw AH10621. * test/modules/arch/linux/README: Describe the watchdog tests. Co-Authored-By: Claude Opus 5 (1M context) GitHub: closes #728 Merge r1937644 from trunk. (cherry picked from commit 06c0a97f628c6f714fffd0d8ceee6595c6e87657) --- changes-entries/systemd-watchdog.txt | 2 + docs/manual/mod/mod_systemd.xml | 25 +++ modules/arch/unix/mod_systemd.c | 69 +++++- test/modules/arch/linux/README | 26 ++- test/modules/arch/linux/env.py | 68 +++++- test/modules/arch/linux/test_001_notify.py | 20 +- test/modules/arch/linux/test_006_watchdog.py | 211 +++++++++++++++++++ 7 files changed, 405 insertions(+), 16 deletions(-) create mode 100644 changes-entries/systemd-watchdog.txt create mode 100644 test/modules/arch/linux/test_006_watchdog.py diff --git a/changes-entries/systemd-watchdog.txt b/changes-entries/systemd-watchdog.txt new file mode 100644 index 00000000000..2fca28a6e4e --- /dev/null +++ b/changes-entries/systemd-watchdog.txt @@ -0,0 +1,2 @@ + *) mod_systemd: Support the systemd watchdog, sending the keep-alive + notification if the service unit sets WatchdogSec=. [Joe Orton] diff --git a/docs/manual/mod/mod_systemd.xml b/docs/manual/mod/mod_systemd.xml index 043ec0bd674..fd324a533f5 100644 --- a/docs/manual/mod/mod_systemd.xml +++ b/docs/manual/mod/mod_systemd.xml @@ -81,6 +81,31 @@ WantedBy=multi-user.target module="core">ExtendedStatus is not disabled in the configuration, run-time load and request statistics are made available in the systemctl status output.

+ +

The systemd watchdog is supported. If the service unit sets + WatchdogSec=, the parent process sends the keep-alive + notification which tells systemd the server is still alive; a server + which stops sending it is terminated and, with a suitable + Restart= setting, restarted. The notification is sent + while the configuration is being read and again once it is loaded, so + that a reload is covered, and periodically from the parent process + while the server runs.

+ +

That periodic notification is sent about every ten seconds, which + is how often the parent process runs the hook it is sent from. A + WatchdogSec= of less than twice that cannot be met, and + would have systemd terminating a server which is working normally; + such a setting is reported as a warning at startup. Use a + WatchdogSec= of at least 20 seconds.

+ + + Adding watchdog supervision to the unit above + 
+[Service]
+WatchdogSec=30
+Restart=on-failure
+
+ diff --git a/modules/arch/unix/mod_systemd.c b/modules/arch/unix/mod_systemd.c index 43b24ef2d0b..6c03acc3a35 100644 --- a/modules/arch/unix/mod_systemd.c +++ b/modules/arch/unix/mod_systemd.c @@ -52,10 +52,37 @@ static apr_uint64_t monotonic_usec(void) return (apr_uint64_t)ts.tv_sec * APR_USEC_PER_SEC + ts.tv_nsec / 1000; } +/* ap_run_monitor() is called once every INTERVAL_OF_WRITABLE_PROBES turns + * of the parent's one second loop in ap_wait_or_timeout(), so that is how + * often a keep-alive notification can be sent, and the shortest watchdog + * timeout which can be met is twice that: sd_watchdog_enabled(3) asks for + * a notification every half of the configured timeout. */ +#define WATCHDOG_INTERVAL_SEC (10) + +/* The WatchdogSec= of the service in microseconds, or zero if the service + * manager is not watching. Set in pre_config, before the first + * notification which could carry a keep-alive. */ +static apr_uint64_t watchdog_usec; + +/* A keep-alive assignment to paste into a notification, or nothing while + * the service manager is not asking for one. Sending WATCHDOG=1 when it + * is not expected is harmless, but saying so only when asked keeps what + * httpd reports the same as what the service was configured for. */ +static const char *watchdog_ping(void) +{ + return watchdog_usec ? "WATCHDOG=1\n" : ""; +} + static int systemd_pre_config(apr_pool_t *pconf, apr_pool_t *plog, apr_pool_t *ptemp) { - apr_uint64_t usec = monotonic_usec(); + apr_uint64_t usec = monotonic_usec(), wd_usec; + + /* Read afresh on each configuration load, since a restart unloads and + * loads the module again, and without unsetting it as server/listen.c + * does for $LISTEN_FDS, which would stop the keep-alive at the first + * reload. */ + watchdog_usec = sd_watchdog_enabled(0, &wd_usec) > 0 ? wd_usec : 0; /* A Type=notify-reload service ignores a reload notification which * does not say when it was sent. */ @@ -63,12 +90,14 @@ static int systemd_pre_config(apr_pool_t *pconf, apr_pool_t *plog, sd_notifyf(0, "RELOADING=1\n" "MONOTONIC_USEC=%" APR_UINT64_T_FMT "\n" - "STATUS=Reading configuration...\n", usec); + "%s" + "STATUS=Reading configuration...\n", usec, watchdog_ping()); } else { - sd_notify(0, - "RELOADING=1\n" - "STATUS=Reading configuration...\n"); + sd_notifyf(0, + "RELOADING=1\n" + "%s" + "STATUS=Reading configuration...\n", watchdog_ping()); } ap_extended_status = 1; return OK; @@ -121,8 +150,28 @@ static int systemd_post_config(apr_pool_t *pconf, apr_pool_t *plog, apr_pool_cleanup_register(pconf, NULL, systemd_stopping, apr_pool_cleanup_null); - sd_notify(0, "READY=1\n" - "STATUS=Configuration loaded.\n"); + /* A timeout the parent cannot meet would have the service manager + * killing a healthy server every WatchdogSec, so say so rather than + * leaving nothing in the log to explain it. */ + if (watchdog_usec + && watchdog_usec / 2 < (apr_uint64_t)WATCHDOG_INTERVAL_SEC + * APR_USEC_PER_SEC) { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, main_server, APLOGNO(10621) + "WatchdogSec is %" APR_UINT64_T_FMT "us, but keep-alive " + "notifications are sent from the parent process only " + "every %ds; configure a WatchdogSec of at least %ds or " + "the service will be killed while it is healthy", + watchdog_usec, WATCHDOG_INTERVAL_SEC, + 2 * WATCHDOG_INTERVAL_SEC); + } + + /* The keep-alive rides along with the notification which ends a + * reload: the configuration is read outside the parent's monitor loop, + * so nothing reports while it is being parsed, and the service manager + * keeps the timeout armed throughout. */ + sd_notifyf(0, "READY=1\n" + "%s" + "STATUS=Configuration loaded.\n", watchdog_ping()); return OK; } @@ -141,6 +190,12 @@ static int systemd_monitor(apr_pool_t *p, server_rec *s) apr_interval_time_t up_time; char bps[5]; + /* Before anything which might decline: reporting the server is alive + * does not depend on there being a status line to report with it. */ + if (watchdog_usec) { + sd_notify(0, "WATCHDOG=1\n"); + } + if (!ap_extended_status) { /* Nothing useful to report with ExtendedStatus disabled. */ return DECLINED; diff --git a/test/modules/arch/linux/README b/test/modules/arch/linux/README index 3894c6158fa..f8037431e91 100644 --- a/test/modules/arch/linux/README +++ b/test/modules/arch/linux/README @@ -17,15 +17,19 @@ to the service manager rather than to a client: activation and not loading it is what disables it. - ap_extended_status forced on in pre_config, so that the monitor hook has request counts to report. + - the watchdog keep-alive, WATCHDOG=1, sent while the service manager + asks for one. Which it does through the environment as well: + WatchdogSec= in the unit becomes $WATCHDOG_USEC, and $WATCHDOG_PID + names the process expected to report. None of that is observable over HTTP, so the tests observe it directly. How the tests run without systemd, and without privileges --------------------------------------------------------- There is no need for a service manager to exercise the protocol. Only -test_005 involves systemd at all; the rest run anywhere, and need nothing -from the systemd package beyond the libsystemd httpd itself is linked -against. +test_005, and the last test of test_006, involve systemd at all; the rest +run anywhere, and need nothing from the systemd package beyond the +libsystemd httpd itself is linked against. test_001_notify.py $NOTIFY_SOCKET is an ordinary AF_UNIX datagram socket the test binds itself (env.NotifyListener). @@ -51,6 +55,22 @@ against. distributions, and doing it directly needs no systemd tooling at all. + test_006_watchdog.py The keep-alive. Mostly the same stand-in socket, + with $WATCHDOG_USEC set in the environment httpd is + started with; env.ForegroundServer runs httpd in the + foreground where $WATCHDOG_PID can name it, which + apachectl cannot do for a parent it has not forked + yet. The last test uses a real unit with + WatchdogSec=, where systemd's own + WatchdogTimestampMonotonic is the evidence that the + pings arrived. + + The keep-alive is sent from the monitor hook, so it + is only as frequent as that hook is: about every ten + seconds. A WatchdogSec shorter than twice that + cannot be met, and mod_systemd says so (AH10621) + rather than letting a healthy server be killed. + test_005_service.py The real thing: a transient Type=notify unit run with "systemd-run --user". This is what checks that systemd holds the unit in "activating" until READY=1 diff --git a/test/modules/arch/linux/env.py b/test/modules/arch/linux/env.py index 0e3b4f6dd9d..7caf74fe996 100644 --- a/test/modules/arch/linux/env.py +++ b/test/modules/arch/linux/env.py @@ -211,6 +211,28 @@ def statuses(self) -> List[str]: NO_MONITOR_TIMEOUT = 15.0 +# The keep-alive notification is sent from the same monitor hook, so waiting +# for one costs the same as waiting for a status report. +WATCHDOG_TIMEOUT = MONITOR_TIMEOUT + +# The shortest WatchdogSec mod_systemd will accept without complaining, which +# is twice the monitor interval: the recommended keep-alive period is half the +# watchdog timeout, and half of anything shorter than this is out of reach of a +# hook which runs every ten seconds. Keep in step with mod_systemd.c. +SYSTEMD_MONITOR_INTERVAL = 10 +MIN_WATCHDOG_SEC = 2 * SYSTEMD_MONITOR_INTERVAL + + +def is_watchdog_ping(msg: Dict[str, str]) -> bool: + """Whether a notification carries the keep-alive ping. + + mod_systemd is free to send WATCHDOG=1 in a datagram of its own or + alongside whatever else it is reporting, so match on the assignment + rather than on the message being only that. + """ + return msg.get('WATCHDOG') == '1' + + # A configuration for a server run directly rather than through apachectl, # sharing the server root, module list and error log with the rest of the # suite but with its own pid file and port. @@ -435,12 +457,15 @@ class TransientService: """ def __init__(self, env: SystemdTestEnv, port: int, - name: str = None, extra: str = ''): + name: str = None, extra: str = '', + properties: List[str] = None): self.env = env self.port = port self.unit = name or f'httpd-test-{os.getpid()}' self.conf_file = write_server_conf(env, 'transient', port, extra=extra) self.pid_file = os.path.join(env.server_logs_dir, 'transient.pid') + # Extra --property arguments for the unit, such as WatchdogSec=. + self.properties = list(properties or []) def read_pid(self) -> Optional[int]: try: @@ -482,6 +507,7 @@ def start(self, timeout: float = 20.0) -> subprocess.CompletedProcess: '--property=KillMode=mixed', f'--property=ExecReload={httpd} -d {self.env.server_dir} ' f'-f {self.conf_file} -k graceful', + *[f'--property={p}' for p in self.properties], httpd, '-DFOREGROUND', '-d', self.env.server_dir, '-f', self.conf_file, ], capture_output=True, text=True, timeout=timeout) @@ -504,3 +530,43 @@ def __enter__(self) -> 'TransientService': def __exit__(self, *args): self.stop() + + +class ForegroundServer(ActivatedServer): + """httpd run directly in the foreground, with extra environment. + + The watchdog protocol is keyed to a process id: sd_watchdog_enabled(3) + ignores $WATCHDOG_USEC unless $WATCHDOG_PID is unset or names the + process reading it. apachectl cannot be used to set that, since the + variable has to name the parent httpd and the pid is not known until it + exists, so the value is assigned in a shell which then exec's httpd in + its own place -- the same trick ActivatedServer uses for $LISTEN_PID. + + Assignments are shell words, so "$$" in a value is the pid httpd will + have. + """ + + def __init__(self, env: SystemdTestEnv, port: int, + name: str = 'foreground', extra: str = '', + setenv: Dict[str, str] = None, + modules_conf: str = 'modules.conf'): + super().__init__(env, port, name=name, extra=extra, + modules_conf=modules_conf) + self.setenv = dict(setenv or {}) + + def args(self, fd: int = None) -> List[str]: + assigns = ' '.join(f'{k}={v}' for k, v in self.setenv.items()) + return [ + 'bash', '-c', + f'export {assigns}; exec "$0" "$@"' if assigns else 'exec "$0" "$@"', + self.env.httpd_bin, '-DFOREGROUND', + '-d', self.env.server_dir, '-f', self.conf_file, + ] + + def start(self) -> 'ForegroundServer': + # A new session, so the whole group can be signalled on the way out + # even when a failed restart leaves children behind. + self.proc = subprocess.Popen( + self.args(), env=self.env.server_env(), start_new_session=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return self diff --git a/test/modules/arch/linux/test_001_notify.py b/test/modules/arch/linux/test_001_notify.py index 76b999fcf26..d79980bc0ac 100644 --- a/test/modules/arch/linux/test_001_notify.py +++ b/test/modules/arch/linux/test_001_notify.py @@ -127,16 +127,26 @@ def test_systemd_001_08_reloading_monotonic(self, env): assert 0 < int(msg['MONOTONIC_USEC']) <= time.clock_gettime_ns( time.CLOCK_MONOTONIC) // 1000 - @pytest.mark.xfail(reason="mod_systemd implements no watchdog keepalive, " - "so a unit using WatchdogSec= would be killed") def test_systemd_001_09_watchdog(self, env): + """A server started with $WATCHDOG_USEC reports that it is alive. + + $WATCHDOG_PID is left unset, which sd_watchdog_enabled(3) takes to + mean any process may report: it has to be, since apachectl starts a + parent whose pid nothing knew in advance. The keep-alive arrives + with the startup notifications rather than only from the monitor + hook ten seconds later, which is what makes this cheap to check. + test_006_watchdog.py covers the rest of the protocol. + """ assert env.apache_stop() == 0 - env.set_httpd_env('WATCHDOG_USEC', '2000000') # ping every 1s + # Long enough that mod_systemd does not object to it (AH10622). + env.set_httpd_env('WATCHDOG_USEC', str(60 * 1000000)) try: env.notify.clear() assert env.apache_restart() == 0 - assert env.notify.wait_for_key('WATCHDOG', timeout=4), \ - "no watchdog keepalive was sent" + msg = env.notify.wait_for_key('WATCHDOG', timeout=4) + assert msg, f"no watchdog keepalive was sent, " \ + f"got {env.notify.messages}" + assert msg['WATCHDOG'] == '1' finally: env.set_httpd_env('WATCHDOG_USEC', None) assert env.apache_restart() == 0 diff --git a/test/modules/arch/linux/test_006_watchdog.py b/test/modules/arch/linux/test_006_watchdog.py new file mode 100644 index 00000000000..d4fa8653bf6 --- /dev/null +++ b/test/modules/arch/linux/test_006_watchdog.py @@ -0,0 +1,211 @@ +import os +import re +import time + +import pytest + +from .env import (ForegroundServer, TransientService, MIN_WATCHDOG_SEC, + NO_MONITOR_TIMEOUT, WATCHDOG_TIMEOUT, is_watchdog_ping) + + +def watchdog_env(usec: int, pid: str = '$$') -> dict: + """The environment a service manager sets for a watched service. + + $WATCHDOG_PID names the process expected to report; "$$" is the shell + which exec's httpd, so it is the pid httpd will run as. + """ + env = {'WATCHDOG_USEC': str(usec)} + if pid is not None: + env['WATCHDOG_PID'] = pid + return env + + +class TestSystemdWatchdog: + """The keep-alive ping of the systemd watchdog protocol. + + A service whose unit sets WatchdogSec= is started with $WATCHDOG_USEC + holding that timeout in microseconds, and $WATCHDOG_PID holding the pid + expected to report. The service must then send "WATCHDOG=1" to the + notification socket more often than the timeout, or systemd puts the + unit into a failed state with Result=watchdog. The timeout is armed + once start-up completes and, as test_006_06 relies on, stays armed + across a reload. + + Nothing here needs systemd: the ping goes to $NOTIFY_SOCKET like every + other notification, so the same stand-in socket observes it. Only the + last test involves a service manager. + """ + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + # These run their own servers on the second port; the shared one + # would only add its notifications to the same socket. + assert env.apache_stop() == 0 + yield + assert env.apache_stop() == 0 + + @pytest.fixture + def server_factory(self, env): + started = [] + + def make(usec=None, pid='$$', extra='', name='watchdog'): + setenv = watchdog_env(usec, pid) if usec is not None else {} + srv = ForegroundServer(env, port=env.http_port2, name=name, + extra=extra, setenv=setenv) + started.append(srv) + env.notify.clear() + srv.start() + assert srv.is_live(), \ + f"server did not come up: {srv.stderr!r}" + return srv + + yield make + for srv in started: + srv.stop() + + def test_systemd_006_01_no_ping_when_unwatched(self, env, server_factory): + """A server the service manager is not watching sends no keep-alive. + + The monitor hook still runs -- its status report is the evidence of + that -- so the absence of a ping is a decision and not silence. + """ + server_factory() + assert env.notify.wait_for_status(r'^Total requests: ', + timeout=WATCHDOG_TIMEOUT), \ + "the monitor hook never ran, so this proves nothing" + assert not [m for m in env.notify.messages if is_watchdog_ping(m)], \ + "a keep-alive was sent with $WATCHDOG_USEC unset" + + def test_systemd_006_02_ping_when_watched(self, env, server_factory): + """With the watchdog enabled the keep-alive is sent.""" + server_factory(usec=60 * 1000000) + assert env.notify.wait_for(is_watchdog_ping, + timeout=WATCHDOG_TIMEOUT), \ + f"no keep-alive within {WATCHDOG_TIMEOUT}s, " \ + f"got {env.notify.messages}" + + def test_systemd_006_03_ping_repeats(self, env, server_factory): + """The keep-alive is periodic, which is the whole point of it: one + ping would satisfy a test but not systemd.""" + server_factory(usec=60 * 1000000) + for n in range(2): + assert env.notify.wait_for(is_watchdog_ping, + timeout=WATCHDOG_TIMEOUT), \ + f"only {n} keep-alives arrived, got {env.notify.messages}" + env.notify.clear() + + def test_systemd_006_04_ping_without_extended_status(self, env, + server_factory): + """The keep-alive does not depend on ExtendedStatus. + + The monitor hook declines early when it has no request counts to + report (test_003_03), and a server whose status line is switched + off must still be reported as alive. + """ + server_factory(usec=60 * 1000000, extra='ExtendedStatus off') + assert env.notify.wait_for_status(r'^Total requests: ', + timeout=NO_MONITOR_TIMEOUT) is None, \ + "ExtendedStatus off did not stop the status report" + assert env.notify.wait_for(is_watchdog_ping, + timeout=WATCHDOG_TIMEOUT), \ + "no keep-alive with ExtendedStatus off" + + def test_systemd_006_05_no_ping_for_another_pid(self, env, server_factory): + """$WATCHDOG_PID naming a different process means the variables were + set for something further up the process tree, and must be ignored. + """ + server_factory(usec=60 * 1000000, pid='1') + assert env.notify.wait_for_status(r'^Total requests: ', + timeout=WATCHDOG_TIMEOUT), \ + "the monitor hook never ran, so this proves nothing" + assert not [m for m in env.notify.messages if is_watchdog_ping(m)], \ + "a keep-alive was sent although $WATCHDOG_PID was another process" + + def test_systemd_006_06_ping_across_reload(self, env, server_factory): + """The keep-alive survives a graceful restart. + + The parent re-reads its configuration in the same process, but + mod_systemd is unloaded and loaded again with it, so anything it + remembered about the watchdog is gone by the time the monitor hook + runs again. systemd keeps the timeout armed throughout. + """ + srv = server_factory(usec=60 * 1000000) + assert env.notify.wait_for(is_watchdog_ping, timeout=WATCHDOG_TIMEOUT) + env.notify.clear() + assert srv.reload() == 0 + assert srv.is_live() + assert env.notify.wait_for(is_watchdog_ping, + timeout=WATCHDOG_TIMEOUT), \ + f"no keep-alive after a reload, got {env.notify.messages}" + + def test_systemd_006_07_ping_covers_the_reload_window(self, env, + server_factory): + """A keep-alive is sent as the configuration is read, and again once + it is loaded. + + Reading the configuration happens outside the parent's monitor loop, + so nothing else reports during it. The watchdog stays armed while + the unit reloads, and a configuration which takes longer to parse + than the timeout would otherwise be killed halfway through. + """ + srv = server_factory(usec=60 * 1000000) + assert env.notify.wait_for(is_watchdog_ping, timeout=WATCHDOG_TIMEOUT) + env.notify.clear() + assert srv.reload() == 0 + assert env.notify.wait_for( + lambda m: 'RELOADING' in m and is_watchdog_ping(m), + timeout=WATCHDOG_TIMEOUT), \ + f"no keep-alive as the configuration was read, " \ + f"got {env.notify.messages}" + assert env.notify.wait_for( + lambda m: m.get('READY') == '1' and is_watchdog_ping(m), + timeout=WATCHDOG_TIMEOUT), \ + f"no keep-alive once the configuration was loaded, " \ + f"got {env.notify.messages}" + + def test_systemd_006_08_short_timeout_warned(self, env, server_factory): + """A WatchdogSec the parent cannot meet is reported. + + The keep-alive is sent from the monitor hook, which runs once every + ten turns of the parent's one second loop. A timeout of a few + seconds cannot be met however the module is written, and failing + silently would leave the server being killed and restarted with + nothing in the log to say why. + """ + server_factory(usec=2 * 1000000) + assert env.httpd_error_log.scan_recent( + re.compile(r'.*AH10621: .*[Ww]atchdog.*'), timeout=10), \ + "no warning about a watchdog timeout that cannot be met" + env.httpd_error_log.ignore_recent(lognos=['AH10621']) + + def test_systemd_006_09_workable_timeout_not_warned(self, env, + server_factory): + """A timeout the parent can meet is not complained about.""" + server_factory(usec=MIN_WATCHDOG_SEC * 1000000) + assert env.notify.wait_for(is_watchdog_ping, timeout=WATCHDOG_TIMEOUT) + with pytest.raises(TimeoutError): + env.httpd_error_log.scan_recent( + re.compile(r'.*AH10621: .*'), timeout=1) + + @pytest.mark.skipif(not TransientService.is_available(), + reason="no per-user systemd manager") + def test_systemd_006_10_service_keeps_watchdog_alive(self, env): + """The real thing: systemd records each keep-alive it receives, and + the unit stays active rather than failing with Result=watchdog.""" + with TransientService(env, port=env.http_port2, + properties=[f'WatchdogSec={MIN_WATCHDOG_SEC}s'], + name=f'httpd-wd-{os.getpid()}') as svc: + assert svc.start().returncode == 0 + assert svc.wait_active() + first = svc.show('WatchdogTimestampMonotonic') + assert first and int(first) > 0, \ + "systemd recorded no keep-alive at all" + end = time.time() + WATCHDOG_TIMEOUT + while time.time() < end: + if svc.show('WatchdogTimestampMonotonic') != first: + break + time.sleep(0.5) + assert svc.show('WatchdogTimestampMonotonic') != first, \ + "systemd received no further keep-alive" + assert svc.show('ActiveState') == 'active' + assert svc.show('Result') == 'success' From 69515939f0d01efb19807ca67b7dc6378e3df64e Mon Sep 17 00:00:00 2001 From: Joe Orton Date: Sun, 30 Aug 2026 10:05:28 +0000 Subject: [PATCH 11/13] * docs/manual/mod/mod_systemd.xml: Document Type=notify-reload, ReloadSignal= and ExecReload=. Co-Authored-By: Claude Opus 5 (1M context) GitHub: PR #729 Merge r1937646 from trunk. (cherry picked from commit 4a71f7211da0e23fc9f3296666b1ceacf519a076) --- docs/manual/mod/mod_systemd.xml | 51 +++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/docs/manual/mod/mod_systemd.xml b/docs/manual/mod/mod_systemd.xml index fd324a533f5..619ce8b5fa0 100644 --- a/docs/manual/mod/mod_systemd.xml +++ b/docs/manual/mod/mod_systemd.xml @@ -32,7 +32,7 @@

This module provides support for systemd integration. It allows httpd to be used in a service with the systemd - Type=notify (see Type=notify or Type=notify-reload (see systemd.service(5) for more information). The module is activated if loaded.

@@ -69,6 +69,53 @@ WantedBy=multi-user.target href="https://www.freedesktop.org/software/systemd/man/systemd.kill.html">systemd.kill(5) for more information.

+

A service manager from systemd 253 onwards offers + Type=notify-reload, which is worth using in preference. + Under Type=notify a systemctl reload + returns as soon as the ExecReload command has sent its + signal, which is before the new configuration has been read, and it + reports success whatever becomes of the restart afterwards. + Type=notify-reload instead holds the reload open until + the server reports it finished, so the command waits for the new + configuration to be in use and fails if it never is. mod_systemd + sends the RELOADING=1 notification the protocol expects + while the configuration is being read, stamped with the + MONOTONIC_USEC the service manager requires, and + READY=1 once it has been loaded.

+ + + Example of a service unit which reloads synchronously + 
+[Service]
+Type=notify-reload
+ReloadSignal=SIGCONT
+ExecStart=/usr/local/apache2/bin/httpd -D FOREGROUND -k start
+ExecReload=/usr/local/apache2/bin/httpd -k graceful
+KillMode=mixed
+
+ + +

The service manager runs ExecReload first, and sends + the signal named by ReloadSignal only once that command + has exited successfully. Keeping ExecReload is what + makes the reload safe: httpd -k graceful parses the new + configuration in a process of its own and exits without signalling + anything if it does not parse, so the reload fails and the running + server carries on with the configuration it already has. Leaving + ExecReload out, and letting the service manager signal + the server directly, gives up that check: the running parent reads + the new configuration itself, and a configuration which does not + parse makes it exit, taking the server down.

+ +

The signal sent after ExecReload has run is then + redundant, so ReloadSignal should name one httpd does + not act on, such as SIGCONT. It matters that it is set: + the default is SIGHUP, which httpd takes as an + ungraceful restart, dropping the connections a reload is + meant to preserve. A unit which does leave out + ExecReload must set ReloadSignal=SIGUSR1, + the signal httpd restarts gracefully on.

+

Systemd socket activation is supported if httpd was built with it. Each Listen port must then be one passed in by systemd; a port which was not is a @@ -99,7 +146,7 @@ WantedBy=multi-user.target WatchdogSec= of at least 20 seconds.

- Adding watchdog supervision to the unit above + Adding watchdog supervision to either unit above 
 [Service]
 WatchdogSec=30

From 470dcccede8efbdc1f3da5bd79a07fa5ee77c5e7 Mon Sep 17 00:00:00 2001
From: Joe Orton 
Date: Sun, 30 Aug 2026 10:05:36 +0000
Subject: [PATCH 12/13] * test/modules/arch/linux/test_007_notify_reload.py:
 New test suite for   Type=notify-reload.

* test/modules/arch/linux/env.py (TransientService): Take the service
  type and whether to configure ExecReload=.
  (TransientService.systemctl): Take a timeout.
  (TransientService.reload, TransientService.rewrite_conf,
  error_log_size, error_log_since, systemd_version): New.

* test/modules/arch/linux/README: Describe the new suite.

Co-Authored-By: Claude Opus 5 (1M context) 
GitHub: PR #729

Merge r1937647 from trunk.
(cherry picked from commit be7efad1eb5868432c382136280bc26e14d32e86)
---
 test/modules/arch/linux/README                |  18 ++-
 test/modules/arch/linux/env.py                |  84 +++++++++--
 .../arch/linux/test_007_notify_reload.py      | 139 ++++++++++++++++++
 3 files changed, 228 insertions(+), 13 deletions(-)
 create mode 100644 test/modules/arch/linux/test_007_notify_reload.py

diff --git a/test/modules/arch/linux/README b/test/modules/arch/linux/README
index f8037431e91..4bdd7af91d1 100644
--- a/test/modules/arch/linux/README
+++ b/test/modules/arch/linux/README
@@ -27,8 +27,8 @@ None of that is observable over HTTP, so the tests observe it directly.
 How the tests run without systemd, and without privileges
 ---------------------------------------------------------
 There is no need for a service manager to exercise the protocol.  Only
-test_005, and the last test of test_006, involve systemd at all; the rest
-run anywhere, and need nothing from the systemd package beyond the
+test_005, the last test of test_006, and test_007 involve systemd at all;
+the rest run anywhere, and need nothing from the systemd package beyond the
 libsystemd httpd itself is linked against.
 
   test_001_notify.py   $NOTIFY_SOCKET is an ordinary AF_UNIX datagram
@@ -78,13 +78,23 @@ libsystemd httpd itself is linked against.
                        reported STATUS= as the unit's status text.
                        Skipped when the user has no systemd manager.
 
+  test_007_notify_reload.py
+                       The reload protocol, as a transient
+                       Type=notify-reload unit: that "systemctl reload"
+                       waits for the new configuration to be in use, that a
+                       configuration which does not parse fails the reload
+                       without disturbing the running server, and that a
+                       reload restarts it gracefully and only once.  Also
+                       skipped where systemd is older than 253, which is
+                       where Type=notify-reload arrived.
+
 The whole package is skipped unless mod_systemd was built, which needs
 configure --enable-systemd.  A static module is enough for everything
 except the test which has to leave mod_systemd out of the configuration;
 that one needs --enable-systemd=shared.
 
-Running the last suite where there is no user session
------------------------------------------------------
+Running the systemd suites where there is no user session
+---------------------------------------------------------
 "systemctl --user" needs a per-user manager, which a login session has but
 a CI container does not; enabling lingering needs privileges.  Where that
 is a problem, run the tests inside a container with systemd as pid 1,
diff --git a/test/modules/arch/linux/env.py b/test/modules/arch/linux/env.py
index 7caf74fe996..ce1c2464f27 100644
--- a/test/modules/arch/linux/env.py
+++ b/test/modules/arch/linux/env.py
@@ -286,6 +286,25 @@ def http_responds(port: int, timeout: float = 2.0) -> bool:
         return False
 
 
+def error_log_size(env: SystemdTestEnv) -> int:
+    """Where the shared error log ends now, so that a later read can take
+    only what one operation wrote."""
+    try:
+        return os.path.getsize(env.httpd_error_log.path)
+    except OSError:
+        return 0
+
+
+def error_log_since(env: SystemdTestEnv, pos: int) -> str:
+    """The error log written since error_log_size() returned pos."""
+    try:
+        with open(env.httpd_error_log.path, errors='replace') as fd:
+            fd.seek(pos)
+            return fd.read()
+    except OSError:
+        return ''
+
+
 class ActivatedServer:
     """An httpd handed a listening socket the way a service manager does.
 
@@ -445,6 +464,24 @@ def __exit__(self, *args):
         self.stop()
 
 
+# Type=notify-reload and ReloadSignal= both arrived in systemd 253.  An
+# unrecognised Type= does not degrade to anything, it stops the unit loading
+# at all, so there is nothing to fall back to and the tests are skipped.
+NOTIFY_RELOAD_VERSION = 253
+
+
+def systemd_version() -> int:
+    """The version of the systemd on this host, or 0 if it cannot be asked.
+    "systemctl --version" opens with "systemd 259 (259.8-1.fc44)"."""
+    try:
+        p = subprocess.run(['systemctl', '--version'], capture_output=True,
+                           text=True, timeout=15)
+    except (OSError, subprocess.TimeoutExpired):
+        return 0
+    m = re.match(r'systemd (\d+)', p.stdout)
+    return int(m.group(1)) if m else 0
+
+
 class TransientService:
     """httpd run as a real transient systemd unit, with systemd-run.
 
@@ -454,17 +491,24 @@ class TransientService:
     tracks MAINPID, and shows the reported STATUS= as the unit's status
     text.  It needs a per-user service manager, which a login session has
     but a bare CI container does not.
+
+    service_type selects what is exercised: "notify" for startup and
+    shutdown, "notify-reload" for the reload protocol on top of them.
     """
 
     def __init__(self, env: SystemdTestEnv, port: int,
                  name: str = None, extra: str = '',
+                 service_type: str = 'notify', exec_reload: bool = True,
                  properties: List[str] = None):
         self.env = env
         self.port = port
         self.unit = name or f'httpd-test-{os.getpid()}'
+        self.service_type = service_type
+        self.exec_reload = exec_reload
         self.conf_file = write_server_conf(env, 'transient', port, extra=extra)
         self.pid_file = os.path.join(env.server_logs_dir, 'transient.pid')
-        # Extra --property arguments for the unit, such as WatchdogSec=.
+        # Extra --property arguments for the unit, such as WatchdogSec= or
+        # ReloadSignal=.
         self.properties = list(properties or [])
 
     def read_pid(self) -> Optional[int]:
@@ -490,9 +534,10 @@ def is_available() -> bool:
         except (OSError, subprocess.TimeoutExpired):
             return False
 
-    def systemctl(self, *args) -> subprocess.CompletedProcess:
+    def systemctl(self, *args,
+                  timeout: float = 60.0) -> subprocess.CompletedProcess:
         return subprocess.run(['systemctl', '--user', *args],
-                              capture_output=True, text=True)
+                              capture_output=True, text=True, timeout=timeout)
 
     def show(self, prop: str) -> str:
         r = self.systemctl('show', '-p', prop, '--value', f'{self.unit}.service')
@@ -500,19 +545,40 @@ def show(self, prop: str) -> str:
 
     def start(self, timeout: float = 20.0) -> subprocess.CompletedProcess:
         httpd = self.env.httpd_bin
+        props = [
+            f'--service-type={self.service_type}',
+            '--property=KillMode=mixed',
+            # A reload which is never reported finished holds the job open
+            # until this elapses, so keep it to the same bound as the start.
+            f'--property=TimeoutStartSec={int(timeout)}',
+        ]
+        if self.exec_reload:
+            props.append(f'--property=ExecReload={httpd} '
+                         f'-d {self.env.server_dir} '
+                         f'-f {self.conf_file} -k graceful')
+        props += [f'--property={p}' for p in self.properties]
         r = subprocess.run([
             'systemd-run', '--user', '--collect', '--quiet',
-            '--unit', self.unit,
-            '--service-type=notify',
-            '--property=KillMode=mixed',
-            f'--property=ExecReload={httpd} -d {self.env.server_dir} '
-            f'-f {self.conf_file} -k graceful',
-            *[f'--property={p}' for p in self.properties],
+            '--unit', self.unit, *props,
             httpd, '-DFOREGROUND',
             '-d', self.env.server_dir, '-f', self.conf_file,
         ], capture_output=True, text=True, timeout=timeout)
         return r
 
+    def reload(self, timeout: float = 60.0) -> subprocess.CompletedProcess:
+        """Ask the manager to reload the unit.  Under Type=notify-reload
+        this returns once the server has reported the reload finished;
+        under Type=notify, as soon as ExecReload= has exited."""
+        return self.systemctl('reload', f'{self.unit}.service',
+                              timeout=timeout)
+
+    def rewrite_conf(self, extra: str = ''):
+        """Replace the configuration the unit reads on its next reload.
+        The path does not change, so ExecStart= and ExecReload= still name
+        it."""
+        self.conf_file = write_server_conf(self.env, 'transient', self.port,
+                                           extra=extra)
+
     def wait_active(self, timeout: float = 20.0) -> bool:
         end = time.time() + timeout
         while time.time() < end:
diff --git a/test/modules/arch/linux/test_007_notify_reload.py b/test/modules/arch/linux/test_007_notify_reload.py
new file mode 100644
index 00000000000..93833299c63
--- /dev/null
+++ b/test/modules/arch/linux/test_007_notify_reload.py
@@ -0,0 +1,139 @@
+import os
+import time
+
+import pytest
+
+from .env import (NOTIFY_RELOAD_VERSION, TransientService, error_log_since,
+                  error_log_size, http_responds, systemd_version)
+
+pytestmark = [
+    pytest.mark.skipif(
+        not TransientService.is_available(),
+        reason="no per-user systemd manager to run a transient service under"),
+    pytest.mark.skipif(
+        systemd_version() < NOTIFY_RELOAD_VERSION,
+        reason=f"Type=notify-reload needs systemd {NOTIFY_RELOAD_VERSION}"),
+]
+
+
+class TestNotifyReload:
+    """httpd as a Type=notify-reload unit.
+
+    Under Type=notify a reload is only whatever ExecReload= does, and
+    systemctl returns as soon as that command exits.  For "httpd -k
+    graceful" that is as soon as the signal has been sent, long before the
+    new configuration is in use, and it reports success however the restart
+    turns out.  Type=notify-reload holds the reload job open until the
+    service sends RELOADING=1 and then READY=1, which mod_systemd sends
+    from pre_config and post_config, so the result is reported once it is
+    known.
+
+    The unit here keeps ExecReload= and sets ReloadSignal=SIGCONT, which
+    httpd ignores.  The manager runs ExecReload= first and sends
+    ReloadSignal= only once it has exited successfully, so:
+
+      - "httpd -k graceful" stays the thing which restarts the server, and
+        because it parses the new configuration in its own process before
+        signalling, a configuration which does not parse fails the reload
+        without the running server being signalled at all.  A bare
+        ReloadSignal= has the running parent read the new configuration and
+        exit if it does not parse.
+
+      - the signal the manager then sends is redundant, so it is pointed at
+        one httpd does not act on.  The default is SIGHUP, which httpd
+        takes as an *ungraceful* restart.
+    """
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        assert env.apache_stop() == 0
+        yield
+        assert env.apache_stop() == 0
+
+    @pytest.fixture
+    def service(self, env) -> TransientService:
+        svc = TransientService(env, port=env.http_port2,
+                               name=f'httpd-reload-{os.getpid()}',
+                               service_type='notify-reload',
+                               properties=['ReloadSignal=SIGCONT'])
+        yield svc
+        svc.stop()
+
+    def test_systemd_007_01_reload_waits_for_the_new_config(self, env, service):
+        """systemctl reload returns only once the reloaded server is
+        serving: the port added to the configuration answers without the
+        test waiting for it, because READY=1 is sent from post_config, by
+        which point the listeners are open."""
+        assert service.start().returncode == 0
+        assert service.wait_active()
+        assert http_responds(env.http_port2)
+
+        service.rewrite_conf(extra=f'Listen {env.proxy_port}')
+        r = service.reload()
+        assert r.returncode == 0, f"reload failed: {r.stderr}"
+        assert http_responds(env.proxy_port, timeout=10.0), \
+            "reload returned before the newly configured port was served"
+        assert http_responds(env.http_port2)
+
+    def test_systemd_007_02_broken_config_fails_the_reload(self, env, service):
+        """A configuration which does not parse fails the reload and leaves
+        the running server alone.  ExecReload= reads it in a process of its
+        own and exits without signalling, and the manager sends no reload
+        signal of its own once that has failed, so the parent never reads
+        it."""
+        assert service.start().returncode == 0
+        assert service.wait_active()
+        pid = int(service.show('MainPID'))
+
+        service.rewrite_conf(extra='ThisDirectiveDoesNotExist on')
+        r = service.reload()
+        assert r.returncode != 0, \
+            "reload of an unparseable configuration reported success"
+        assert service.show('ActiveState') == 'active', \
+            "a failed reload took the unit out of active"
+        # The parent exits when a restart re-reads a configuration which
+        # does not parse, and its children go on serving for a while
+        # afterwards, so answering a request is not on its own proof that
+        # the server survived: check the process the manager tracks.
+        assert int(service.show('MainPID')) == pid, \
+            "the parent was restarted despite the configuration not parsing"
+        assert http_responds(env.http_port2)
+
+    def test_systemd_007_03_reload_restarts_gracefully_once(self, env, service):
+        """One reload is one graceful restart, and no ungraceful one.
+        Leaving ReloadSignal= at its SIGHUP default is what this catches:
+        httpd restarts ungracefully on SIGHUP, dropping the connections a
+        reload is supposed to keep."""
+        if env.mpm_module not in ('mpm_event', 'mpm_worker'):
+            pytest.skip(f"{env.mpm_module} does not log the graceful restart")
+        assert service.start().returncode == 0
+        assert service.wait_active()
+
+        pos = error_log_size(env)
+        assert service.reload().returncode == 0
+        # The restart is logged before the configuration is re-read, so it
+        # is written by the time READY=1 ends the reload; give the
+        # redundant signal which follows a moment to land as well.
+        time.sleep(2)
+        log = error_log_since(env, pos)
+        assert log.count("Attempting to restart") == 0, \
+            f"the reload restarted the server ungracefully:\n{log}"
+        assert log.count("Doing graceful restart") == 1, \
+            f"expected one graceful restart, log said:\n{log}"
+
+    def test_systemd_007_04_reload_repeats(self, env, service):
+        """Reloading twice works.  The manager ignores a RELOADING=1 which
+        is not stamped later than the reload it asked for, so a server
+        which got MONOTONIC_USEC wrong could report the first reload and
+        then leave the second to time out."""
+        assert service.start().returncode == 0
+        assert service.wait_active()
+        pid = int(service.show('MainPID'))
+
+        for i in range(2):
+            r = service.reload()
+            assert r.returncode == 0, f"reload {i + 1} failed: {r.stderr}"
+            assert service.show('ActiveState') == 'active'
+            assert int(service.show('MainPID')) == pid, \
+                "a graceful restart replaced the parent process"
+        assert http_responds(env.http_port2)

From 76e34c4dd5bdb378b67e5d2c54c1bdfe3da3b811 Mon Sep 17 00:00:00 2001
From: Stefan Eissing 
Date: Mon, 28 Sep 2026 15:09:41 +0100
Subject: [PATCH 13/13] * test/pyhttpd/conf/httpd.conf.template: Set
 DefaultRuntimeDir and   PidFile.

Merge the test/pyhttpd/conf/httpd.conf.template part of r1909769 from trunk.
(cherry picked from commit bdd49d384558257ee89384b5a380f0f042f6d604)
---
 test/pyhttpd/conf/httpd.conf.template | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/pyhttpd/conf/httpd.conf.template b/test/pyhttpd/conf/httpd.conf.template
index 255b88ad05f..92cef2e06d2 100644
--- a/test/pyhttpd/conf/httpd.conf.template
+++ b/test/pyhttpd/conf/httpd.conf.template
@@ -1,6 +1,9 @@
 ServerName localhost
 ServerRoot "${server_dir}"
 
+DefaultRuntimeDir logs
+PidFile httpd.pid
+
 Include "conf/modules.conf"
 
 DocumentRoot "${server_dir}/htdocs"