diff --git a/.gitignore b/.gitignore index 1e684823db175..8ab90f323cfa8 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,7 @@ CMakeSettings.json #Ducktape /results +/.ducktests-demo .ducktape *.pyc /tests/venv diff --git a/modules/ducktests/README.md b/modules/ducktests/README.md index 3d56ca33f292d..822a0c6352cc7 100644 --- a/modules/ducktests/README.md +++ b/modules/ducktests/README.md @@ -129,9 +129,8 @@ To locally simulate validation matrices across distinct target runtimes (e.g., P pyenv install 3.9 pyenv shell 3.8 3.9 ``` -3. Install `tox` and run the validation suite: +3. Run the validation suite: ```bash - pip install tox tox tox -r -e codestyle,py3 ``` @@ -236,6 +235,60 @@ You can target specific cross-product version compatibility combinations inside --global-json '{"safepoint_log_enabled": true}' ``` +### Demo Mode (Breakpoints) + +Scenarios can be frozen at named breakpoints, so a cluster can be shown to an audience in exactly that state and then resumed. Ducktape runs the test inside `ducker01` with stdin closed, so the keyboard lives in a second terminal. + +Terminal 1 - run the test with the `demo_pause` global: +```bash +./docker/run_tests.sh -n 12 -gj '{"demo_pause": "*"}' \ + -t ./ignitetest/tests/mdc/partition_resilience_test.py::MdcPartitionResilienceTest.test_mdc_cluster_partition_resilience +``` + +Terminal 2 - drive the breakpoints: +```bash +python docker/demo_console.py +``` + +At every breakpoint the console prints the step, the elapsed time, every service node with its liveness, the live network state (netem delays and partition drops as they are actually applied), and ready-to-paste commands for entering nodes and reading their logs and configs. `Enter` continues, `c` runs the rest unattended, `a` aborts the test. + +While paused the cluster keeps running and any network impairment stays in effect, so nodes can be inspected freely: +```bash +./docker/ducker-ignite ssh ducker03 +docker exec ducker03 bash -c "tail -n 50 /mnt/service/logs/ignite*.log" +docker exec ducker03 cat /mnt/service/config/ignite-config.xml +``` + +The console is optional - the test communicates through files under `/.ducktests-demo`, which is shared with the host by the same bind mount that carries the repository into the containers: +```bash +cat .ducktests-demo/paused.txt # the banner of the breakpoint currently held +touch .ducktests-demo/continue-3 # resume breakpoint 3 +touch .ducktests-demo/continue-all # resume and skip the remaining breakpoints +touch .ducktests-demo/abort # fail the test and tear down +``` + +Both sides find that directory on their own, so by default nothing has to be configured. `demo_pause_dir` overrides it - but the two sides name the same directory differently, since the test runs inside `ducker01`, where the repository is mounted at `/opt/ignite-dev`, while the console runs on the host. Override it and the console has to be pointed at the host side of it: +```bash +./docker/run_tests.sh -gj '{"demo_pause": "*", "demo_pause_dir": "/opt/ignite-dev/.demo"}' -t ./ignitetest/tests/ + +python docker/demo_console.py -d .demo +``` + +Breakpoints are added to a test with `self.pause("name", mdc, net)` and cost nothing when the global is absent, which is how they stay in the tests without affecting CI. Run one test at a time in demo mode (no `--max-parallel`): a single control directory holds one breakpoint at a time. + +A held test reports nothing back to ducktape, which kills a session it has heard nothing from for `--test-runner-timeout` (30 minutes by default) - and that budget is spent by the whole test, setup included, not by the breakpoint alone. Breakpoints therefore auto-continue while the runner is still waiting, shortening themselves below `demo_pause_timeout_sec` when there is not enough of the budget left and saying so in the test log. For a demo that needs longer, raise the runner timeout too (milliseconds): +```bash +./docker/run_tests.sh -n 12 --test-runner-timeout 7200000 \ + -gj '{"demo_pause": "*", "demo_pause_timeout_sec": 1800}' \ + -t ./ignitetest/tests/mdc/partition_resilience_test.py +``` + +| Global Parameter Key | Definition | Example Configuration | +|---------------------|------------|----------------------| +| **demo_pause** | Which breakpoints stop the scenario. Absent or `false` disables them all (the default); `true` or `"*"` stops at every one; a list or comma separated string stops only at the named ones, matched case insensitively. | ```{"demo_pause": "split-brain,healed"}``` | +| **demo_pause_timeout_sec** | How long one breakpoint may hold the scenario before it resumes on its own. Default is 600, and it is capped by what is left of `--test-runner-timeout`. | ```{"demo_pause_timeout_sec": 1800}``` | +| **demo_pause_dir** | Control directory shared with the host, named as the test sees it - inside the containers the repository is `/opt/ignite-dev`. Default is `/.ducktests-demo`; anything else has to be passed to the console as well, with `-d` and the host path. | ```{"demo_pause_dir": "/opt/ignite-dev/.demo"}``` | + ### Security Settings ```bash # Enable built-in authentication overrides diff --git a/modules/ducktests/tests/checks/support/demo_pause_control.py b/modules/ducktests/tests/checks/support/demo_pause_control.py new file mode 100644 index 0000000000000..97d5d31b676e4 --- /dev/null +++ b/modules/ducktests/tests/checks/support/demo_pause_control.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The host side of a demo breakpoint, for checks that drive one. + +A held breakpoint blocks the thread that reached it, so everything the host does - reading the +published banner, dropping a resume file - has to happen from another one while the check +itself sits inside :meth:`ignitetest.utils.pause.DemoPause.pause`. + +The protocol itself is spoken through :class:`ignitetest.utils.pause_control.ControlDir`, the +same class the test and ``docker/demo_console.py`` use - a check that hand rolled the file +names would stop checking the protocol and start checking its own copy of it. +""" + +import threading +import time +from contextlib import contextmanager + +from ignitetest.utils.pause import DEMO_PAUSE_TIMEOUT_SEC, DemoPause +from ignitetest.utils.pause_control import ControlDir + +from checks.support.ducktape_doubles import FakeLogger + +# Stands for the test a breakpoint was reached in; breakpoints report it to the host. +TEST_NAME = "check.CheckPause.check_something" + +# Far longer than the fraction of a second a check actually holds a breakpoint for, and far +# shorter than the framework's own default: a resume that never arrives has to fail the check +# that expected it rather than hold the suite for ten minutes. +RESUME_TIMEOUT_SEC = 30 + + +def new_demo_pause(control_dir, started_at=None, runner_timeout_sec=None, **test_globals): + """ + :return: A DemoPause over the given control directory, logging into a FakeLogger its + ``logger`` attribute hands back to the check. + """ + test_globals.setdefault(DEMO_PAUSE_TIMEOUT_SEC, RESUME_TIMEOUT_SEC) + + return DemoPause(FakeLogger(), test_globals, TEST_NAME, control_dir=str(control_dir), + started_at=started_at, runner_timeout_sec=runner_timeout_sec) + + +def resume_with(control_dir, name, delay_sec=.05): + """ + Creates a resume file from another thread, the way the host does while the test blocks. + """ + control = ControlDir(control_dir) + + timer = threading.Timer(delay_sec, lambda: control.resume(name)) + timer.daemon = True + timer.start() + + +@contextmanager +def published_status(control_dir, resume=None, timeout_sec=30): + """ + Reads the published breakpoint while the check blocks on it, the way the host console + does, and optionally resumes it. + + Polls for the file rather than reading it once after a fixed delay: a breakpoint that is + only held for a fraction of a second - which is what these checks hold them for - would + otherwise be a race against the machine the checks happen to run on. + + :param resume: Name of the resume file to create once the breakpoint has been read, None + to leave it held. + :return: A dict, empty on entry and filled with the published breakpoint by the time the + block is left. + """ + control = ControlDir(control_dir) + published = {} + + def read(): + deadline = time.monotonic() + timeout_sec + + while time.monotonic() < deadline: + status = control.read_status() + + if status is not None: + published.update(status) + + break + + time.sleep(.01) + + if resume: + control.resume(resume) + + reader = threading.Thread(target=read, daemon=True) + reader.start() + + try: + yield published + finally: + reader.join(timeout_sec) diff --git a/modules/ducktests/tests/checks/support/ducktape_doubles.py b/modules/ducktests/tests/checks/support/ducktape_doubles.py new file mode 100644 index 0000000000000..7a8621e583106 --- /dev/null +++ b/modules/ducktests/tests/checks/support/ducktape_doubles.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Stand-ins for the ducktape objects a test is handed - a logger, nodes, services and the +registry they are collected in - for checks of framework code that only reads them. + +Each double is as poor as the real thing is at the point of use, so that a check fails on +code reaching for more than a test actually offers it. +""" + +from types import SimpleNamespace + +from ignitetest.services.utils.path import IgnitePathAware + + +class FakeLogger: + """ + Collects what the code under check would have logged. + """ + def __init__(self): + self.messages = [] + + def info(self, msg): + """Records an info message.""" + self.messages.append(msg) + + def warn(self, msg): + """Records a warning.""" + self.messages.append(msg) + + debug = info + error = warn + + +def fake_nodes(*hostnames): + """ + :return: Nodes carrying the account attributes that ducktape's do. + """ + return [SimpleNamespace(account=SimpleNamespace(hostname=host, externally_routable_ip=host)) + for host in hostnames] + + +class FakeService: + """ + Stands in for a non-Ignite service of the test registry, e.g. a zookeeper one: it carries + paths of its own, which code following the Ignite services must not hand out for Ignite + nodes. + """ + log_dir = "/mnt/service/zk-logs" + config_file = "/mnt/service/zookeeper.properties" + + def __init__(self, *hostnames): + self.nodes = fake_nodes(*hostnames) + + def who_am_i(self, node): + """Names the node the way a ducktape service does.""" + return f"{self.__class__.__name__}-{node.account.hostname}" + + +class FakeIgniteService(IgnitePathAware): + """ + Stands in for an Ignite service, with the real path layout behind it. + """ + def __init__(self, *hostnames): + self.nodes = fake_nodes(*hostnames) + + def who_am_i(self, node): + """Names the node the way a ducktape service does.""" + return f"{self.__class__.__name__}-{node.account.hostname}" + + @property + def product(self): + return "ignite-dev" + + @property + def globals(self): + return {} + + +class FakeBrokenService: + """ + Stands in for a service that can no longer answer for the nodes it still holds, the way a + ducktape one does once a node has been freed from it: ``who_am_i`` goes through ``idx()``, + which raises for a node the service does not own. + """ + def __init__(self, *hostnames): + self.nodes = fake_nodes(*hostnames) + + def who_am_i(self, node): + """Fails the way a ducktape service does for a node it does not own.""" + raise RuntimeError(f"Could not find node {node}") + + +class FakeRegistry: + """ + Stands in for ducktape's ServiceRegistry, which is what a test hands to the framework: it + is iterable and nothing else, so code reading it may not index it or ask it for a length. + """ + def __init__(self, *services): + self._services = services + + def __iter__(self): + return iter(self._services) diff --git a/modules/ducktests/tests/checks/utils/check_ignite_spec.py b/modules/ducktests/tests/checks/utils/check_ignite_spec.py index a3c8e52104939..b60fce333e18f 100644 --- a/modules/ducktests/tests/checks/utils/check_ignite_spec.py +++ b/modules/ducktests/tests/checks/utils/check_ignite_spec.py @@ -16,6 +16,7 @@ """ Checks Spec class that describes config and command line to start Ignite-aware service. """ +import os from unittest.mock import Mock import pytest @@ -90,10 +91,15 @@ def check_boolean_options__go_after_default_ones_and_overwrite_them__if_passed_v def check_colon_options__go_after_default_ones_and_overwrite_them__if_passed_via_jvm_opt(service): service.log_dir = "/default-path" + + # The default is built with os.path.join, which follows the control machine rather than the + # node - so the expectation is joined the same way instead of being written out, or the check + # would only hold where the separator happens to be "/". + default_gc_log = ("-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:" + f"{os.path.join(service.log_dir, 'gc.log')}:uptime,time,level,tags") + spec = IgniteApplicationSpec(service, jvm_opts=["-Xlog:gc:/some-non-default-path/gc.log"]) + assert "-Xlog:gc:/some-non-default-path/gc.log" in spec.jvm_opts - assert "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags" \ - in spec.jvm_opts - assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > \ - spec.jvm_opts.index( - "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags") + assert default_gc_log in spec.jvm_opts + assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > spec.jvm_opts.index(default_gc_log) diff --git a/modules/ducktests/tests/checks/utils/pause/check_banner.py b/modules/ducktests/tests/checks/utils/pause/check_banner.py new file mode 100644 index 0000000000000..d32443388dee0 --- /dev/null +++ b/modules/ducktests/tests/checks/utils/pause/check_banner.py @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks the banner a held demo breakpoint renders: how long the scenario has been running, the +nodes it is made of, and the commands offered for looking into them. +""" + +import os +import time + +from ignitetest.utils.pause import ALL, DemoPause, _resume_command +from ignitetest.utils.pause_control import ControlDir, repo_root + +from checks.support.demo_pause_control import new_demo_pause, published_status +from checks.support.ducktape_doubles import FakeBrokenService, FakeIgniteService, FakeRegistry, FakeService + + +def check_elapsed_is_counted_from_test_start(tmp_path): + """ + Check that the banner counts from the start of the test rather than from the first + breakpoint: the setup phase of a multi-node scenario is minutes long, and a demo that + reports t+00:00 after it hides exactly the part worth showing. + """ + demo = new_demo_pause(tmp_path, started_at=time.monotonic() - 600, demo_pause=ALL, demo_pause_timeout_sec=.3) + + with published_status(tmp_path) as published: + demo.pause("split-brain") + + assert published["elapsed_sec"] >= 600 + assert any("t+10:00 since test start" in line for line in published["banner"]) + + +def check_banner_is_rendered_from_the_service_registry(tmp_path): + """ + Check that the banner is built by iterating the services alone: what a test passes is + ducktape's ServiceRegistry, which supports nothing else. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3) + + with published_status(tmp_path) as published: + demo.pause("split-brain", services=FakeRegistry(FakeService("ducker02"), FakeIgniteService("ducker03"))) + + banner = "\n".join(published["banner"]).replace("\\", "/") + + assert "FakeService-ducker02" in banner + assert "FakeIgniteService-ducker03" in banner + assert "/mnt/service/logs/ignite*.log" in banner, "the hints must still follow the Ignite service" + assert "ducker02 ducker03" in banner + + +def check_a_service_that_cannot_answer_is_degraded_not_raised(tmp_path): + """ + Check that a service which cannot answer for its nodes costs the banner those lines and + nothing more. A breakpoint only observes the cluster, so one that throws while rendering + would fail the scenario at exactly the point the demo was added to show. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3) + + with published_status(tmp_path) as published: + demo.pause("split-brain", + services=FakeRegistry(FakeBrokenService("ducker02"), FakeIgniteService("ducker03"))) + + banner = "\n".join(published["banner"]) + + assert "FakeBrokenService-ducker02" in banner, "the node must still be named, without asking its service" + assert "RuntimeError" in banner, "and what could not be read must say so" + assert "FakeIgniteService-ducker03" in banner, "a service after the broken one must still be listed" + assert "ducker02 ducker03" in banner, "and every node must still be offered by the hints" + + +def check_the_resume_command_is_named_where_it_will_be_typed(): + """ + Check that the command a breakpoint offers is one its reader can actually run. The banner + is rendered inside ducker01, where the repository is mounted at /opt/ignite-dev, and the + console prints it verbatim on the host, where that path does not exist - so the shared + repository root is the only anchor the two sides have in common. + """ + inside = ControlDir(os.path.join(repo_root(), ".ducktests-demo")) + + command = _resume_command(inside, 3) + + assert command == "touch .ducktests-demo/continue-3 (from the repository root)" + assert repo_root() not in command, "an absolute path here names a directory the reader may not have" + + +def check_the_resume_command_falls_back_to_the_whole_path(): + """ + Check that a control directory outside the repository is still named in full: there is no + shared anchor left to make it relative to, so a whole path is the honest answer. + """ + outside = ControlDir(os.path.join(os.path.dirname(repo_root()), "elsewhere", "demo")) + + command = _resume_command(outside, 3) + + assert command.endswith(os.path.join("elsewhere", "demo", "continue-3")) + assert "from the repository root" not in command + + +def check_hints_follow_the_ignite_services(): + """ + Check that the copy-pasteable commands name the Ignite paths even when a service of + another kind was registered first, as the zookeeper discovery scenarios do. + """ + # noinspection PyProtectedMember + hints = "\n".join(DemoPause._hints_section([FakeService("ducker02"), # pylint: disable=protected-access + FakeIgniteService("ducker03")])) + + # The service paths come from os.path.join, which follows the control machine rather than + # the nodes - a check that runs on Windows would otherwise see its separators. + hints = hints.replace("\\", "/") + + assert "/mnt/service/config/ignite-config.xml" in hints + assert "/mnt/service/logs/ignite*.log" in hints + assert "zookeeper.properties" not in hints + assert "zk-logs" not in hints + + # Every node is still offered, whichever service it belongs to. + assert "ducker02 ducker03" in hints diff --git a/modules/ducktests/tests/checks/utils/pause/check_control.py b/modules/ducktests/tests/checks/utils/pause/check_control.py new file mode 100644 index 0000000000000..e84c83478a62a --- /dev/null +++ b/modules/ducktests/tests/checks/utils/pause/check_control.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks the file protocol a demo breakpoint and the host speak over the control directory: +what a held breakpoint publishes, what resumes it, and what is swept up afterwards. +""" + +import os + +import pytest + +from ignitetest.utils.pause import ALL +from ignitetest.utils.pause_control import ABORT, CONTINUE_ALL, STATUS_JSON, STATUS_TXT, continue_file + +from checks.support.demo_pause_control import TEST_NAME, new_demo_pause, published_status, resume_with + + +def check_publishes_and_consumes_status(tmp_path): + """ + Check the published breakpoint - what the host reads - and that the test cleans it up + once resumed, so a stale banner never outlives the pause it describes. + """ + demo = new_demo_pause(tmp_path, demo_pause=True) + + with published_status(tmp_path, resume=continue_file(1)) as published: + demo.pause("split-brain", services=[]) + + assert published["seq"] == 1 + assert published["run"] == demo.run + assert published["name"] == "split-brain" + assert published["test"] == TEST_NAME + assert any("PAUSED 1 split-brain" in line for line in published["banner"]) + + for leftover in (STATUS_JSON, STATUS_TXT, continue_file(1)): + assert not os.path.exists(str(tmp_path / leftover)), leftover + + +def check_continue_all_skips_the_rest(tmp_path): + """ + Check that continue-all resumes the current breakpoint and disables every later one, so + a demo can be cut short without restarting the scenario. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL) + + resume_with(tmp_path, CONTINUE_ALL) + + demo.pause("split-brain") + + assert demo.seq == 1 + assert not demo.enabled + + demo.pause("healed") + + assert demo.seq == 1, "breakpoints after continue-all must not stop the scenario" + assert not os.path.exists(str(tmp_path / CONTINUE_ALL)) + + +def check_abort_fails_the_test(tmp_path): + """ + Check that abort ends the scenario through an assertion, so ducktape tears the cluster + down instead of leaving it running. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL) + + resume_with(tmp_path, ABORT) + + with pytest.raises(AssertionError, match="split-brain"): + demo.pause("split-brain") + + assert not os.path.exists(str(tmp_path / ABORT)) + assert not os.path.exists(str(tmp_path / STATUS_JSON)) + + +def check_a_control_directory_that_cannot_be_made_does_not_fail_the_scenario(tmp_path): + """ + Check that a control directory which cannot even be created costs the demo and nothing + else. It is a bind mount of the host repository, so it can be read only or owned by + another user - neither of which says anything about the cluster under test, and a + breakpoint must not be what turns a passing run red. + """ + # A file where the directory should go: the portable way to make os.makedirs fail. + (tmp_path / "in-the-way").write_text("not a directory", encoding="utf-8") + + demo = new_demo_pause(tmp_path / "in-the-way" / "control", demo_pause=ALL) + + demo.pause("split-brain") + + assert demo.seq == 0, "the scenario must have carried straight on" + assert not demo.enabled, "and the later breakpoints must not try it again" + assert any("control directory cannot be used" in msg for msg in demo.logger.messages), demo.logger.messages + + +def check_a_breakpoint_that_cannot_be_published_is_skipped(tmp_path): + """ + Check the same where the directory exists but the banner cannot be written. Blocking would + be no better than raising here: nothing reached the host, so there would be nothing on + screen to resume, and the scenario would sit there for the whole timeout. + """ + # A directory where the banner goes makes the write fail wherever these checks run. The + # sweep cannot remove it either, so it is still in the way when the breakpoint publishes. + os.mkdir(str(tmp_path / STATUS_TXT)) + + demo = new_demo_pause(tmp_path, demo_pause=ALL) + + demo.pause("split-brain") + + assert not demo.enabled + assert not os.path.exists(str(tmp_path / STATUS_JSON)), "half a breakpoint must not be left published" + assert any("control directory cannot be used" in msg for msg in demo.logger.messages), demo.logger.messages + + +def check_stale_resume_file_is_cleared(tmp_path): + """ + Check that a resume file left by a previous run does not skip the first breakpoint of + this one - the control directory outlives a test, its contents must not. + """ + open(str(tmp_path / continue_file(1)), "w").close() + open(str(tmp_path / STATUS_TXT), "w").close() + + demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3) + + demo.pause("split-brain") + + assert demo.seq == 1 + assert any("timed out" in msg for msg in demo.logger.messages), \ + "the stale file must have been cleared, leaving the breakpoint to time out" diff --git a/modules/ducktests/tests/checks/utils/pause/check_control_dir.py b/modules/ducktests/tests/checks/utils/pause/check_control_dir.py new file mode 100644 index 0000000000000..e5bb01c1ab6da --- /dev/null +++ b/modules/ducktests/tests/checks/utils/pause/check_control_dir.py @@ -0,0 +1,208 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks the control directory itself - the files a paused test and the host exchange, and the +polling that waits for them - with no breakpoint on top of it. +""" + +import os +import time + +from ignitetest.utils.pause_control import ABORT, CONTINUE_ALL, ControlDir, STATUS_JSON, STATUS_TXT, continue_file + + +def check_taking_a_file_consumes_it(tmp_path): + """ + Check that taking a control file reports it and removes it: the test is the only party + that deletes what the host wrote, so taking one is what acknowledges it. + """ + control = ControlDir(tmp_path) + + assert not control.take(ABORT), "nothing was written yet" + + control.resume(ABORT) + + assert control.exists(ABORT) + assert control.take(ABORT) + assert not control.exists(ABORT), "a taken file must not be left for the next breakpoint" + assert not control.take(ABORT) + + +def check_sweep_spares_a_held_breakpoint(tmp_path): + """ + Check the asymmetry the two sides need: both clear the resume files an earlier run left, + but only the test drops the banner as well. The console is just as likely to have been + started against a test that is already holding one. + """ + control = ControlDir(tmp_path) + + control.publish(["PAUSED 1 split-brain"], {"seq": 1}) + control.resume(continue_file(3)) + control.resume(CONTINUE_ALL) + + control.sweep() + + assert not control.exists(continue_file(3)) + assert not control.exists(CONTINUE_ALL) + assert control.read_status() is not None, "the console must not withdraw a breakpoint it did not hold" + + control.sweep(status=True) + + assert control.read_status() is None + assert not control.exists(STATUS_TXT) + + +def check_sweeping_a_directory_that_is_not_there(tmp_path): + """ + Check that sweeping is safe before anything has been created: without the global no + breakpoint ever makes the directory, and the console may well be started first. + """ + control = ControlDir(tmp_path / "never-created") + + control.sweep(status=True) + + assert control.read_status() is None + assert not os.path.exists(control.path), "sweeping must not create what it was asked to clean" + + +def check_sweep_clears_what_an_interrupted_write_left(tmp_path): + """ + Check that the temporary name a write goes through is swept too, whichever file it + belonged to. Every control file is matched by prefix for this reason: one matched exactly + would leave its ".tmp" in a directory that nothing else ever visits. + """ + control = ControlDir(tmp_path) + + for interrupted in (continue_file(1), CONTINUE_ALL, ABORT, STATUS_TXT, STATUS_JSON): + control.write(interrupted + ".tmp", "") + + control.sweep(status=True) + + assert os.listdir(control.path) == [] + + +def check_a_resume_file_lands_without_a_temporary(tmp_path): + """ + Check that resuming creates the file and nothing besides: a resume file is empty, so an + atomic replace would buy nothing and leave a transient behind for the sweep to know about. + """ + control = ControlDir(tmp_path) + + control.resume(ABORT) + + assert os.listdir(control.path) == [ABORT] + + +def check_publishing_round_trips(tmp_path): + """ + Check that what is published is what a reader gets back, and that withdrawing it leaves + nothing of either file behind. + """ + control = ControlDir(tmp_path) + + control.publish(["PAUSED 1 split-brain", " test some.Test"], {"seq": 1, "name": "split-brain"}) + + assert control.read_status() == {"seq": 1, "name": "split-brain"} + + with open(control.file(STATUS_TXT), encoding="utf-8") as file: + assert file.read() == "PAUSED 1 split-brain\n test some.Test\n" + + control.clear_status() + + assert control.read_status() is None + assert not control.exists(STATUS_TXT) + assert not control.exists(STATUS_JSON) + + +def check_unreadable_status_reads_as_nothing_published(tmp_path): + """ + Check that a status file caught half written reads as "not paused" rather than raising: + the host polls this in a loop and simply comes back. + """ + control = ControlDir(tmp_path) + + control.write(STATUS_JSON, '{"seq": 1') + + assert control.read_status() is None + + +def check_awaiting_takes_the_file_that_arrived(tmp_path): + """ + Check the wait ends on the file that appears, and hands back which one it was. + """ + control = ControlDir(tmp_path) + + control.resume(continue_file(1)) + + assert control.await_any([ABORT, continue_file(1)], 5) == continue_file(1) + assert not control.exists(continue_file(1)), "the wait must consume what ended it" + + +def check_awaiting_honours_the_order_it_was_given(tmp_path): + """ + Check that the first name wins when several are already there: abort has to beat a + continue that landed in the same interval, or a demo would be resumed instead of ended. + """ + control = ControlDir(tmp_path) + + control.resume(CONTINUE_ALL) + control.resume(ABORT) + + assert control.await_any([ABORT, CONTINUE_ALL], 5) == ABORT + assert control.exists(CONTINUE_ALL), "only the file that ended the wait may be consumed" + + +def check_awaiting_gives_up(tmp_path): + """ + Check that a wait nobody answers ends on its own rather than holding the scenario until + ducktape kills it. + """ + control = ControlDir(tmp_path) + + started_at = time.monotonic() + + assert control.await_any([ABORT], .3) is None + assert time.monotonic() - started_at >= .3, "it must have waited for what it was given" + + +def check_awaiting_ticks_without_being_told_how_often(tmp_path): + """ + Check that a caller who wants to hear that the wait is still running need not also pick an + interval - asking for one and getting a TypeError out of a held breakpoint would be a poor + way to find out that the two arguments go together. + """ + control = ControlDir(tmp_path) + + ticks = [] + + assert control.await_any([ABORT], .6, tick=ticks.append) is None + assert ticks, "a wait longer than one poll must report itself" + + +def check_awaiting_reports_that_it_is_still_waiting(tmp_path): + """ + Check that the caller is ticked while the wait runs, which is how a held breakpoint says + in the test log that it is paused rather than stuck - without this class having to know + what a log is. + """ + control = ControlDir(tmp_path) + + ticks = [] + + assert control.await_any([ABORT], 1.2, tick=ticks.append, tick_sec=.01) is None + assert ticks, "a wait longer than the tick interval must report itself" + assert all(0 < left <= 1.2 for left in ticks), ticks + assert ticks == sorted(ticks, reverse=True), "each tick must report less time left than the last" diff --git a/modules/ducktests/tests/checks/utils/pause/check_selector.py b/modules/ducktests/tests/checks/utils/pause/check_selector.py new file mode 100644 index 0000000000000..f01c568562693 --- /dev/null +++ b/modules/ducktests/tests/checks/utils/pause/check_selector.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks which demo breakpoints stop a scenario: how the ``demo_pause`` global is read, and +which of the breakpoints in the test it then selects. +""" + +import os + +from ignitetest.utils.pause import ALL, parse_selector +from ignitetest.utils.pause_control import continue_file + +from checks.support.demo_pause_control import new_demo_pause, resume_with + + +def check_selector_parsing(): + """ + Check that every shape the demo_pause global can arrive in is understood: -g passes it as + a string, -gj as whatever the json holds. + """ + for disabled in (None, False, "", "false", "off", "0", [], " "): + assert parse_selector(disabled) is None, disabled + + for every in (True, "*", "all", "true", "ON", "1"): + assert parse_selector(every) == ALL, every + + assert parse_selector("split-brain") == {"split-brain"} + assert parse_selector("split-brain, healed ,") == {"split-brain", "healed"} + assert parse_selector(["split-brain", "healed"]) == {"split-brain", "healed"} + + # The global is typed by hand, the names live in the test source - the two meet case insensitively. + assert parse_selector("Split-Brain, HEALED") == {"split-brain", "healed"} + assert parse_selector(["Split-Brain"]) == {"split-brain"} + + +def check_names_are_matched_case_insensitively(tmp_path): + """ + Check that a breakpoint is found however the global spells it. + """ + demo = new_demo_pause(tmp_path, demo_pause="Split-Brain") + + resume_with(tmp_path, continue_file(1)) + + demo.pause("split-brain") + + assert demo.seq == 1, "the global must not have to repeat the case of the name in the test" + + +def check_disabled_leaves_no_trace(tmp_path): + """ + Check that without the global a breakpoint is a plain return: it must not block, and it + must not even create the control directory, since every test carries breakpoints in CI. + """ + control_dir = tmp_path / "control" + + demo = new_demo_pause(control_dir) + + assert not demo.enabled + + demo.pause("split-brain") + + assert not os.path.exists(str(control_dir)) + assert demo.seq == 0 + + +def check_selected_breakpoints_only(tmp_path): + """ + Check that only the named breakpoints stop the scenario. + """ + demo = new_demo_pause(tmp_path, demo_pause="split-brain") + + demo.pause("cluster-up") + demo.pause("healed") + + assert demo.seq == 0, "an unnamed breakpoint must not stop the scenario" + + resume_with(tmp_path, continue_file(1)) + + demo.pause("split-brain") + + assert demo.seq == 1 diff --git a/modules/ducktests/tests/checks/utils/pause/check_timeout.py b/modules/ducktests/tests/checks/utils/pause/check_timeout.py new file mode 100644 index 0000000000000..e47b922281f39 --- /dev/null +++ b/modules/ducktests/tests/checks/utils/pause/check_timeout.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks how long a demo breakpoint may hold a scenario: the timeout the demo asks for, and +ducktape's --test-runner-timeout budget that only ever shortens it. +""" + +import os + +from ignitetest.utils.pause import ALL, RUNNER_TIMEOUT_MARGIN_SEC +from ignitetest.utils.pause_control import STATUS_JSON + +from checks.support.demo_pause_control import new_demo_pause, published_status + + +def check_timeout_resumes_on_its_own(tmp_path): + """ + Check that a forgotten breakpoint gives up rather than holding the scenario until + ducktape kills it. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3) + + demo.pause("split-brain") + + assert demo.seq == 1 + assert demo.enabled, "a timed out breakpoint must not disable the later ones" + assert not os.path.exists(str(tmp_path / STATUS_JSON)) + + +def check_timeout_stays_within_the_runner_budget(tmp_path): + """ + Check that a breakpoint gives up while ducktape's runner is still waiting: it hears + nothing from a paused test, and killing the client takes the whole session down instead + of just cutting the demo short. The requested timeout only ever shrinks. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=3600, + runner_timeout_sec=RUNNER_TIMEOUT_MARGIN_SEC + .3) + + demo.pause("split-brain") + + assert demo.seq == 1 + assert any("timed out" in msg for msg in demo.logger.messages), \ + "the breakpoint must not outsit the runner budget it was given" + assert any("--test-runner-timeout" in msg for msg in demo.logger.messages), \ + "shortening a breakpoint must say what to raise to keep it" + + +def check_timeout_is_left_alone_within_the_runner_budget(tmp_path): + """ + Check that the budget only ever caps the requested timeout - a demo that fits must be + held for exactly as long as it asked for. + """ + demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3, runner_timeout_sec=1800) + + with published_status(tmp_path) as published: + demo.pause("split-brain") + + assert published["timeout_sec"] == .3 + assert not any("--test-runner-timeout" in msg for msg in demo.logger.messages) diff --git a/modules/ducktests/tests/conftest.py b/modules/ducktests/tests/conftest.py new file mode 100644 index 0000000000000..76fa9af5b7e85 --- /dev/null +++ b/modules/ducktests/tests/conftest.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Deliberately empty - pytest imports this file for its side effect alone, which is to put the +directory it sits in on ``sys.path``. + +That is the whole of it: importing a conftest adds *its own* directory, and pytest on its own +adds only each check file's. Two things depend on this one being the directory that gets +added, so moving this file down into ``checks/`` - which is where it looks like it belongs - +breaks both: + + - ``from checks.support... import ...``. The checks live in ``checks/``, their helper + modules apart from them in ``checks/support/``, and the name ``checks`` resolves only + while its parent is on the path. A conftest in ``checks/`` would add ``checks/`` and the + import would have to become a bare ``support...`` instead. + + - ``import ignitetest``, under tox. ``[tox] skipsdist = True`` reads to tox 4 as + ``no_package``, which silently drops the ``usedevelop`` install, so the tox environment + has no installed ignitetest at all and reaches the checkout through this path entry + alone. Take this file away and every check file fails at collection, not just the ones + importing ``checks.support``. Outside tox the editable install covers it, which is why + the breakage only shows in one of the two ways the checks are run. + +The second reason is a workaround, not a design: once the packaging is fixed - a +``pyproject.toml`` and a PEP 517 editable build, so ``usedevelop`` works again - only the +first remains, and this file could then reasonably move into ``checks/``. + +Nothing below ``checks/`` carries an ``__init__.py`` on purpose: ``setup.py`` collects the +distribution with ``find_packages()``, which only finds directories that have one, so the +checks and their helpers stay out of the installed ``ignitetest`` package without ``setup.py`` +having to name them, and ``checks.support`` resolves as a namespace package instead. +""" diff --git a/modules/ducktests/tests/docker/demo_console.py b/modules/ducktests/tests/docker/demo_console.py new file mode 100644 index 0000000000000..6f4b6c8c6a85a --- /dev/null +++ b/modules/ducktests/tests/docker/demo_console.py @@ -0,0 +1,194 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Host side of the demo breakpoints - run it in a second terminal, next to the one running +``run_tests.sh``, when a test is started with the ``demo_pause`` global: + + ./docker/run_tests.sh -gj '{"demo_pause": "*"}' -t ./ignitetest/tests/ + + python docker/demo_console.py + +Ducktape runs the test with stdin on /dev/null inside the ``ducker01`` container, so this is +where the keyboard lives. The console itself is deliberately dumb: the test renders the +banner and this only prints it and writes back a resume file. Everything it does can be done +by hand instead - ``cat .ducktests-demo/paused.txt``, then ``touch .ducktests-demo/continue-3``. + +Standard library only: it runs on the host, outside the ducktests virtualenv. +""" + +import argparse +import importlib.util +import os +import sys +import time + +# Speak the protocol through the framework's own ControlDir rather than restating it here: +# the two sides of a shared directory have to agree file for file, and a second copy of it +# is a second thing to keep in step. +# +# The host has no ducktape and no installed ignitetest, so the module is loaded by path - +# importing ignitetest.utils.pause_control would pull in the package __init__ chain and its +# ducktape imports. pause_control itself is standard library only, which is what makes it +# loadable like this; ignitetest.utils.pause, which holds what the files mean, is not. +_TESTS_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)) +_PAUSE_CONTROL_PY = os.path.join(_TESTS_DIR, "ignitetest", "utils", "pause_control.py") + +_SPEC = importlib.util.spec_from_file_location("ignitetest_pause_control", _PAUSE_CONTROL_PY) +pause = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(pause) + +POLL_SEC = .3 + +KEYS = """ + [Enter] continue [c] continue, skipping the rest [a] abort the test + [q] leave the console (the test stays paused) +""" + + +def breakpoint_key(status): + """ + :return: What identifies the published breakpoint. Not the sequence number on its own: + that one is per test, so it restarts at 1 for every test of a session, and a + run that died while paused leaves behind a banner numbered like a live one. + """ + return status.get("run"), status.get("seq") + + +def clear_stale(control): + """ + Removes resume files left behind by an earlier run, which would otherwise skip the first + breakpoint of this one. The test clears them too, on its side, at its first breakpoint. + + Only ever done while nothing is published: a console is just as likely to be started + against a test that is already holding a breakpoint, and a resume file that was written + for that one - by hand, or by a console that has just been closed - is the host's answer + to it rather than a leftover. The published breakpoint itself is left alone either way, + a stale banner is told apart by its run id. + + :return: Whether the sweep was performed. + """ + if control.read_status() is not None: + return False + + control.sweep() + + return True + + +def prompt(control, seq): + """ + Asks what to do with the breakpoint that is currently published. + + :return: False when the console should stop, True to wait for the next breakpoint. + """ + while True: + try: + answer = input(" > ").strip().lower() + except EOFError: + return False + + if answer in ("", "n", "next"): + control.resume(pause.continue_file(seq)) + + return True + + if answer in ("c", "continue", "all"): + control.resume(pause.CONTINUE_ALL) + + print(" continuing, remaining breakpoints skipped") + + return False + + if answer in ("a", "abort"): + control.resume(pause.ABORT) + + print(" aborting the test") + + return False + + if answer in ("q", "quit", "exit"): + print(f" leaving the test paused, resume it with:\n" + f" touch {control.file(pause.continue_file(seq))}") + + return False + + print(KEYS) + + +def main(): + """ + Waits for breakpoints and drives them until the test is resumed for good. + """ + parser = argparse.ArgumentParser(description="Drives the ducktests demo breakpoints.") + parser.add_argument("-d", "--control-dir", default=pause.default_control_dir(), + help="control directory shared with the test, defaults to " + f"/{pause.CONTROL_DIR_NAME}") + + args = parser.parse_args() + control = pause.ControlDir(args.control_dir) + + swept = clear_stale(control) + + print(f"Demo console, watching {control.path}") + + if swept: + print("Waiting for the first breakpoint... (Ctrl-C to leave)") + else: + print("A breakpoint is already held, joining it as it is (Ctrl-C to leave)") + + last_key, resumed_at = None, None + + while True: + status = control.read_status() + + if status is None: + # The test removes its status files as it resumes, so this is also what tells the + # console that the breakpoint it has just driven is over and the next one - which + # may well repeat its number, in the next test of the session - is a new one. + last_key = None + + time.sleep(POLL_SEC) + + continue + + if breakpoint_key(status) == last_key: + time.sleep(POLL_SEC) + + continue + + last_key = breakpoint_key(status) + + print() + + if resumed_at is not None: + print(f" ({time.monotonic() - resumed_at:.0f}s since the previous breakpoint)") + + print("\n".join(status.get("banner", []))) + print(KEYS) + + if not prompt(control, status.get("seq")): + return + + resumed_at = time.monotonic() + + print(" resumed, waiting for the next breakpoint...") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130) diff --git a/modules/ducktests/tests/docker/requirements-dev.txt b/modules/ducktests/tests/docker/requirements-dev.txt index f868e43d67466..8ce0758663a4a 100644 --- a/modules/ducktests/tests/docker/requirements-dev.txt +++ b/modules/ducktests/tests/docker/requirements-dev.txt @@ -16,3 +16,4 @@ -r requirements.txt pytest==6.2.5 flake8==6.1.0 +tox diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index 511c106701a29..b23ca856dcd7b 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -84,6 +84,10 @@ The options are as follows: --image Set custom docker image to run tests on. +--test-runner-timeout + Milliseconds ducktape waits for a sign of life from a running test before killing the + session, 1800000 by default. + EOF exit 0 } @@ -131,6 +135,7 @@ while [[ $# -ge 1 ]]; do --subnet) SUBNET="--subnet $2"; shift 2;; --jdk) JDK_VERSION="$2"; shift 2;; --image) IMAGE_NAME="$2"; shift 2;; + --test-runner-timeout) TEST_RUNNER_TIMEOUT="$2"; shift 2;; -f|--force) FORCE=$1; shift;; *) break;; esac @@ -169,5 +174,9 @@ if [[ -n "$REPEAT" ]]; then DUCKTAPE_OPTIONS="$DUCKTAPE_OPTIONS --repeat $REPEAT" fi +if [[ -n "$TEST_RUNNER_TIMEOUT" ]]; then + DUCKTAPE_OPTIONS="$DUCKTAPE_OPTIONS --test-runner-timeout $TEST_RUNNER_TIMEOUT" +fi + "$SCRIPT_DIR"/ducker-ignite test $TC_PATHS "$DUCKTAPE_OPTIONS" \ || die "ducker-ignite test failed" diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index d37b9c045fa1c..a9ccbe4496d3c 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -195,6 +195,36 @@ def network_registry(self) -> Dict[str, List]: return registry + def describe(self) -> List[str]: + """ + Describes the cluster per data center for a demo breakpoint banner + (see :meth:`ignitetest.utils.ignite_test.IgniteTest.pause`). The generic banner sees + a flat list of services, which is where the DC each node belongs to gets lost. + + Structure only - which node is up is what the banner's own service section reports, + and it pays an SSH probe per node to find out. + + :return: Section lines, the first one being the section title. + """ + lines = ["DATA CENTERS"] + + for dc in DCS: + roles = [(label, [node.account.hostname for svc in services for node in svc.nodes]) + for label, services in (("server", [self.servers[dc]] if dc in self.servers else []), + ("runner", self.runners.get(dc, [])), + ("loader", self.loaders.get(dc, [])), + ("extra", self.extras.get(dc, [])))] + + # A DC that holds nothing is not named at all: an empty header reads as a DC whose + # nodes have gone, which is exactly what a partition demo is being watched for. + if not any(hosts for _, hosts in roles): + continue + + lines.append(f" {dc}") + lines.extend(f" {label:<7} {' '.join(hosts)}" for label, hosts in roles if hosts) + + return lines + def thin_client_addresses(self) -> List[str]: """ :return: Thin client addresses of all server nodes across all DCs. diff --git a/modules/ducktests/tests/ignitetest/services/network_group/manager.py b/modules/ducktests/tests/ignitetest/services/network_group/manager.py index 3f90b7f9450a1..c06884b37267b 100644 --- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py +++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py @@ -297,6 +297,29 @@ def _log_network(self, log_tag: str): """ self.logger.debug(f"Network State Overview: [START][{log_tag}]") + # The per-node SSH probes flood the debug log with their own command output. Collect + # first, print contiguously after: the overview must stay readable as one block. + for node_status in self._probe_network(): + self.logger.debug(node_status) + + self.logger.debug(f"Network State Overview: [END][{log_tag}]") + + def describe(self): + """ + Describes the live network state for a demo breakpoint banner + (see :meth:`ignitetest.utils.ignite_test.IgniteTest.pause`). + + :return: Section lines, the first one being the section title. + """ + return ["NETWORK"] + [f" {status}" for status in self._probe_network()] + + def _probe_network(self): + """ + Probes the actual network state of every node - the applied netem constraints, what + they are filtered onto, and the partition drops - rather than what was asked for. + + :return: One line per node, grouped by network group. + """ entries = [(group, svc, node) for group, services in self.network_group_registry.items() for svc in services @@ -322,12 +345,7 @@ def _log_network(self, log_tag: str): node_statuses.append(f"[{group:<4}] {svc.who_am_i(node):<45}[{node_ip}] : " f"{constraints}{targets_str}{partition_str}") - # The per-node SSH probes above flood the debug log with their own command output. - # Collect first, print contiguously after: the overview must stay readable as one block. - for node_status in node_statuses: - self.logger.debug(node_status) - - self.logger.debug(f"Network State Overview: [END][{log_tag}]") + return node_statuses def _to_network_probe_cmd(self, node) -> str: """ diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py index 2ac3929805558..e67d807caeaa0 100644 --- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py +++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py @@ -63,17 +63,23 @@ def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms) as net: mdc.start_servers() + self.pause("cluster-up", mdc, net) + mdc.generate_data(DC_1, CACHE_NAME, 0, 100, backups=BACKUPS) mdc.generate_data(DC_2, CACHE_NAME, 100, 200, backups=BACKUPS) mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + self.pause("data-loaded", mdc, net) + net.enable_network_partition(DC_1, DC_2) sleep(SPLIT_SETTLE_SECS) mdc.verify_split_brain() + self.pause("split-brain", mdc, net) + # All data written before the split is readable in both halves. mdc.check_data(DC_1, CACHE_NAME, 0, 200) mdc.check_data(DC_2, CACHE_NAME, 0, 200) @@ -82,6 +88,8 @@ def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency mdc.check_put_admissibility(DC_1, CACHE_NAME, True) mdc.check_put_admissibility(DC_2, CACHE_NAME, False) + self.pause("secondary-read-only", mdc, net) + net.disable_network_partition(DC_1, DC_2) # Split-brain does not self-heal: the read-only half rejoins via restart. @@ -94,6 +102,8 @@ def test_mdc_cluster_partition_resilience(self, ignite_version, cross_dc_latency mdc.control(DC_1).idle_verify(CACHE_NAME) + self.pause("healed", mdc, net) + mdc.verify_servers_log_clean() mdc.stop_servers() diff --git a/modules/ducktests/tests/ignitetest/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/utils/ignite_test.py index 1feda7859ea99..82562ef5941bf 100644 --- a/modules/ducktests/tests/ignitetest/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/utils/ignite_test.py @@ -23,6 +23,7 @@ from ducktape.tests.test import Test, TestContext from ignitetest.services.utils.ducktests_service import DucktestsService +from ignitetest.utils.pause import DemoPause # globals: JFR_ENABLED = "jfr_enabled" @@ -66,6 +67,46 @@ def __init__(self, test_context): super().__init__(test_context=test_context) + self.__demo_pause = None + + # Stamped here rather than at the first breakpoint: it is what demo breakpoints count + # their elapsed time from, and what they measure the runner budget against, and both + # of those mean the start of the test - setup included. + self.__started_at = monotonic() + + def pause(self, name, *describers): + """ + Holds the scenario at a named demo breakpoint until it is resumed from the host, so + that the cluster can be shown in exactly this state. Does nothing at all unless the + `demo_pause` global names this breakpoint - see :mod:`ignitetest.utils.pause`. + + Must be called from the test body: :meth:`tearDown` kills every service, so a + breakpoint placed after the body would only ever show a dead cluster. + + :param name: Breakpoint name, matched against the `demo_pause` global. + :param describers: Objects exposing `describe() -> list of str`, each contributing a + section to the banner shown while paused, on top of the service list every + breakpoint reports. Any fixture a test drives can implement it. + """ + if self.__demo_pause is None: + self.__demo_pause = DemoPause(self.logger, self.test_context.globals, self.test_context.test_name, + started_at=self.__started_at, + runner_timeout_sec=self.__runner_timeout_sec()) + + self.__demo_pause.pause(name, describers, self.test_context.services) + + def __runner_timeout_sec(self): + """ + :return: Ducktape's `--test-runner-timeout` in seconds, None when the session carries + none. A breakpoint has to give up inside it, since the runner kills a test + client it hears nothing from for that long - see + :meth:`ignitetest.utils.pause.DemoPause._budgeted_timeout`. + """ + session_context = getattr(self.test_context, "session_context", None) + timeout_ms = getattr(session_context, "test_runner_timeout", None) + + return timeout_ms / 1000 if timeout_ms else None + @property def available_cluster_size(self): # noinspection PyUnresolvedReferences diff --git a/modules/ducktests/tests/ignitetest/utils/pause.py b/modules/ducktests/tests/ignitetest/utils/pause.py new file mode 100644 index 0000000000000..05fd17eb78757 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/utils/pause.py @@ -0,0 +1,506 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Demo breakpoints: freezing a scenario at a named point so a live audience can be shown the +cluster in that exact state. + +Ducktape runs the test inside the ``ducker01`` container with stdin on /dev/null, so a test +cannot read a keypress. What it can do is share files with the host: ``ducker-ignite`` bind +mounts the whole Ignite repository into every container, so a control directory below the +repository root is visible to the test and to the host at the same time. + +The directory itself, and the file protocol over it, are +:class:`ignitetest.utils.pause_control.ControlDir`. This module holds only what those files +*mean*: which breakpoint stops the scenario, what the banner says, and that ``abort`` ends +the test. + +Between breakpoints both sides sweep the directory for files an earlier run left behind, +which would otherwise skip the next breakpoint: the test at its first one (see +:meth:`DemoPause._prepare`), the console at startup - and only while nothing is published, so +a resume file meant for a breakpoint that is currently held is never swept. + +``docker/demo_console.py`` is the host side of it, but nothing depends on it: reading +``paused.txt`` and touching ``continue-`` by hand works just as well. + +Globals: + + demo_pause - absent or false disables every breakpoint (the default, so tests are + unaffected in CI); true or "*" stops at all of them; a list or a comma separated string + stops only at the named ones, matched case insensitively. + + demo_pause_timeout_sec - how long a single breakpoint may hold the scenario before it + resumes on its own, 600 by default. Capped by what is left of ducktape's + ``--test-runner-timeout``, see :meth:`DemoPause._budgeted_timeout`. + + demo_pause_dir - control directory, ``/.ducktests-demo`` by default. +""" + +import os +import time + +from ignitetest.services.utils.path import IgnitePathAware +from ignitetest.utils.pause_control import ABORT, CONTINUE_ALL, ControlDir, continue_file, default_control_dir, \ + repo_root + +# globals: +DEMO_PAUSE = "demo_pause" +DEMO_PAUSE_TIMEOUT_SEC = "demo_pause_timeout_sec" +DEMO_PAUSE_DIR = "demo_pause_dir" + +# Well below ducktape's own --test-runner-timeout (1800s), which a breakpoint must not +# outsit - see _budgeted_timeout(). +DEFAULT_TIMEOUT_SEC = 600 + +# Kept free of the runner budget, so that resuming a breakpoint at the very last moment +# still leaves the scenario time to reach its next event. +RUNNER_TIMEOUT_MARGIN_SEC = 60 + +# Timestamps the demo in the test log, so that a run can be read back afterwards and it is +# visible that the scenario is held rather than stuck. Deliberately NOT a keepalive towards +# ducktape: the test logger writes to files and to stdout only, while the runner listens for +# zmq events that just the runner client itself emits - _budgeted_timeout() is what keeps a +# paused test within the runner's patience. +HEARTBEAT_SEC = 15 + +# Every breakpoint name matches. +ALL = "*" + +_WIDTH = 100 + + +def parse_selector(value): + """ + Interprets the ``demo_pause`` global. + + Names are matched case insensitively, both here and in :meth:`DemoPause._stops_at`: the + global is typed by hand next to a test whose breakpoint names are written in the source. + + :return: None when demo pausing is disabled, :data:`ALL` to stop at every breakpoint, + or the set of breakpoint names to stop at, lower cased. + """ + if value is None or value is False: + return None + + if value is True: + return ALL + + if isinstance(value, (list, tuple, set, frozenset)): + names = {str(name).strip().lower() for name in value} + names.discard("") + + return names or None + + if isinstance(value, str): + text = value.strip().lower() + + if text in ("", "false", "no", "off", "0"): + return None + + if text in (ALL, "all", "true", "yes", "on", "1"): + return ALL + + names = {name.strip() for name in text.split(",")} + names.discard("") + + return names or None + + return ALL if value else None + + +def _fmt_duration(seconds): + """ + :return: Duration as mm:ss, or hh:mm:ss once it no longer fits. + """ + seconds = max(int(seconds), 0) + + if seconds >= 3600: + return f"{seconds // 3600}:{seconds // 60 % 60:02d}:{seconds % 60:02d}" + + return f"{seconds // 60:02d}:{seconds % 60:02d}" + + +def _resume_command(control, seq): + """ + :return: The command that resumes a breakpoint by hand, named where it will be typed. + + Relative to the repository root rather than absolute: the banner is rendered inside + ``ducker01``, where the repository is mounted at ``/opt/ignite-dev``, and read on the host, + where that path does not exist - so an absolute one would be a command that cannot work + for the person it is offered to. Both sides see the same repository through the bind + mount, which makes the relative form the one string that holds for both, and it is the + form README documents. + """ + path = control.file(continue_file(seq)) + + try: + relative = os.path.relpath(path, repo_root()) + except ValueError: + # A control directory on another Windows drive than the repository: nothing shared to + # be relative to, so the full path is all there is to offer. + relative = path + + if relative.startswith(os.pardir) or relative == path: + return f"touch {path}" + + return f"touch {relative.replace(os.sep, '/')} (from the repository root)" + + +def _node_host(node): + """ + :return: Hostname of the node, None when it carries no account to read one from. + """ + return getattr(getattr(node, "account", None), "hostname", None) + + +def _node_addr(node): + """ + :return: The node's routable address when it adds anything to the name the banner already + carries - under ducker the two are the same string. + + Deliberately not resolved to an IP: a name that does not resolve costs a DNS round trip + per node, and a breakpoint that takes seconds to print its banner defeats the point. The + NETWORK section resolves addresses where they actually matter. + """ + addr = node.account.externally_routable_ip + + return "" if addr == node.account.hostname else f"[{addr}]" + + +def _node_state(service, node): + """ + :return: Liveness of the node as far as its service can tell, "?" when the probe itself + failed - a breakpoint must never fail the scenario it is only observing. + """ + alive = getattr(service, "alive", None) + + if alive is None: + return "" + + try: + return "UP" if alive(node) else "DOWN" + except Exception: # pylint: disable=broad-except + return "?" + + +def _node_line(service, node): + """ + :return: The node's line of the SERVICES section, degraded to the name that can be read + without asking the service when the service itself cannot answer for the node - + ``who_am_i`` goes through ``idx()``, which raises for a node the service no + longer owns. + + Like :func:`_node_state`, this lets no reading failure out: a breakpoint must never fail + the scenario it is only observing, least of all while rendering the banner it was added + for. + """ + try: + return f" {service.who_am_i(node):<58} {_node_addr(node):<17} {_node_state(service, node)}".rstrip() + except Exception as ex: # pylint: disable=broad-except + name = f"{type(service).__name__}-{_node_host(node) or '?'}" + + return f" {name:<58} ({type(ex).__name__})" + + +class DemoPause: + """ + Holds a scenario at named breakpoints. + + Disabled unless the ``demo_pause`` global says otherwise, in which case :meth:`pause` is + a plain return and nothing is written anywhere. + """ + def __init__(self, logger, test_globals, test_name, control_dir=None, started_at=None, + runner_timeout_sec=None): + """ + :param started_at: Monotonic timestamp the test itself started at, which is both what + the banner counts from and what the runner budget is spent from. Defaults to + now, which is only right when the first breakpoint is the start of the test. + :param runner_timeout_sec: Ducktape's ``--test-runner-timeout`` in seconds, None when + unknown, in which case no breakpoint is cut short by it. + """ + self.logger = logger + self.test_name = test_name + + self.names = parse_selector(test_globals.get(DEMO_PAUSE)) + self.timeout_sec = float(test_globals.get(DEMO_PAUSE_TIMEOUT_SEC, DEFAULT_TIMEOUT_SEC)) + self.runner_timeout_sec = runner_timeout_sec + + self.control = ControlDir(control_dir or test_globals.get(DEMO_PAUSE_DIR) or default_control_dir()) + + self.seq = 0 + + # Identifies this run of this test to the host console, which has no other way of + # telling a breakpoint of the current run from one left published by a run that + # died while paused: seq alone restarts at 1 for every test. + self.run = f"{os.getpid()}-{int(time.time())}" + + self._started_at = time.monotonic() if started_at is None else started_at + self._prepared = False + self._continue_all = False + self._unusable = False + + @property + def enabled(self): + """ + :return: Whether any breakpoint of this test can stop the scenario. + """ + return self.names is not None and not self._continue_all and not self._unusable + + def pause(self, name, describers=(), services=()): + """ + Blocks the scenario at the named breakpoint until the host resumes it. + + A control directory that cannot be used costs the demo and nothing else - see + :meth:`_give_up`. + + :param name: Breakpoint name, matched against the ``demo_pause`` global. + :param describers: Objects exposing ``describe() -> list of str``, each contributing + a section to the banner. The first line of a section is its title. + :param services: Services to list in the banner, normally the test's whole registry. + """ + if not self._stops_at(name): + return + + try: + self._prepare() + + self.seq += 1 + + timeout_sec = self._budgeted_timeout() + + banner = self._render(name, describers, services, timeout_sec) + + self._publish(name, banner, timeout_sec) + except OSError as ex: + self._give_up(name, ex) + + return + + self.logger.info(f"Demo breakpoint reached [seq={self.seq}, name={name}, dir={self.control.path}]") + + self._await_resume(name, timeout_sec) + + def _give_up(self, name, error): + """ + Turns the remaining breakpoints off, after the control directory turned out to be + unusable. + + A breakpoint observes a scenario; it must not be what ends one. The directory is a + bind mount of the host repository, so it can be read only, be owned by another user or + be full - none of which says anything about the cluster under test, and all of which + would otherwise fail a run that was about to pass. Blocking would be no better than + raising: a breakpoint whose banner never reached the host would hold the scenario for + its whole timeout with nothing on screen to resume it. + + Every later breakpoint would fail the same way, so they are dropped here rather than + reported again at each one. + """ + self._unusable = True + + self.logger.warn(f"Demo breakpoints disabled, the control directory cannot be used " + f"[dir={self.control.path}, error={error}, seq={self.seq}, name={name}, " + f"test={self.test_name}]") + + def _stops_at(self, name): + if not self.enabled: + return False + + return self.names == ALL or name.strip().lower() in self.names + + def _budgeted_timeout(self): + """ + :return: How long this breakpoint may actually hold the scenario. + + ``demo_pause_timeout_sec`` is what the demo asks for, the runner budget is what it is + allowed. Ducktape's runner kills a test client it has received no event from for + ``--test-runner-timeout`` and takes the whole session down with it, and a paused test + sends no events - so a breakpoint has to give up while the runner is still waiting. + The budget is spent from the start of the test rather than from the breakpoint, hence + a long setup, or a long earlier pause, leaves less of it for this one. + """ + if self.runner_timeout_sec is None: + return self.timeout_sec + + left = self.runner_timeout_sec - (time.monotonic() - self._started_at) - RUNNER_TIMEOUT_MARGIN_SEC + + if left >= self.timeout_sec: + return self.timeout_sec + + self.logger.warn(f"Demo breakpoint held for at most {_fmt_duration(max(left, 0))} instead of the requested " + f"{_fmt_duration(self.timeout_sec)}: what is left of ducktape's --test-runner-timeout " + f"({_fmt_duration(self.runner_timeout_sec)}) after {_fmt_duration(self.elapsed_sec)} of " + f"this test. Raise --test-runner-timeout for a longer demo " + f"[seq={self.seq}, test={self.test_name}]") + + return max(left, 0.0) + + @property + def elapsed_sec(self): + """ + :return: Seconds since the test started. + """ + return time.monotonic() - self._started_at + + def _prepare(self): + """ + Readies the control directory, once per test: a resume file left by a previous run + would skip the very first breakpoint of this one. + """ + if self._prepared: + return + + self.control.prepare() + + self._prepared = True + + def _render(self, name, describers, services, timeout_sec): + """ + :return: The banner as a list of lines. + """ + elapsed = f" t+{_fmt_duration(self.elapsed_sec)} since test start" + auto = f"auto-continue in {_fmt_duration(timeout_sec)} " + + lines = [ + "=" * _WIDTH, + f" PAUSED {self.seq} {name}", + f" test {self.test_name}", + (elapsed + auto.rjust(max(_WIDTH - len(elapsed), 1))).rstrip(), + ] + + for section in [self._services_section(services)] + [self._section(d) for d in describers]: + if section: + lines.append("-" * _WIDTH) + lines.extend(section) + + lines.append("-" * _WIDTH) + lines.extend(self._hints_section(services)) + + lines.append("-" * _WIDTH) + lines.append(" continue: [Enter] in the demo console") + lines.append(f" or {_resume_command(self.control, self.seq)}") + lines.append("=" * _WIDTH) + + return lines + + def _section(self, describer): + try: + return list(describer.describe()) + except Exception as ex: # pylint: disable=broad-except + self.logger.warn(f"Demo breakpoint describer failed [describer={describer}, error={ex}]") + + return [] + + @staticmethod + def _services_section(services): + lines = ["SERVICES"] + + for service in services: + for node in service.nodes: + lines.append(_node_line(service, node)) + + if len(lines) == 1: + lines.append(" (none)") + + return lines + + @staticmethod + def _hints_section(services): + """ + Node logs live only on the nodes while the test runs - ducktape copies them into the + results directory at teardown - so every hint goes through the node container. + """ + # A node the banner could not name is left out rather than allowed to fail the hints: + # the commands are offered for the nodes that can be entered, and one unreadable node + # must not cost the demo the rest of them. + hosts = sorted({_node_host(node) for service in services for node in service.nodes} - {None}) + + log_dir = "/mnt/service/logs" + config_file = "/mnt/service/config/ignite-config.xml" + + # One set of copy-pasteable commands for a service list that is not homogeneous, so + # they follow the Ignite services: a ZookeeperService or a KafkaService registered + # ahead of them - which the discovery and CDC scenarios do - carries paths of its own + # and would have the banner name a zookeeper.properties for nodes that never had one. + for service in [s for s in services if isinstance(s, IgnitePathAware)] or list(services): + try: + svc_log_dir = getattr(service, "log_dir", None) + svc_config_file = getattr(service, "config_file", None) + except Exception: # pylint: disable=broad-except + continue + + log_dir = svc_log_dir or log_dir + config_file = svc_config_file or config_file + + break + + # Node paths, joined by hand: they are always POSIX, os.path.join is not when the + # control machine happens to be Windows. + ignite_log = f"{log_dir.rstrip('/')}/ignite*.log" + console_log = f"{log_dir.rstrip('/')}/console.log" + + return [ + f" nodes {' '.join(hosts) if hosts else '(none)'}", + " shell ./docker/ducker-ignite ssh ", + f" logs docker exec bash -c \"tail -n 50 {ignite_log}\"", + f" console docker exec tail -n 50 {console_log}", + f" config docker exec cat {config_file}", + ] + + def _publish(self, name, banner, timeout_sec): + """ + Publishes the breakpoint for the host: the banner to print, plus what a reader needs + to tell this pause from any other. + """ + self.control.publish(banner, { + "run": self.run, + "seq": self.seq, + "name": name, + "test": self.test_name, + "elapsed_sec": round(self.elapsed_sec, 1), + "timeout_sec": timeout_sec, + "banner": banner + }) + + def _await_resume(self, name, timeout_sec): + """ + Holds the scenario until the host resumes the breakpoint, or until it gives up on its + own. + + What each resume file means lives here rather than in the control directory: it is the + only part of the protocol that knows there is a scenario to end. + """ + def still_waiting(left_sec): + self.logger.info(f"Still paused at demo breakpoint [seq={self.seq}, name={name}, " + f"held={_fmt_duration(timeout_sec - left_sec)}, " + f"left={_fmt_duration(left_sec)}]") + + taken = self.control.await_any([ABORT, CONTINUE_ALL, continue_file(self.seq)], timeout_sec, + tick=still_waiting, tick_sec=HEARTBEAT_SEC) + + # Whatever ended the wait, the banner describes a breakpoint that is over. + self.control.clear_status() + + if taken == ABORT: + raise AssertionError(f"Demo aborted at breakpoint [seq={self.seq}, name={name}]") + + if taken == CONTINUE_ALL: + self._continue_all = True + + self.logger.info(f"Demo resumed, remaining breakpoints skipped [seq={self.seq}, name={name}]") + elif taken is None: + self.logger.warn(f"Demo breakpoint timed out after {timeout_sec}s, resuming " + f"[seq={self.seq}, name={name}]") + else: + self.logger.info(f"Demo resumed [seq={self.seq}, name={name}]") diff --git a/modules/ducktests/tests/ignitetest/utils/pause_control.py b/modules/ducktests/tests/ignitetest/utils/pause_control.py new file mode 100644 index 0000000000000..9c9a679bca60e --- /dev/null +++ b/modules/ducktests/tests/ignitetest/utils/pause_control.py @@ -0,0 +1,244 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The directory a paused test and the host talk over, and the file protocol they talk in. + +The protocol is one sided while a breakpoint is held - the host only ever creates files, the +test is the only party that deletes them - so no step of a held breakpoint can race with the +host: + + - the test publishes ``paused.txt`` (a rendered banner) and ``paused.json`` (the same + content as data) and then blocks; + - the host creates ``continue-``, ``continue-all`` or ``abort``; + - the test consumes that file, removes it along with its own status files, and proceeds. + +Both sides drive it through :class:`ControlDir`, which owns the mechanics alone - paths, +atomic writes, sweeping, polling. What a file *means* is the caller's business: +:mod:`ignitetest.utils.pause` decides that ``abort`` fails the scenario, and +``docker/demo_console.py`` decides which one to write. + +Standard library only, and deliberately free of every ignitetest import: the console loads +this module by path, on a host that has neither ducktape nor ignitetest installed. +""" + +import json +import os +import time + +CONTROL_DIR_NAME = ".ducktests-demo" + +STATUS_TXT = "paused.txt" +STATUS_JSON = "paused.json" +CONTINUE_PREFIX = "continue-" +CONTINUE_ALL = "continue-all" +ABORT = "abort" + +# The control directory is polled rather than watched: it is a bind mount shared with the +# host, where inotify is not dependable. +POLL_SEC = .5 + + +def repo_root(): + """ + :return: Path of the Ignite repository root, derived from this module's own location + (``/modules/ducktests/tests/ignitetest/utils/pause_control.py``), so that a + fork checked out elsewhere resolves its own root. + """ + return os.path.abspath(os.path.join(os.path.dirname(__file__), *[os.pardir] * 5)) + + +def default_control_dir(): + """ + :return: Path of the control directory shared between the test and the host. + """ + return os.path.join(repo_root(), CONTROL_DIR_NAME) + + +def continue_file(seq): + """ + :return: Name of the file that resumes the breakpoint with the given sequence number. + """ + return f"{CONTINUE_PREFIX}{seq}" + + +class ControlDir: + """ + One directory, shared between a paused test and the host driving it. + + Every method here is mechanics. Nothing in this class knows what a breakpoint is, which + is what lets the test, the console and the checks all speak the protocol through the same + code rather than through three copies of it that have to agree. + """ + def __init__(self, path): + self._path = str(path) + + @property + def path(self): + """ + :return: Path of the directory itself, as both sides name it. + """ + return self._path + + def file(self, name): + """ + :return: Path of a control file, for reporting it to someone who will type it. + """ + return os.path.join(self._path, name) + + def exists(self, name): + """ + :return: Whether the control file is there. + """ + return os.path.exists(self.file(name)) + + def take(self, name): + """ + Consumes a control file: the test is the only party that removes what the host wrote, + so taking it is what acknowledges it. + + :return: Whether the file was there to take. + """ + if not self.exists(name): + return False + + self.remove(name) + + return True + + def remove(self, name): + """ + Removes a control file, if it is still there - the other side may have just swept it. + """ + try: + os.remove(self.file(name)) + except OSError: + pass + + def write(self, name, content): + """ + Writes a control file whole: it lands through a temporary name, so the other side + never reads a half written one. + """ + path = self.file(name) + tmp = path + ".tmp" + + with open(tmp, "w", encoding="utf-8") as file: + file.write(content) + + os.replace(tmp, path) + + def prepare(self): + """ + Creates the directory and clears anything an earlier run left in it: a stale resume + file would skip the very first breakpoint of this one. + """ + os.makedirs(self._path, exist_ok=True) + + self.sweep(status=True) + + def sweep(self, status=False): + """ + Removes what an earlier run left behind. + + :param status: Whether to drop a published banner as well. The test does, at its first + breakpoint. The console must not: it is just as likely to have been started + against a test that is already holding one. + """ + if not os.path.isdir(self._path): + return + + # Matched by prefix rather than by name: an interrupted write leaves the temporary + # ``.tmp`` it goes through behind, and a sweep is the only thing that ever + # visits this directory without knowing what it expects to find. + for name in os.listdir(self._path): + stale = name.startswith(CONTINUE_PREFIX) or name.startswith(ABORT) + + if status: + stale = stale or name.startswith(STATUS_TXT) or name.startswith(STATUS_JSON) + + if stale: + self.remove(name) + + def publish(self, banner, payload): + """ + Publishes a held breakpoint, banner first as text and then as data, so that whoever + polls for the data never finds it ahead of the text it describes. + """ + self.write(STATUS_TXT, "\n".join(banner) + "\n") + self.write(STATUS_JSON, json.dumps(payload, indent=2)) + + def read_status(self): + """ + :return: The published breakpoint, or None when nothing is published. A missing or + half written file simply reads as "not paused" and is retried by the caller. + """ + try: + with open(self.file(STATUS_JSON), encoding="utf-8") as file: + return json.load(file) + except (OSError, ValueError): + return None + + def clear_status(self): + """ + Withdraws the published breakpoint, so a stale banner never outlives the pause it + describes. + """ + for name in (STATUS_TXT, STATUS_JSON): + self.remove(name) + + def resume(self, name): + """ + Writes a resume file. The test consumes and removes it. + + Created outright rather than through :meth:`write`: a resume file is empty, so there + is no half written state for an atomic replace to hide, and the temporary name that + replace goes through would be one more transient to leave lying about. + """ + with open(self.file(name), "w", encoding="utf-8"): + pass + + def await_any(self, names, timeout_sec, tick=None, tick_sec=POLL_SEC): + """ + Polls until one of the named files appears, and takes it. + + :param names: Names to watch, in priority order - the first one present wins when + several land between two polls. + :param tick: Called with the seconds left, every ``tick_sec`` that passes without a + file. This is how a caller reports that it is still waiting without this class + having to know what it would report to. + :param tick_sec: How often to do that, by default as often as the directory is looked + at - which is as often as there is anything new to say. + :return: The name that ended the wait, None when the timeout ran out first. + """ + deadline = time.monotonic() + timeout_sec + next_tick = time.monotonic() + tick_sec if tick else None + + while True: + for name in names: + if self.take(name): + return name + + now = time.monotonic() + + if now >= deadline: + return None + + if next_tick is not None and now >= next_tick: + next_tick = now + tick_sec + + tick(deadline - now) + + time.sleep(POLL_SEC) diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index e0fa4a74df8aa..c40e563eb269c 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -21,6 +21,12 @@ usedevelop = True envdir = {toxworkdir}/.virtualenvs/ignite-ducktests-{envname} deps = -r{toxinidir}/docker/requirements-dev.txt install_command = pip install --extra-index-url https://pypi.org/simple {opts} {packages} +# pytest builds its tmp_path root below a per user directory, and getpass.getuser() has no +# password database to fall back on when the name is missing from the environment - which is +# what a Windows control machine runs into, since tox passes neither variable by default. +passenv = + USER + USERNAME commands = pytest {env:PYTESTARGS:} {posargs} [testenv:codestyle]