From 569e8679dff2f55d500af17eeaaa8a3c30867a28 Mon Sep 17 00:00:00 2001 From: Roger Luethi Date: Tue, 9 Jun 2026 06:57:49 +0200 Subject: [PATCH 1/4] Return failure on reconciler output timeout `osism sync inventory` waits for a terminal marker in the reconciler task's output stream. If no further output arrives before `--task-timeout`, the command logs an error but currently returns no failure status, so callers can treat the timed-out command as successful. This timeout is normally five minutes, but callers can override it. The deploy that exposed the reconciler lock race used `--task-timeout 1800` and therefore waited the full 30 minutes before reaching this path. Return a non-zero result when the output wait times out so automation can detect the failure, following the exit-code convention established by #2313. This does not fix the missing terminal marker itself; later commits address the task-side causes. AI-assisted: Codex/GPT 5.5 Signed-off-by: Roger Luethi --- osism/commands/reconciler.py | 1 + tests/unit/commands/test_reconciler.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/unit/commands/test_reconciler.py diff --git a/osism/commands/reconciler.py b/osism/commands/reconciler.py index d559d8e6c..4f3fdbd41 100644 --- a/osism/commands/reconciler.py +++ b/osism/commands/reconciler.py @@ -64,6 +64,7 @@ def take_action(self, parsed_args): logger.error( f"Timeout while waiting for further output of task {t.task_id} (sync inventory)" ) + return 1 else: logger.info( f"Task {t.task_id} (sync inventory) is running in background. No more output." diff --git a/tests/unit/commands/test_reconciler.py b/tests/unit/commands/test_reconciler.py new file mode 100644 index 000000000..356148edb --- /dev/null +++ b/tests/unit/commands/test_reconciler.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the ``osism reconciler`` commands.""" + +from unittest.mock import MagicMock, patch + +from osism.commands import reconciler + + +def test_sync_returns_nonzero_on_task_timeout(): + cmd = reconciler.Sync(MagicMock(), MagicMock()) + parsed_args = cmd.get_parser("test").parse_args([]) + + with patch("osism.commands.reconciler.utils.check_task_lock_and_exit"), patch( + "osism.tasks.reconciler.run.delay", return_value=MagicMock() + ), patch( + "osism.commands.reconciler.utils.fetch_task_output", + side_effect=TimeoutError, + ): + result = cmd.take_action(parsed_args) + + assert result == 1 From 7b426298fceb7b2de3c84d6b6d0d74fcfac00cdd Mon Sep 17 00:00:00 2001 From: Roger Luethi Date: Tue, 9 Jun 2026 06:57:58 +0200 Subject: [PATCH 2/4] Retry contended reconciler syncs `osism sync inventory` dispatches an explicit reconciler task and waits for that task's output stream to publish a terminal marker. During a deploy, seven reconciler tasks were dispatched in a short burst. Five failed to acquire the reconciler lock within 20 seconds, returned `None` as successful Celery tasks, and never published a terminal marker. The CLI waiting on one of those tasks consequently waited until its configured `--task-timeout 1800` expired 30 minutes later. Failing immediately on contention would stop the wait but would not fulfil the requested sync: the current lock holder may have started before the caller's inventory change existed. Retry the same Celery task up to five times at fixed five-second intervals so the waiting caller continues following the correct task ID and the requested reconciliation can still run. Publish a non-terminal status before each retry. Besides explaining the delay, each status resets `fetch_task_output`'s inactivity deadline. After retry exhaustion, publish a non-zero terminal result and re-raise the Celery failure. `osism sync inventory --no-wait` returns immediately and dispatches the same task with output publication disabled. Retry these background tasks as well so contention does not silently discard the requested sync. They publish no retry or terminal stream messages because no caller is waiting, but remain Celery failures if retries are exhausted. AI-assisted: Codex/GPT 5.5 Signed-off-by: Roger Luethi --- osism/tasks/reconciler.py | 94 +++++++++++++++++++++-------- tests/unit/tasks/test_reconciler.py | 81 +++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 24 deletions(-) create mode 100644 tests/unit/tasks/test_reconciler.py diff --git a/osism/tasks/reconciler.py b/osism/tasks/reconciler.py index 1764c8f04..87a7263f7 100644 --- a/osism/tasks/reconciler.py +++ b/osism/tasks/reconciler.py @@ -5,6 +5,7 @@ import subprocess from celery import Celery +from celery.exceptions import MaxRetriesExceededError from loguru import logger from osism import settings, utils from osism.tasks import Config @@ -12,6 +13,10 @@ app = Celery("reconciler") app.config_from_object(Config) +LOCK_RETRY_MAX_RETRIES = 5 +LOCK_RETRY_DELAY = 5 +LOCK_TIMEOUT_RC = 1 + @app.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): @@ -24,7 +29,46 @@ def setup_periodic_tasks(sender, **kwargs): ) -@app.task(bind=True, name="osism.tasks.reconciler.run") +def _push_task_output_best_effort(task_id, line): + try: + utils.push_task_output(task_id, line) + except Exception: + logger.exception(f"Failed to publish output for reconciler task {task_id}") + + +def _finish_task_output_best_effort(task_id, rc): + try: + utils.finish_task_output(task_id, rc=rc) + except Exception: + logger.exception(f"Failed to finish output for reconciler task {task_id}") + + +def _retry_after_lock_timeout(task, publish): + if publish and task.request.retries < LOCK_RETRY_MAX_RETRIES: + _push_task_output_best_effort( + task.request.id, + f"Reconciler busy; retrying lock acquisition in {LOCK_RETRY_DELAY}s\n", + ) + + try: + raise task.retry(countdown=LOCK_RETRY_DELAY) + except MaxRetriesExceededError: + message = ( + "Reconciler lock could not be acquired after " + f"{LOCK_RETRY_MAX_RETRIES + 1} attempts\n" + ) + logger.error(message.rstrip()) + if publish: + _push_task_output_best_effort(task.request.id, message) + _finish_task_output_best_effort(task.request.id, LOCK_TIMEOUT_RC) + raise + + +@app.task( + bind=True, + name="osism.tasks.reconciler.run", + max_retries=LOCK_RETRY_MAX_RETRIES, +) def run(self, publish=True): # Check if tasks are locked before execution utils.check_task_lock_and_exit() @@ -34,36 +78,38 @@ def run(self, publish=True): auto_release_time=60, ) - if lock.acquire(timeout=20): - logger.info("RUN /run.sh") + if not lock.acquire(timeout=20): + return _retry_after_lock_timeout(self, publish) - env = os.environ.copy() + logger.info("RUN /run.sh") - p = subprocess.Popen( - "/run.sh", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - ) + env = os.environ.copy() - if publish: - for line in io.TextIOWrapper(p.stdout, encoding="utf-8"): - utils.push_task_output(self.request.id, line) + p = subprocess.Popen( + "/run.sh", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) - rc = p.wait(timeout=60) + if publish: + for line in io.TextIOWrapper(p.stdout, encoding="utf-8"): + utils.push_task_output(self.request.id, line) - if publish: - utils.finish_task_output(self.request.id, rc=rc) + rc = p.wait(timeout=60) - from pottery import ReleaseUnlockedLock + if publish: + utils.finish_task_output(self.request.id, rc=rc) - try: - lock.release() - except ReleaseUnlockedLock: - logger.warning( - "Lock auto-released before explicit release (auto_release_time exceeded)" - ) + from pottery import ReleaseUnlockedLock + + try: + lock.release() + except ReleaseUnlockedLock: + logger.warning( + "Lock auto-released before explicit release (auto_release_time exceeded)" + ) @app.task(bind=True, name="osism.tasks.reconciler.run_on_change") diff --git a/tests/unit/tasks/test_reconciler.py b/tests/unit/tasks/test_reconciler.py new file mode 100644 index 000000000..3cc6ceb60 --- /dev/null +++ b/tests/unit/tasks/test_reconciler.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import MagicMock + +import pytest +from celery.exceptions import MaxRetriesExceededError, Retry + +from osism.tasks import reconciler + + +def _task(*, retries=0, retry_exception=None): + task = MagicMock() + task.request.id = "task-1" + task.request.retries = retries + task.retry.side_effect = retry_exception or Retry() + return task + + +def test_lock_timeout_retries_same_task_with_status(mocker): + task = _task() + push = mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + + with pytest.raises(Retry): + reconciler._retry_after_lock_timeout(task, publish=True) + + push.assert_called_once_with( + "task-1", + f"Reconciler busy; retrying lock acquisition in {reconciler.LOCK_RETRY_DELAY}s\n", + ) + finish.assert_not_called() + task.retry.assert_called_once_with(countdown=reconciler.LOCK_RETRY_DELAY) + + +def test_retry_status_publication_failure_does_not_prevent_retry(mocker): + task = _task() + mocker.patch( + "osism.tasks.reconciler.utils.push_task_output", + side_effect=RuntimeError("redis unavailable"), + ) + with pytest.raises(Retry): + reconciler._retry_after_lock_timeout(task, publish=True) + + task.retry.assert_called_once_with(countdown=reconciler.LOCK_RETRY_DELAY) + + +def test_lock_retry_exhaustion_publishes_failure_and_raises(mocker): + task = _task( + retries=reconciler.LOCK_RETRY_MAX_RETRIES, + retry_exception=MaxRetriesExceededError("exhausted"), + ) + push = mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + with pytest.raises(MaxRetriesExceededError): + reconciler._retry_after_lock_timeout(task, publish=True) + + push.assert_called_once() + assert "could not be acquired" in push.call_args.args[1] + finish.assert_called_once_with("task-1", rc=reconciler.LOCK_TIMEOUT_RC) + + +def test_publish_false_retries_without_stream_output(mocker): + task = _task() + push = mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + with pytest.raises(Retry): + reconciler._retry_after_lock_timeout(task, publish=False) + + push.assert_not_called() + finish.assert_not_called() + + +def test_publish_false_exhaustion_raises_without_stream_output(mocker): + task = _task(retry_exception=MaxRetriesExceededError("exhausted")) + push = mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + with pytest.raises(MaxRetriesExceededError): + reconciler._retry_after_lock_timeout(task, publish=False) + + push.assert_not_called() + finish.assert_not_called() From 7fe54278a2171db7965baeb685ca0d32ef31927a Mon Sep 17 00:00:00 2001 From: Roger Luethi Date: Mon, 8 Jun 2026 21:39:03 +0200 Subject: [PATCH 3/4] Guarantee reconciler terminal output The lock-timeout path is not the only way an awaited reconciler task can finish without publishing the terminal marker consumed by `fetch_task_output`. Failures starting `/run.sh`, decoding its output, or publishing output can bypass `finish_task_output`. The existing 60-second process wait can also raise `TimeoutExpired`, and `check_task_lock_and_exit()` can raise `SystemExit` before a lock exists. In each case the CLI otherwise waits until its configured task timeout, which was 30 minutes in the deploy that exposed this class of failure. Run explicit reconciliation through a guarded helper. For every non-retry failure, publish a best-effort explanatory message and non-zero terminal result. Kill and reap a subprocess that exceeds its process wait, and always release an acquired lock. Preserve normal Celery `Retry` and retry-exhaustion control flow so a contention retry is not reported as an ordinary task failure. Terminal publication remains best-effort because Redis may itself be the failing component. AI-assisted: Codex/GPT 5.5 Signed-off-by: Roger Luethi --- osism/tasks/reconciler.py | 113 +++++++++++++++++++--------- tests/unit/tasks/test_reconciler.py | 102 +++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 37 deletions(-) diff --git a/osism/tasks/reconciler.py b/osism/tasks/reconciler.py index 87a7263f7..1d299252a 100644 --- a/osism/tasks/reconciler.py +++ b/osism/tasks/reconciler.py @@ -5,7 +5,7 @@ import subprocess from celery import Celery -from celery.exceptions import MaxRetriesExceededError +from celery.exceptions import MaxRetriesExceededError, Retry from loguru import logger from osism import settings, utils from osism.tasks import Config @@ -43,6 +43,33 @@ def _finish_task_output_best_effort(task_id, rc): logger.exception(f"Failed to finish output for reconciler task {task_id}") +def _publish_failure_best_effort(task_id, exc): + _push_task_output_best_effort(task_id, f"Reconciler failed: {exc}\n") + _finish_task_output_best_effort(task_id, 1) + + +def _release_lock_best_effort(lock): + from pottery import ReleaseUnlockedLock + + try: + lock.release() + except ReleaseUnlockedLock: + logger.warning( + "Lock auto-released before explicit release (auto_release_time exceeded)" + ) + except Exception: + logger.exception("Failed to release reconciler lock") + + +def _terminate_process_best_effort(process): + try: + if process.poll() is None: + process.kill() + process.wait() + except Exception: + logger.exception("Failed to terminate reconciler subprocess") + + def _retry_after_lock_timeout(task, publish): if publish and task.request.retries < LOCK_RETRY_MAX_RETRIES: _push_task_output_best_effort( @@ -64,52 +91,64 @@ def _retry_after_lock_timeout(task, publish): raise -@app.task( - bind=True, - name="osism.tasks.reconciler.run", - max_retries=LOCK_RETRY_MAX_RETRIES, -) -def run(self, publish=True): - # Check if tasks are locked before execution - utils.check_task_lock_and_exit() +def _execute_reconciler(task, publish): + lock = None + lock_acquired = False + process = None - lock = utils.create_redlock( - key="lock_osism_tasks_reconciler_run", - auto_release_time=60, - ) + try: + utils.check_task_lock_and_exit() - if not lock.acquire(timeout=20): - return _retry_after_lock_timeout(self, publish) + lock = utils.create_redlock( + key="lock_osism_tasks_reconciler_run", + auto_release_time=60, + ) - logger.info("RUN /run.sh") + if not lock.acquire(timeout=20): + return _retry_after_lock_timeout(task, publish) - env = os.environ.copy() + lock_acquired = True + logger.info("RUN /run.sh") - p = subprocess.Popen( - "/run.sh", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - ) + process = subprocess.Popen( + "/run.sh", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=os.environ.copy(), + ) - if publish: - for line in io.TextIOWrapper(p.stdout, encoding="utf-8"): - utils.push_task_output(self.request.id, line) + if publish: + for line in io.TextIOWrapper(process.stdout, encoding="utf-8"): + utils.push_task_output(task.request.id, line) - rc = p.wait(timeout=60) + rc = process.wait(timeout=60) - if publish: - utils.finish_task_output(self.request.id, rc=rc) + if publish: + utils.finish_task_output(task.request.id, rc=rc) - from pottery import ReleaseUnlockedLock + return rc + except (Retry, MaxRetriesExceededError): + raise + except BaseException as exc: + if process is not None: + _terminate_process_best_effort(process) + logger.exception(f"Reconciler task {task.request.id} failed") + if publish: + _publish_failure_best_effort(task.request.id, exc) + raise + finally: + if lock_acquired: + _release_lock_best_effort(lock) - try: - lock.release() - except ReleaseUnlockedLock: - logger.warning( - "Lock auto-released before explicit release (auto_release_time exceeded)" - ) + +@app.task( + bind=True, + name="osism.tasks.reconciler.run", + max_retries=LOCK_RETRY_MAX_RETRIES, +) +def run(self, publish=True): + return _execute_reconciler(self, publish) @app.task(bind=True, name="osism.tasks.reconciler.run_on_change") diff --git a/tests/unit/tasks/test_reconciler.py b/tests/unit/tasks/test_reconciler.py index 3cc6ceb60..e124cebd5 100644 --- a/tests/unit/tasks/test_reconciler.py +++ b/tests/unit/tasks/test_reconciler.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 +from subprocess import TimeoutExpired from unittest.mock import MagicMock import pytest @@ -79,3 +80,104 @@ def test_publish_false_exhaustion_raises_without_stream_output(mocker): push.assert_not_called() finish.assert_not_called() + + +def test_execute_reconciler_publishes_success_and_releases_lock(mocker): + task = _task() + lock = MagicMock() + lock.acquire.return_value = True + process = MagicMock() + process.stdout = [b"line\n"] + process.wait.return_value = 0 + mocker.patch("osism.tasks.reconciler.utils.check_task_lock_and_exit") + mocker.patch("osism.tasks.reconciler.utils.create_redlock", return_value=lock) + mocker.patch("osism.tasks.reconciler.subprocess.Popen", return_value=process) + mocker.patch("osism.tasks.reconciler.io.TextIOWrapper", return_value=["line\n"]) + push = mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + + result = reconciler._execute_reconciler(task, publish=True) + + assert result == 0 + push.assert_called_once_with("task-1", "line\n") + finish.assert_called_once_with("task-1", rc=0) + lock.release.assert_called_once_with() + + +def test_execute_reconciler_popen_failure_publishes_and_releases(mocker): + task = _task() + lock = MagicMock() + lock.acquire.return_value = True + mocker.patch("osism.tasks.reconciler.utils.check_task_lock_and_exit") + mocker.patch("osism.tasks.reconciler.utils.create_redlock", return_value=lock) + mocker.patch( + "osism.tasks.reconciler.subprocess.Popen", + side_effect=OSError("cannot start"), + ) + push = mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + + with pytest.raises(OSError, match="cannot start"): + reconciler._execute_reconciler(task, publish=True) + + assert "cannot start" in push.call_args.args[1] + finish.assert_called_once_with("task-1", rc=1) + lock.release.assert_called_once_with() + + +def test_execute_reconciler_timeout_kills_process(mocker): + task = _task() + lock = MagicMock() + lock.acquire.return_value = True + process = MagicMock() + process.stdout = [] + process.wait.side_effect = [TimeoutExpired("/run.sh", 60), 0] + process.poll.return_value = None + mocker.patch("osism.tasks.reconciler.utils.check_task_lock_and_exit") + mocker.patch("osism.tasks.reconciler.utils.create_redlock", return_value=lock) + mocker.patch("osism.tasks.reconciler.subprocess.Popen", return_value=process) + mocker.patch("osism.tasks.reconciler.io.TextIOWrapper", return_value=[]) + mocker.patch("osism.tasks.reconciler.utils.push_task_output") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + + with pytest.raises(TimeoutExpired): + reconciler._execute_reconciler(task, publish=True) + + process.kill.assert_called_once_with() + finish.assert_called_once_with("task-1", rc=1) + lock.release.assert_called_once_with() + + +def test_execute_reconciler_task_lock_system_exit_publishes_failure(mocker): + task = _task() + mocker.patch( + "osism.tasks.reconciler.utils.check_task_lock_and_exit", + side_effect=SystemExit(1), + ) + create_lock = mocker.patch("osism.tasks.reconciler.utils.create_redlock") + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + + with pytest.raises(SystemExit): + reconciler._execute_reconciler(task, publish=True) + + create_lock.assert_not_called() + finish.assert_called_once_with("task-1", rc=1) + + +def test_execute_reconciler_does_not_convert_retry_to_failure(mocker): + task = _task() + lock = MagicMock() + lock.acquire.return_value = False + mocker.patch("osism.tasks.reconciler.utils.check_task_lock_and_exit") + mocker.patch("osism.tasks.reconciler.utils.create_redlock", return_value=lock) + mocker.patch( + "osism.tasks.reconciler._retry_after_lock_timeout", + side_effect=Retry(), + ) + finish = mocker.patch("osism.tasks.reconciler.utils.finish_task_output") + + with pytest.raises(Retry): + reconciler._execute_reconciler(task, publish=True) + + finish.assert_not_called() + lock.release.assert_not_called() From 97e2a8f7d06c8a9891ca36b51d5dba8b9466d9fd Mon Sep 17 00:00:00 2001 From: Roger Luethi Date: Mon, 8 Jun 2026 21:48:39 +0200 Subject: [PATCH 4/4] Serialize reconciler executions Explicit `osism sync inventory` tasks and periodic reconciler tasks use different lock keys even though both execute `/run.sh`. They can therefore run concurrently and rewrite the same generated inventory files. Contention retries for explicit tasks do not prevent this because the periodic task's different lock is always independent. Use one execution lock key for both task types. Explicit sync requests continue retrying when the shared lock is held so their requested reconciliation eventually runs. Periodic tasks continue coalescing by skipping a run when they cannot acquire the lock; a later scheduled run will reconcile the changes. Keep the existing 60-second auto-release time and all other execution behavior unchanged. Extending the lease for reconciliations that run longer than 60 seconds is outside this commit's scope. AI-assisted: Codex/GPT 5.5 Signed-off-by: Roger Luethi --- osism/tasks/reconciler.py | 5 +++-- tests/unit/tasks/test_reconciler.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/osism/tasks/reconciler.py b/osism/tasks/reconciler.py index 1d299252a..8cab5f1a1 100644 --- a/osism/tasks/reconciler.py +++ b/osism/tasks/reconciler.py @@ -16,6 +16,7 @@ LOCK_RETRY_MAX_RETRIES = 5 LOCK_RETRY_DELAY = 5 LOCK_TIMEOUT_RC = 1 +RECONCILER_LOCK_KEY = "lock_osism_tasks_reconciler_execution" @app.on_after_configure.connect @@ -100,7 +101,7 @@ def _execute_reconciler(task, publish): utils.check_task_lock_and_exit() lock = utils.create_redlock( - key="lock_osism_tasks_reconciler_run", + key=RECONCILER_LOCK_KEY, auto_release_time=60, ) @@ -154,7 +155,7 @@ def run(self, publish=True): @app.task(bind=True, name="osism.tasks.reconciler.run_on_change") def run_on_change(self): lock = utils.create_redlock( - key="lock_osism_tasks_reconciler_run_on_change", + key=RECONCILER_LOCK_KEY, auto_release_time=60, ) diff --git a/tests/unit/tasks/test_reconciler.py b/tests/unit/tasks/test_reconciler.py index e124cebd5..133bd1055 100644 --- a/tests/unit/tasks/test_reconciler.py +++ b/tests/unit/tasks/test_reconciler.py @@ -181,3 +181,32 @@ def test_execute_reconciler_does_not_convert_retry_to_failure(mocker): finish.assert_not_called() lock.release.assert_not_called() + + +def test_explicit_and_periodic_runs_use_shared_lock(mocker): + task = _task() + lock = MagicMock() + lock.acquire.return_value = False + create_lock = mocker.patch( + "osism.tasks.reconciler.utils.create_redlock", + return_value=lock, + ) + mocker.patch("osism.tasks.reconciler.utils.check_task_lock_and_exit") + mocker.patch( + "osism.tasks.reconciler._retry_after_lock_timeout", + side_effect=Retry(), + ) + + with pytest.raises(Retry): + reconciler._execute_reconciler(task, publish=True) + reconciler.run_on_change.run() + + execution_locks = [ + item + for item in create_lock.call_args_list + if item.kwargs.get("auto_release_time") == 60 + ] + assert len(execution_locks) == 2 + assert all( + item.kwargs["key"] == reconciler.RECONCILER_LOCK_KEY for item in execution_locks + )