From cb8c3e2304cc207b9880432d6716f0342f142035 Mon Sep 17 00:00:00 2001 From: johnnylawdgb Date: Tue, 11 Aug 2026 10:54:46 +0000 Subject: [PATCH] oracle: fix shutdown race in signing orchestrator OracleSigningOrchestrator registers itself as a validation interface subscriber, and Shutdown() destroys it immediately after Stop(). Stop() only calls UnregisterValidationInterface(), which is non-blocking and can return while a notification is still in flight, so a queued BlockConnected callback can run on the destroyed object: OnBlockConnected() -> CleanupOldSessions() locks m_sessions_mutex on freed memory. Locking a destroyed std::mutex is undefined behaviour, and the two runtimes express it differently: on glibc the scheduler thread parks in a futex and never wakes, so Shutdown() blocks behind it forever in CScheduler::stop(); on libc++ it aborts, which surfaces as exit code -6. -stopatheight makes the collision near-certain, because StartShutdown() is called from KernelNotifications::blockTip() during block connection, so a BlockConnected notification for that same block is already queued when shutdown begins. That is why rpc_blockchain.py fails intermittently in CI, on either platform depending on load. The window is not specific to -stopatheight: any shutdown that begins while a block is connecting can hit it. Drain the queue with SyncWithValidationInterfaceQueue() before destroying the orchestrator, per the contract documented in validationinterface.h. The use-after-free is a microsecond-wide race, so the added test pins the contract that prevents it instead: Shutdown() must not return while a queued notification is still pending. Without the drain that check fails and the suite additionally takes a SIGSEGV as the freed orchestrator is touched. --- src/Makefile.test.include | 1 + src/oracle/signing_orchestrator.cpp | 7 ++++ src/test/oracle_shutdown_tests.cpp | 56 +++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 src/test/oracle_shutdown_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 18e44eb419..08ccaf58ad 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -108,6 +108,7 @@ DIGIBYTE_TESTS =\ test/digidollar_oracle_feeds_wave11_tests.cpp \ test/digidollar_hot_path_logging_tests.cpp \ test/musig2_net_processing_tests.cpp \ + test/oracle_shutdown_tests.cpp \ test/rh50_oracle_keyset_alignment_tests.cpp \ test/rh51_checkphase3_v1_split_tests.cpp \ test/rh52_bip34_scriptnum_escape_tests.cpp \ diff --git a/src/oracle/signing_orchestrator.cpp b/src/oracle/signing_orchestrator.cpp index cfeeb64c76..d7f3665726 100644 --- a/src/oracle/signing_orchestrator.cpp +++ b/src/oracle/signing_orchestrator.cpp @@ -843,6 +843,13 @@ void OracleSigningOrchestrator::Shutdown() { if (g_signing_orchestrator) { g_signing_orchestrator->Stop(); + // Stop() only unregisters, which is non-blocking and can return while a + // notification is still in flight on the scheduler thread. Drain the + // queue before destroying the object that callback is about to touch, + // otherwise it locks m_sessions_mutex on freed memory: the scheduler + // thread then blocks forever (and Shutdown() blocks behind it in + // CScheduler::stop) or aborts, depending on the C++ runtime. + SyncWithValidationInterfaceQueue(); g_signing_orchestrator.reset(); } } diff --git a/src/test/oracle_shutdown_tests.cpp b/src/test/oracle_shutdown_tests.cpp new file mode 100644 index 0000000000..ca02634c66 --- /dev/null +++ b/src/test/oracle_shutdown_tests.cpp @@ -0,0 +1,56 @@ +// Copyright (c) 2026 The DigiByte Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// Shutdown-ordering test for the MuSig2 signing orchestrator. +// +// OracleSigningOrchestrator is a validation interface subscriber, and its +// Shutdown() destroys the object. UnregisterValidationInterface() is +// non-blocking and can return while a notification is still in flight, so +// destroying without draining the queue lets a queued BlockConnected callback +// run on freed memory (OnBlockConnected -> CleanupOldSessions locks +// m_sessions_mutex), which deadlocks the scheduler thread on glibc and aborts +// on libc++. +// +// The use-after-free itself is a microsecond-wide race and cannot be asserted +// on directly, so this pins the contract that prevents it: Shutdown() must not +// return while a queued notification is still pending. + +#include +#include +#include + +#include + +#include + +#include +#include + +BOOST_FIXTURE_TEST_SUITE(oracle_shutdown_tests, TestingSetup) + +BOOST_AUTO_TEST_CASE(shutdown_waits_for_in_flight_validation_callbacks) +{ + if (!g_signing_orchestrator) { + OracleSigningOrchestrator::Initialize(); + } + BOOST_REQUIRE(g_signing_orchestrator); + + // Occupy the validation interface queue with work that outlives a + // non-draining Shutdown(). A queued orchestrator notification behaves the + // same way; this stands in for one without depending on chain activity. + std::atomic callback_finished{false}; + CallFunctionInValidationInterfaceQueue([&callback_finished] { + UninterruptibleSleep(std::chrono::milliseconds{250}); + callback_finished = true; + }); + + OracleSigningOrchestrator::Shutdown(); + + // Without the drain, Shutdown() returns while the callback is still + // pending -- and the next queued notification would touch an orchestrator + // that no longer exists. + BOOST_CHECK(callback_finished.load()); +} + +BOOST_AUTO_TEST_SUITE_END()