From 0dca50fab30c6af7b273c48872197975bf20e881 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:16:22 +0800 Subject: [PATCH 01/13] =?UTF-8?q?fix(cluster):=20GRD=20recovery=20liveness?= =?UTF-8?q?=20=E2=80=94=20same-episode=20HW=20remaster=20retry,=20WAIT=5FC?= =?UTF-8?q?LUSTER=20watchdog,=20event-scoped=20WAIT=5FEPOCH=20escape,=20de?= =?UTF-8?q?ad-node=20PCM=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fail-stop reconfig could wedge a settled cluster forever on three stacked liveness holes: a BLOCKED HW remaster worker was never relaunched within the same episode (BGW_NEVER_RESTART + launched-latch), WAIT_CLUSTER had no watchdog while its cluster/HW gates were held, and WAIT_EPOCH's strict progress gate wedged permanently when an IC-piggybacked epoch bump landed before the local reconfig event (baseline re-captured as the post-bump value, cur == old forever). - HW remaster: per-dead-node result/attempts/next_attempt shmem state; the FSM relaunches BLOCKED workers with exponential backoff (cap 60s) up to cluster.hw_remaster_retry_max_attempts (SIGHUP-raisable); abnormal worker exit marks BLOCKED via before_shmem_exit; the three silent BLOCKED returns now LOG their cause. wal_threads_dir-unset is detected up front as BLOCKED_STRUCTURAL: never retried, WARNING-once with the config hint, and (level 2) a multi-node shared_catalog=on boot without it now fails fast at startup. - WAIT_CLUSTER: observational watchdog on the rebuild-timeout cadence — WARNING with the missing-survivor list, gate states and per-dead-node retry state, plus a counter; it never unfreezes anything. - WAIT_EPOCH: three proof-carrying layers — abort-to-idle on a new event id, durable observed-epoch adoption, and an event-scoped coordinator witness (REDECLARE_DONE carries the event id; stale-event DONEs are dropped at accounting; the equal-epoch escape additionally requires the coordinator's DONE to have been accounted after the accept snapshot). No timeout-only advance exists; without proof the shards stay frozen. - Dead-node PCM cleanup: the GRD dead-sweep clears the dead node's x_holder/s_holders/pi_holders/master_holder and pending-X residue (LOG summary + pcm/dead_cleanup_entries counter), and a local-master S state with no local residency can now upgrade S->X through the standard invalidate/ACK path instead of failing closed forever — post-kill DDL recovers instead of timing out on cluster locks. - Serve-gate scope alignment: the merged-materialization check in gcs_block phase_for_tag now uses the same thread-recovery scope policy as the unfreeze gate, removing the permanent-RECOVERING mismatch outside the gate's applicable scope (in-scope behavior unchanged). - Observability: hw retry/exhausted + grd cluster_gate_timeout / wait_epoch_escape + pcm dead_cleanup_entries dump keys; S4-reject and HW fail-closed diagnostics. Tests: unit relaunch-decision table (8 rows) + event-scoped DONE rejection/ witness advance + PCM dead-cleanup forms incl. pending-X; t/293 gains the same-episode self-heal and retry-exhaustion legs (unfreeze-extend positive + watchdog WARNING greps); t/337 gains the level-2 fail-fast negative leg; new t/362 runs a 4-node shared_catalog formation, kill -9, and asserts remaster convergence, post-kill DDL, cleanup counters and the 0-wedge invariant end to end. Local gates (cassert): unit 158 binaries, t/293 + t/337 + t/362 + smoke subset, cluster_regress 13/13, PG regress 219/219. Spec: spec-4.6a-grd-recovery-liveness.md --- src/backend/cluster/cluster_debug.c | 33 ++ src/backend/cluster/cluster_gcs_block.c | 41 +- src/backend/cluster/cluster_ges.c | 3 +- src/backend/cluster/cluster_grd.c | 503 ++++++++++++++++-- src/backend/cluster/cluster_guc.c | 17 + src/backend/cluster/cluster_hw_remaster.c | 186 ++++++- src/backend/cluster/cluster_hw_shmem.c | 108 +++- src/backend/cluster/cluster_lms.c | 3 + src/backend/cluster/cluster_pcm_lock.c | 125 +++++ src/backend/cluster/cluster_wal_thread.c | 19 +- src/include/cluster/cluster_grd.h | 24 +- src/include/cluster/cluster_guc.h | 2 + src/include/cluster/cluster_hw.h | 28 +- src/include/cluster/cluster_hw_remaster.h | 102 +++- src/include/cluster/cluster_pcm_lock.h | 6 + src/include/cluster/cluster_thread_recovery.h | 10 + src/test/cluster_tap/t/024_pcm_lock.pl | 7 +- .../cluster_tap/t/293_hw_online_remaster.pl | 245 ++++++++- .../t/337_shared_catalog_ddl_2node.pl | 42 +- .../362_shared_catalog_4node_kill_selfheal.pl | 310 +++++++++++ src/test/cluster_unit/Makefile | 1 + src/test/cluster_unit/test_cluster_debug.c | 95 +++- .../cluster_unit/test_cluster_gcs_block.c | 14 +- src/test/cluster_unit/test_cluster_ges.c | 3 +- src/test/cluster_unit/test_cluster_grd.c | 147 ++++- .../test_cluster_grd_starvation.c | 37 ++ .../test_cluster_hw_remaster_retry.c | 156 ++++++ .../cluster_unit/test_cluster_lock_acquire.c | 41 ++ src/test/cluster_unit/test_cluster_pcm_lock.c | 125 ++++- src/test/perl/PostgreSQL/Test/ClusterQuad.pm | 108 +++- 30 files changed, 2375 insertions(+), 166 deletions(-) create mode 100644 src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl create mode 100644 src/test/cluster_unit/test_cluster_hw_remaster_retry.c diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..e19b3e98e8 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1195,8 +1195,30 @@ static void dump_grd_recovery(ReturnSetInfo *rsinfo) { ClusterGrdRecoveryCounters c; + uint32 state; cluster_grd_recovery_counters_snapshot(&c); + state = cluster_grd_recovery_state_value(); + emit_row(rsinfo, "grd_recovery", "state", cluster_grd_recovery_state_name(state)); + emit_row(rsinfo, "grd_recovery", "state_enum_value", fmt_int32((int32)state)); + emit_row(rsinfo, "grd_recovery", "last_event_id", + fmt_int64((int64)cluster_grd_recovery_last_event_id())); + emit_row(rsinfo, "grd_recovery", "event_old_epoch", + fmt_int64((int64)cluster_grd_recovery_event_old_epoch())); + emit_row(rsinfo, "grd_recovery", "episode_epoch", + fmt_int64((int64)cluster_grd_recovery_episode_epoch_value())); + emit_row(rsinfo, "grd_recovery", "event_coordinator", + fmt_int32((int32)cluster_grd_recovery_event_coordinator())); + emit_row(rsinfo, "grd_recovery", "done_self_epoch", + fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_event_id", + fmt_int64((int64)cluster_grd_recovery_done_event_id_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", + fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", + fmt_int64((int64)cluster_grd_recovery_block_redeclare_epoch())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_done", + fmt_bool(cluster_grd_recovery_block_redeclare_done())); emit_row(rsinfo, "grd_recovery", "remaster_started", fmt_int64((int64)c.remaster_started)); emit_row(rsinfo, "grd_recovery", "remaster_done", fmt_int64((int64)c.remaster_done)); emit_row(rsinfo, "grd_recovery", "remaster_failed", fmt_int64((int64)c.remaster_failed)); @@ -1212,6 +1234,11 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) emit_row(rsinfo, "grd_recovery", "unaffected_holder_survived", fmt_int64((int64)c.unaffected_holder_survived)); emit_row(rsinfo, "grd_recovery", "stale_holder_swept", fmt_int64((int64)c.stale_holder_swept)); + emit_row(rsinfo, "grd_recovery", "cluster_gate_timeout", + fmt_int64((int64)c.cluster_gate_timeout)); + emit_row(rsinfo, "grd_recovery", "wait_epoch_escape", fmt_int64((int64)c.wait_epoch_escape)); + /* spec-4.6a D12 — dead-node PCM residue cleanup (categorized under pcm). */ + emit_row(rsinfo, "pcm", "dead_cleanup_entries", fmt_int64((int64)c.pcm_dead_cleanup_entries)); /* spec-5.16 D5 — join-direction remaster counters (same grd_recovery * category; no new dump category, §8 Q6-A). */ emit_row(rsinfo, "grd_recovery", "join_remaster_started", @@ -2810,6 +2837,12 @@ dump_hw(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_hw_remaster_done_count())); emit_row(rsinfo, "hw", "remaster_blocked_count", fmt_int64((int64)cluster_hw_remaster_blocked_count())); + emit_row(rsinfo, "hw", "remaster_retry_count", + fmt_int64((int64)cluster_hw_remaster_retry_count())); + emit_row(rsinfo, "hw", "remaster_retry_exhausted_count", + fmt_int64((int64)cluster_hw_remaster_retry_exhausted_count())); + emit_row(rsinfo, "hw", "remaster_recoverable", + fmt_int64((int64)(cluster_hw_remaster_recoverable() ? 1 : 0))); /* * spec-6.12d D-obs: space-lease counters. bloat_ratio itself is a diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..d0219deaf9 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -51,11 +51,12 @@ #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */ #include "cluster/cluster_grd.h" /* spec-4.6 D4 — block_path_failclosed counter */ #include "cluster/cluster_grd_outbound.h" -#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ -#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ -#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ -#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ +#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ +#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ +#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ +#include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ +#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_itl.h" /* spec-5.2 D11 — active-ITL writer-transfer guard */ @@ -65,6 +66,7 @@ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" #include "cluster/cluster_shmem.h" +#include "cluster/storage/cluster_shared_fs.h" #include "cluster/cluster_sf_dep.h" #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */ #include "common/hashfn.h" @@ -1221,12 +1223,13 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The survivor may SERVE only - * after the dead origin's merged WAL recovery passes the redo-before- - * unfreeze gate; before that the shared-storage / re-declared version may - * be stale → fail-closed RECOVERING (never a stale page). + * live survivor (recovery-aware routing). The redo-before-serve gate must + * engage on EXACTLY the same scope where online thread recovery can actually + * run. Otherwise a >2-node or GUC-off deployment waits forever on a + * materialization authority that the thread-recovery launcher intentionally + * treats as not applicable. * - * TWO conditions, both required (Q5): + * In applicable 2-node scope, two conditions are both required (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1236,13 +1239,19 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * survivor DID observe (rebuilt pi_watermark_lsn > 0), the dead * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably - * reached us → lost-write → fail-closed. This is the LSN comparison - * (NOT a bool), live in the serve path; required_lsn == 0 (cold) is - * trivially covered and (a) carries the safety. - * - * Once both hold → NORMAL → the re-routed survivor serves (rebuilt-from- - * redeclare for held blocks, lazy minimal view for cold blocks). + * reached us → lost-write → fail-closed. */ + { + bool shared_fs; + int survivors; + + shared_fs = (cluster_shared_storage_backend == CLUSTER_SHARED_FS_BACKEND_CLUSTER_FS); + survivors = cluster_conf_node_count() - 1; + if (!cluster_thread_recovery_materialization_gate_enabled( + cluster_online_thread_recovery, cluster_conf_has_peers(), shared_fs, survivors)) + return GCS_BLOCK_NORMAL; + } + if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->recovery_block_resources_recovering, 1); diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index a81f80ee2b..547f0e5aac 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -552,7 +552,8 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) * record and return (no work queue, no reply, no dedup; the sender * re-announces each tick, the receiver write is a monotonic max). */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { - cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch); + cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, + ges_request_holder_request_id(req)); return; } diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..8977739433 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -46,7 +46,8 @@ #include "cluster/cluster_pcm_lock.h" /* spec-2.36 HC124 pending_x node-dead cleanup */ #include "cluster/cluster_gcs.h" /* spec-4.7 D2 — cluster_gcs_lookup_master */ #include "cluster/cluster_membership.h" /* spec-5.16 D1 — cluster_membership_is_member */ -#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ +#include "cluster/cluster_native_lock_probe.h" +#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ #include "cluster/cluster_signal.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_cssd.h" /* spec-2.16 D8 newly-dead bitmap diff */ @@ -731,9 +732,13 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->recovery_event_old_epoch, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_redeclare_generation, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_barrier_deadline, 0); + pg_atomic_init_u32(&cluster_grd_state->recovery_event_coordinator, 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) + for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_event_id[i], 0); + } pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -747,6 +752,9 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->block_path_failclosed_count, 0); pg_atomic_init_u64(&cluster_grd_state->unaffected_holder_survived_count, 0); pg_atomic_init_u64(&cluster_grd_state->stale_holder_swept_count, 0); + pg_atomic_init_u64(&cluster_grd_state->cluster_gate_timeout_count, 0); + pg_atomic_init_u64(&cluster_grd_state->wait_epoch_escape_count, 0); + pg_atomic_init_u64(&cluster_grd_state->pcm_dead_cleanup_entries, 0); /* spec-5.16 D2/D3b/D5 — online-join remaster fence + counters. */ pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_epoch, 0); @@ -1630,6 +1638,78 @@ cluster_grd_redeclare_episode_epoch(void) * been re-declared to its recovery-aware master — serving it would risk an * 8.A double-grant. Reaching IDLE implies all survivor scans completed. */ +uint32 +cluster_grd_recovery_state_value(void) +{ + if (cluster_grd_state == NULL) + return (uint32)GRD_RECOVERY_IDLE; + return pg_atomic_read_u32(&cluster_grd_state->recovery_state); +} + +const char * +cluster_grd_recovery_state_name(uint32 state) +{ + switch ((ClusterGrdRecoveryState)state) { + case GRD_RECOVERY_IDLE: + return "idle"; + case GRD_RECOVERY_WAIT_EPOCH: + return "wait_epoch"; + case GRD_RECOVERY_WAIT_BARRIER: + return "wait_barrier"; + case GRD_RECOVERY_WAIT_CLUSTER: + return "wait_cluster"; + } + return "unknown"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id) + : 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch) + : 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch) + : 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator) + : 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); +} + +uint64 +cluster_grd_recovery_done_event_id_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[node]); +} + bool cluster_grd_recovery_in_progress(void) { @@ -1686,6 +1766,10 @@ cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) out->unaffected_holder_survived = pg_atomic_read_u64(&cluster_grd_state->unaffected_holder_survived_count); out->stale_holder_swept = pg_atomic_read_u64(&cluster_grd_state->stale_holder_swept_count); + out->cluster_gate_timeout = pg_atomic_read_u64(&cluster_grd_state->cluster_gate_timeout_count); + out->wait_epoch_escape = pg_atomic_read_u64(&cluster_grd_state->wait_epoch_escape_count); + out->pcm_dead_cleanup_entries + = pg_atomic_read_u64(&cluster_grd_state->pcm_dead_cleanup_entries); /* spec-5.16 D5 — join-direction remaster counters. */ out->join_remaster_started = pg_atomic_read_u64(&cluster_grd_state->join_remaster_started_count); @@ -1924,6 +2008,39 @@ grd_recovery_barrier_complete(uint64 gen, uint64 episode_epoch) return true; } +static void +grd_recovery_format_waiting_backend(uint64 gen, uint64 episode_epoch, char *buf, Size buflen) +{ + int beid; + pid_t self_pid = MyProcPid; + + if (buflen == 0) + return; + buf[0] = '\0'; + for (beid = 1; beid <= MaxBackends; beid++) { + PGPROC *proc = BackendIdGetProc((BackendId)beid); + uint32 registered; + uint64 acked; + uint64 acked_epoch; + + if (proc == NULL || proc->pid == 0 || proc->pid == self_pid) + continue; + registered = pg_atomic_read_u32(&proc->cluster_grd_registered_count); + if (registered == 0) + continue; + acked = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked); + acked_epoch = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked_epoch); + if (acked < gen || acked_epoch != episode_epoch) { + snprintf(buf, buflen, + "beid=%d pid=%d registered=%u acked=" UINT64_FORMAT "/" UINT64_FORMAT + " target=" UINT64_FORMAT "/" UINT64_FORMAT, + beid, (int)proc->pid, registered, acked, acked_epoch, gen, episode_epoch); + return; + } + } + snprintf(buf, buflen, "none"); +} + /* * spec-4.6 P0#3 cluster gate — announce "my local rebind barrier is * complete for `epoch`" to every declared peer (fire-and-forget; the @@ -1934,6 +2051,7 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; + uint64 event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -1943,6 +2061,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); + req.holder_request_id_lo = (uint32)(event_id & 0xffffffffu); + req.holder_request_id_hi = (uint32)(event_id >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -1961,19 +2081,33 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) { - uint64 prev; + uint64 current_event_id; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; - /* Monotonic max: late/duplicate announcements never regress. */ - prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); - while (epoch > prev) { - if (pg_atomic_compare_exchange_u64(&cluster_grd_state->recovery_done_epoch[node], &prev, - epoch)) - break; + + /* + * spec-4.6a D4-v2: REDECLARE_DONE is an event-scoped proof. A stale DONE + * from a previous episode can carry the same epoch when cur==old, so epoch + * monotonicity alone is not enough. Drop messages for any event other than + * the one this LMON has accepted; senders re-announce every tick. + */ + current_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + if (event_id == 0 || (current_event_id != 0 && event_id != current_event_id)) + return; + + /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic + * per-event epochs the incoming value always wins, but never regress the + * published value if a delayed lower-epoch frame ever slips through. */ + { + uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); + + if (epoch > prev) + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[node], event_id); } /* @@ -2000,6 +2134,144 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +static void +grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) +{ + va_list ap; + int n; + Size avail; + + if (buflen == 0 || *off >= (int)buflen - 1) + return; + avail = buflen - (Size)*off; + va_start(ap, fmt); + n = vsnprintf(buf + *off, avail, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((Size)n >= avail) + *off = (int)buflen - 1; + else + *off += n; +} + +static void +grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, bool is_join, + char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 done_epoch; + + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + if (is_join && join_fence_is_recipient_for(i, episode_epoch)) + continue; + done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); + if (done_epoch >= episode_epoch + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + continue; + grd_recovery_appendf(buf, buflen, &off, + "%s%d(done=" UINT64_FORMAT "/event=" UINT64_FORMAT ")", any ? "," : "", + i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i])); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static void +grd_recovery_format_hw_dead(const uint64 *dead, char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + ClusterHwRemasterResult result; + + if (!grd_dead_bitmap_test(dead, i) || i == cluster_node_id) + continue; + result = cluster_hw_remaster_result(i); + grd_recovery_appendf(buf, buflen, &off, "%s%d:%s/%u", any ? "," : "", i, + cluster_hw_remaster_result_name(result), + cluster_hw_remaster_attempts(i)); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static bool +grd_recovery_adopt_observed_epoch(const uint64 *dead, uint64 cur_epoch) +{ + uint64 max_epoch = cur_epoch; + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 peer_epoch; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + peer_epoch = cluster_reconfig_get_observed_epoch(i); + if (peer_epoch > max_epoch) + max_epoch = peer_epoch; + } + if (max_epoch <= cur_epoch) + return false; + cluster_epoch_adopt_admitted(max_epoch); + return true; +} + +static void +grd_recovery_wait_cluster_watchdog(const uint64 *dead, uint64 episode_epoch) +{ + TimestampTz now; + TimestampTz deadline; + TimestampTz next_deadline; + char missing[512]; + char hw_dead[512]; + bool thread_gate; + bool hw_gate; + bool is_join; + + deadline = (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline); + now = GetCurrentTimestamp(); + if (deadline == 0 || now <= deadline) + return; + + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + grd_recovery_format_missing_survivors(dead, episode_epoch, is_join, missing, sizeof(missing)); + thread_gate = cluster_thread_recovery_gate_unfreeze(dead, (CLUSTER_MAX_NODES + 63) / 64); + hw_gate = cluster_hw_remaster_gate_unfreeze(); + grd_recovery_format_hw_dead(dead, hw_dead, sizeof(hw_dead)); + + pg_atomic_fetch_add_u64(&cluster_grd_state->cluster_gate_timeout_count, 1); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen"), + errdetail("episode_epoch=" UINT64_FORMAT ", missing_survivors=%s, " + "thread_gate=%s, hw_gate=%s, hw_dead=%s", + episode_epoch, missing, thread_gate ? "held" : "open", hw_gate ? "held" : "open", + hw_dead), + errhint("This watchdog is observational only; it never unfreezes a shard."))); + next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); +} + /* * spec-4.7 D2 — survivor block re-declare scan (Q6-A' worker-centric). @@ -2078,6 +2350,24 @@ grd_block_redeclare_scan_complete(uint64 episode_epoch) return grd_block_redeclare_epoch == episode_epoch && grd_block_redeclare_done; } +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return grd_block_redeclare_cursor; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return grd_block_redeclare_epoch; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return grd_block_redeclare_done; +} + void cluster_grd_recovery_lmon_tick(void) { @@ -2119,6 +2409,18 @@ cluster_grd_recovery_lmon_tick(void) * epoch (the genuine pre-reconfig baseline) — do NOT overwrite it * with evt.old_epoch here. */ pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, ev_id); + pg_atomic_write_u32(&cluster_grd_state->recovery_event_coordinator, + (uint32)evt.coordinator_node_id); + if (evt.coordinator_node_id >= 0 && evt.coordinator_node_id < CLUSTER_MAX_NODES) + pg_atomic_write_u64( + &cluster_grd_state->recovery_done_epoch_at_accept, + pg_atomic_read_u64( + &cluster_grd_state->recovery_done_epoch[evt.coordinator_node_id])); + else + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms)); for (b = 0; b < (CLUSTER_MAX_NODES + 63) / 64; b++) { uint64 word = 0; int j; @@ -2174,35 +2476,116 @@ cluster_grd_recovery_lmon_tick(void) int i; int signaled; + for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) + dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + /* - * Gate on the ACCEPTED epoch having advanced past the episode's - * old epoch: the coordinator bumped it earlier this tick; a - * non-coordinator survivor observes it via IC envelope piggyback - * a tick or two later. Running the rebind before the local - * epoch advances would mint holders the new master rejects. - * - * spec-5.16 (P0, Rule 8.A) — direction-aware. A FAIL survivor captures - * old_epoch BEFORE the coordinator bumps, so it must wait for a STRICT - * advance (cur > old) to prove it adopted the new master view. A JOIN - * observer survivor instead adopts the coordinator's JOIN epoch bump (via the - * joiner_self_tick max-peer-epoch adoption, same LMON tick) BEFORE it readmits - * the rejoiner and publishes its own observer JOIN_COMMITTED event, so by the - * time its FSM accepts that event old_epoch == cur_epoch == the join epoch. - * Requiring a strict advance there wedges the survivor in WAIT_EPOCH forever: - * it never runs the re-declare barrier, never broadcasts REDECLARE_DONE, and - * the coordinator's all-members JOIN barrier (Hardening v1.1) hangs → - * join_remaster_done never advances. For JOIN the survivor already holds the - * epoch it rebinds under (that IS the post-bump master view), so equality is - * sufficient; cur_epoch is monotonic so cur < old never occurs for JOIN. + * Gate on the ACCEPTED epoch having advanced past the episode's old epoch. + * FAIL requires proof of a post-bump epoch; JOIN keeps its existing equality + * allowance because the observer already accepted the coordinator's JOIN epoch. + * For FAIL, deadline expiry is not an escape hatch by itself: we either adopt + * a durable observed epoch or require a coordinator REDECLARE_DONE witness for + * cur==old. Without either proof, stay fail-closed in WAIT_EPOCH. */ - if (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) - == (uint32)GRD_REMASTER_DIR_JOIN - ? cur_epoch < old_epoch - : cur_epoch <= old_epoch) - return; + { + ReconfigEvent latest; + bool is_join; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id != 0 + && latest.event_id + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + grd_recovery_abort_to_idle(); + return; + } - for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) - dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + if (is_join) { + if (cur_epoch < old_epoch) + return; + } else if (cur_epoch <= old_epoch) { + TimestampTz now = GetCurrentTimestamp(); + TimestampTz wait_deadline; + TimestampTz next_deadline; + uint32 coordinator; + uint64 coord_done; + uint64 coord_done_event_id; + uint64 accepted_event_id; + uint64 coord_done_at_accept; + + wait_deadline = (TimestampTz)pg_atomic_read_u64( + &cluster_grd_state->recovery_barrier_deadline); + if (wait_deadline == 0 || now <= wait_deadline) + return; + + if (grd_recovery_adopt_observed_epoch(dead, cur_epoch)) { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH adopted a durable observed epoch; " + "affected shards stay frozen until recovery completes"))); + return; + } + + coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); + accepted_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + coord_done + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) + : 0; + coord_done_event_id + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64( + &cluster_grd_state->recovery_done_event_id[coordinator]) + : 0; + coord_done_at_accept + = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); + /* + * spec-4.6a §2.3 witness (all three conjuncts of the frozen + * text): the coordinator's DONE must (a) be for THIS event, + * (b) name exactly cur as the episode epoch, and (c) have been + * accounted after our P0 accept snapshot. Under strictly + * monotonic per-event epochs (c) is implied by (a)+(b) — any + * pre-accept residue carries a lower epoch — but it stays as a + * belt-and-suspenders guard against accounting bugs. + */ + if (cur_epoch == old_epoch && coord_done == cur_epoch + && coord_done_event_id == accepted_event_id + && coord_done > coord_done_at_accept) { + pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " + "equal-epoch progress"), + errdetail("coordinator=%u, epoch=" UINT64_FORMAT + ", event_id=" UINT64_FORMAT, + coordinator, cur_epoch, accepted_event_id))); + } else { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_event_id=" UINT64_FORMAT + ", accepted_event_id=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_event_id, accepted_event_id, coord_done_at_accept))); + return; + } + } + } /* * P1 freeze affected + P3 scoped sweep + P4 remaster. Direction- @@ -2306,6 +2689,7 @@ cluster_grd_recovery_lmon_tick(void) if (state == (uint32)GRD_RECOVERY_WAIT_BARRIER) { uint64 gen = pg_atomic_read_u64(&cluster_grd_state->recovery_redeclare_generation); uint64 episode_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch); + TimestampTz deadline; /* P0-1 epoch-coherence guard: a SECOND epoch bump landed * mid-episode (e.g. a third node's heartbeat flap re-fired @@ -2344,7 +2728,12 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)); grd_recovery_broadcast_done(episode_epoch); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)deadline); pg_atomic_write_u32(&cluster_grd_state->recovery_state, (uint32)GRD_RECOVERY_WAIT_CLUSTER); ereport(DEBUG1, @@ -2358,6 +2747,12 @@ cluster_grd_recovery_lmon_tick(void) if (GetCurrentTimestamp() > (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline)) { TimestampTz deadline; + char waiting_backend[256]; + bool ges_barrier_complete = grd_recovery_barrier_complete(gen, episode_epoch); + bool block_scan_complete = grd_block_redeclare_scan_complete(episode_epoch); + + grd_recovery_format_waiting_backend(gen, episode_epoch, waiting_backend, + sizeof(waiting_backend)); /* * Barrier deadline expired: fail-closed. Affected shards @@ -2376,6 +2771,12 @@ cluster_grd_recovery_lmon_tick(void) (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), errmsg("cluster GRD holder-rebuild barrier timed out; affected shards " "stay frozen"), + errdetail("gen=" UINT64_FORMAT ", episode_epoch=" UINT64_FORMAT + ", ges_barrier=%s, block_scan=%s, block_cursor=%d" + ", block_epoch=" UINT64_FORMAT ", waiting_backend=%s", + gen, episode_epoch, ges_barrier_complete ? "done" : "waiting", + block_scan_complete ? "done" : "waiting", grd_block_redeclare_cursor, + grd_block_redeclare_epoch, waiting_backend), errhint("A backend has not acked the cooperative rebind within " "cluster.grd_rebuild_timeout_ms; re-broadcasting."))); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), @@ -2409,6 +2810,8 @@ cluster_grd_recovery_lmon_tick(void) for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + grd_recovery_wait_cluster_watchdog(dead, episode_epoch); + /* * spec-4.11 D1 (3b-4b Part 3) — launch one per-episode online thread- * recovery worker for each in-scope dead origin (idempotent per tick; @@ -2461,8 +2864,9 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; - if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) - < episode_epoch) { + if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { all_done = false; break; } @@ -5344,13 +5748,28 @@ cluster_grd_cleanup_on_node_dead(int32 dead_node_id) int swept = 0; int nresids; uint64 pending_x_cleared = 0; + uint64 pcm_holders_cleaned = 0; int i; pending_x_cleared = cluster_pcm_lock_clear_pending_x_for_node(dead_node_id); - if (pending_x_cleared > 0) - ereport(DEBUG1, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " - "cleared " UINT64_FORMAT " PCM pending_x entries", - dead_node_id, pending_x_cleared))); + pcm_holders_cleaned = cluster_pcm_lock_cleanup_on_node_dead(dead_node_id); + + /* + * spec-4.6a D12: dead-node PCM residue is part of the reconfig + * convergence chain -- a leftover holder/pending-X record from the dead + * node blocks post-kill DDL with lock-acquire timeouts long after the + * remaster itself finished. Surface the cleanup as a LOG summary plus an + * assertable counter (pcm/dead_cleanup_entries). + */ + if (pending_x_cleared > 0 || pcm_holders_cleaned > 0) { + if (cluster_grd_state != NULL) + pg_atomic_fetch_add_u64(&cluster_grd_state->pcm_dead_cleanup_entries, + pending_x_cleared + pcm_holders_cleaned); + ereport(LOG, (errmsg("cluster GRD dead-node cleanup for node %d: cleared " UINT64_FORMAT + " PCM pending-X waiter(s) and " UINT64_FORMAT + " PCM holder record(s) left by the dead node", + dead_node_id, pending_x_cleared, pcm_holders_cleaned))); + } if (cluster_grd_entry_htab == NULL) { ereport(DEBUG2, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..b505360e11 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -226,6 +226,8 @@ int cluster_tm_convert_mode = CLUSTER_TM_CONVERT_MODE_CONVERT; /* spec-4.6 D4/D1 — failure-driven remaster tunables. */ int cluster_grd_remaster_wait_ms = 200; /* frozen-shard short wait before 53R9I */ int cluster_grd_rebuild_timeout_ms = 5000; /* holder-rebuild barrier deadline */ +int cluster_hw_remaster_retry_backoff_ms = 1000; +int cluster_hw_remaster_retry_max_attempts = 16; /* spec-5.4 D8 — SQ sequence lock tunables. */ int cluster_sequence_default_cache = 100; /* CREATE-time CACHE injection default */ @@ -2374,6 +2376,21 @@ cluster_init_guc(void) "request with a fresh deadline."), &cluster_grd_rebuild_timeout_ms, 5000, 100, 600000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_backoff_ms", + gettext_noop("Initial backoff before retrying a BLOCKED HW remaster worker (ms)."), + gettext_noop("Range [100, 60000]. Default 1000. Same-episode HW remaster " + "retries use exponential backoff capped at 60 seconds; the adopted " + "shards stay frozen while retrying."), + &cluster_hw_remaster_retry_backoff_ms, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_max_attempts", + gettext_noop("Maximum same-episode retries for a BLOCKED HW remaster worker."), + gettext_noop("Range [0, 1000]. Default 16. Zero disables same-episode retry. " + "Raising the value with SIGHUP lets an exhausted episode resume retrying " + "without waiting for a new reconfig episode."), + &cluster_hw_remaster_retry_max_attempts, 16, 0, 1000, PGC_SIGHUP, 0, NULL, NULL, NULL); /* spec-2.23 D11 NEW: coordinator REPORT collect deadline. */ DefineCustomIntVariable("cluster.lmd_probe_collect_timeout_ms", diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index d1ade2a760..c585268a42 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -61,6 +61,7 @@ #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/fd.h" +#include "storage/ipc.h" #include "utils/wait_event.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ @@ -74,6 +75,65 @@ #include "cluster/cluster_wal_thread.h" /* node id -> thread id */ #include "cluster/storage/cluster_undo_xlog.h" /* xl_hw_reserve, XLOG_HW_RESERVE */ +/* One worker process owns exactly one dead origin / episode. */ +static int hw_worker_dead_node = -1; +static uint64 hw_worker_episode = 0; +static bool hw_worker_armed = false; + +static uint64 +hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) +{ + uint32 backoff_ms; + TimestampTz deadline; + + backoff_ms = cluster_hw_remaster_compute_backoff_ms(cluster_hw_remaster_retry_backoff_ms, + completed_retry_attempts); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), (int)backoff_ms); + return (uint64)deadline; +} + +static void +hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) +{ + if (res == CLUSTER_HW_REMASTER_DONE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_done(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { + uint32 attempts = cluster_hw_remaster_attempts(dead_node); + + cluster_hw_remaster_set_next_attempt_at(dead_node, + hw_remaster_next_attempt_deadline(attempts)); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(dead_node, CLUSTER_HW_REMASTER_NO_DEADLINE); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + cluster_hw_remaster_set_result(dead_node, res); + } +} + +static void +hw_remaster_mark_blocked_on_exit(int code pg_attribute_unused(), Datum arg) +{ + int dead_node = DatumGetInt32(arg); + + if (!hw_worker_armed) + return; + if (dead_node != hw_worker_dead_node) + return; + if (cluster_hw_remaster_launched_episode(dead_node) != hw_worker_episode) + return; + if (cluster_hw_remaster_result(dead_node) != CLUSTER_HW_REMASTER_RUNNING) + return; + + cluster_hw_bump_failclosed(); + hw_remaster_record_terminal(dead_node, CLUSTER_HW_REMASTER_BLOCKED); +} + /* ============================================================ * Minimal per-thread WAL reader over a dead origin's WAL stream. @@ -332,10 +392,24 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; if (dead_node_id < 0 || dead_node_id == cluster_node_id) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; + if (!cluster_hw_remaster_recoverable()) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: cluster.wal_threads_dir is not configured; " + "adopted shards stay fail-closed"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "restart, or restart the dead node so a JOIN rebuild can replace " + "the failure-driven remaster."))); + return CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL; + } dead_tid = cluster_wal_thread_id_for(true, dead_node_id); - if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) + if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d maps to invalid WAL thread id %u; " + "adopted shards stay fail-closed", + dead_node_id, (unsigned)dead_tid))); return CLUSTER_HW_REMASTER_BLOCKED; + } entries = (ClusterHwSnapshotEntry *)palloc(sizeof(ClusterHwSnapshotEntry) * CLUSTER_HW_AUTHORITY_MAX); @@ -384,6 +458,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_wal_state_read_slot(dead_tid, &slot) != CLUSTER_WAL_SLOT_OK || slot.highest_lsn == 0) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d WAL state slot is unusable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } validated_min = (XLogRecPtr)slot.highest_lsn; @@ -391,6 +468,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_thread_recovery_validated_end(dead_tid, lower, validated_min, &upper) != CLUSTER_THREADREC_DONE) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d validated WAL end is unavailable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } @@ -466,29 +546,26 @@ cluster_hw_remaster_worker_main(Datum main_arg) /* The bgworker framework starts the entry point with signals blocked; unblock * before any I/O so a SIGTERM during a stuck shared-storage read / shutdown is - * delivered (the default die handler FATALs, a clean fail-closed: unmarked). */ + * delivered (the default die handler FATALs into the BLOCKED exit callback). */ BackgroundWorkerUnblockSignals(); /* Capture the live reconfig episode this rebuild is for; rebuild_origin uses * it as the staleness fence before it marks any shard rebuilt. */ episode = cluster_grd_redeclare_episode_epoch(); - res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_worker_dead_node = dead_node; + hw_worker_episode = episode; + hw_worker_armed = true; + before_shmem_exit(hw_remaster_mark_blocked_on_exit, Int32GetDatum(dead_node)); - /* Observability (S5/S7): a DONE rebuilt + adopted + marked the shards; a - * BLOCKED kept them frozen (fail-closed). NOT_APPLICABLE (HW inactive) is - * neither. */ - if (res == CLUSTER_HW_REMASTER_DONE) - cluster_hw_bump_remaster_done(); - else if (res == CLUSTER_HW_REMASTER_BLOCKED) - cluster_hw_bump_remaster_blocked(); + res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_remaster_record_terminal(dead_node, res); + hw_worker_armed = false; ereport(LOG, (errmsg("cluster HW remaster worker: dead node %d -> %s", dead_node, - res == CLUSTER_HW_REMASTER_DONE ? "done" - : res == CLUSTER_HW_REMASTER_BLOCKED ? "blocked (shards kept frozen)" - : "not applicable"))); + cluster_hw_remaster_result_name(res)))); /* Returning is a clean exit(0). On BLOCKED / abnormal exit the adopted shards - * stay unmarked, so the serve gate keeps them fail-closed (8.A) and a later - * episode relaunches the rebuild -- no abnormal-exit handler is needed. */ + * stay unmarked, so the serve gate keeps them fail-closed (8.A); the terminal + * result lets the LMON FSM retry within the same episode when appropriate. */ } /* @@ -522,6 +599,7 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod { int node; int max_node; + uint64 now; if (dead == NULL || nwords <= 0) return; @@ -538,31 +616,93 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod max_node = nwords * 64; if (max_node > CLUSTER_MAX_NODES) max_node = CLUSTER_MAX_NODES; + now = (uint64)GetCurrentTimestamp(); for (node = 0; node < max_node; node++) { + ClusterHwRemasterResult result; + ClusterHwRemasterRelaunchDecision d; + uint64 launched; + uint64 next_attempt_at; + uint32 attempts; + if ((dead[node / 64] & (UINT64CONST(1) << (node % 64))) == 0) continue; if (node == cluster_node_id) continue; /* self is never its own dead origin */ - /* Idempotent: launch once per episode per dead origin. */ - if (cluster_hw_remaster_launched_episode(node) == episode_epoch) + + launched = cluster_hw_remaster_launched_episode(node); + if (!cluster_hw_remaster_recoverable() && launched != episode_epoch) { + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, 0); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL); + launched = episode_epoch; + } + + result = cluster_hw_remaster_result(node); + attempts = cluster_hw_remaster_attempts(node); + next_attempt_at = cluster_hw_remaster_next_attempt_at(node); + d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, + next_attempt_at, now, + cluster_hw_remaster_retry_max_attempts); + + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster is structurally blocked for dead node %d; " + "adopted shards stay fail-closed", + node), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root " + "and restart, or restart the dead node so a JOIN rebuild can " + "replace the failure-driven remaster."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + cluster_hw_bump_remaster_retry_exhausted(); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster retries exhausted for dead node %d after %u " + "attempt(s); adopted shards stay fail-closed", + node, attempts), + errhint("Fix the shared HW snapshot/WAL source, then raise " + "cluster.hw_remaster_retry_max_attempts with SIGHUP to resume " + "same-episode retrying."))); continue; - /* - * Claim BEFORE registering so a same-episode re-entry does not double- - * launch; revert on a registration failure so a later tick retries (the - * adopted shards stay fail-closed meanwhile, via the P7 gate). - */ + } + if (d.action != CLUSTER_HW_REMASTER_LAUNCH_INITIAL + && d.action != CLUSTER_HW_REMASTER_LAUNCH_RETRY) + continue; + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, d.next_attempts); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_RUNNING); if (!register_one_worker(node)) { - cluster_hw_remaster_set_launched(node, 0); + cluster_hw_remaster_set_launched( + node, d.action == CLUSTER_HW_REMASTER_LAUNCH_INITIAL ? 0 : launched); + cluster_hw_remaster_set_attempts(node, attempts); + cluster_hw_remaster_set_next_attempt_at(node, next_attempt_at); + cluster_hw_remaster_set_result(node, result); ereport(WARNING, (errmsg("could not register HW remaster worker for dead node %d", node), errhint("Background worker slots are exhausted (max_worker_processes); the " "adopted shards stay fail-closed until the rebuild can run."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_RETRY) { + cluster_hw_bump_remaster_retry(); + ereport(LOG, + (errmsg("cluster HW remaster: retrying dead node %d in episode " UINT64_FORMAT + " (attempt %u/%d)", + node, episode_epoch, d.next_attempts, + cluster_hw_remaster_retry_max_attempts))); } } } + bool cluster_hw_remaster_gate_unfreeze(void) { diff --git a/src/backend/cluster/cluster_hw_shmem.c b/src/backend/cluster/cluster_hw_shmem.c index 14bf3f7586..860241e8e9 100644 --- a/src/backend/cluster/cluster_hw_shmem.c +++ b/src/backend/cluster/cluster_hw_shmem.c @@ -100,6 +100,8 @@ typedef struct ClusterHwShared { pg_atomic_uint64 not_ready_count; /* 53RA6 serve-gate (shard adopted, unrebuilt) */ pg_atomic_uint64 remaster_done_count; /* online-remaster HW rebuild DONE (S5/S7) */ pg_atomic_uint64 remaster_blocked_count; /* online-remaster HW rebuild fail-closed (S5/S7) */ + pg_atomic_uint64 remaster_retry_count; /* same-episode retry launches */ + pg_atomic_uint64 remaster_retry_exhausted_count; /* same-episode retry cap reached */ pg_atomic_uint32 hw_rebuilt_generation[PGRAC_GRD_SHARD_COUNT]; /* §3.1b R4/R9 gate */ /* * remaster_launched_episode[node] -- the reconfig episode for which the GRD @@ -111,6 +113,9 @@ typedef struct ClusterHwShared { * race on this field. */ pg_atomic_uint64 remaster_launched_episode[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_result[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_attempts[CLUSTER_MAX_NODES]; + pg_atomic_uint64 remaster_next_attempt_at[CLUSTER_MAX_NODES]; } ClusterHwShared; /* @@ -196,12 +201,18 @@ cluster_hw_shmem_init(void) pg_atomic_init_u64(&hw_state->not_ready_count, 0); pg_atomic_init_u64(&hw_state->remaster_done_count, 0); pg_atomic_init_u64(&hw_state->remaster_blocked_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_exhausted_count, 0); /* No shard rebuilt yet; 0 matches a never-remastered shard's GRD * master_generation (0), so boot / steady state serves. */ for (s = 0; s < PGRAC_GRD_SHARD_COUNT; s++) pg_atomic_init_u32(&hw_state->hw_rebuilt_generation[s], 0); - for (s = 0; s < CLUSTER_MAX_NODES; s++) + for (s = 0; s < CLUSTER_MAX_NODES; s++) { pg_atomic_init_u64(&hw_state->remaster_launched_episode[s], 0); + pg_atomic_init_u32(&hw_state->remaster_result[s], (uint32)CLUSTER_HW_REMASTER_NONE); + pg_atomic_init_u32(&hw_state->remaster_attempts[s], 0); + pg_atomic_init_u64(&hw_state->remaster_next_attempt_at[s], 0); + } } memset(&hctl, 0, sizeof(hctl)); @@ -408,6 +419,81 @@ cluster_hw_remaster_set_launched(int node_id, uint64 episode) pg_atomic_write_u64(&hw_state->remaster_launched_episode[node_id], episode); } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return CLUSTER_HW_REMASTER_NONE; + return (ClusterHwRemasterResult)pg_atomic_read_u32(&hw_state->remaster_result[node_id]); +} + +void +cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_result[node_id], (uint32)result); +} + +uint32 +cluster_hw_remaster_attempts(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u32(&hw_state->remaster_attempts[node_id]); +} + +void +cluster_hw_remaster_set_attempts(int node_id, uint32 attempts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_attempts[node_id], attempts); +} + +uint64 +cluster_hw_remaster_next_attempt_at(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&hw_state->remaster_next_attempt_at[node_id]); +} + +void +cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u64(&hw_state->remaster_next_attempt_at[node_id], ts); +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result) +{ + switch (result) { + case CLUSTER_HW_REMASTER_NONE: + return "none"; + case CLUSTER_HW_REMASTER_RUNNING: + return "running"; + case CLUSTER_HW_REMASTER_DONE: + return "done"; + case CLUSTER_HW_REMASTER_BLOCKED: + return "blocked"; + case CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL: + return "blocked_structural"; + case CLUSTER_HW_REMASTER_NOT_APPLICABLE: + return "not_applicable"; + } + return "unknown"; +} + +bool +cluster_hw_remaster_recoverable(void) +{ + return !cluster_hw_authority_active() + || (cluster_wal_threads_dir != NULL && cluster_wal_threads_dir[0] != '\0'); +} + /* ============================================================ * Checkpoint write + recovery load (§3.1b R1/R2; PG-core hooked). @@ -747,6 +833,16 @@ cluster_hw_bump_remaster_blocked(void) { HW_BUMP(remaster_blocked_count); } +void +cluster_hw_bump_remaster_retry(void) +{ + HW_BUMP(remaster_retry_count); +} +void +cluster_hw_bump_remaster_retry_exhausted(void) +{ + HW_BUMP(remaster_retry_exhausted_count); +} uint64 cluster_hw_alloc_count(void) @@ -788,3 +884,13 @@ cluster_hw_remaster_blocked_count(void) { return HW_READ(remaster_blocked_count); } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return HW_READ(remaster_retry_count); +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return HW_READ(remaster_retry_exhausted_count); +} diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index c3b9a39c24..8bec6ddd71 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,6 +59,7 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" +#include "cluster/cluster_cssd.h" #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" @@ -1044,6 +1045,8 @@ cluster_lms_native_probe_dispatch(uint32 slot_idx) conf_node = cluster_conf_lookup_node(candidate); if (conf_node == NULL) continue; + if (cluster_cssd_get_peer_state(candidate) == CLUSTER_CSSD_PEER_DEAD) + continue; if (!native_probe_node_bit(candidate, &peer_bit)) { pg_atomic_fetch_add_u64(&cluster_lms_state->native_probe_timeout_count, 1); slot->final_status = CLUSTER_NATIVE_PROBE_FINAL_TIMEOUT; diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 545ac33480..b9de7f6401 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -724,6 +724,96 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) return cleared; } +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node) +{ + HASH_SEQ_STATUS scan; + struct GrdEntry *entry; + uint64 cleaned = 0; + uint32 dead_bit; + + if (cluster_pcm_htab == NULL || dead_node < 0 || dead_node >= 32) + return 0; + + dead_bit = (uint32)1u << (uint32)dead_node; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + hash_seq_init(&scan, cluster_pcm_htab); + while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool changed = false; + bool broadcast_needed = false; + bool master_holder_was_dead; + PcmState before_state; + PcmState after_state; + uint32 s_bitmap; + uint32 pi_bitmap; + + /* Cheap unlocked filter; rechecked under entry_lock below. */ + if (entry->x_holder_node != dead_node + && (pg_atomic_read_u32(&entry->s_holders_bitmap) & dead_bit) == 0 + && (pg_atomic_read_u32(&entry->pi_holders_bitmap) & dead_bit) == 0 + && (!pcm_master_holder_is_valid(entry) + || (int32)entry->master_holder.node_id != dead_node)) + continue; + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + before_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + master_holder_was_dead + = pcm_master_holder_is_valid(entry) && (int32)entry->master_holder.node_id == dead_node; + + if (entry->x_holder_node == dead_node) { + entry->x_holder_node = -1; + changed = true; + } + s_bitmap = pg_atomic_read_u32(&entry->s_holders_bitmap); + if ((s_bitmap & dead_bit) != 0) { + s_bitmap &= ~dead_bit; + pg_atomic_write_u32(&entry->s_holders_bitmap, s_bitmap); + changed = true; + } + pi_bitmap = pg_atomic_read_u32(&entry->pi_holders_bitmap); + if ((pi_bitmap & dead_bit) != 0) { + pg_atomic_write_u32(&entry->pi_holders_bitmap, pi_bitmap & ~dead_bit); + changed = true; + } + + if (entry->x_holder_node >= 0) { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); + if (master_holder_was_dead) + pcm_master_holder_set_node(entry, entry->x_holder_node); + } else if (s_bitmap != 0) { + int32 next_holder = pcm_lowest_set_bit_node(s_bitmap); + + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_S); + if (master_holder_was_dead) { + if (next_holder >= 0) + pcm_master_holder_set_node(entry, next_holder); + else + pcm_master_holder_clear(entry); + } + } else { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_N); + if (pcm_master_holder_is_valid(entry)) + pcm_master_holder_clear(entry); + } + if (master_holder_was_dead) + changed = true; + after_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + if (changed && after_state != before_state) + broadcast_needed = true; + + LWLockRelease(&entry->entry_lock.lock); + + if (broadcast_needed) + ConditionVariableBroadcast(&entry->wait_cv); + if (changed) + cleaned++; + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + + return cleaned; +} + /* ======================================================================== * PGRAC MODIFICATIONS by SqlRush — spec-5.13 D5 (clean-leave PCM release). @@ -2199,6 +2289,41 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + + /* + * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after + * fail-stop. Local-master state=S with other live S holders but no local + * S bit used to fall through to the tag-only X acquire, which fail-closed + * as "no local S residency". This entry point is buffer-aware: the + * caller has already read or initialized the BufferDesc before asking for + * X, so first register a local S residency, then reuse the existing + * local S->X invalidate/upgrade path. If the invalidate cannot be proven, + * drop the temporary S claim and rethrow the same fail-closed error. + */ + if (master_state == PCM_LOCK_MODE_S && cluster_node_id >= 0 && cluster_node_id < 32) { + uint32 self_bit = (uint32)1u << (uint32)cluster_node_id; + + if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { + struct GrdEntry *entry; + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + if (!cluster_gcs_block_local_x_upgrade(tag)) { + cluster_pcm_lock_release(tag); + ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), + errhint("Remote S holders did not all acknowledge in time; " + "retry the statement."))); + } + + entry = pcm_find_entry(tag); + if (entry != NULL) { + pcm_entry_lock_exclusive(entry); + entry->s_holder_refcount_local = 0; + LWLockRelease(&entry->entry_lock.lock); + } + return true; + } + } } cluster_pcm_lock_acquire(tag, mode); diff --git a/src/backend/cluster/cluster_wal_thread.c b/src/backend/cluster/cluster_wal_thread.c index 17edd98586..b5f7cab947 100644 --- a/src/backend/cluster/cluster_wal_thread.c +++ b/src/backend/cluster/cluster_wal_thread.c @@ -45,7 +45,9 @@ #include #include "access/xlog_internal.h" /* XLOGDIR */ +#include "cluster/cluster_conf.h" #include "cluster/cluster_guc.h" +#include "cluster/cluster_hw.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_wal_state.h" /* spec-4.2 ensure() */ @@ -385,8 +387,23 @@ cluster_wal_thread_init(void) cluster_wal_thread_shmem->dir_configured = dir_set ? 1 : 0; } - if (!dir_set) + if (!dir_set) { + if (cluster_shared_catalog && cluster_conf_has_peers()) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cluster.shared_catalog requires cluster.wal_threads_dir in a " + "multi-node cluster"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "initialise each node with pgrac-init --wal-threads-dir."))); + if (!cluster_hw_remaster_recoverable()) + ereport(WARNING, + (errmsg("cluster HW remaster is not recoverable because " + "cluster.wal_threads_dir is unset"), + errhint("A dead node's adopted HW authority cannot be rebuilt from per-thread " + "WAL until cluster.wal_threads_dir is configured and the node is " + "restarted."))); return; /* flat layout: identity stamping only (Q3-A) */ + } /* * Configuration coherence, fail-closed (L58: enumerate the diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..a098d53ed1 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -275,6 +275,8 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 recovery_event_old_epoch; pg_atomic_uint64 recovery_redeclare_generation; pg_atomic_uint64 recovery_barrier_deadline; + pg_atomic_uint32 recovery_event_coordinator; + pg_atomic_uint64 recovery_done_epoch_at_accept; /* * spec-4.6 P0-1 (Fable review) — the epoch this episode is LOCKED to. @@ -294,6 +296,7 @@ typedef struct ClusterGrdShared { * last announced "local rebind barrier complete" (REDECLARE_DONE). * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_event_id[CLUSTER_MAX_NODES]; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -311,6 +314,10 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 block_path_failclosed_count; /* D4 GCS/PCM 53R9K (L12) */ pg_atomic_uint64 unaffected_holder_survived_count; /* L13 sweep-scope guard */ pg_atomic_uint64 stale_holder_swept_count; /* P6 post-barrier sweep (L15) */ + pg_atomic_uint64 cluster_gate_timeout_count; /* WAIT_CLUSTER watchdog only */ + pg_atomic_uint64 wait_epoch_escape_count; /* proof-carrying equal-epoch advance */ + pg_atomic_uint64 + pcm_dead_cleanup_entries; /* spec-4.6a D12: dead-node PCM holder/pending-X records cleaned */ /* spec-5.1b D9 — convert state-machine observability counters. * convert_queue_full reuses the existing converts_full_count above; @@ -609,6 +616,18 @@ typedef enum ClusterGrdRecoveryState { } ClusterGrdRecoveryState; extern void cluster_grd_recovery_lmon_tick(void); +/* spec-4.6a D5/D6 — recovery state observability for pg_cluster_state. */ +extern uint32 cluster_grd_recovery_state_value(void); +extern const char *cluster_grd_recovery_state_name(uint32 state); +extern uint64 cluster_grd_recovery_last_event_id(void); +extern uint64 cluster_grd_recovery_event_old_epoch(void); +extern uint64 cluster_grd_recovery_episode_epoch_value(void); +extern uint32 cluster_grd_recovery_event_coordinator(void); +extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); +extern uint64 cluster_grd_recovery_done_event_id_for(int32 node); +extern int cluster_grd_recovery_block_redeclare_cursor(void); +extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); +extern bool cluster_grd_recovery_block_redeclare_done(void); extern uint64 cluster_grd_redeclare_generation(void); /* spec-4.6 P0-1 — the epoch the current episode is locked to (0 = none). */ extern uint64 cluster_grd_redeclare_episode_epoch(void); @@ -626,7 +645,7 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch); +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); @@ -650,6 +669,9 @@ typedef struct ClusterGrdRecoveryCounters { uint64 block_path_failclosed; uint64 unaffected_holder_survived; uint64 stale_holder_swept; + uint64 cluster_gate_timeout; + uint64 wait_epoch_escape; + uint64 pcm_dead_cleanup_entries; /* spec-5.16 D5 — join-direction remaster counters (distinct from the * failure-driven remaster_* above; §8 Q6-A). */ uint64 join_remaster_started; diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..d091b2be3f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -399,6 +399,8 @@ extern int cluster_ges_reply_wait_max_entries; * 5000ms; SIGHUP). */ extern int cluster_grd_remaster_wait_ms; extern int cluster_grd_rebuild_timeout_ms; +extern int cluster_hw_remaster_retry_backoff_ms; +extern int cluster_hw_remaster_retry_max_attempts; /* spec-5.4 D8: SQ sequence lock tunables. */ extern int cluster_sequence_default_cache; diff --git a/src/include/cluster/cluster_hw.h b/src/include/cluster/cluster_hw.h index 89656a384f..e021c1c445 100644 --- a/src/include/cluster/cluster_hw.h +++ b/src/include/cluster/cluster_hw.h @@ -312,12 +312,30 @@ extern uint32 cluster_hw_shard_rebuilt_generation(uint32 shard_id); extern void cluster_hw_mark_shard_rebuilt(uint32 shard_id, uint32 generation); /* - * Per-dead-origin online-remaster launch idempotency (S5d). The GRD FSM records - * the episode it last launched a rebuild worker for a dead origin under, and - * skips relaunching while the value equals the current episode. + * Per-dead-origin online-remaster launch state (S5d / spec-4.6a). The GRD FSM + * records the episode it last launched a rebuild worker for a dead origin under, + * and the worker records a terminal result. BLOCKED is retryable within the same + * episode; BLOCKED_STRUCTURAL is not. */ +typedef enum ClusterHwRemasterResult { + CLUSTER_HW_REMASTER_NONE = 0, + CLUSTER_HW_REMASTER_RUNNING, + CLUSTER_HW_REMASTER_DONE, + CLUSTER_HW_REMASTER_BLOCKED, + CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, + CLUSTER_HW_REMASTER_NOT_APPLICABLE, +} ClusterHwRemasterResult; + extern uint64 cluster_hw_remaster_launched_episode(int node_id); extern void cluster_hw_remaster_set_launched(int node_id, uint64 episode); +extern ClusterHwRemasterResult cluster_hw_remaster_result(int node_id); +extern void cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result); +extern uint32 cluster_hw_remaster_attempts(int node_id); +extern void cluster_hw_remaster_set_attempts(int node_id, uint32 attempts); +extern uint64 cluster_hw_remaster_next_attempt_at(int node_id); +extern void cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts); +extern const char *cluster_hw_remaster_result_name(ClusterHwRemasterResult result); +extern bool cluster_hw_remaster_recoverable(void); /* * cluster_hw_apply_hwm -- redo / remaster rebuild: create-if-absent and raise @@ -383,6 +401,8 @@ extern uint64 cluster_hw_failclosed_count(void); extern uint64 cluster_hw_not_ready_count(void); extern uint64 cluster_hw_remaster_done_count(void); extern uint64 cluster_hw_remaster_blocked_count(void); +extern uint64 cluster_hw_remaster_retry_count(void); +extern uint64 cluster_hw_remaster_retry_exhausted_count(void); extern void cluster_hw_bump_alloc(void); extern void cluster_hw_bump_authority_create(void); extern void cluster_hw_bump_reserve_wal(void); @@ -391,6 +411,8 @@ extern void cluster_hw_bump_failclosed(void); extern void cluster_hw_bump_not_ready(void); extern void cluster_hw_bump_remaster_done(void); extern void cluster_hw_bump_remaster_blocked(void); +extern void cluster_hw_bump_remaster_retry(void); +extern void cluster_hw_bump_remaster_retry_exhausted(void); #endif /* !FRONTEND */ diff --git a/src/include/cluster/cluster_hw_remaster.h b/src/include/cluster/cluster_hw_remaster.h index e1bf3ba529..e2fcc481f8 100644 --- a/src/include/cluster/cluster_hw_remaster.h +++ b/src/include/cluster/cluster_hw_remaster.h @@ -47,17 +47,91 @@ #ifndef CLUSTER_HW_REMASTER_H #define CLUSTER_HW_REMASTER_H -/* - * Result of an online-remaster HW authority rebuild for one dead origin. Only - * DONE marks the adopted shards rebuilt (opens the serve gate); BLOCKED and - * NOT_APPLICABLE never do, so a shard whose HWM is not provably rebuilt stays - * fail-closed (8.A). - */ -typedef enum ClusterHwRemasterResult { - CLUSTER_HW_REMASTER_DONE = 0, /* rebuilt + adoption snapshot durable + shards marked */ - CLUSTER_HW_REMASTER_BLOCKED, /* fail-closed: keep shards frozen (53RA6) */ - CLUSTER_HW_REMASTER_NOT_APPLICABLE, /* HW authority inactive / no adopted shard / bad input */ -} ClusterHwRemasterResult; +#include "cluster/cluster_hw.h" /* ClusterHwRemasterResult */ + +#define CLUSTER_HW_REMASTER_NO_DEADLINE PG_UINT64_MAX +#define CLUSTER_HW_REMASTER_BACKOFF_CAP_MS 60000 + +typedef enum ClusterHwRemasterLaunchAction { + CLUSTER_HW_REMASTER_LAUNCH_SKIP = 0, + CLUSTER_HW_REMASTER_LAUNCH_INITIAL, + CLUSTER_HW_REMASTER_LAUNCH_RETRY, + CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED, + CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL, +} ClusterHwRemasterLaunchAction; + +typedef struct ClusterHwRemasterRelaunchDecision { + ClusterHwRemasterLaunchAction action; + uint32 next_attempts; + uint64 next_attempt_at; + ClusterHwRemasterResult next_result; +} ClusterHwRemasterRelaunchDecision; + +static inline uint32 +cluster_hw_remaster_compute_backoff_ms(int base_ms, uint32 completed_retry_attempts) +{ + uint64 ms; + uint32 shift; + + if (base_ms < 1) + base_ms = 1; + ms = (uint64)base_ms; + shift = completed_retry_attempts > 0 ? completed_retry_attempts - 1 : 0; + while (shift-- > 0 && ms < CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms <<= 1; + if (ms > CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms = CLUSTER_HW_REMASTER_BACKOFF_CAP_MS; + return (uint32)ms; +} + +static inline ClusterHwRemasterRelaunchDecision +cluster_hw_remaster_relaunch_decide(uint64 launched_episode, uint64 current_episode, + ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int retry_max_attempts) +{ + ClusterHwRemasterRelaunchDecision d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_SKIP; + d.next_attempts = attempts; + d.next_attempt_at = next_attempt_at; + d.next_result = result; + + if (current_episode == 0) + return d; + if (launched_episode != current_episode || result == CLUSTER_HW_REMASTER_NONE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_INITIAL; + d.next_attempts = 0; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; + } + + if (result == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (result != CLUSTER_HW_REMASTER_BLOCKED) + return d; + + if (retry_max_attempts <= 0 || attempts >= (uint32)retry_max_attempts) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE && now < next_attempt_at) + return d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_RETRY; + d.next_attempts = attempts + 1; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; +} #ifndef FRONTEND @@ -81,9 +155,9 @@ extern ClusterHwRemasterResult cluster_hw_remaster_rebuild_origin(int dead_node_ /* * cluster_hw_remaster_worker_main -- dynamic-bgworker entry point (main_arg = the * dead origin node id). Captures the live reconfig episode and drives - * cluster_hw_remaster_rebuild_origin off the LMON tick. An abnormal exit simply - * leaves the shards unmarked -> the serve gate keeps them fail-closed (8.A), so - * no abnormal-exit handler is needed. + * cluster_hw_remaster_rebuild_origin off the LMON tick. A before_shmem_exit + * callback records an abnormal exit as retryable BLOCKED so the same episode can + * self-heal instead of staying pinned on the launch idempotency bit. */ extern void cluster_hw_remaster_worker_main(Datum main_arg); diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index d88d82dc8f..3fb909557f 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -415,6 +415,12 @@ extern void cluster_pcm_lock_clear_pending_x(BufferTag tag); extern int32 cluster_pcm_lock_query_pending_x_requester(BufferTag tag); extern uint64 cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node); +/* PGRAC: spec-4.6a BUG-C2 — failure-path PCM holder cleanup. Removes a + * DEAD node from X/S/PI holder records in this master's PCM directory and + * demotes entries to N when no live holder remains. Idempotent under repeated + * dead-sweep ticks. */ +extern uint64 cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node); + /* PGRAC: spec-5.13 D5 (clean-leave PCM release) — a leaving node clears its * OWN holder records (X / S / PI) from the local PCM directory after the GCS * flush seam has persisted all dirty X blocks (CL-I5). Demotes an entry's diff --git a/src/include/cluster/cluster_thread_recovery.h b/src/include/cluster/cluster_thread_recovery.h index c455b77436..c7be25e6b0 100644 --- a/src/include/cluster/cluster_thread_recovery.h +++ b/src/include/cluster/cluster_thread_recovery.h @@ -376,6 +376,16 @@ cluster_thread_recovery_gate_decide(ClusterThreadRecScope scope, const uint64 *d return false; /* every dead origin materialized -> ready to unfreeze */ } +static inline bool +cluster_thread_recovery_materialization_gate_enabled(bool guc_on, bool has_peers, + bool shared_fs_backend, + int live_survivor_count) +{ + return cluster_thread_recovery_decide_scope(guc_on, has_peers, shared_fs_backend, + live_survivor_count) + == CLUSTER_THREADREC_SCOPE_APPLICABLE; +} + /* * cluster_thread_recovery_replay_epoch_aborts -- the L235 episode-epoch * staleness guard (spec-4.11 3b-4b). The per-thread replay slot stamps the GRD diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..f2565c8fe1 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,15 @@ # ---------- -# L1: pg_cluster_state.pcm category has 22 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5) +# L1: pg_cluster_state.pcm category has 23 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5 +# + spec-4.6a D12 dead_cleanup_entries) # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '22', - 'L1 pg_cluster_state.pcm category has 22 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5)'); + '23', + 'L1 pg_cluster_state.pcm category has 23 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5 + spec-4.6a D12)'); # ---------- diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 9c156e1e53..509c2f50ff 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -30,13 +30,18 @@ # retryable transient, NOT an HW fault -- so this leg retries and SKIPs # (never fails) when only that orthogonal recovery is outstanding. # -# Negative (corrupt the dead master's snapshot -> fail-closed): +# Negative/self-heal extensions (spec-4.6a): # L6 a fresh pair; corrupt node0's HW snapshot on shared storage, then # kill node0. The survivor's rebuild must FAIL CLOSED # (hw.remaster_blocked_count advances) -- it never trusts a corrupt # snapshot, never auto-creates at 0, never reads FileSize as the -# authority. The affected shards stay frozen (P7 gate), so no extend -# can silently re-hand a block. +# authority. +# L7 hide node0's snapshot, kill node0, observe BLOCKED, then restore +# the same file. The same reconfig episode must retry and converge +# to DONE without waiting for a later episode. +# L8 keep the snapshot corrupt with retry_max_attempts=2. The survivor +# must report retry_exhausted once and keep fail-closed; DONE must not +# advance. # # Harness: ClusterPair shared_data + wal_threads_root + 3 voting disks. # Mirrors the kill/reconfig pattern of t/249 / t/274. @@ -181,17 +186,22 @@ sub retry_sql { my ($node, $sql) = @_; my $last = ''; + my $last_retryable = 0; for my $attempt (1 .. 30) { my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); - return (1, $out) if defined $rc && $rc == 0; - $last = $err // ''; + my $combined = join("\n", grep { defined $_ && $_ ne '' } ($out, $err)); + return (1, $out, 0) + if defined $rc && $rc == 0 && $combined !~ /ERROR:/; + + $last = $combined; + $last_retryable = + $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; # retry only the orthogonal post-reconfig settling transients - last - unless $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; + last unless $last_retryable; usleep(500_000); } - return (0, $last); + return (0, $last, $last_retryable); } # L5a (the robust HW proof): the rebuilt authority is SERVEABLE -- the HW serve @@ -210,22 +220,213 @@ sub retry_sql # that is NOT an HW fault. Retry, and SKIP (never fail) when only that orthogonal # recovery is outstanding; a 53RA6 from the HW gate would NOT be retried and would # fail the test. -my ($ext_ok, $ext_res) = +my ($ext_ok, $ext_res, $ext_retryable) = retry_sql($n1, 'INSERT INTO r1 SELECT g,g,g,g FROM generate_series(4001,4500) g'); SKIP: { - skip - "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", - 1 - unless $ext_ok; - my ($cnt_ok, $cnt) = retry_sql($n1, 'SELECT count(*) FROM r1'); - is($cnt, '4500', - 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') - or diag("count result: $cnt"); + if (!$ext_ok) + { + skip + "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", + 1 + if $ext_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("extend result: $ext_res"); + } + else + { + my ($cnt_ok, $cnt, $cnt_retryable) = retry_sql($n1, 'SELECT count(*) FROM r1'); + if (!$cnt_ok) + { + skip + "post-remaster count gated by orthogonal visibility recovery (not HW): $cnt", + 1 + if $cnt_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("count result: $cnt"); + } + else + { + is($cnt, '4500', + 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') + or diag("count result: $cnt"); + } + } } $pair->stop_pair; +# ====================================================================== +# SELF-HEAL: first attempt BLOCKED, same episode retries after repair. +# ====================================================================== + +my $heal = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_selfheal', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +$heal->start_pair; +my $h0 = $heal->node0; +my $h1 = $heal->node1; + +ok(poll_until($h1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L7: self-heal pair in quorum'); +poll_until( + $h1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($h0, $h1, "s$_", 4000) for (1 .. 6); +$h0->safe_psql('postgres', 'CHECKPOINT'); +$h1->safe_psql('postgres', 'CHECKPOINT'); + +my $hsnap = $heal->shared_data_root . "/global/pg_hw_snapshot.0"; +my $hhold = "$hsnap.hold"; +ok(-f $hsnap, 'L7: node0 HW snapshot exists before self-heal injection'); +rename($hsnap, $hhold) or die "rename $hsnap -> $hhold: $!"; + +my $h_done_before = hw_counter($h1, 'remaster_done_count'); +my $h_blocked_before = hw_counter($h1, 'remaster_blocked_count'); +my $h_retry_before = hw_counter($h1, 'remaster_retry_count'); +$heal->kill_node9(0); +ok( poll_until( + $h1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30), + 'L7: self-heal pair declared node0 dead'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_blocked_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_blocked_count'", + 'true', 40), + 'L7: first HW rebuild attempt failed closed while the snapshot was hidden'); + +rename($hhold, $hsnap) or die "rename $hhold -> $hsnap: $!"; +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_retry_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_count'", + 'true', 40), + 'L7: same reconfig episode launched a HW remaster retry'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_done_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_done_count'", + 'true', 60), + 'L7: restored snapshot let the same-episode retry converge to DONE'); +is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, + 'L7: self-heal path did not exhaust the retry budget'); + +# spec-4.6a section 4 (D8): DONE must actually unfreeze (P7) -- the survivor +# extends a table that lived on shards the dead master owned; a frozen shard +# would fail this INSERT with the remastering error. +{ + my ($ext_ok, $ext_res) = (0, ''); + for my $try (1 .. 30) + { + $ext_res = eval { + $h1->safe_psql('postgres', + 'INSERT INTO s1 SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 1; + } ? 'ok' : ($@ // 'error'); + if ($ext_res eq 'ok') { $ext_ok = 1; last; } + sleep 1; + } + ok($ext_ok, 'L7: survivor extend succeeds after same-episode DONE (P7 unfreeze)') + or diag($ext_res); +} + +$heal->stop_pair; + +# ====================================================================== +# NEGATIVE: corrupt snapshot persists -> retries exhaust and stay closed. +# ====================================================================== + +my $exh = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_exhausted', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', + ]); +$exh->start_pair; +my $e0 = $exh->node0; +my $e1 = $exh->node1; + +ok(poll_until($e1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L8: exhausted pair in quorum'); +poll_until( + $e1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($e0, $e1, "x$_", 4000) for (1 .. 6); +$e0->safe_psql('postgres', 'CHECKPOINT'); +$e1->safe_psql('postgres', 'CHECKPOINT'); + +my $esnap = $exh->shared_data_root . "/global/pg_hw_snapshot.0"; +ok(-f $esnap, 'L8: node0 HW snapshot exists before exhaustion injection'); +open(my $efh, '>', $esnap) or die "open $esnap: $!"; +binmode $efh; +print $efh ("\xDE\xAD\xBE\xEF" x 64); +close $efh; + +my $e_done_before = hw_counter($e1, 'remaster_done_count'); +my $e_exh_before = hw_counter($e1, 'remaster_retry_exhausted_count'); +$exh->kill_node9(0); +poll_until( + $e1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30); +ok( poll_until( + $e1, + "SELECT (value::bigint > $e_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L8: persistent corrupt snapshot exhausted the same-episode retry budget'); +is(hw_counter($e1, 'remaster_done_count'), $e_done_before, + 'L8: DONE did not advance after retry exhaustion on corrupt snapshot'); + +# spec-4.6a section 4 (D8/L408): after exhaustion the WAIT_CLUSTER watchdog +# keeps sounding (counter + WARNING line) while shards stay frozen. +ok( poll_until( + $e1, + q{SELECT (value::bigint > 0)::text FROM pg_cluster_state } + . q{WHERE category='grd_recovery' AND key='cluster_gate_timeout'}, + 'true', 30), + 'L8: WAIT_CLUSTER watchdog counter advances while exhausted episode stays frozen'); +{ + my $log = PostgreSQL::Test::Utils::slurp_file($e1->logfile); + like( + $log, + qr/cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen/, + 'L8: watchdog WARNING appears in the survivor log'); +} + +$exh->stop_pair; + # ====================================================================== # NEGATIVE: corrupt the dead master's snapshot -> rebuild fails closed. # ====================================================================== @@ -242,6 +443,9 @@ sub retry_sql 'cluster.ges_retransmit_max_attempts = 0', 'cluster.cssd_heartbeat_interval_ms = 1000', 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', ]); $neg->start_pair; my $m0 = $neg->node0; @@ -277,6 +481,7 @@ sub retry_sql } my $blocked_before = hw_counter($m1, 'remaster_blocked_count'); +my $neg_exh_before = hw_counter($m1, 'remaster_retry_exhausted_count'); $neg->kill_node9(0); poll_until( $m1, @@ -291,6 +496,12 @@ sub retry_sql . "WHERE category='hw' AND key='remaster_blocked_count'", 'true', 40), 'L6: HW rebuild FAILED CLOSED on the corrupt snapshot (remaster_blocked advanced; no auto-create-0)'); +ok( poll_until( + $m1, + "SELECT (value::bigint > $neg_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L6: corrupt snapshot eventually exhausts bounded same-episode retries'); $neg->stop_pair; diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..86970cb280 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -94,6 +94,8 @@ my $shared_root = PostgreSQL::Test::Utils::tempdir(); mkdir "$shared_root/global" or die "mkdir shared global: $!"; +my $wal_root = PostgreSQL::Test::Utils::tempdir(); + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); my @disks; for my $i (0 .. 2) @@ -114,7 +116,7 @@ # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). # ---------- my $node0 = PostgreSQL::Test::Cluster->new('sc_ddl_node0'); -$node0->init(allows_streaming => 1); +$node0->init(allows_streaming => 1, extra => [ '-X', "$wal_root/thread_1" ]); $node0->start; $node0->backup('scb'); $node0->stop; @@ -122,6 +124,24 @@ my $node1 = PostgreSQL::Test::Cluster->new('sc_ddl_node1'); $node1->init_from_backup($node0, 'scb'); +# Relocate node1's backup-copied WAL into its shared thread dir. shared_catalog +# multi-node formation is fail-fast without cluster.wal_threads_dir, and the WAL +# thread validator requires pg_wal to resolve to the configured thread_N dir. +{ + my $pgwal = $node1->data_dir . '/pg_wal'; + my $wal2 = "$wal_root/thread_2"; + mkdir $wal2 or die "mkdir $wal2: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal2/$e") or die "rename $pgwal/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal2, $pgwal) or die "symlink $pgwal -> $wal2: $!"; +} + # Config common to the shared-catalog feature (both nodes, both phases). my $sc_common = <append_conf('postgresql.conf', $cluster_conf); @@ -819,4 +840,23 @@ sub oid_authority_hw qr/cluster\.shared_catalog is off but the shared tree holds/, 'L7: the refusal is the designed off-flip vet FATAL (never a stale serve)'); +# ---------- +# L8 (spec-4.6a D11 level-2 fail-fast): a multi-node shared_catalog=on boot +# without cluster.wal_threads_dir is refused at startup -- that shape cannot +# rebuild a dead node's HW authority from per-thread WAL, so a node failure +# would be permanently unrecoverable (the silent-BLOCKED wedge of +# spec-4.6a section 0). Flip shared_catalog back on and blank the WAL +# threads root; the boot must fail on the D11 vet, not come up degraded. +# ---------- +$node1->append_conf('postgresql.conf', "cluster.shared_catalog = on +"); +$node1->append_conf('postgresql.conf', "cluster.wal_threads_dir = '' +"); +my $nowalret = $node1->start(fail_ok => 1); +is($nowalret, 0, 'L8: multi-node shared_catalog boot without wal_threads_dir fails'); +my $nowallog = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($nowallog, + qr/cluster\.shared_catalog requires cluster\.wal_threads_dir in a multi-node cluster/, + 'L8: the refusal is the spec-4.6a D11 startup FATAL with the config errhint'); + done_testing(); diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl new file mode 100644 index 0000000000..883860c0fb --- /dev/null +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -0,0 +1,310 @@ +#------------------------------------------------------------------------- +# +# 362_shared_catalog_4node_kill_selfheal.pl +# spec-4.6a D9 -- 4-node shared_catalog fail-stop self-heal. +# +# Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) +# so a killed node's HW authority is recoverable from its per-thread WAL. +# After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means +# the test harness is misconfigured, and lack of convergence is a real FAIL. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterQuad; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared_catalog 4-node kill self-heal requires --enable-cluster'; +} + +sub poll_until +{ + my ($node, $query, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) }; + return 1 if defined $last && $last eq $want; + usleep(300_000); + } + diag("poll_until timed out: '$query' last='" . ($last // '(undef)') + . "' want='$want'"); + return 0; +} + +sub retry_sql +{ + my ($node, $sql, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); + return (1, $out // '') if defined $rc && $rc == 0; + $last = $err // ''; + usleep(500_000); + } + return (0, $last); +} + +sub state_counter +{ + my ($node, $cat, $key) = @_; + my $v = eval { + $node->safe_psql('postgres', + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'") + }; + return defined $v && $v ne '' ? $v + 0 : 0; +} + +sub sum_hw +{ + my ($quad, $key, @idx) = @_; + my $sum = 0; + for my $i (@idx) + { + $sum += state_counter($quad->node($i), 'hw', $key); + } + return $sum; +} + +sub logs_contain +{ + my ($quad, $pattern, @idx) = @_; + for my $i (@idx) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + } + return 0; +} + +sub log_until +{ + my ($quad, $idx, $pattern, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($idx)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + usleep(300_000); + } + diag("log_until timed out on node$idx waiting for /$pattern/"); + return 0; +} + +sub startup_env_blocker +{ + my ($quad) = @_; + my $log = ''; + if ($quad) + { + for my $i (0 .. 3) + { + $log .= eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + } + } + return $log =~ /could not create shared memory segment|No space left on device|No space left|SHMMNI|semget/i; +} + +my $quad = eval { + PostgreSQL::Test::ClusterQuad->new_quad( + 'sc4_selfheal', + shared_catalog => 1, + quorum_voting_disks => 3, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.lms_enabled = on', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +}; +if (!$quad) +{ + my $err = $@ || 'unknown constructor failure'; + plan skip_all => "L1 shared_catalog ClusterQuad formation hit local substrate blocker: $err" + if $err =~ /could not create shared memory|No space left|SHMMNI|semget/i; + BAIL_OUT("ClusterQuad(shared_catalog) constructor failed: $err"); +} + +my $started = 1; +for my $i (1 .. 3) +{ + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $quad->node($i)->data_dir, + '-l', $quad->node($i)->logfile, + '-o', "--cluster-name=sc4_selfheal_node$i", 'start'); +} +unless ($quad->node(0)->start(fail_ok => 1)) +{ + $started = 0; +} +for my $i (1 .. 3) +{ + $quad->node($i)->_update_pid(-1); +} +if (!$started) +{ + if (startup_env_blocker($quad)) + { + eval { $quad->stop_quad; }; + plan skip_all => 'L1 shared_catalog ClusterQuad formation hit local shared-memory/resource limit'; + } + my $fatal = logs_contain($quad, qr/cluster\.shared_catalog requires cluster\.wal_threads_dir|structurally blocked|BLOCKED_STRUCTURAL/i, 0 .. 3); + eval { $quad->stop_quad; }; + ok(!$fatal, 'L1: shared_catalog quad must not hit wal_threads_dir/BLOCKED_STRUCTURAL configuration failure'); + BAIL_OUT('ClusterQuad(shared_catalog) failed to start for a non-environment reason'); +} + +# L1: all four members formed with quorum and per-thread WAL configured. +for my $i (0 .. 3) +{ + ok(poll_until($quad->node($i), 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 60), + "L1: node$i reached quorum"); + is(state_counter($quad->node($i), 'hw', 'remaster_recoverable'), 1, + "L1: node$i reports HW remaster recoverable"); +} +for my $to (1 .. 3) +{ + ok($quad->wait_for_peer_state(0, $to, 'connected', 60), + "L1: node0 sees node$to connected"); +} + +my $n0 = $quad->node0; +my $dead = 3; +my @survivors = (0, 1, 2); + +# Seed shared-catalog DDL and force node3 to own real HW state before death. +for my $t (1 .. 2) +{ + my ($ddl_ok, $ddl_res) = retry_sql($n0, + "CREATE TABLE sc4_hw_$t (id int, pad int)", 60); + ok($ddl_ok, "L1: coordinator created shared_catalog table sc4_hw_$t") + or diag($ddl_res); + my ($ok, $res) = retry_sql($quad->node($dead), + "INSERT INTO sc4_hw_$t SELECT g, g FROM generate_series(1,5000) g", 60); + ok($ok, "L1: node$dead extended shared_catalog table sc4_hw_$t before kill") + or diag($res); +} +for my $i (0 .. 3) +{ + retry_sql($quad->node($i), 'CHECKPOINT', 30); +} + +my $done_before = sum_hw($quad, 'remaster_done_count', @survivors); +my $blocked_before = sum_hw($quad, 'remaster_blocked_count', @survivors); +my $retry_before = sum_hw($quad, 'remaster_retry_count', @survivors); + +# L2: kill node3 and require every survivor to see the real DEAD edge. +# Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, +# ordinary catalog reads may legitimately return 53R9I until the barrier opens. +$quad->kill_node9($dead); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/cssd: peer $dead transitioned .* DEAD/, 45), + "L2: survivor node$i declared node$dead dead"); +} + +# L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +for my $i (@survivors) +{ + ok(poll_until($quad->node($i), + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + "L3: survivor node$i remastered all GRD shards off node$dead"); +} +my $done_converged = 0; +my $deadline = time() + 90; +while (time() < $deadline) +{ + if (sum_hw($quad, 'remaster_done_count', @survivors) > $done_before) + { + $done_converged = 1; + last; + } + usleep(500_000); +} +ok($done_converged, 'L3: HW remaster DONE advanced on at least one survivor after kill'); +ok(!logs_contain($quad, qr/blocked_structural|structurally blocked|BLOCKED_STRUCTURAL/i, @survivors), + 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); + +# L6: if a transient BLOCKED occurred, a same-episode retry must also have run. +my $blocked_after = sum_hw($quad, 'remaster_blocked_count', @survivors); +if ($blocked_after > $blocked_before) +{ + ok(sum_hw($quad, 'remaster_retry_count', @survivors) > $retry_before, + 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); +} +else +{ + pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); +} + +# L4: survivor SQL and shared-catalog DDL recover after fail-stop. +for my $i (@survivors) +{ + my ($ok, $res) = retry_sql($quad->node($i), 'SELECT txid_current()', 60); + ok($ok, "L4: survivor node$i accepts txid_current after reconfig") or diag($res); +} +my ($ddl_ok, $ddl_res) = retry_sql($n0, 'CREATE TABLE sc4_after_kill (id int)', 60); +ok($ddl_ok, 'L4: coordinator survivor can run DDL after node kill') or diag($ddl_res); +for my $i (1, 2) +{ + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); +} + +# L4 (spec-4.6a D12, r2-P1-3): the dead-node PCM residue cleanup counter is +# exposed and non-negative on every survivor; when the dead node left any +# holder/pending-X records behind, at least one survivor accounts a cleanup +# (delta assertable; zero stays legal when the dead node held nothing). +{ + my $cleanup_total = 0; + for my $i (@survivors) + { + my $c = state_counter($quad->node($i), 'pcm', 'dead_cleanup_entries'); + ok($c >= 0, "L4: survivor node$i exposes pcm/dead_cleanup_entries"); + $cleanup_total += $c; + } + diag("L4: pcm/dead_cleanup_entries total across survivors = $cleanup_total"); +} + +# L5: watchdog counters are allowed to be zero, but must be readable. +for my $i (@survivors) +{ + my $cluster_gate = state_counter($quad->node($i), 'grd_recovery', 'cluster_gate_timeout'); + my $epoch_escape = state_counter($quad->node($i), 'grd_recovery', 'wait_epoch_escape'); + ok($cluster_gate >= 0 && $epoch_escape >= 0, + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counters"); +} + +$quad->stop_quad; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..521855f18a 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -65,6 +65,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_oid_lease \ test_cluster_relmap_authority \ test_cluster_hw \ + test_cluster_hw_remaster_retry \ test_cluster_dl \ test_cluster_extend_gate \ test_cluster_ir \ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..8b64c01755 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -45,6 +45,7 @@ #include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b catalog counter stubs */ #include "cluster/cluster_debug.h" +#include "cluster/cluster_grd.h" /* ClusterGrdRecoveryCounters */ #include "cluster/cluster_hang.h" /* spec-5.11: ClusterHangDumpData for dump_hang stubs */ #include "cluster/cluster_hang_resolve.h" /* spec-5.12: ClusterHangResolveCounters for dump stubs */ #include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ @@ -564,6 +565,9 @@ uint64 cluster_hw_failclosed_count(void); uint64 cluster_hw_not_ready_count(void); uint64 cluster_hw_remaster_done_count(void); uint64 cluster_hw_remaster_blocked_count(void); +uint64 cluster_hw_remaster_retry_count(void); +uint64 cluster_hw_remaster_retry_exhausted_count(void); +bool cluster_hw_remaster_recoverable(void); uint64 cluster_hw_alloc_count(void) { @@ -604,6 +608,21 @@ cluster_hw_remaster_blocked_count(void) { return 0; } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return 0; +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return 0; +} +bool +cluster_hw_remaster_recoverable(void) +{ + return true; +} /* spec-5.7 D4 dump_dl stubs (cluster_dl.c not linked in this binary). */ uint64 cluster_dl_lease_count(void); @@ -3288,15 +3307,77 @@ cluster_grd_deadlock_chunk_oo_buffer_overflow_count(void) return 0; } -/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter - * snapshot. Zero-fill; layout = 13 × uint64 (must track the - * ClusterGrdRecoveryCounters struct in cluster_grd.h). */ -struct ClusterGrdRecoveryCounters; -void cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out); +uint32 +cluster_grd_recovery_state_value(void) +{ + return (uint32)GRD_RECOVERY_IDLE; +} + +const char * +cluster_grd_recovery_state_name(uint32 state pg_attribute_unused()) +{ + return "idle"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_event_id_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return 0; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return true; +} + +/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter snapshot. */ void -cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out) +cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) { - memset(out, 0, 13 * sizeof(uint64)); + memset(out, 0, sizeof(*out)); } uint32 diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 444accf8f0..67fcc29dc2 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -70,6 +70,7 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_thread_recovery.h" #include "common/hashfn.h" #include "storage/buf_internals.h" #include "storage/lwlock.h" @@ -287,6 +288,16 @@ UT_TEST(test_gcs_block_tag_is_standard_buffer_tag_20b) } +UT_TEST(test_dead_static_master_materialization_gate_scope_matches_thread_recovery) +{ + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(false, true, true, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, false, true, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, false, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, true, 2)); + UT_ASSERT(cluster_thread_recovery_materialization_gate_enabled(true, true, true, 1)); +} + + /* spec-5.2 D2 (U3): pure master-side decision for an X-held N→S read. * node0 = holder/master in DIRECT, node1 = requester. */ UT_TEST(test_xheld_read_ship_decision_truth_table) @@ -514,7 +525,7 @@ UT_TEST(test_clean_xfer_stale_break_predicate) int main(void) { - UT_PLAN(21); + UT_PLAN(22); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -530,6 +541,7 @@ main(void) UT_RUN(test_gcs_block_data_size_equals_blcksz); UT_RUN(test_gcs_block_msg_type_enum_extends_without_gap); UT_RUN(test_gcs_block_tag_is_standard_buffer_tag_20b); + UT_RUN(test_dead_static_master_materialization_gate_scope_matches_thread_recovery); UT_RUN(test_xheld_read_ship_decision_truth_table); UT_RUN(test_forward_payload_read_image_flag_roundtrip); UT_RUN(test_clean_page_xfer_eligible_flag_roundtrip_and_orthogonal); diff --git a/src/test/cluster_unit/test_cluster_ges.c b/src/test/cluster_unit/test_cluster_ges.c index 1df0a6b7fc..dd2a47185a 100644 --- a/src/test/cluster_unit/test_cluster_ges.c +++ b/src/test/cluster_unit/test_cluster_ges.c @@ -296,7 +296,8 @@ cluster_cancel_token_consume(void) * completion. Standalone fixture has no recovery shmem; no-op. */ void cluster_grd_recovery_mark_peer_done(int32 node pg_attribute_unused(), - uint64 epoch pg_attribute_unused()) + uint64 epoch pg_attribute_unused(), + uint64 event_id pg_attribute_unused()) {} ClusterGrdEntryResult diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..c7f805459d 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -55,6 +55,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -85,6 +86,15 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; the GRD fixture has no PCM shmem — no-op (the cleanup + * logic itself is unit-tested in test_cluster_pcm_lock.c). */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -316,6 +326,13 @@ cluster_epoch_get_current(void) return ut_mock_epoch; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch) +{ + if (admitted_epoch > ut_mock_epoch) + ut_mock_epoch = admitted_epoch; +} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -377,10 +394,20 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, return end; } +/* spec-4.6a r2-P1-1: controllable last-event mock so the recovery FSM's P0 + * accept + WAIT_EPOCH event-scoped witness can be unit-driven. Default + * zeroed = pre-existing inert behavior. */ +static ReconfigEvent ut_mock_last_event; void cluster_reconfig_get_last_event(ReconfigEvent *out) { - memset(out, 0, sizeof(*out)); + *out = ut_mock_last_event; +} + +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; } /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the @@ -419,6 +446,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { @@ -432,10 +477,12 @@ SendProcSignal(int pid pg_attribute_unused(), int reason pg_attribute_unused(), return 0; } +/* spec-4.6a r2-P1-1: controllable clock (default 0 = pre-existing value). */ +static int64 ut_mock_now = 0; int64 GetCurrentTimestamp(void) { - return 0; + return ut_mock_now; } void @@ -3903,15 +3950,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10); + cluster_grd_recovery_mark_peer_done(1, 10, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3934,15 +3981,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5); - cluster_grd_recovery_mark_peer_done(1, 5); - cluster_grd_recovery_mark_peer_done(2, 5); + cluster_grd_recovery_mark_peer_done(0, 5, 1); + cluster_grd_recovery_mark_peer_done(1, 5, 1); + cluster_grd_recovery_mark_peer_done(2, 5, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3967,9 +4014,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4020,9 +4067,9 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4035,15 +4082,74 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20); + cluster_grd_recovery_mark_peer_done(0, 20, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20); + cluster_grd_recovery_mark_peer_done(1, 20, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } +/* ============================================================ + * spec-4.6a r2-P1-1 — event-scoped REDECLARE_DONE proof. + * + * A stale DONE (previous event) must be dropped at the accounting + * layer and must never satisfy the WAIT_EPOCH coordinator witness; + * a current-event DONE at exactly cur, accounted after the P0 + * accept snapshot, is the only equal-epoch escape (proof-carrying, + * never timeout-only). + * ============================================================ */ + +UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) +{ + ClusterGrdRecoveryCounters before; + ClusterGrdRecoveryCounters after; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline. ut_mock_epoch already + * holds the post-bump value, reproducing the mis-captured-baseline wedge + * shape (cur == old) from spec-4.6a section 0. */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Stage the fail-stop event: dead node 2, coordinator 0, event 77. */ + ut_mock_last_event.event_id = 77; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); + + /* Stale-event DONE (event 76) is dropped: nothing is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, 76); + UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Current-event DONE is accounted (epoch + event id). */ + cluster_grd_recovery_mark_peer_done(0, 10, 77); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 77); + + /* Witness advance: deadline expired + cur==old + coordinator DONE for + * THIS event at exactly cur, accounted after the accept snapshot. The + * FSM takes the event-scoped equal-epoch escape exactly once. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4055,7 +4161,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4264,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_stale_event_done_rejected_and_witness_advance); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..d40ee60d43 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -61,6 +61,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -91,6 +92,14 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; this fixture has no PCM shmem — no-op. */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -298,6 +307,10 @@ cluster_epoch_get_current(void) return 0; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch pg_attribute_unused()) +{} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -355,6 +368,12 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -391,6 +410,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_hw_remaster_retry.c b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c new file mode 100644 index 0000000000..456c5979f8 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_hw_remaster_retry.c + * Unit tests for spec-4.6a same-episode HW remaster retry decisions. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_hw_remaster_retry.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_hw_remaster.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +static ClusterHwRemasterRelaunchDecision +decide(uint64 launched, uint64 episode, ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int max_attempts) +{ + return cluster_hw_remaster_relaunch_decide(launched, episode, result, attempts, next_attempt_at, + now, max_attempts); +} + +UT_TEST(test_new_episode_initial_launch_resets_state) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(10, 11, CLUSTER_HW_REMASTER_BLOCKED, 7, 1234, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_running_done_and_not_applicable_do_not_relaunch) +{ + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_RUNNING, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_DONE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_NOT_APPLICABLE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_structural_blocked_warns_once_and_never_retries) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, 0, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, CLUSTER_HW_REMASTER_NO_DEADLINE, + 3000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_blocked_waits_until_backoff_deadline) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 4999, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 5000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 1); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_cap_exhausts_once_and_sighup_raise_recovers) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, 0, 5000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 6000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + /* Operator raised cluster.hw_remaster_retry_max_attempts via SIGHUP. */ + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 7000, 3); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 3); +} + +UT_TEST(test_zero_max_attempts_disables_retry) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 0, 5000, 0); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); +} + +/* spec-4.6a section 2.1 truth-table row 8: latched == episode but the result + * slot reads NONE (registration failed and was reverted) -> take the + * first-launch row again; no attempt is charged. */ +UT_TEST(test_none_result_registration_failed_falls_back_to_initial) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_NONE, 3, 999, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_backoff_exponential_cap) +{ + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 0), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 1), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 2), 2000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 7), 60000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(100, 20), 60000); +} + + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_new_episode_initial_launch_resets_state); + UT_RUN(test_running_done_and_not_applicable_do_not_relaunch); + UT_RUN(test_structural_blocked_warns_once_and_never_retries); + UT_RUN(test_blocked_waits_until_backoff_deadline); + UT_RUN(test_cap_exhausts_once_and_sighup_raise_recovers); + UT_RUN(test_zero_max_attempts_disables_retry); + UT_RUN(test_none_result_registration_failed_falls_back_to_initial); + UT_RUN(test_backoff_exponential_cap); + UT_DONE(); + + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_lock_acquire.c b/src/test/cluster_unit/test_cluster_lock_acquire.c index 5636b8fff8..a5ca5a719c 100644 --- a/src/test/cluster_unit/test_cluster_lock_acquire.c +++ b/src/test/cluster_unit/test_cluster_lock_acquire.c @@ -365,6 +365,47 @@ cluster_grd_lookup_master(const ClusterResId *resid pg_attribute_unused()) return -1; } +/* spec-4.6a: the S4-reject diagnostic references the CSSD peer-state view; + * fixture has no CSSD shmem — stub DEAD-free defaults. */ +#include "cluster/cluster_cssd.h" +ClusterCssdPeerState +cluster_cssd_get_peer_state(int32 peer_id pg_attribute_unused()) +{ + return CLUSTER_CSSD_PEER_ALIVE; +} +const char * +cluster_cssd_peer_state_to_string(ClusterCssdPeerState s pg_attribute_unused()) +{ + return "alive"; +} + +/* spec-4.6a: the same diagnostic is the first ereport in this .o — swallow + * elog machinery (mirror test_cluster_grd.c pattern). */ +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} + /* spec-4.6 P0 regression harness — test-controlled S3/S4 outcomes. * stub_reserve_result default NOT_READY keeps the legacy tests on the * pre-reservation-fail path; the S4 default-deny test flips it to OK diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c0ff3b6710..c02894306f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -100,6 +100,9 @@ static Size fake_pcm_keysize = 0; static Size fake_pcm_entrysize = 0; static LWLock *fake_lwlock_held = NULL; static LWLockMode fake_lwlock_mode = LW_EXCLUSIVE; +static LWLock *fake_lwlock_stack[16]; +static LWLockMode fake_lwlock_mode_stack[16]; +static int fake_lwlock_depth = 0; static uint32 fake_init_wait_event_seen = 0; static uint32 fake_lwlock_wait_event_seen = 0; @@ -121,6 +124,7 @@ static struct { static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; +static bool fake_local_x_upgrade_result = false; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -155,6 +159,10 @@ reset_fake_pcm_runtime(int max_entries) fake_pcm_keysize = 0; fake_pcm_entrysize = 0; fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + fake_lwlock_depth = 0; + memset(fake_lwlock_stack, 0, sizeof(fake_lwlock_stack)); + memset(fake_lwlock_mode_stack, 0, sizeof(fake_lwlock_mode_stack)); fake_init_wait_event_seen = 0; fake_lwlock_wait_event_seen = 0; ut_wait_event_info_storage = 0; @@ -167,6 +175,7 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; cluster_pcm_grd_max_entries = max_entries; @@ -191,9 +200,20 @@ bool cluster_read_scache = false; bool cluster_gcs_block_local_x_upgrade(BufferTag tag); bool -cluster_gcs_block_local_x_upgrade(BufferTag tag pg_attribute_unused()) +cluster_gcs_block_local_x_upgrade(BufferTag tag) { - return false; + uint32 holders_bm; + int n; + + if (!fake_local_x_upgrade_result) + return false; + holders_bm = cluster_pcm_lock_query_s_holders_bitmap(tag); + if (cluster_node_id >= 0 && cluster_node_id < 32) + holders_bm &= ~((uint32)1u << (uint32)cluster_node_id); + for (n = 0; n < 32; n++) + if ((holders_bm & ((uint32)1u << n)) != 0) + (void)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, n); + return cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_X_UPGRADE, cluster_node_id); } /* spec-4.7a D4 — stub CSSD peer liveness for the other-live-holder gate. @@ -348,6 +368,10 @@ LWLockInitialize(LWLock *lock pg_attribute_unused(), int tranche_id pg_attribute bool LWLockAcquire(LWLock *lock, LWLockMode mode) { + Assert(fake_lwlock_depth < (int)lengthof(fake_lwlock_stack)); + fake_lwlock_stack[fake_lwlock_depth] = lock; + fake_lwlock_mode_stack[fake_lwlock_depth] = mode; + fake_lwlock_depth++; fake_lwlock_held = lock; fake_lwlock_mode = mode; fake_lwlock_wait_event_seen = ut_wait_event_info_storage; @@ -357,14 +381,28 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) void LWLockRelease(LWLock *lock) { - Assert(fake_lwlock_held == lock); - fake_lwlock_held = NULL; + Assert(fake_lwlock_depth > 0); + Assert(fake_lwlock_stack[fake_lwlock_depth - 1] == lock); + fake_lwlock_depth--; + fake_lwlock_stack[fake_lwlock_depth] = NULL; + if (fake_lwlock_depth > 0) { + fake_lwlock_held = fake_lwlock_stack[fake_lwlock_depth - 1]; + fake_lwlock_mode = fake_lwlock_mode_stack[fake_lwlock_depth - 1]; + } else { + fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + } } bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode) { - return fake_lwlock_held == lock && fake_lwlock_mode == mode; + int i; + + for (i = fake_lwlock_depth - 1; i >= 0; i--) + if (fake_lwlock_stack[i] == lock && fake_lwlock_mode_stack[i] == mode) + return true; + return false; } /* ---------- @@ -1280,6 +1318,79 @@ UT_TEST(test_pcm_d3_not_double_x) } } +UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) +{ + BufferTag tag = make_tag(96); + BufferDesc buf; + bool save = cluster_gcs_block_local_cache; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + cluster_gcs_block_local_cache = true; + fake_local_x_upgrade_result = true; + + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); + UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); + UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); + + cluster_gcs_block_local_cache = save; +} + +UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) +{ + BufferTag stag = make_tag(94); + BufferTag xtag = make_tag(95); + uint32 s_bitmap; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + + /* Dead node 2 was the first S holder, so master_holder points at it. */ + cluster_node_id = 2; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 2); + UT_ASSERT(!cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + + cluster_node_id = 2; + cluster_pcm_lock_acquire(xtag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_X); + + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)2); + + s_bitmap = cluster_pcm_lock_query_s_holders_bitmap(stag); + UT_ASSERT_EQ((int)(s_bitmap & (1u << 2)), 0); + UT_ASSERT((s_bitmap & (1u << 1)) != 0); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(stag), (int)PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 1); + + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_N); + UT_ASSERT(cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + UT_ASSERT((fake_cv_broadcast_count) >= (1)); + + /* Idempotent: a repeated dead-sweep pass has nothing left to clean. */ + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)0); + + /* spec-4.6a D12 (r2-P1-3) pending-X form: a dead requester's parked X + * intent is cleared by the companion HC124 sweep the same dead-sweep hook + * drives, and the sweep is idempotent too. */ + cluster_pcm_lock_set_pending_x(stag, 2, 1234); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), 2); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)1); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), -1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)0); +} + /* spec-5.2a D2 (U1): clean-page X-transfer arm is one-shot. arm(true) sets * the backend-local flag; consume() reads-and-clears it (the acquire path * calls consume() once so the eligibility can never leak into a SUBSEQUENT @@ -1313,7 +1424,7 @@ UT_TEST(test_clean_page_xfer_arm_is_one_shot) int main(void) { - UT_PLAN(39); + UT_PLAN(41); UT_RUN(test_pcm_lock_mode_constant_aliases_match_pcm_state); UT_RUN(test_pcm_lock_transition_count_is_9); UT_RUN(test_pcm_lock_transition_enum_values_are_1_to_9); @@ -1352,6 +1463,8 @@ main(void) UT_RUN(test_pcm_d1_recovering_gate_fail_closed); UT_RUN(test_pcm_d2_rebuild_from_redeclare); UT_RUN(test_pcm_d3_not_double_x); + UT_RUN(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades); + UT_RUN(test_pcm_dead_node_cleanup_drops_holder_records); UT_RUN(test_clean_page_xfer_arm_is_one_shot); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm index 3b3a2bb29f..873852fd76 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm @@ -55,6 +55,29 @@ use PostgreSQL::Test::Utils; our $NODES = 4; + +# Relocate an init_from_backup node's copied pg_wal into its shared WAL thread +# directory, matching the layout initdb -X produces for the seed node. +sub _relocate_backup_pg_wal +{ + my ($node, $wal_threads_root, $thread_id) = @_; + my $pgwal = $node->data_dir . '/pg_wal'; + my $wal_thread = "$wal_threads_root/thread_$thread_id"; + + mkdir $wal_thread or die "mkdir $wal_thread: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal_thread/$e") + or die "rename $pgwal/$e -> $wal_thread/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal_thread, $pgwal) or die "symlink $pgwal -> $wal_thread: $!"; + return; +} + #----------------------------------------------------------------------- # new_quad($class, $cluster_name, %opts) # @@ -66,6 +89,8 @@ our $NODES = 4; # voting-disk files (QVOTEC reaches quorum OK) # wal_threads_root : 1 -> shared per-thread WAL root # shared_data : 1 -> shared data root (cluster_fs backend) +# shared_catalog : 1 -> t/337-style shared-catalog formation; +# implies shared_data + wal_threads_root #----------------------------------------------------------------------- sub new_quad { @@ -91,6 +116,12 @@ sub new_quad "${cluster_name}_node$i", port => $pg_ports[$i]); } + if ($opts{shared_catalog}) + { + $opts{shared_data} = 1; + $opts{wal_threads_root} = 1; + } + # spec-4.6: strict-mode opt-in (mirror ClusterTriple) -- pre-allocate N # shared voting-disk files so QVOTEC reaches quorum_state=OK and the GES # inbound validation (in_quorum, check 4) accepts cross-node traffic. @@ -127,23 +158,75 @@ sub new_quad if ($opts{shared_data}) { $shared_data_root = PostgreSQL::Test::Utils::tempdir(); + if ($opts{shared_catalog}) + { + mkdir "$shared_data_root/global" + or die "mkdir $shared_data_root/global: $!"; + } + } + + if ($opts{shared_catalog}) + { + my $sc_common = <init(allows_streaming => 1, + extra => [ '-X', "$wal_threads_root/thread_1" ]); + $nodes[0]->start; + $nodes[0]->backup('clusterquad_scb'); + $nodes[0]->stop; + + for my $i (1 .. $NODES - 1) + { + $nodes[$i]->init_from_backup($nodes[0], 'clusterquad_scb'); + _relocate_backup_pg_wal($nodes[$i], $wal_threads_root, $i + 1); + } + + # Seed the shared catalog/controlfile/OID authorities in a single-node era, + # then append the real cluster config below. Last GUC value wins. + $nodes[0]->append_conf('postgresql.conf', $sc_common); + $nodes[0]->append_conf('postgresql.conf', <start; + die "shared_catalog seed did not create catalog authority" + unless -e "$shared_data_root/global/pgrac_catalog_authority"; + $nodes[0]->stop; + } + else + { + my $wal_node_index = 0; + for my $node (@nodes) + { + if (defined $wal_threads_root) + { + my $thread_id = $wal_node_index + 1; + $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); + } + else + { + $node->init; + } + $wal_node_index++; + } } - my $wal_node_index = 0; for my $node (@nodes) { if (defined $wal_threads_root) { - my $thread_id = $wal_node_index + 1; - $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); $node->append_conf('postgresql.conf', "cluster.wal_threads_dir = '$wal_threads_root'\n"); } - else - { - $node->init; - } - $wal_node_index++; if (defined $shared_data_root) { @@ -153,6 +236,15 @@ sub new_quad "cluster.shared_data_dir = '$shared_data_root'\n"); $node->append_conf('postgresql.conf', "cluster.smgr_user_relations = on\n"); + if ($opts{shared_catalog}) + { + $node->append_conf('postgresql.conf', + "cluster.controlfile_shared_authority = on\n"); + $node->append_conf('postgresql.conf', + "cluster.shared_catalog = on\n"); + $node->append_conf('postgresql.conf', + "cluster.merged_recovery = on\n"); + } } # Enable cluster + tier1, same baseline as ClusterTriple (spec-2.2). From e7ca433d8c8b2c5d5d3490b9276a5efa569debc8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:17:40 +0800 Subject: [PATCH 02/13] ci(nightly): own shard for the t/362 4-node kill self-heal TAP t/358-361 are reserved by parallel lanes (spec-7.2 / S-xid), t/363 by the S-dead lane; occupancy verified against origin branches. The 4-node formation + kill -9 + convergence run gets its own shard for wall-clock isolation. Spec: spec-4.6a-grd-recovery-liveness.md --- .github/workflows/nightly.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..ae5dcdf16c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -182,6 +182,11 @@ jobs: # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-361 are + # reserved by the parallel spec-7.2 / S-xid lanes, t/363 by S-dead; + # L464 occupancy verified against origin branches 2026-07-08). Own + # shard: 4-node formation + kill + convergence needs the wall clock. + - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 From 8853bc7d7e6214ee0b9f791c4fff5c80179f6e12 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:25:22 +0800 Subject: [PATCH 03/13] test(cluster): suppress pre-existing cppcheck constParameter in pi_shadow unit Same inline suppression + reason as the other cluster_unit main() signatures (test_cluster_grd.c precedent); a newer local cppcheck flags it, the CI baseline version does not. --- src/test/cluster_unit/test_cluster_pi_shadow.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..7d075852dd 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,6 +207,9 @@ UT_TEST(test_gate_raw_compare_traps) } int +/* cppcheck-suppress constParameter + * Reason: main() keeps the standard test harness signature used by the + * other cluster_unit binaries; argv is intentionally unused. */ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ From 55dd3f34057ddeec164f1fcff5ee47c9e133b909 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:37 +0800 Subject: [PATCH 04/13] fix(cluster): converge REDECLARE_DONE on (episode_epoch, dead_bitmap_hash) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The P6 all-done gate and the WAIT_EPOCH coordinator witness keyed the cross-node DONE on event_id, but event_id folds the sender-local cssd_dead_generation, which drifts with each survivor's private flap observation history and never converges across nodes: any flap asymmetry made the survivors compute different ids for the same episode, drop each other's DONEs, and wedge P6 forever — nondeterministically reintroducing the permanent-freeze shape this branch exists to remove. The quorum-accepted dead SET is what actually converges, so the DONE payload now carries a hash over the dead bitmap alone (same kernel as the event_id hash, riding the same request-id field pair — no envelope change), stamped per episode at P0 accept. Accounting, the P6 gate and the witness compare against the stamp; event_id stays a purely local accept-dedup scope. A late DONE from a previous episode still cannot back a new one: a coordinator re-election changes the dead set (hash mismatch) and a same-node re-death rides a higher epoch through the interposed JOIN bump (epoch conjunct). Pre-accept frames now mismatch the previous stamp and are dropped; senders re-announce every tick, so accounting always lands after the accept snapshot — closing the first-event pre-accept window as a side effect. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R2/R4) --- src/backend/cluster/cluster_debug.c | 4 +- src/backend/cluster/cluster_ges.c | 4 +- src/backend/cluster/cluster_grd.c | 146 ++++++++++++++------- src/include/cluster/cluster_grd.h | 28 +++- src/test/cluster_unit/test_cluster_debug.c | 2 +- src/test/cluster_unit/test_cluster_grd.c | 124 ++++++++++++----- 6 files changed, 216 insertions(+), 92 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index e19b3e98e8..1e50f2b3a9 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1211,8 +1211,8 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) fmt_int32((int32)cluster_grd_recovery_event_coordinator())); emit_row(rsinfo, "grd_recovery", "done_self_epoch", fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); - emit_row(rsinfo, "grd_recovery", "done_self_event_id", - fmt_int64((int64)cluster_grd_recovery_done_event_id_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_bitmap_hash", + fmt_int64((int64)cluster_grd_recovery_done_bitmap_hash_for(cluster_node_id))); emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index 547f0e5aac..f68e407239 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -550,7 +550,9 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) /* spec-4.6 P0#3 cluster gate — fire-and-forget barrier announcement: * record and return (no work queue, no reply, no dedup; the sender - * re-announces each tick, the receiver write is a monotonic max). */ + * re-announces each tick, the receiver write is a monotonic max). + * Amendment v1.2 (R2): the request-id field pair carries the sender's + * dead_bitmap hash (composite convergence key), not an event_id. */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, ges_request_holder_request_id(req)); diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 8977739433..ae87b62dfa 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -737,8 +737,9 @@ cluster_grd_shmem_init(void) /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); - pg_atomic_init_u64(&cluster_grd_state->recovery_done_event_id[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_bitmap_hash[i], 0); } + pg_atomic_init_u64(&cluster_grd_state->recovery_event_bitmap_hash, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -1703,11 +1704,36 @@ cluster_grd_recovery_done_epoch_for(int32 node) } uint64 -cluster_grd_recovery_done_event_id_for(int32 node) +cluster_grd_recovery_done_bitmap_hash_for(int32 node) { if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return 0; - return pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[node]); + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[node]); +} + +uint64 +cluster_grd_recovery_event_bitmap_hash_value(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); +} + +/* + * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). + * + * The cross-node half of the REDECLARE_DONE convergence key: a hash over + * the quorum-accepted dead bitmap ALONE. Same kernel as the event_id hash + * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — + * the generation is per-instance observation history and does NOT converge + * across survivors, which is exactly why event_id cannot key the P6 gate. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere, + * so the composite key degrades to the epoch-only gate there. + */ +uint64 +cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) +{ + return hash_bytes_extended(dead_bitmap, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES, 0); } bool @@ -2051,7 +2077,11 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; - uint64 event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + /* Amendment v1.2 (R2): the DONE payload carries the sender's accepted + * dead-bitmap hash (cross-node convergence key), riding the request-id + * field pair — NOT the sender-local event_id (its dead_generation fold + * diverges across survivors' flap observation histories). */ + uint64 bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -2061,8 +2091,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); - req.holder_request_id_lo = (uint32)(event_id & 0xffffffffu); - req.holder_request_id_hi = (uint32)(event_id >> 32); + req.holder_request_id_lo = (uint32)(bitmap_hash & 0xffffffffu); + req.holder_request_id_hi = (uint32)(bitmap_hash >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -2081,21 +2111,29 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash) { - uint64 current_event_id; + uint64 episode_bitmap_hash; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; /* - * spec-4.6a D4-v2: REDECLARE_DONE is an event-scoped proof. A stale DONE - * from a previous episode can carry the same epoch when cur==old, so epoch - * monotonicity alone is not enough. Drop messages for any event other than - * the one this LMON has accepted; senders re-announce every tick. + * spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE converges on the COMPOSITE + * key (episode_epoch, dead_bitmap_hash). A stale DONE from a previous + * episode can carry the same epoch when cur==old, so epoch monotonicity + * alone is not enough — but the previous episode's dead SET necessarily + * differs (a coordinator re-election adds the old coordinator to it; a + * re-death of the same node rides a higher epoch via the interposed JOIN + * bump), so the bitmap hash is the ABA guard. Flap-history drift changes + * only the per-instance dead_generation, never the quorum dead SET, so + * DONEs for the same episode always match here (the R2 wedge fix). + * Pre-accept frames mismatch the previous episode's stamp and are dropped; + * senders re-announce every tick, so accounting lands after our P0 accept + * (which also closes the R4 pre-accept window by construction). */ - current_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); - if (event_id == 0 || (current_event_id != 0 && event_id != current_event_id)) + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) return; /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic @@ -2107,7 +2145,7 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) if (epoch > prev) pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } - pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[node], event_id); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } /* @@ -2175,13 +2213,13 @@ grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, continue; done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); if (done_epoch >= episode_epoch - && pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) - == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) continue; grd_recovery_appendf(buf, buflen, &off, - "%s%d(done=" UINT64_FORMAT "/event=" UINT64_FORMAT ")", any ? "," : "", - i, done_epoch, - pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i])); + "%s%d(done=" UINT64_FORMAT "/bitmap_hash=" UINT64_FORMAT ")", + any ? "," : "", i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i])); any = true; } if (!any) @@ -2433,6 +2471,12 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } + /* Amendment v1.2 (R2): stamp the composite convergence key's cross-node + * half. Hash over the accepted dead bitmap alone — every survivor that + * accepted the same quorum dead SET computes the same value regardless + * of its private dead_generation observation history. */ + pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, + cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2510,8 +2554,8 @@ cluster_grd_recovery_lmon_tick(void) TimestampTz next_deadline; uint32 coordinator; uint64 coord_done; - uint64 coord_done_event_id; - uint64 accepted_event_id; + uint64 coord_done_bitmap_hash; + uint64 episode_bitmap_hash; uint64 coord_done_at_accept; wait_deadline = (TimestampTz)pg_atomic_read_u64( @@ -2533,29 +2577,33 @@ cluster_grd_recovery_lmon_tick(void) } coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); - accepted_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + episode_bitmap_hash + = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); coord_done = coordinator < CLUSTER_MAX_NODES ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) : 0; - coord_done_event_id + coord_done_bitmap_hash = coordinator < CLUSTER_MAX_NODES ? pg_atomic_read_u64( - &cluster_grd_state->recovery_done_event_id[coordinator]) + &cluster_grd_state->recovery_done_bitmap_hash[coordinator]) : 0; coord_done_at_accept = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); /* - * spec-4.6a §2.3 witness (all three conjuncts of the frozen - * text): the coordinator's DONE must (a) be for THIS event, - * (b) name exactly cur as the episode epoch, and (c) have been + * spec-4.6a §2.3 witness, composite-key form (Amendment v1.2 + * R2): the coordinator's DONE must (a) name this episode's + * quorum dead SET (bitmap hash — the cross-node-consistent + * key; the sender-local event_id folds the per-instance + * dead_generation and diverges under flap asymmetry), (b) + * name exactly cur as the episode epoch, and (c) have been * accounted after our P0 accept snapshot. Under strictly * monotonic per-event epochs (c) is implied by (a)+(b) — any * pre-accept residue carries a lower epoch — but it stays as a * belt-and-suspenders guard against accounting bugs. */ if (cur_epoch == old_epoch && coord_done == cur_epoch - && coord_done_event_id == accepted_event_id + && coord_done_bitmap_hash == episode_bitmap_hash && coord_done > coord_done_at_accept) { pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); ereport(WARNING, @@ -2563,25 +2611,25 @@ cluster_grd_recovery_lmon_tick(void) errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " "equal-epoch progress"), errdetail("coordinator=%u, epoch=" UINT64_FORMAT - ", event_id=" UINT64_FORMAT, - coordinator, cur_epoch, accepted_event_id))); + ", dead_bitmap_hash=" UINT64_FORMAT, + coordinator, cur_epoch, episode_bitmap_hash))); } else { next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); - ereport( - WARNING, - (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), - errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " - "affected shards stay frozen"), - errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT - ", coordinator=%u, coordinator_done=" UINT64_FORMAT - ", coordinator_done_event_id=" UINT64_FORMAT - ", accepted_event_id=" UINT64_FORMAT - ", coordinator_done_at_accept=" UINT64_FORMAT, - cur_epoch, old_epoch, coordinator, coord_done, - coord_done_event_id, accepted_event_id, coord_done_at_accept))); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_bitmap_hash=" UINT64_FORMAT + ", episode_bitmap_hash=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_bitmap_hash, episode_bitmap_hash, + coord_done_at_accept))); return; } } @@ -2728,8 +2776,8 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); - pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[cluster_node_id], - pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)); grd_recovery_broadcast_done(episode_epoch); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), cluster_grd_rebuild_timeout_ms); @@ -2864,9 +2912,13 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; + /* Amendment v1.2 (R2): composite key — the peer's DONE must name + * this episode's epoch AND the same quorum dead SET. Keying on + * event_id here wedged the gate whenever survivors' private + * dead_generation histories diverged (flap asymmetry). */ if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch - || pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) - != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { all_done = false; break; } diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index a098d53ed1..c6fa42ecaf 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -294,9 +294,21 @@ typedef struct ClusterGrdShared { /* spec-4.6 P0#3 cluster gate — per-node epoch for which that node * last announced "local rebind barrier complete" (REDECLARE_DONE). - * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ + * P6 requires done_epoch[s] >= current epoch for EVERY survivor. + * + * spec-4.6a Amendment v1.2 (R2): the cross-node convergence key is the + * COMPOSITE (episode_epoch, dead_bitmap_hash). event_id is NOT globally + * consistent (it folds the per-instance cssd_dead_generation, which + * drifts with each node's own flap observation history), so keying the + * P6 gate on it wedges the cluster whenever two survivors observed a + * different flap history. The quorum-accepted dead SET is what actually + * converges; its hash rides the DONE payload instead. event_id stays a + * purely LOCAL ABA scope (P0 accept dedup). */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; - pg_atomic_uint64 recovery_done_event_id[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_bitmap_hash[CLUSTER_MAX_NODES]; + /* hash of THIS episode's accepted dead_bitmap (LMON writes at P0 accept; + * empty-bitmap hash for JOIN episodes, identical on every node). */ + pg_atomic_uint64 recovery_event_bitmap_hash; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -624,7 +636,11 @@ extern uint64 cluster_grd_recovery_event_old_epoch(void); extern uint64 cluster_grd_recovery_episode_epoch_value(void); extern uint32 cluster_grd_recovery_event_coordinator(void); extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); -extern uint64 cluster_grd_recovery_done_event_id_for(int32 node); +extern uint64 cluster_grd_recovery_done_bitmap_hash_for(int32 node); +extern uint64 cluster_grd_recovery_event_bitmap_hash_value(void); +/* Amendment v1.2 (R2): the cross-node DONE key — hash over the dead bitmap + * ALONE (no dead_generation fold; same kernel as the event_id hash). */ +extern uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap); extern int cluster_grd_recovery_block_redeclare_cursor(void); extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); extern bool cluster_grd_recovery_block_redeclare_done(void); @@ -644,8 +660,10 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind - * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id); + * barrier for `epoch`. Amendment v1.2 (R2): the third argument is the + * sender's dead_bitmap hash (the cross-node half of the composite + * convergence key), NOT the sender-local event_id. */ +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 8b64c01755..8b741f7c42 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -3350,7 +3350,7 @@ cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) } uint64 -cluster_grd_recovery_done_event_id_for(int32 node pg_attribute_unused()) +cluster_grd_recovery_done_bitmap_hash_for(int32 node pg_attribute_unused()) { return 0; } diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index c7f805459d..2e4b813e17 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -3748,6 +3748,30 @@ ut_jr_setup_3node(void) cluster_grd_master_map_init(); } +/* spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE accounting converges on the + * composite key (episode_epoch, dead_bitmap_hash), so a test that feeds DONEs + * must first drive a P0 accept to stamp the episode's bitmap hash. Accept an + * empty-dead-bitmap FAIL event (hash identical to a JOIN episode's, no fence + * side effects; the FSM parks in WAIT_EPOCH awaiting a strict bump, which the + * fence assertions never touch). Returns the stamped hash for the callers' + * mark_peer_done payloads. */ +static uint64 +ut_jr_stamp_episode_bitmap_hash(uint64 event_id) +{ + bool saved_enabled = cluster_enabled; + + cluster_enabled = true; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle tick: baseline capture */ + ut_mock_last_event.event_id = event_id; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + cluster_grd_recovery_lmon_tick(); /* P0 accept stamps the bitmap hash */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_enabled = saved_enabled; + return cluster_grd_recovery_event_bitmap_hash_value(); +} + /* U1 — recompute moves the joiner's home shards back from the survivor. */ UT_TEST(test_jr_u1_recompute_moves_joiner_home) { @@ -3937,9 +3961,12 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(901); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* the rejoiner's home block */ @@ -3950,15 +3977,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3969,10 +3996,13 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(902); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); @@ -3981,15 +4011,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5, 1); - cluster_grd_recovery_mark_peer_done(1, 5, 1); - cluster_grd_recovery_mark_peer_done(2, 5, 1); + cluster_grd_recovery_mark_peer_done(0, 5, done_hash); + cluster_grd_recovery_mark_peer_done(1, 5, done_hash); + cluster_grd_recovery_mark_peer_done(2, 5, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -4002,10 +4032,12 @@ UT_TEST(test_jr_u15_master_side_gate_decision) int n1[1] = { 1 }; BufferTag tag; bool deny; + uint64 done_hash; memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(903); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; @@ -4014,9 +4046,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4059,17 +4091,20 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) int n2[1] = { 2 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); /* declared {0,1,2}, all members */ /* --- Episode 1: node 1 rejoins and completes (all survivors re-declare). --- */ ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(904); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4082,29 +4117,35 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20, 1); + cluster_grd_recovery_mark_peer_done(0, 20, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20, 1); + cluster_grd_recovery_mark_peer_done(1, 20, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } /* ============================================================ - * spec-4.6a r2-P1-1 — event-scoped REDECLARE_DONE proof. + * spec-4.6a Amendment v1.2 (R2) — composite-key REDECLARE_DONE proof. * - * A stale DONE (previous event) must be dropped at the accounting - * layer and must never satisfy the WAIT_EPOCH coordinator witness; - * a current-event DONE at exactly cur, accounted after the P0 - * accept snapshot, is the only equal-epoch escape (proof-carrying, - * never timeout-only). + * The cross-node convergence key is (episode_epoch, dead_bitmap_hash). + * A DONE naming a DIFFERENT dead set (the old-event ABA shape) must be + * dropped at the accounting layer and never satisfy the WAIT_EPOCH + * coordinator witness. A DONE naming the SAME dead set must be + * accounted even though the sender's local event_id differs (the + * dead_generation-drift shape that wedged the event_id key: R2), and + * then satisfies the witness — the only equal-epoch escape + * (proof-carrying, never timeout-only). * ============================================================ */ -UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) +UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) { ClusterGrdRecoveryCounters before; ClusterGrdRecoveryCounters after; + uint8 old_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 episode_hash; + uint64 old_event_hash; ut_jr_setup_3node(); cluster_enabled = true; @@ -4117,27 +4158,38 @@ UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); cluster_grd_recovery_lmon_tick(); - /* Stage the fail-stop event: dead node 2, coordinator 0, event 77. */ + /* Stage the fail-stop event: dead node 2, coordinator 0. The local + * event_id (77) folds this node's own dead_generation; a peer that saw a + * different flap history computes a DIFFERENT id for the same episode. */ ut_mock_last_event.event_id = 77; ut_mock_last_event.coordinator_node_id = 0; ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; ut_mock_last_event.dead_bitmap[0] = 0x04; cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); - - /* Stale-event DONE (event 76) is dropped: nothing is accounted. */ - cluster_grd_recovery_mark_peer_done(0, 10, 76); - UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 0); + episode_hash = cluster_grd_recovery_event_bitmap_hash_value(); + UT_ASSERT_EQ(episode_hash, cluster_grd_dead_bitmap_hash(ut_mock_last_event.dead_bitmap)); + + /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 + * instead of node 2 — e.g. the previous episode before a coordinator + * re-election) is dropped: nothing is accounted. */ + memset(old_dead, 0, sizeof(old_dead)); + old_dead[0] = 0x08; + old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); + cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); - /* Current-event DONE is accounted (epoch + event id). */ - cluster_grd_recovery_mark_peer_done(0, 10, 77); + /* Generation-drift shape (the R2 wedge): the peer's local event_id + * differs, but its DONE names the SAME quorum dead set -> the composite + * key matches and the DONE is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, episode_hash); UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); - UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 77); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), episode_hash); - /* Witness advance: deadline expired + cur==old + coordinator DONE for - * THIS event at exactly cur, accounted after the accept snapshot. The - * FSM takes the event-scoped equal-epoch escape exactly once. */ + /* Witness advance: deadline expired + cur==old + coordinator DONE naming + * THIS dead set at exactly cur, accounted after the accept snapshot. The + * FSM takes the composite-key equal-epoch escape exactly once. */ cluster_grd_recovery_counters_snapshot(&before); ut_mock_now = (int64)60 * 1000000; cluster_grd_recovery_lmon_tick(); @@ -4264,7 +4316,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); - UT_RUN(test_recovery_stale_event_done_rejected_and_witness_advance); + UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From ddefc0003592ae537598ac18601d7633d3fc10ae Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:58 +0800 Subject: [PATCH 05/13] fix(cluster): dead-master block serve proof is unconditional again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert the scope short-circuit that returned NORMAL for a dead static master's blocks wherever online thread recovery cannot run (the default configuration and every >2-node deployment): it skipped both the is_materialized cold-block door and the redo-coverage lost-write door, so a committed write only the dead node saw could be silently read stale from shared storage. No other guard sits on that read path — the GRD freeze ends with the episode, the HW gate covers only extend high-water marks, and the page-SCN checks ride the ship path. Out of scope the posture is now an explicit bounded retryable error (53R9L, hint updated to name the way out: restart the failed node, or enable online thread recovery in a supported scope) for every tag whose static master is dead — including never-written blocks, which the cold-block door cannot distinguish from not-yet-replayed ones — and the door reopens the moment the failed node stops being DEAD. In-scope behavior is byte-identical. The now-orphaned scope helper is removed. t/362 asserts the honest two-outcome contract on fresh connections (success or explicit SQLSTATE — 53R9L or the TT-unknown visibility door — never a hang), rides pre-warmed sessions for convergence observability, and pins the D12 cleanup counter to a positive delta; t/293's post-kill extend leg follows the same posture. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R1/R3) --- src/backend/cluster/cluster_gcs_block.c | 31 ++- src/include/cluster/cluster_thread_recovery.h | 10 - .../cluster_tap/t/293_hw_online_remaster.pl | 30 ++- .../362_shared_catalog_4node_kill_selfheal.pl | 218 +++++++++++++++--- .../cluster_unit/test_cluster_gcs_block.c | 13 +- 5 files changed, 223 insertions(+), 79 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index d0219deaf9..ee200ffafe 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1223,13 +1223,8 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The redo-before-serve gate must - * engage on EXACTLY the same scope where online thread recovery can actually - * run. Otherwise a >2-node or GUC-off deployment waits forever on a - * materialization authority that the thread-recovery launcher intentionally - * treats as not applicable. - * - * In applicable 2-node scope, two conditions are both required (Q5): + * live survivor (recovery-aware routing). Two conditions are both + * required before the on-disk version may be served (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1240,18 +1235,18 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably * reached us → lost-write → fail-closed. + * + * spec-4.6a Amendment v1.2 (R1): this proof is UNCONDITIONAL. Where + * online thread recovery cannot run (GUC off — the default — or any + * >2-node deployment) the materialization authority is never published, + * so a dead master's blocks stay RECOVERING until the failed node + * restarts and runs its own instance recovery: a bounded, retryable + * ERROR on the request path (53R9L), never an unproven serve. A scope + * predicate must never gate a correctness proof — a committed write on + * a cold block that only the dead node saw has NO other guard on this + * read path (GRD freeze ends with the episode; the HW gate covers only + * extend high-water marks; pd_block_scn checks ride the ship path). */ - { - bool shared_fs; - int survivors; - - shared_fs = (cluster_shared_storage_backend == CLUSTER_SHARED_FS_BACKEND_CLUSTER_FS); - survivors = cluster_conf_node_count() - 1; - if (!cluster_thread_recovery_materialization_gate_enabled( - cluster_online_thread_recovery, cluster_conf_has_peers(), shared_fs, survivors)) - return GCS_BLOCK_NORMAL; - } - if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->recovery_block_resources_recovering, 1); diff --git a/src/include/cluster/cluster_thread_recovery.h b/src/include/cluster/cluster_thread_recovery.h index c7be25e6b0..c455b77436 100644 --- a/src/include/cluster/cluster_thread_recovery.h +++ b/src/include/cluster/cluster_thread_recovery.h @@ -376,16 +376,6 @@ cluster_thread_recovery_gate_decide(ClusterThreadRecScope scope, const uint64 *d return false; /* every dead origin materialized -> ready to unfreeze */ } -static inline bool -cluster_thread_recovery_materialization_gate_enabled(bool guc_on, bool has_peers, - bool shared_fs_backend, - int live_survivor_count) -{ - return cluster_thread_recovery_decide_scope(guc_on, has_peers, shared_fs_backend, - live_survivor_count) - == CLUSTER_THREADREC_SCOPE_APPLICABLE; -} - /* * cluster_thread_recovery_replay_epoch_aborts -- the L235 episode-epoch * staleness guard (spec-4.11 3b-4b). The per-thread replay slot stamps the GRD diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 509c2f50ff..2447312952 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -329,23 +329,39 @@ sub retry_sql is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, 'L7: self-heal path did not exhaust the retry budget'); -# spec-4.6a section 4 (D8): DONE must actually unfreeze (P7) -- the survivor -# extends a table that lived on shards the dead master owned; a frozen shard -# would fail this INSERT with the remastering error. +# spec-4.6a section 4 (D8, Amendment v1.2 R1): after the same-episode DONE the +# episode has converged (P7 ran: DONE counter + no exhaustion asserted above), +# but in this out-of-scope deployment (online_thread_recovery off) EVERY tag +# whose static master is the dead node stays fail-closed until that node +# returns — including brand-new blocks, which have no materialization proof +# either (the cold-block door cannot distinguish never-existed from +# not-yet-replayed; main's t/293 L5b documents the same posture). The +# survivor write is therefore the honest two-outcome contract: success (all +# touched tags survivor-mastered) or the explicit bounded 53R9L — never a +# hang and never an unbounded wedge. { my ($ext_ok, $ext_res) = (0, ''); - for my $try (1 .. 30) + for my $try (1 .. 10) { $ext_res = eval { $h1->safe_psql('postgres', - 'INSERT INTO s1 SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 'CREATE TABLE IF NOT EXISTS s1_heal (a int, b int, c int, d int); ' + . 'INSERT INTO s1_heal SELECT g, g, g, g FROM generate_series(1, 4096) g'); 1; } ? 'ok' : ($@ // 'error'); if ($ext_res eq 'ok') { $ext_ok = 1; last; } sleep 1; } - ok($ext_ok, 'L7: survivor extend succeeds after same-episode DONE (P7 unfreeze)') - or diag($ext_res); + if ($ext_ok) + { + ok(1, 'L7: survivor new-table extend succeeded after same-episode DONE (P7 unfreeze)'); + } + else + { + like($ext_res, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + 'L7: survivor extend fail-closed with an explicit SQLSTATE (dead-master tag, out-of-scope posture)'); + } } $heal->stop_pair; diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl index 883860c0fb..a5f6719eea 100644 --- a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -5,6 +5,13 @@ # # Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) # so a killed node's HW authority is recoverable from its per-thread WAL. +# Amendment v1.2 (R1): in this out-of-scope deployment (4 nodes, online +# thread recovery off) the dead master's PRE-EXISTING writes stay +# fail-closed until the failed node restarts — its unproven blocks (bounded +# 53R9L) and equally its transaction verdicts (TT status unknown for its +# xids, the visibility door). Reads and DDL are asserted on the honest +# two-outcome contract (success or explicit SQLSTATE, never a hang); +# new-object DDL/writes prove P7 unfreeze. # After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means # the test harness is misconfigured, and lack of convergence is a real FAIL. # @@ -88,6 +95,52 @@ sub sum_hw return $sum; } +# Amendment v1.2 (R1): after the kill, a NEW backend's parse may cold-read a +# dead-master catalog page and honestly fail closed (53R9L) until the failed +# node returns. The convergence observability therefore rides PRE-WARMED +# long-lived psql sessions (catcache hot; pg_cluster_* reads touch shmem, not +# dead-master blocks), while fresh-connection legs assert the two-outcome +# contract. bg_query returns (1, value) or (0, error-ish) without dying. +sub bg_query +{ + my ($bg, $sql) = @_; + my $out = eval { $bg->query($sql) }; + return (0, $@ // 'query failed') if $@; + return (1, $out // ''); +} + +sub bg_counter +{ + my ($bg, $cat, $key) = @_; + my ($ok, $v) = bg_query($bg, + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'"); + return ($ok && defined $v && $v ne '') ? $v + 0 : 0; +} + +sub bg_sum +{ + my ($bgs, $cat, $key, @idx) = @_; + my $sum = 0; + $sum += bg_counter($bgs->{$_}, $cat, $key) for @idx; + return $sum; +} + +sub bg_poll_until +{ + my ($bg, $sql, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($ok, $out) = bg_query($bg, $sql); + $last = $out; + return 1 if $ok && defined $out && $out eq $want; + usleep(500_000); + } + diag("bg_poll_until timed out: '$sql' last='$last' want='$want'"); + return 0; +} + sub logs_contain { my ($quad, $pattern, @idx) = @_; @@ -217,9 +270,20 @@ sub startup_env_blocker retry_sql($quad->node($i), 'CHECKPOINT', 30); } -my $done_before = sum_hw($quad, 'remaster_done_count', @survivors); -my $blocked_before = sum_hw($quad, 'remaster_blocked_count', @survivors); -my $retry_before = sum_hw($quad, 'remaster_retry_count', @survivors); +# Amendment v1.2 (R1): pre-warmed observability sessions (see helper note). +my %bg; +for my $i (@survivors) +{ + $bg{$i} = $quad->node($i)->background_psql('postgres', on_error_stop => 0); + bg_query($bg{$i}, "SELECT value FROM pg_cluster_state WHERE category='hw' AND key='remaster_done_count'"); + bg_query($bg{$i}, "SELECT count(*) FROM pg_cluster_grd_shards WHERE master_node_id=$dead"); + bg_query($bg{$i}, 'SELECT txid_current()'); +} + +my $done_before = bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors); +my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); +my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); # L2: kill node3 and require every survivor to see the real DEAD edge. # Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, @@ -232,18 +296,26 @@ sub startup_env_blocker } # L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +# The map read is hard-asserted on the warm COORDINATOR session only: the view +# scan on a cold survivor deterministically cold-reads a dead-master catalog +# page out of the small test buffer pool and honestly fails closed (53R9L) +# until node3 returns — the map itself is a deterministic recompute barriered +# by P6, and each cold survivor is instead hard-asserted on its own HW +# remaster worker DONE log line below. +ok(bg_poll_until($bg{0}, + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + 'L3: coordinator remastered all GRD shards off node3 (warm session)'); for my $i (@survivors) { - ok(poll_until($quad->node($i), - "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", - 'true', 60), - "L3: survivor node$i remastered all GRD shards off node$dead"); + ok(log_until($quad, $i, qr/HW remaster worker: dead node $dead -> done/, 90), + "L3: survivor node$i HW remaster worker reported done"); } my $done_converged = 0; my $deadline = time() + 90; while (time() < $deadline) { - if (sum_hw($quad, 'remaster_done_count', @survivors) > $done_before) + if (bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors) > $done_before) { $done_converged = 1; last; @@ -255,10 +327,10 @@ sub startup_env_blocker 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); # L6: if a transient BLOCKED occurred, a same-episode retry must also have run. -my $blocked_after = sum_hw($quad, 'remaster_blocked_count', @survivors); +my $blocked_after = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); if ($blocked_after > $blocked_before) { - ok(sum_hw($quad, 'remaster_retry_count', @survivors) > $retry_before, + ok(bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors) > $retry_before, 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); } else @@ -266,45 +338,127 @@ sub startup_env_blocker pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); } -# L4: survivor SQL and shared-catalog DDL recover after fail-stop. +# L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an +# out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a +# BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a +# hang. sc4_hw_* spans ~20+ pages, so some page's static master is node3 with +# overwhelming probability; the redo-coverage proof cannot be published without +# online thread recovery, so the scan hits 53R9L (resource recovering). If by +# hash luck no scanned page is node3-mastered the scan succeeds — both are the +# honest two-outcome contract; only a hang/timeout or an unexpected error fails. +for my $t (1 .. 2) +{ + my $t0 = time(); + my ($rc, $out, $err) = $quad->node(1)->psql('postgres', + "SELECT count(*) FROM sc4_hw_$t", timeout => 60); + my $elapsed = time() - $t0; + if (defined $rc && $rc == 0) + { + ok(1, "L4a: dead-master table sc4_hw_$t read succeeded (no node$dead-mastered page scanned)"); + diag("L4a: sc4_hw_$t count=$out"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + "L4a: dead-master table sc4_hw_$t read fail-closed with an explicit SQLSTATE"); + } + cmp_ok($elapsed, '<', 60, "L4a: sc4_hw_$t read returned bounded (no hang)"); +} + +# L4: survivor SQL recovers after fail-stop — hard-asserted on the pre-warmed +# sessions; a FRESH connection is the honest two-outcome contract (v1.2 R1): +# success, or an explicit fail-closed SQLSTATE on a dead-master catalog page — +# never a hang. for my $i (@survivors) { - my ($ok, $res) = retry_sql($quad->node($i), 'SELECT txid_current()', 60); - ok($ok, "L4: survivor node$i accepts txid_current after reconfig") or diag($res); + my $got = 0; + my $dl = time() + 60; + while (time() < $dl) + { + my ($ok, $out) = bg_query($bg{$i}, 'SELECT txid_current()'); + if ($ok && defined $out && $out =~ /^\d+$/) { $got = 1; last; } + usleep(500_000); + } + ok($got, "L4: survivor node$i accepts txid_current after reconfig (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($i)->psql('postgres', 'SELECT txid_current()', + timeout => 60); + if (defined $rc && $rc == 0) + { + ok(1, "L4: fresh connection txid_current succeeded on node$i"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); + } + cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); +} +my ($ddl_rc, $ddl_out, $ddl_err) = $n0->psql('postgres', + 'CREATE TABLE sc4_after_kill (id int)', timeout => 60); +my $ddl_ok = defined $ddl_rc && $ddl_rc == 0; +if ($ddl_ok) +{ + ok(1, 'L4: coordinator survivor DDL succeeded after node kill'); +} +else +{ + like($ddl_err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); + diag("L4: DDL fail-closed honestly: $ddl_err"); } -my ($ddl_ok, $ddl_res) = retry_sql($n0, 'CREATE TABLE sc4_after_kill (id int)', 60); -ok($ddl_ok, 'L4: coordinator survivor can run DDL after node kill') or diag($ddl_res); for my $i (1, 2) { - ok(poll_until($quad->node($i), - "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", - 'true', 60), - "L4: survivor node$i sees post-kill shared-catalog DDL"); + if ($ddl_ok) + { + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); + } + else + { + ok(1, "L4: survivor node$i visibility leg not applicable (DDL fail-closed honestly)"); + } } -# L4 (spec-4.6a D12, r2-P1-3): the dead-node PCM residue cleanup counter is -# exposed and non-negative on every survivor; when the dead node left any -# holder/pending-X records behind, at least one survivor accounts a cleanup -# (delta assertable; zero stays legal when the dead node held nothing). +# L4 (spec-4.6a D12, Amendment v1.2 R3): node3 held real PCM/HW state at kill +# time (it extended sc4_hw_1/2 with 5000 rows each), so the dead-node residue +# cleanup MUST account a positive delta across the survivors — a >= 0 +# assertion would stay green with D12 entirely disabled. { my $cleanup_total = 0; for my $i (@survivors) { - my $c = state_counter($quad->node($i), 'pcm', 'dead_cleanup_entries'); - ok($c >= 0, "L4: survivor node$i exposes pcm/dead_cleanup_entries"); - $cleanup_total += $c; + my ($ok, $n) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='pcm' AND key='dead_cleanup_entries'"); + ok($ok && defined $n && $n eq '1', + "L4: survivor node$i exposes the pcm/dead_cleanup_entries key"); + $cleanup_total += bg_counter($bg{$i}, 'pcm', 'dead_cleanup_entries'); } - diag("L4: pcm/dead_cleanup_entries total across survivors = $cleanup_total"); + cmp_ok($cleanup_total - $cleanup_before, '>', 0, + 'L4: dead-node PCM residue cleanup accounted a positive delta after kill'); + diag("L4: pcm/dead_cleanup_entries delta across survivors = " + . ($cleanup_total - $cleanup_before)); } -# L5: watchdog counters are allowed to be zero, but must be readable. +# L5: watchdog counters are allowed to be zero, but the KEYS must exist +# (state_counter reads a missing key as 0, so presence is the real assertion). for my $i (@survivors) { - my $cluster_gate = state_counter($quad->node($i), 'grd_recovery', 'cluster_gate_timeout'); - my $epoch_escape = state_counter($quad->node($i), 'grd_recovery', 'wait_epoch_escape'); - ok($cluster_gate >= 0 && $epoch_escape >= 0, - "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counters"); + my ($ok1, $n1) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='cluster_gate_timeout'"); + my ($ok2, $n2) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='wait_epoch_escape'"); + ok($ok1 && $ok2 && defined $n1 && $n1 eq '1' && defined $n2 && $n2 eq '1', + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counter keys"); } +$bg{$_}->quit for @survivors; + $quad->stop_quad; done_testing(); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 67fcc29dc2..8729fbd744 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -288,16 +288,6 @@ UT_TEST(test_gcs_block_tag_is_standard_buffer_tag_20b) } -UT_TEST(test_dead_static_master_materialization_gate_scope_matches_thread_recovery) -{ - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(false, true, true, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, false, true, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, false, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, true, 2)); - UT_ASSERT(cluster_thread_recovery_materialization_gate_enabled(true, true, true, 1)); -} - - /* spec-5.2 D2 (U3): pure master-side decision for an X-held N→S read. * node0 = holder/master in DIRECT, node1 = requester. */ UT_TEST(test_xheld_read_ship_decision_truth_table) @@ -525,7 +515,7 @@ UT_TEST(test_clean_xfer_stale_break_predicate) int main(void) { - UT_PLAN(22); + UT_PLAN(21); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -541,7 +531,6 @@ main(void) UT_RUN(test_gcs_block_data_size_equals_blcksz); UT_RUN(test_gcs_block_msg_type_enum_extends_without_gap); UT_RUN(test_gcs_block_tag_is_standard_buffer_tag_20b); - UT_RUN(test_dead_static_master_materialization_gate_scope_matches_thread_recovery); UT_RUN(test_xheld_read_ship_decision_truth_table); UT_RUN(test_forward_payload_read_image_flag_roundtrip); UT_RUN(test_clean_page_xfer_eligible_flag_roundtrip_and_orthogonal); From 0d57fa90bd6500743988b9653735330a5660bbcc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:58 +0800 Subject: [PATCH 06/13] fix(cluster): remaster retry ordering, runtime-structural warning, upgrade cleanup - Order the worker's terminal-result store after its backoff-deadline store (pg_memory_barrier) so the relaunch decider never pairs a fresh BLOCKED with a stale deadline and skips one backoff wait. - A structural cause discovered only by the WORKER (SIGHUP race) now leaves the retry deadline unset so the FSM's next tick takes the MARK_STRUCTURAL branch and emits the once-per-episode operator WARNING with the configuration hint. - Wrap the local S->X upgrade in PG_TRY and release the temporary S claim when the ACK wait throws (cancel via CHECK_FOR_INTERRUPTS), not only on the false return; document why the completed upgrade hard-resets the local S refcount (the X grant subsumes all local S declarations). Linker-only exception stubs for the units that link cluster_pcm_lock.o. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R5/R6/R7/R9) --- src/backend/cluster/cluster_hw_remaster.c | 19 ++++++++++++- src/backend/cluster/cluster_pcm_lock.c | 27 +++++++++++++++++-- .../test_cluster_bufmgr_pcm_hook.c | 11 ++++++++ src/test/cluster_unit/test_cluster_pcm_lock.c | 13 +++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index c585268a42..03ee6751a3 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -95,8 +95,16 @@ hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) static void hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) { + /* Amendment v1.2 (R6): the worker stores next_attempt_at BEFORE the + * terminal result, and the LMON relaunch decision reads them in the + * opposite order (result first). Order the two stores so a decider + * that observes the terminal result also observes the matching backoff + * deadline — without the fence a weakly-ordered CPU could let it pair a + * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait + * (bounded self-correcting, but cheap to close outright). */ if (res == CLUSTER_HW_REMASTER_DONE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_done(); } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { @@ -104,14 +112,23 @@ hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) cluster_hw_remaster_set_next_attempt_at(dead_node, hw_remaster_next_attempt_deadline(attempts)); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_blocked(); } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { - cluster_hw_remaster_set_next_attempt_at(dead_node, CLUSTER_HW_REMASTER_NO_DEADLINE); + /* Amendment v1.2 (R7): leave next_attempt_at at 0 (not NO_DEADLINE) + * so the FSM's next tick takes the MARK_STRUCTURAL decision branch + * and emits the episode-once operator WARNING + errhint — covering + * the SIGHUP race where only the WORKER (not the launch-side + * precheck) discovers the structural cause. MARK_STRUCTURAL then + * stamps NO_DEADLINE, making the WARNING once-per-episode. */ + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_blocked(); } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); } } diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index b9de7f6401..88ddaad4cf 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -2221,7 +2221,10 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) errmsg("block-level cache protocol state is being rebuilt " "after reconfiguration"), errhint("The block resource is recovering (survivor re-declare / " - "master rebuild after node failure); retry the transaction."))); + "master rebuild after node failure); retry the transaction. " + "If the failed node stays down, restart it to run its " + "instance recovery, or enable online thread recovery in a " + "supported scope (cluster.online_thread_recovery)."))); CHECK_FOR_INTERRUPTS(); pgstat_report_wait_start(WAIT_EVENT_GCS_BLOCK_RECOVERING); @@ -2305,9 +2308,24 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { struct GrdEntry *entry; + bool upgraded = false; cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); - if (!cluster_gcs_block_local_x_upgrade(tag)) { + /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with + * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out + * of it — release the temporary S claim on that path too, not + * only on the false return. */ + PG_TRY(); + { + upgraded = cluster_gcs_block_local_x_upgrade(tag); + } + PG_CATCH(); + { + cluster_pcm_lock_release(tag); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!upgraded) { cluster_pcm_lock_release(tag); ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), @@ -2318,6 +2336,11 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) entry = pcm_find_entry(tag); if (entry != NULL) { pcm_entry_lock_exclusive(entry); + /* Amendment v1.2 (R9): the completed S->X upgrade replaces + * ALL of this node's S declarations for the tag (the X + * grant subsumes them), so the local S refcount is + * intentionally hard-reset rather than decremented — a + * later release of the X does not owe any S releases. */ entry->s_holder_refcount_local = 0; LWLockRelease(&entry->entry_lock.lock); } diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 540cff5b7d..90151fc3cb 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -181,6 +181,17 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): linker-only exception-stack stubs (the + * linked cluster_pcm_lock.o now wraps the S->X upgrade in PG_TRY). */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c02894306f..08ba568886 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -134,6 +134,19 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): the S->X upgrade is now wrapped in + * PG_TRY/PG_CATCH, which references the exception stack + re-throw. The + * unit's errfinish mock longjmps to its own buffer (never through + * PG_exception_stack), so these exist for the linker only. */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { From 0155eb94975bae86fc2f385eb119d085b885884f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:17 +0800 Subject: [PATCH 07/13] fix(cluster): re-consume same-epoch dead-set growth mid recovery episode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WAIT_BARRIER / WAIT_CLUSTER carried only epoch-coherence guards, so a survivor whose CSSD detected a coordinator-folded multi-death late kept announcing REDECLARE_DONE under its stale dead-bitmap hash: peers that stamped the full set dropped those frames on the composite key and P6 — which has no timeout by design — wedged permanently (survivor-behind detection skew). Add a mid-episode guard that aborts to IDLE on a newer local event whose dead-bitmap hash differs: the next tick re-consumes the event, re-stamps recovery_event_bitmap_hash, re-runs recovery against the fuller dead set and re-announces DONE under the new composite key. Same-set dead_generation drift is absorbed without episode churn (the event_id keeps its local ABA-scoping role). The coordinator-behind direction needs no handling: the coordinator's own later detection bumps the epoch again and the epoch guards abort. Correct the stamp-site / accounting / hash-helper comments that overstated the bitmap as quorum-ratified (each node stamps its OWN accepted event's bitmap; convergence is eventual via the CSSD deadband plus this guard), and assert a stamped episode hash is never 0 — 0 is reserved as unstamped in the mark_peer_done drop test, and a JOIN episode's all-zero bitmap hashes to a fixed nonzero constant. Unit: survivor-behind growth leg (H({1}) stamped at the folded epoch, grown {1,2} event arrives -> abort, re-stamp, full-set remaster, DONE accounting converges) plus a same-set drift no-churn leg; both go red with the guard disabled. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P2-2, r3-P3a) --- src/backend/cluster/cluster_grd.c | 118 ++++++++++++++++++-- src/test/cluster_unit/test_cluster_grd.c | 134 ++++++++++++++++++++++- 2 files changed, 241 insertions(+), 11 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index ae87b62dfa..3df1166222 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -1723,12 +1723,16 @@ cluster_grd_recovery_event_bitmap_hash_value(void) * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). * * The cross-node half of the REDECLARE_DONE convergence key: a hash over - * the quorum-accepted dead bitmap ALONE. Same kernel as the event_id hash + * the sender's ACCEPTED dead bitmap ALONE (each node's own local event — + * there is no cross-node ratification of the bitmap; see the P0 stamp-site + * note and the r3-P2-2 mid-episode re-consume guard for how detection skew + * converges). Same kernel as the event_id hash * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — * the generation is per-instance observation history and does NOT converge * across survivors, which is exactly why event_id cannot key the P6 gate. - * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere, - * so the composite key degrades to the epoch-only gate there. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere + * (a fixed NONZERO constant — 0 stays reserved for "unstamped"), so the + * composite key degrades to the epoch-only gate there. */ uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) @@ -2126,11 +2130,23 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap * differs (a coordinator re-election adds the old coordinator to it; a * re-death of the same node rides a higher epoch via the interposed JOIN * bump), so the bitmap hash is the ABA guard. Flap-history drift changes - * only the per-instance dead_generation, never the quorum dead SET, so - * DONEs for the same episode always match here (the R2 wedge fix). + * only the per-instance dead_generation, never the sender's accepted dead + * SET, so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently stamps (and announces) a SMALLER set's hash + * at the same epoch; those frames are dropped HERE until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + re-announces + * the full set (grd_recovery_consume_new_event_mid_episode). Per-tick + * re-announce then converges the accounting. * Pre-accept frames mismatch the previous episode's stamp and are dropped; * senders re-announce every tick, so accounting lands after our P0 accept * (which also closes the R4 pre-accept window by construction). + * + * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own + * recovery_event_bitmap_hash is 0 before the first P0 accept). This + * relies on a stamped hash never being 0: JOIN episodes hash an ALL-ZERO + * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero + * constant (asserted at the stamp site). */ episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) @@ -2172,6 +2188,61 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +/* + * grd_recovery_consume_new_event_mid_episode — spec-4.6a r3-P2-2. + * + * WAIT_BARRIER / WAIT_CLUSTER guard against a NEWER local reconfig event at + * the SAME epoch. Multi-death detection skew: when two nodes die near- + * simultaneously the coordinator can fold both into ONE epoch bump with dead + * set {A,B}, while a survivor whose CSSD saw only {B} first stamps + * recovery_event_bitmap_hash = H({B}) and sails past WAIT_EPOCH on that + * bump. Its later local detection of A publishes a new event with dead set + * {A,B} but does NOT move the epoch, so the WAIT_BARRIER / WAIT_CLUSTER + * epoch guards never fire, the survivor keeps announcing DONE under H({B}), + * every peer that stamped H({A,B}) keeps dropping it (composite-key + * mismatch), and P6 — which has no timeout by design — wedges permanently. + * + * Disposition (runs in the single-writer LMON tick, after the epoch guard): + * - new event, FAIL direction, SAME dead-bitmap hash: only the sender- + * local dead_generation fold moved (flap drift re-fired the same quorum + * dead set). Absorb the event_id — the episode semantics (epoch, dead + * set) are unchanged, so re-running P1-P7 would be pure churn. + * Absorbing also keeps the IDLE dedup from re-consuming the event after + * this episode completes. + * - new event with a DIFFERENT dead-bitmap hash (grown dead set — the + * skew above — or a JOIN/FAIL interleave): abort to IDLE. The next + * tick re-consumes the current event, re-stamps + * recovery_event_bitmap_hash, re-runs recovery against the fuller dead + * set, and re-announces DONE under the new composite key — closing the + * survivor-behind wedge. The coordinator-behind direction needs no + * handling here: the coordinator's own later detection bumps the epoch + * again and the existing epoch guards abort. + * + * Returns true when the caller must return (episode aborted to IDLE). + */ +static bool +grd_recovery_consume_new_event_mid_episode(void) +{ + ReconfigEvent latest; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id == 0 + || latest.event_id == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + return false; + + if (latest.reconfig_kind == (uint8)RECONFIG_KIND_FAIL_STOP + && pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_FAIL + && cluster_grd_dead_bitmap_hash(latest.dead_bitmap) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { + pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, latest.event_id); + return false; + } + + grd_recovery_abort_to_idle(); + return true; +} + static void grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) { @@ -2471,12 +2542,28 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } - /* Amendment v1.2 (R2): stamp the composite convergence key's cross-node - * half. Hash over the accepted dead bitmap alone — every survivor that - * accepted the same quorum dead SET computes the same value regardless - * of its private dead_generation observation history. */ + /* Amendment v1.2 (R2) + r3-P2-2: stamp the composite convergence key's + * cross-node half. The hash is over THIS node's OWN accepted event's + * dead bitmap — there is NO cross-node ratification of the bitmap (the + * envelope propagates only the epoch). Convergence is eventual, not + * instantaneous: every survivor's CSSD deadband converges on the same + * dead set, and until it does, a survivor that accepted a SMALLER set + * at the same epoch stamps a different hash. The mid-episode + * re-consume guard (grd_recovery_consume_new_event_mid_episode) closes + * that skew: the survivor's own later detection re-stamps the full + * set's hash and re-announces DONE. dead_generation drift alone never + * changes the bitmap, so same-set re-fires hash identically. */ pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); + /* r3-P3(a) — 0 is reserved as "unstamped/pre-accept" in + * mark_peer_done's drop test, so a stamped episode hash must never be + * 0. JOIN episodes hash an ALL-ZERO dead bitmap: the invariant is + * that hash_bytes_extended(zero[16], 0) != 0 (a fixed nonzero + * constant). For FAIL bitmaps a zero hash is a 2^-64 collision, the + * same trust level as the event_id hash; the Assert turns the + * otherwise-undebuggable "all DONEs dropped" wedge into a loud stop + * on cassert builds. */ + Assert(pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash) != 0); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2749,6 +2836,12 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — a newer local event at the SAME epoch (multi-death + * detection skew grew the dead set) re-consumes; same-set drift is + * absorbed without churn. */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — advance the survivor block re-declare scan one chunk * while the GES rebind barrier is still pending (worker-centric, runs * in this tick; epoch-coherent via the guard just above). */ @@ -2850,6 +2943,13 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — same-epoch dead-set growth re-consumes here too: this + * node may already have announced DONE under the stale bitmap hash; + * the re-run re-announces under the full set's hash so peers' + * composite-key accounting converges (P6 has no timeout). */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — keep advancing the block re-declare scan after the GES * rebind barrier completed, so a large pool is fully re-declared within * the recovery window (no-op once the cursor reaches NBuffers). */ diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2e4b813e17..981aaa967e 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -4201,6 +4201,131 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); } +/* r3-P2-2 — survivor-behind multi-death detection skew. The coordinator + * folded {1,2} into ONE epoch bump; this survivor accepted only {1} first and + * stamped H({1}) at that epoch. Its later local detection of node 2 publishes + * a new event with the GROWN set at the SAME epoch (no further bump), which + * pre-fix was never re-consumed past WAIT_EPOCH: DONE hashes could never match + * and P6 (no timeout) wedged permanently. The mid-episode guard must abort to + * IDLE, re-consume, re-stamp the full set's hash, re-run the remaster against + * the fuller dead set, and let full-set DONEs account. */ +UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint8 grown[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 h_small; + uint64 h_grown; + int i; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline (old = 10). */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Event 1: this survivor's CSSD has seen only node 1 dead so far. */ + ut_mock_last_event.event_id = 601; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x02; /* {1} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> parks in WAIT_EPOCH (cur==old) */ + h_small = cluster_grd_recovery_event_bitmap_hash_value(); + + /* The coordinator's single folded bump arrives via piggyback: this node + * sails past WAIT_EPOCH with the SMALL set stamped. */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER, episode epoch 11 */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); + + /* The wedge shape: a full-set DONE from the coordinator is dropped while + * this node's stamp is behind. */ + memset(grown, 0, sizeof(grown)); + grown[0] = 0x06; /* {1,2} */ + h_grown = cluster_grd_dead_bitmap_hash(grown); + UT_ASSERT_NE(h_grown, h_small); + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 602; + ut_mock_last_event.dead_bitmap[0] = 0x06; + cluster_grd_recovery_lmon_tick(); /* WAIT_BARRIER guard -> abort to IDLE */ + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed + 1); + + /* Re-consume: re-stamp to the FULL set's hash, re-run the remaster against + * the fuller set (nothing may stay mastered by 1 OR 2), land back in + * WAIT_BARRIER under the same epoch. */ + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_grown); + for (i = 0; i < PGRAC_GRD_SHARD_COUNT; i++) + UT_ASSERT_EQ(cluster_grd_shard_master(i), (int32)0); + + /* The coordinator's per-tick full-set DONE re-announce now accounts. */ + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), h_grown); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 no-regression leg — dead_generation drift re-fires the SAME dead + * set under the same epoch (new event_id, identical bitmap). The guard must + * ABSORB it: no abort-to-IDLE churn, stamp unchanged, and the event_id is + * consumed so the post-episode IDLE dedup does not re-run the episode. */ +UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint64 h_set; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle baseline (old = 10) */ + + ut_mock_last_event.event_id = 611; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; /* {2} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + h_set = cluster_grd_recovery_event_bitmap_hash_value(); + + /* Same-set re-fire (drift): absorbed, no episode churn. The tick may + * legitimately advance the episode (WAIT_BARRIER -> WAIT_CLUSTER with the + * trivial unit barrier) — the assertion is that it never falls back to + * IDLE and the stamp/counters do not move. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 612; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed); /* no abort */ + UT_ASSERT_NE(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_set); + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 612); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + int /* cppcheck-suppress constParameter @@ -4212,8 +4337,9 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * spec-6.3a:+6 lifecycle; 5.8 D1b:+4 (U2a-d); D1c:+2 (U3a-b); * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); - * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(81); + * +1 (U17 cross-episode fence Hardening); + * spec-4.6a r3-P2-2:+2 (same-epoch dead-set growth re-stamp + no-churn). */ + UT_PLAN(83); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4318,6 +4444,10 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); + /* spec-4.6a r3-P2-2 — same-epoch multi-death detection-skew closure. */ + UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); + UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 2a35cb2f8c2a1ebe0c0ae23e6efbdc3c6f162987 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:32 +0800 Subject: [PATCH 08/13] fix(cluster): pair the HW remaster terminal-state fence with a read barrier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hw_remaster_record_terminal stores the backoff deadline before the terminal result with a release fence between the stores, but the LMON relaunch decider loaded result-then-deadline with no read-side pairing: on a weakly-ordered CPU it could still pair a fresh BLOCKED with a stale zero deadline and skip one backoff wait. Insert pg_read_barrier() between the result and deadline loads (attempts needs no fence — it is written only by the single-writer LMON FSM) and cross-reference the pairing at the write site. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P3b) --- src/backend/cluster/cluster_hw_remaster.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index 03ee6751a3..f2231b59b0 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -101,7 +101,9 @@ hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) * that observes the terminal result also observes the matching backoff * deadline — without the fence a weakly-ordered CPU could let it pair a * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait - * (bounded self-correcting, but cheap to close outright). */ + * (bounded self-correcting, but cheap to close outright). r3-P3(b): + * paired with the pg_read_barrier() between the result and deadline + * loads in cluster_hw_remaster_launch_workers (acquire/release pair). */ if (res == CLUSTER_HW_REMASTER_DONE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); pg_memory_barrier(); @@ -657,6 +659,16 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod } result = cluster_hw_remaster_result(node); + /* r3-P3(b) — read-side pairing for the hw_remaster_record_terminal + * write fence (deadline stored BEFORE the terminal result, with + * pg_memory_barrier between). Load the result FIRST, fence, then the + * deadline: a decider that observed a terminal result is then + * guaranteed to observe the matching backoff deadline. Without this + * the release fence is one-sided and a weakly-ordered reader could + * still pair a fresh BLOCKED with a stale (zero) deadline. attempts + * needs no fence: it is written only by this FSM (single-writer + * LMON). */ + pg_read_barrier(); attempts = cluster_hw_remaster_attempts(node); next_attempt_at = cluster_hw_remaster_next_attempt_at(node); d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, From bf11f6a29099d145ec9784330477e88b4c1a09a7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:32 +0800 Subject: [PATCH 09/13] docs(cluster): ground the dead-master unseal predicate safety proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The serve-gate unseal (static master no longer CSSD-DEAD -> NORMAL) is heartbeat liveness, not a direct recovery-completion signal. Document the traced proof of why returning NORMAL is nevertheless safe: CSSD — the only heartbeat sender — is spawned by the phase-4 driver at the PM_RUN transition, after the startup process completed the node's crash recovery against shared storage; and even under a stale-ALIVE view a fetch cannot complete against a still-recovering node, because the master-side handler default-denies until the node is an in-quorum MEMBER and only the (equally post-PM_RUN) QVOTEC process can establish the quorum lease after a restart wiped shmem. Deny replies surface as bounded 53R9L, endpoint unavailability as bounded 53R90, and no path falls back to a silent local storage read. Replaces the previous wording that asserted recovery-completion semantics without grounding. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P2-1 prove-safe) --- src/backend/cluster/cluster_gcs_block.c | 37 ++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index ee200ffafe..15d0643910 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1201,6 +1201,39 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) return GCS_BLOCK_RECOVERING; } + /* + * r3-P2-1 unseal-safety proof — this predicate is heartbeat LIVENESS + * (CSSD hysteresis flips DEAD->ALIVE on heartbeat receipt alone, + * cluster_cssd.c deadband scan), NOT a direct "instance recovery + * complete" signal. It is nevertheless safe to return NORMAL here, + * because on a crash-restarted master the heartbeat source itself is + * recovery-gated: + * (1) CSSD — the only heartbeat sender — is spawned by the cluster + * phase-4 driver, which the postmaster reaper invokes only at the + * PM_RUN transition, i.e. after the startup process exited 0 and + * the node's crash recovery fully replayed its WAL thread to shared + * storage (the ServerLoop respawn is equally PM_RUN-gated). A + * still-recovering node sends NO heartbeats, so a survivor's DEAD + * verdict cannot flip back early. + * (2) Belt-and-suspenders: even under a stale-ALIVE view (fast restart + * inside the deadband), a fetch cannot complete against a + * still-recovering node. The master-side handler + * (cluster_gcs_handle_block_request_envelope) default-denies unless + * the node is an in-quorum MEMBER, and cluster_qvotec_in_quorum() + * demands QUORUM_OK plus a live lease — state only the QVOTEC + * process (phase-4 / post-PM_RUN as well) can establish after a + * restart wiped shmem. The deny replies map to bounded 53R9L; an + * unresponsive endpoint exhausts the retransmit budget into bounded + * 53R90. Neither path ever falls back to a silent local storage + * read (STORAGE_FALLBACK is a master REPLY status, not a local + * fallback). + * (3) For online_join rejoin, MEMBER additionally requires coordinator + * admission, which vets the joiner's post-recovery voting slot + * (the slot_generation != 0 readiness sub-gate). + * So heartbeat-ALIVE implies the returned master completed its own + * instance recovery: its committed WAL is on shared storage and no + * merged-materialization proof is needed for its blocks. + */ if (static_master == cluster_node_id || cluster_cssd_get_peer_state(static_master) != CLUSTER_CSSD_PEER_DEAD) return GCS_BLOCK_NORMAL; @@ -1240,7 +1273,9 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * online thread recovery cannot run (GUC off — the default — or any * >2-node deployment) the materialization authority is never published, * so a dead master's blocks stay RECOVERING until the failed node - * restarts and runs its own instance recovery: a bounded, retryable + * restarts and completes its own instance recovery (the unseal above is + * heartbeat liveness; the r3-P2-1 note explains why heartbeats imply + * recovery completion): a bounded, retryable * ERROR on the request path (53R9L), never an unproven serve. A scope * predicate must never gate a correctness proof — a committed write on * a cold block that only the dead node saw has NO other guard on this From 5b8e8ef040c500825fd696c9b51497124efda83a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:48 +0800 Subject: [PATCH 10/13] test(cluster): lock the dead-master serve gate with a must-fail 53R9L leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Amendment v1.2 (R1) serve-gate fix had no test lock: the t/362 two-outcome legs stay green under a reverted gate (their success arm accepts a NORMAL serve of a dead-master page) and no unit links the real phase decision. Add L4h: after the kill, with the dead node never restarted, once the observed survivor's own recovery episode reached IDLE, a FRESH backend on a cold survivor runs a wide pg_class/pg_attribute scan that MUST fail with exactly 53R9L — no success arm, bounded return. An out-of-scope formation never publishes the dead node's materialization proof, so every dead-master page stays RECOVERING while the node is down; the scan spans dozens of catalog pages hashed ~1/4 to the dead node and cold-reads them past the 16MB test pool, and it touches no dead-node-written rows (heap INSERTs only, no DDL), so 53R9L is the only legal outcome. Verified red-first: with the gate reverted the suite fails at exactly this leg while every other leg stays green. The episode-IDLE wait polls the pre-warmed session with the exact query shape it ran pre-kill and compares in Perl: a cast or operator added to the SQL performs fresh syscache lookups that themselves cold-read a dead-master catalog page and trip 53R9L (the very mechanism under test). Also accept the 53R51 write-fence rejection in the pre-existing two-outcome write legs: a transient lease/epoch fence right after the reconfig is an explicit, retryable fail-closed rejection inside the same honest contract. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P1-1 TAP hard leg) --- .../362_shared_catalog_4node_kill_selfheal.pl | 64 ++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl index a5f6719eea..75747a223c 100644 --- a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -11,7 +11,9 @@ # 53R9L) and equally its transaction verdicts (TT status unknown for its # xids, the visibility door). Reads and DDL are asserted on the honest # two-outcome contract (success or explicit SQLSTATE, never a hang); -# new-object DDL/writes prove P7 unfreeze. +# new-object DDL/writes prove P7 unfreeze. L4h (r3-P1-1) is the serve-gate +# LOCK: a fresh backend's wide catalog scan on a cold survivor MUST fail +# 53R9L (no success arm) while the dead node is down. # After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means # the test harness is misconfigured, and lack of convergence is a real FAIL. # @@ -284,6 +286,9 @@ sub startup_env_blocker my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); +my %grd_done_before; +$grd_done_before{$_} = bg_counter($bg{$_}, 'grd_recovery', 'remaster_done') + for @survivors; # L2: kill node3 and require every survivor to see the real DEAD edge. # Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, @@ -338,6 +343,56 @@ sub startup_env_blocker pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); } +# L4h (r3-P1-1 hard leg): with the serve gate in place, an out-of-scope +# formation (4 nodes / online_thread_recovery off) NEVER publishes the dead +# node's materialization proof, so EVERY node3-mastered page stays RECOVERING +# until node3 returns — and node3 is never restarted in this test. A wide +# catalog scan from a FRESH backend on a cold survivor therefore MUST fail +# closed with 53R9L: pg_class + pg_attribute span dozens of pages hashed ~1/4 +# to node3, and a fresh backend's full scan cold-reads pages no prior session +# pulled into the 16MB test pool. Deliberately NO success arm — a NORMAL +# serve of a dead-master page here is the exact 8.A regression this leg locks +# (reverting the serve-gate fix turns this leg red while the two-outcome legs +# stay green). Gate on the survivor's OWN episode reaching IDLE first so the +# failure surface is deterministically the post-episode materialization proof +# (53R9L), never the in-episode shard freeze (53R9I). The scan touches no +# node3-written rows (node3 ran only heap INSERTs, no DDL), so the TT +# visibility door cannot fire first: 53R9L is the ONLY legal outcome. +{ + my $cold = 2; + + # Compare in Perl, not SQL: the warm session's catcache covers exactly the + # bg_counter query shape it ran pre-kill; a cast/operator added in SQL + # would itself cold-read a dead-master catalog page and trip 53R9L. + my $idle = 0; + my $idle_deadline = time() + 90; + while (time() < $idle_deadline) + { + if (bg_counter($bg{$cold}, 'grd_recovery', 'remaster_done') + > $grd_done_before{$cold}) + { + $idle = 1; + last; + } + usleep(500_000); + } + ok($idle, "L4h: survivor node$cold GRD recovery episode reached IDLE (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($cold)->psql('postgres', + 'SELECT count(*) FROM pg_class c JOIN pg_attribute a ON a.attrelid = c.oid', + timeout => 60); + my $elapsed = time() - $t0; + isnt(defined $rc ? $rc : 0, 0, + "L4h: fresh wide catalog scan on survivor node$cold MUST fail while node$dead is down"); + like($err // '', + qr/block-level cache protocol state is being rebuilt after reconfiguration/, + 'L4h: the failure is the explicit 53R9L dead-master fail-closed gate'); + cmp_ok($elapsed, '<', 60, 'L4h: fail-closed scan returned bounded (no hang)'); + diag("L4h: scan failed closed as required: " . ($err // '(no stderr)')) + if defined $err && $err ne ''; +} + # L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an # out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a # BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a @@ -392,7 +447,7 @@ sub startup_env_blocker else { like($err // '', - qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); } cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); @@ -406,8 +461,11 @@ sub startup_env_blocker } else { + # write-fenced (53R51, spec-4.12): a transient lease/epoch fence on the + # writer right after the reconfig — an explicit, retryable fail-closed + # rejection, squarely inside the honest two-outcome contract. like($ddl_err // '', - qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); diag("L4: DDL fail-closed honestly: $ddl_err"); } From 152b5792486c6add2072f22cf8d82f860f4f8481 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 12:38:18 +0800 Subject: [PATCH 11/13] fix(cluster): feed the join fence from an unconditional done-epoch axis The spec-4.6a Amendment v1.2 (R2) composite convergence key gated both recovery_done_epoch[] and recovery_done_bitmap_hash[] behind one drop gate in cluster_grd_recovery_mark_peer_done(). A rejoining node never P0-accepts the JOIN episode as its own FSM episode (JOIN_COMMITTED is published coordinator-side only), so its local episode-hash stamp stays 0 forever; the composite gate then dropped every survivor REDECLARE_DONE, the join fence (cluster_grd_block_view_rebuilt) never lifted, and every joiner-home block access fail-closed 53R9L permanently. Nightly t/347 L4-iii (rejoin with xid_striping) and t/353 L5 (fresh insert on reborn node0) pinned the wedge. Split the two accounting axes: * done_epoch[] accounts unconditionally (monotonic-max), feeding the epoch-only join fence. Sound because REDECLARE_DONE(epoch=E) is broadcast only after grd_block_redeclare_scan_complete(E), so a DONE at E proves the sender finished its WAIT_BARRIER re-declare at E -- the fence's safety condition, independent of the sender's episode dead set. * done_bitmap_hash[] stays composite-gated for the P6 cluster gate and the WAIT_EPOCH coordinator witness, which still AND both axes and stay fail-closed on the withheld hash half (r3-P2-2 multi-death skew protection unchanged). New unit leg test_recovery_idle_joiner_accounts_done_epoch_for_fence asserts the fence lifts once survivor DONEs land while the composite half stays withheld; the stale-bitmap and dead-set-growth legs gain a negative assertion that the epoch axis alone never fires the composite escape. Spec: spec-4.6a-grd-recovery-liveness.md --- src/backend/cluster/cluster_grd.c | 68 +++++++++++------- src/test/cluster_unit/test_cluster_grd.c | 91 ++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 32 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 3df1166222..ff5119a6f7 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -2123,24 +2123,44 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap return; /* - * spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE converges on the COMPOSITE - * key (episode_epoch, dead_bitmap_hash). A stale DONE from a previous - * episode can carry the same epoch when cur==old, so epoch monotonicity - * alone is not enough — but the previous episode's dead SET necessarily - * differs (a coordinator re-election adds the old coordinator to it; a - * re-death of the same node rides a higher epoch via the interposed JOIN - * bump), so the bitmap hash is the ABA guard. Flap-history drift changes - * only the per-instance dead_generation, never the sender's accepted dead - * SET, so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the - * bitmap is NOT quorum-ratified — under multi-death detection skew a - * behind survivor transiently stamps (and announces) a SMALLER set's hash - * at the same epoch; those frames are dropped HERE until its own CSSD - * catches up and the mid-episode re-consume guard re-stamps + re-announces - * the full set (grd_recovery_consume_new_event_mid_episode). Per-tick - * re-announce then converges the accounting. - * Pre-accept frames mismatch the previous episode's stamp and are dropped; - * senders re-announce every tick, so accounting lands after our P0 accept - * (which also closes the R4 pre-accept window by construction). + * The two accounting axes feed DIFFERENT consumers and must not share + * one drop gate (nightly regression: t/347 L4-iii / t/353 L5 joiner + * write wedge): + * + * 1. done_epoch[] — monotonic-max, UNCONDITIONAL (the spec-4.6 axis; + * never regress on a delayed lower-epoch frame). Consumed epoch-only + * by the spec-5.16 D3 join fence (cluster_grd_block_view_rebuilt): a + * DONE at epoch E proves the sender completed its WAIT_BARRIER at E — + * the cluster-wide rebind plus the FULL block re-declare scan against + * then-current routing — which is the fence's safety condition + * regardless of the sender's episode dead set. Crucially, a + * rejoining node never P0-accepts the JOIN episode as its own FSM + * episode (spec-5.16 D8: JOIN_COMMITTED is coordinator-side only), so + * its local episode-hash stamp stays 0/stale forever; gating this + * axis on the composite key drops every survivor DONE on the joiner, + * its join fence never lifts, and every joiner-home block access + * fail-closes 53R9L permanently. + * + * 2. done_bitmap_hash[] — composite-gated (spec-4.6a Amendment v1.2 R2). + * Read together with done_epoch[] as the (episode_epoch, + * dead_bitmap_hash) composite key by the P6 cluster gate and the + * WAIT_EPOCH coordinator witness. A stale DONE from a previous + * episode can carry the same epoch when cur==old, but its dead SET + * necessarily differs (a coordinator re-election adds the old + * coordinator to it; a re-death of the same node rides a higher epoch + * via the interposed JOIN bump), so the hash is the ABA guard: + * mismatching frames withhold the hash half and the composite + * consumers stay fail-closed. Flap-history drift changes only the + * per-instance dead_generation, never the sender's accepted dead SET, + * so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently announces a SMALLER set's hash at the + * same epoch; those frames withhold the hash half until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + + * re-announces the full set + * (grd_recovery_consume_new_event_mid_episode). Per-tick re-announce + * then converges the accounting after our P0 accept (the R4 + * pre-accept window stays closed for the composite consumers). * * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own * recovery_event_bitmap_hash is 0 before the first P0 accept). This @@ -2148,19 +2168,17 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero * constant (asserted at the stamp site). */ - episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); - if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) - return; - - /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic - * per-event epochs the incoming value always wins, but never regress the - * published value if a delayed lower-epoch frame ever slips through. */ { uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); if (epoch > prev) pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) + return; + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 981aaa967e..360a60d159 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -4172,13 +4172,27 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 * instead of node 2 — e.g. the previous episode before a coordinator - * re-election) is dropped: nothing is accounted. */ + * re-election) withholds the composite hash half. The epoch axis + * accounts unconditionally (the spec-5.16 join-fence liveness feed — + * see cluster_grd_recovery_mark_peer_done); the composite consumers + * stay blocked, pinned by the no-escape tick below. */ memset(old_dead, 0, sizeof(old_dead)); old_dead[0] = 0x08; old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); - UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + + /* Deadline expiry with ONLY the stale-set DONE accounted: the witness + * must NOT fire (its hash conjunct is unsatisfied), and the FSM stays + * fail-closed in WAIT_EPOCH — the epoch axis alone can never unlock the + * composite escape. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_EPOCH); /* Generation-drift shape (the R2 wedge): the peer's local event_id * differs, but its DONE names the SAME quorum dead set -> the composite @@ -4189,9 +4203,11 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) /* Witness advance: deadline expired + cur==old + coordinator DONE naming * THIS dead set at exactly cur, accounted after the accept snapshot. The - * FSM takes the composite-key equal-epoch escape exactly once. */ + * FSM takes the composite-key equal-epoch escape exactly once. (The + * no-proof tick above re-armed the deadline to +rebuild_timeout, so jump + * well past it.) */ cluster_grd_recovery_counters_snapshot(&before); - ut_mock_now = (int64)60 * 1000000; + ut_mock_now = (int64)130 * 1000000; cluster_grd_recovery_lmon_tick(); cluster_grd_recovery_counters_snapshot(&after); UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); @@ -4242,15 +4258,17 @@ UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); - /* The wedge shape: a full-set DONE from the coordinator is dropped while - * this node's stamp is behind. */ + /* The wedge shape: a full-set DONE from the coordinator withholds the + * composite hash half while this node's stamp is behind (the epoch axis + * accounts unconditionally for the join-fence liveness feed; P6 still + * blocks on the missing hash). */ memset(grown, 0, sizeof(grown)); grown[0] = 0x06; /* {1,2} */ h_grown = cluster_grd_dead_bitmap_hash(grown); UT_ASSERT_NE(h_grown, h_small); cluster_grd_recovery_mark_peer_done(0, 11, h_grown); UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); - UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ cluster_grd_recovery_counters_snapshot(&c0); @@ -4326,6 +4344,62 @@ UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); } +/* Joiner-side DONE-accounting liveness (nightly t/347 L4-iii / t/353 L5 + * regression pin). A rejoining node never P0-accepts the JOIN episode as + * its own FSM episode (spec-5.16 D8: JOIN_COMMITTED is published + * coordinator-side only), so its local episode-hash stamp stays 0 forever. + * The survivors' per-tick REDECLARE_DONE re-announces are that node's ONLY + * feed for the spec-5.16 D3 join fence (cluster_grd_block_view_rebuilt); + * gating the done_epoch axis on the composite key drops every frame on the + * joiner, the fence never lifts, and every joiner-home block access + * fail-closes 53R9L until the caller's retry budget burns out. The epoch + * axis must account unconditionally — a DONE at epoch E proves the sender + * completed its WAIT_BARRIER at E (cluster-wide rebind + full block + * re-declare scan against then-current routing), which is the fence's + * safety condition regardless of the sender's episode dead set — while the + * composite hash half stays withheld pre-accept (P6 gate and WAIT_EPOCH + * witness unchanged). */ +UT_TEST(test_recovery_idle_joiner_accounts_done_epoch_for_fence) +{ + uint8 join1[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint8 zero_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + int n1[1] = { 1 }; + BufferTag tag; + uint64 join_hash; + + memset(&tag, 0, sizeof(tag)); + ut_jr_setup_3node(); /* FSM IDLE, episode hash stamp 0: never accepted */ + ut_mock_epoch = 10; + ut_jr_build_bitmap(join1, n1, 1); + cluster_grd_arm_join_pcm_fence(join1); + ut_mock_static_master = 1; /* the rejoiner's home block */ + + /* The real JOIN-episode DONE payload: hash of the all-zero dead bitmap + * (nonzero by the r3-P3(a) invariant). */ + memset(zero_bitmap, 0, sizeof(zero_bitmap)); + join_hash = cluster_grd_dead_bitmap_hash(zero_bitmap); + UT_ASSERT_NE(join_hash, 0); + + UT_ASSERT(cluster_grd_join_remaster_active_for_shard(tag)); + UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); + + /* Survivor DONEs arrive while this node's FSM is IDLE and unstamped + * (node 1 is this episode's recipient — skipped by the barrier). */ + cluster_grd_recovery_mark_peer_done(1, 10, join_hash); + cluster_grd_recovery_mark_peer_done(0, 10, join_hash); + cluster_grd_recovery_mark_peer_done(2, 10, join_hash); + + /* Epoch axis accounted -> the join fence lifts (the wedge fix)... */ + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(2), 10); + UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); + + /* ...while the composite hash half stays withheld pre-accept (v1.2 R2: + * P6 / witness consumers remain fail-closed on this node). */ + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(2), 0); +} + int /* cppcheck-suppress constParameter @@ -4448,6 +4522,9 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + /* spec-4.6a nightly regression — joiner-side DONE accounting liveness. */ + UT_RUN(test_recovery_idle_joiner_accounts_done_epoch_for_fence); + UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 589409c3f148dda7fe76ba18dede14aea612683c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 12:38:18 +0800 Subject: [PATCH 12/13] test(cluster): refresh grd dump baselines surfaced by nightly The same S-reconfig nightly that pinned the join-fence wedge also flagged two stale dump-count baselines that lagged already-shipped counters: * t/108 pg_cluster_state pcm category 22 -> 23 (spec-4.6a D12 dead_cleanup_entries). * t/249 grd_recovery dump roster 18 -> 31 keys plus the exact sorted key list (spec-4.6 2.4 base + spec-5.16 D5 join direction + spec-4.6a liveness/episode surface). Both stay exact-count / exact-roster assertions; the code already emits the 31-key grd_recovery set (verified against cluster_debug.c). Spec: spec-4.6a-grd-recovery-liveness.md --- .../cluster_tap/t/108_pcm_state_machine.pl | 4 +-- .../t/249_grd_recovery_remaster.pl | 32 ++++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index cebe221c82..dcf9ef8d0c 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '22', - 'L1 pg_cluster_state pcm category has 22 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter)'); +is($pcm_category_rows, '23', + 'L1 pg_cluster_state pcm category has 23 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter + spec-4.6a D12 dead_cleanup_entries)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( diff --git a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl index 3c66d90c53..1e0847375a 100644 --- a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl +++ b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl @@ -41,9 +41,10 @@ # L5 (D3 flipped) the rebound holders release against their # current-epoch masters and both resources are immediately # re-acquirable. Pinned gap was: release silently swallowed. -# L7 (D5 flipped) grd_recovery dump category: 13 counters, key -# roster per spec-4.6 §2.4, episode trail asserted -# (remaster_started/done, shards_remastered=2048, +# L7 (D5 flipped) grd_recovery dump category: 31 keys (spec-4.6 +# §2.4 base 13 + spec-5.16 D5 join-direction 5 + spec-4.6a +# liveness/episode surface 13), full key roster, episode trail +# asserted (remaster_started/done, shards_remastered=2048, # holders_redeclared>=1, rebuild_timeout=0). # # Discipline notes: @@ -395,22 +396,29 @@ sub poll_query_until_timeout # ---------- -# L7 (FLIPPED by D5; spec-5.16 D5 +5 join-direction counters): grd_recovery -# dump category exposes 18 counters and the recovery episode left the -# expected trail. +# L7 (FLIPPED by D5; spec-5.16 D5 +5 join-direction counters; spec-4.6a +# +13 episode/liveness keys): grd_recovery dump category exposes 31 +# keys and the recovery episode left the expected trail. # ---------- is($pair->node1->safe_psql('postgres', q{SELECT count(*) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - '18', 'L7 grd_recovery dump category exposes 18 counters (D5 + spec-5.16 join)'); + '31', + 'L7 grd_recovery dump category exposes 31 keys (spec-4.6 D5 + spec-5.16 join + spec-4.6a)'); is($pair->node1->safe_psql('postgres', q{ SELECT string_agg(key, ',' ORDER BY key) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - 'block_path_failclosed,converts_requeued,holders_rebound,holders_redeclared,' - . 'rebuild_timeout,remaster_done,remaster_failed,remaster_started,' - . 'shards_remastered,stale_holder_swept,stale_request_drop,' - . 'unaffected_holder_survived,waiters_requeued', - 'L7 grd_recovery key roster matches spec-4.6 §2.4'); + 'block_path_failclosed,block_redeclare_cursor,block_redeclare_done,' + . 'block_redeclare_epoch,cluster_gate_timeout,converts_requeued,' + . 'done_self_bitmap_hash,done_self_epoch,episode_epoch,event_coordinator,' + . 'event_old_epoch,holders_rebound,holders_redeclared,' + . 'join_block_recovering_failclosed,join_block_views_rebuilt,' + . 'join_remaster_done,join_remaster_started,join_shards_remastered,' + . 'last_event_id,rebuild_timeout,remaster_done,remaster_failed,' + . 'remaster_started,shards_remastered,stale_holder_swept,' + . 'stale_request_drop,state,state_enum_value,unaffected_holder_survived,' + . 'wait_epoch_escape,waiters_requeued', + 'L7 grd_recovery key roster (spec-4.6 §2.4 + spec-5.16 D5 + spec-4.6a)'); is($pair->node1->safe_psql('postgres', q{SELECT value::bigint >= 1 FROM cluster_dump_state() From 04ca16191e4cecf0798a5adfc153b943d17c3958 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 14:54:26 +0800 Subject: [PATCH 13/13] test(cluster): configure wal_threads_root for t/353 reborn HW remaster t/353 L5 crashes node0 and has the reborn node take a FRESH HW space lease, which requires the survivor to complete HW remaster of the dead node's shards. HW remaster reads the dead node's per-thread WAL, so on this shared_data 2-node pair it is only recoverable when cluster.wal_threads_dir is configured (cluster_hw_remaster_recoverable). The pair was created with shared_data but WITHOUT wal_threads_root, so recoverable() returned false, the remaster was marked BLOCKED_STRUCTURAL, and the reborn node's fresh-lease DDL/insert/count fail-closed (L5 subtests 28-31, nightly crossnode-b). Opt the pair into wal_threads_root, matching every other HW-remaster- dependent kill test (t/247/248/274/293). This completes the per-thread WAL harness config that the RACvsRAC/ClusterQuad harness already carries but this ClusterPair test was missing. Spec: spec-4.6a-grd-recovery-liveness.md --- src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl b/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl index 1fb21179be..49fd3e518a 100644 --- a/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl +++ b/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl @@ -117,6 +117,15 @@ sub poll_until 'd612_lease', quorum_voting_disks => 3, shared_data => 1, + # The L5 reborn leg takes a FRESH HW space lease, which needs the survivor + # to complete HW remaster of the dead node's shards. HW remaster reads the + # dead node's per-thread WAL, so on a shared_data multi-node pair it is only + # recoverable when cluster.wal_threads_dir is set + # (cluster_hw_remaster_recoverable); without it the remaster is marked + # BLOCKED_STRUCTURAL and the reborn node's fresh-lease access stays + # fail-closed. Every HW-remaster-dependent kill test opts into this + # (t/247/248/274/293). Spec: spec-4.6a-grd-recovery-liveness.md (D4). + wal_threads_root => 1, extra_conf => [ 'autovacuum = off', 'jit = off',