fix(cluster): GRD recovery liveness — same-episode remaster retry + proof-carrying WAIT_EPOCH + dead-node PCM cleanup (spec-4.6a, BUG-B/C2)

.github/workflows/nightly.yml

@@ -182,6 +182,11 @@ jobs:
           # the same commit; see docs/code-review-checklist.md).
           - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false }
           - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false }
+          # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-361 are
+          # reserved by the parallel spec-7.2 / S-xid lanes, t/363 by S-dead;
+          # L464 occupancy verified against origin branches 2026-07-08).  Own
+          # shard: 4-node formation + kill + convergence needs the wall clock.
+          - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false }
     steps:
       - name: Checkout
         uses: actions/checkout@v4

src/backend/cluster/cluster_debug.c

@@ -1195,8 +1195,30 @@ static void
 dump_grd_recovery(ReturnSetInfo *rsinfo)
 {
 	ClusterGrdRecoveryCounters c;
+	uint32 state;
 
 	cluster_grd_recovery_counters_snapshot(&c);
+	state = cluster_grd_recovery_state_value();
+	emit_row(rsinfo, "grd_recovery", "state", cluster_grd_recovery_state_name(state));
+	emit_row(rsinfo, "grd_recovery", "state_enum_value", fmt_int32((int32)state));
+	emit_row(rsinfo, "grd_recovery", "last_event_id",
+			 fmt_int64((int64)cluster_grd_recovery_last_event_id()));
+	emit_row(rsinfo, "grd_recovery", "event_old_epoch",
+			 fmt_int64((int64)cluster_grd_recovery_event_old_epoch()));
+	emit_row(rsinfo, "grd_recovery", "episode_epoch",
+			 fmt_int64((int64)cluster_grd_recovery_episode_epoch_value()));
+	emit_row(rsinfo, "grd_recovery", "event_coordinator",
+			 fmt_int32((int32)cluster_grd_recovery_event_coordinator()));
+	emit_row(rsinfo, "grd_recovery", "done_self_epoch",
+			 fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id)));
+	emit_row(rsinfo, "grd_recovery", "done_self_bitmap_hash",
+			 fmt_int64((int64)cluster_grd_recovery_done_bitmap_hash_for(cluster_node_id)));
+	emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor",
+			 fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor()));
+	emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch",
+			 fmt_int64((int64)cluster_grd_recovery_block_redeclare_epoch()));
+	emit_row(rsinfo, "grd_recovery", "block_redeclare_done",
+			 fmt_bool(cluster_grd_recovery_block_redeclare_done()));
 	emit_row(rsinfo, "grd_recovery", "remaster_started", fmt_int64((int64)c.remaster_started));
 	emit_row(rsinfo, "grd_recovery", "remaster_done", fmt_int64((int64)c.remaster_done));
 	emit_row(rsinfo, "grd_recovery", "remaster_failed", fmt_int64((int64)c.remaster_failed));
@@ -1212,6 +1234,11 @@ dump_grd_recovery(ReturnSetInfo *rsinfo)
 	emit_row(rsinfo, "grd_recovery", "unaffected_holder_survived",
 			 fmt_int64((int64)c.unaffected_holder_survived));
 	emit_row(rsinfo, "grd_recovery", "stale_holder_swept", fmt_int64((int64)c.stale_holder_swept));
+	emit_row(rsinfo, "grd_recovery", "cluster_gate_timeout",
+			 fmt_int64((int64)c.cluster_gate_timeout));
+	emit_row(rsinfo, "grd_recovery", "wait_epoch_escape", fmt_int64((int64)c.wait_epoch_escape));
+	/* spec-4.6a D12 — dead-node PCM residue cleanup (categorized under pcm). */
+	emit_row(rsinfo, "pcm", "dead_cleanup_entries", fmt_int64((int64)c.pcm_dead_cleanup_entries));
 	/* spec-5.16 D5 — join-direction remaster counters (same grd_recovery
 	 * category; no new dump category, §8 Q6-A). */
 	emit_row(rsinfo, "grd_recovery", "join_remaster_started",
@@ -2810,6 +2837,12 @@ dump_hw(ReturnSetInfo *rsinfo)
 			 fmt_int64((int64)cluster_hw_remaster_done_count()));
 	emit_row(rsinfo, "hw", "remaster_blocked_count",
 			 fmt_int64((int64)cluster_hw_remaster_blocked_count()));
+	emit_row(rsinfo, "hw", "remaster_retry_count",
+			 fmt_int64((int64)cluster_hw_remaster_retry_count()));
+	emit_row(rsinfo, "hw", "remaster_retry_exhausted_count",
+			 fmt_int64((int64)cluster_hw_remaster_retry_exhausted_count()));
+	emit_row(rsinfo, "hw", "remaster_recoverable",
+			 fmt_int64((int64)(cluster_hw_remaster_recoverable() ? 1 : 0)));
 
 	/*
 	 * spec-6.12d D-obs: space-lease counters.  bloat_ratio itself is a

src/backend/cluster/cluster_gcs_block.c

@@ -51,11 +51,12 @@
 #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */
 #include "cluster/cluster_grd.h"			 /* spec-4.6 D4 — block_path_failclosed counter */
 #include "cluster/cluster_grd_outbound.h"
-#include "cluster/cluster_membership.h"		/* spec-5.16 D3b — is_member master-side gate */
-#include "cluster/cluster_qvotec.h"			/* spec-5.16 D3b — in_quorum master-side gate */
-#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */
-#include "cluster/cluster_xnode_profile.h"	/* spec-5.59 D2/D3/D4 profiling buckets */
-#include "cluster/cluster_xnode_lever.h"	/* spec-6.12a — downgrade counters */
+#include "cluster/cluster_membership.h"		 /* spec-5.16 D3b — is_member master-side gate */
+#include "cluster/cluster_qvotec.h"			 /* spec-5.16 D3b — in_quorum master-side gate */
+#include "cluster/cluster_recovery_merge.h"	 /* spec-4.7 D5 — recovered_through redo gate */
+#include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */
+#include "cluster/cluster_xnode_profile.h"	 /* spec-5.59 D2/D3/D4 profiling buckets */
+#include "cluster/cluster_xnode_lever.h"	 /* spec-6.12a — downgrade counters */
 #include "cluster/cluster_guc.h"
 #include "cluster/cluster_inject.h"
 #include "cluster/cluster_itl.h" /* spec-5.2 D11 — active-ITL writer-transfer guard */
@@ -65,6 +66,7 @@
 #include "cluster/cluster_lmon.h"
 #include "cluster/cluster_pcm_lock.h"
 #include "cluster/cluster_shmem.h"
+#include "cluster/storage/cluster_shared_fs.h"
 #include "cluster/cluster_sf_dep.h"
 #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */
 #include "common/hashfn.h"
@@ -1199,6 +1201,39 @@ cluster_gcs_block_phase_for_tag(BufferTag tag)
 		return GCS_BLOCK_RECOVERING;
 	}
 
+	/*
+	 * r3-P2-1 unseal-safety proof — this predicate is heartbeat LIVENESS
+	 * (CSSD hysteresis flips DEAD->ALIVE on heartbeat receipt alone,
+	 * cluster_cssd.c deadband scan), NOT a direct "instance recovery
+	 * complete" signal.  It is nevertheless safe to return NORMAL here,
+	 * because on a crash-restarted master the heartbeat source itself is
+	 * recovery-gated:
+	 *  (1) CSSD — the only heartbeat sender — is spawned by the cluster
+	 *      phase-4 driver, which the postmaster reaper invokes only at the
+	 *      PM_RUN transition, i.e. after the startup process exited 0 and
+	 *      the node's crash recovery fully replayed its WAL thread to shared
+	 *      storage (the ServerLoop respawn is equally PM_RUN-gated).  A
+	 *      still-recovering node sends NO heartbeats, so a survivor's DEAD
+	 *      verdict cannot flip back early.
+	 *  (2) Belt-and-suspenders: even under a stale-ALIVE view (fast restart
+	 *      inside the deadband), a fetch cannot complete against a
+	 *      still-recovering node.  The master-side handler
+	 *      (cluster_gcs_handle_block_request_envelope) default-denies unless
+	 *      the node is an in-quorum MEMBER, and cluster_qvotec_in_quorum()
+	 *      demands QUORUM_OK plus a live lease — state only the QVOTEC
+	 *      process (phase-4 / post-PM_RUN as well) can establish after a
+	 *      restart wiped shmem.  The deny replies map to bounded 53R9L; an
+	 *      unresponsive endpoint exhausts the retransmit budget into bounded
+	 *      53R90.  Neither path ever falls back to a silent local storage
+	 *      read (STORAGE_FALLBACK is a master REPLY status, not a local
+	 *      fallback).
+	 *  (3) For online_join rejoin, MEMBER additionally requires coordinator
+	 *      admission, which vets the joiner's post-recovery voting slot
+	 *      (the slot_generation != 0 readiness sub-gate).
+	 * So heartbeat-ALIVE implies the returned master completed its own
+	 * instance recovery: its committed WAL is on shared storage and no
+	 * merged-materialization proof is needed for its blocks.
+	 */
 	if (static_master == cluster_node_id
 		|| cluster_cssd_get_peer_state(static_master) != CLUSTER_CSSD_PEER_DEAD)
 		return GCS_BLOCK_NORMAL;
@@ -1221,12 +1256,8 @@ cluster_gcs_block_phase_for_tag(BufferTag tag)
 
 	/*
 	 * spec-4.7 D7 + D5 — static master is DEAD;  the block is remastered to a
-	 * live survivor (recovery-aware routing).  The survivor may SERVE only
-	 * after the dead origin's merged WAL recovery passes the redo-before-
-	 * unfreeze gate;  before that the shared-storage / re-declared version may
-	 * be stale → fail-closed RECOVERING (never a stale page).
-	 *
-	 * TWO conditions, both required (Q5):
+	 * live survivor (recovery-aware routing).  Two conditions are both
+	 * required before the on-disk version may be served (Q5):
 	 *  (a) is_materialized(origin):  the dead origin's merged replay completed
 	 *      (publish is atomic at end-of-replay with the max EndRecPtr).  This
 	 *      is the cold-block safety door — a block NO survivor observed has no
@@ -1236,12 +1267,20 @@ cluster_gcs_block_phase_for_tag(BufferTag tag)
 	 *      survivor DID observe (rebuilt pi_watermark_lsn > 0), the dead
 	 *      origin's recovered_lsn must reach that observed page_lsn — else the
 	 *      dead node wrote a version a survivor saw but whose WAL never durably
-	 *      reached us → lost-write → fail-closed.  This is the LSN comparison
-	 *      (NOT a bool), live in the serve path;  required_lsn == 0 (cold) is
-	 *      trivially covered and (a) carries the safety.
+	 *      reached us → lost-write → fail-closed.
 	 *
-	 * Once both hold → NORMAL → the re-routed survivor serves (rebuilt-from-
-	 * redeclare for held blocks, lazy minimal view for cold blocks).
+	 * spec-4.6a Amendment v1.2 (R1):  this proof is UNCONDITIONAL.  Where
+	 * online thread recovery cannot run (GUC off — the default — or any
+	 * >2-node deployment) the materialization authority is never published,
+	 * so a dead master's blocks stay RECOVERING until the failed node
+	 * restarts and completes its own instance recovery (the unseal above is
+	 * heartbeat liveness; the r3-P2-1 note explains why heartbeats imply
+	 * recovery completion): a bounded, retryable
+	 * ERROR on the request path (53R9L), never an unproven serve.  A scope
+	 * predicate must never gate a correctness proof — a committed write on
+	 * a cold block that only the dead node saw has NO other guard on this
+	 * read path (GRD freeze ends with the episode; the HW gate covers only
+	 * extend high-water marks; pd_block_scn checks ride the ship path).
 	 */
 	if (!cluster_merged_instance_is_materialized(static_master)) {
 		if (ClusterGcsBlock != NULL)

src/backend/cluster/cluster_ges.c

@@ -550,9 +550,12 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload)
 
 	/* spec-4.6 P0#3 cluster gate — fire-and-forget barrier announcement:
 	 * record and return (no work queue, no reply, no dedup;  the sender
-	 * re-announces each tick, the receiver write is a monotonic max). */
+	 * re-announces each tick, the receiver write is a monotonic max).
+	 * Amendment v1.2 (R2): the request-id field pair carries the sender's
+	 * dead_bitmap hash (composite convergence key), not an event_id. */
 	if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) {
-		cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch);
+		cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch,
+											ges_request_holder_request_id(req));
 		return;
 	}