From f28bd4e38d26507de4ab61d2dba5eda17b3f6de0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 09:00:15 +0800 Subject: [PATCH 01/58] perf(cluster): wake LMON on GES work-queue enqueue A locally-mastered GES request enqueued into the work queue had no wakeup: LMON only drained it when it woke for another reason, worst case a full heartbeat interval (<= 1s). Mirror the outbound-ring enqueue family and SetLatch the drain consumer right after publishing the slot (publish-before-signal). Spec: spec-7.2-ic-data-plane-decoupling.md (D1) --- src/backend/cluster/cluster_grd_work_queue.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/backend/cluster/cluster_grd_work_queue.c b/src/backend/cluster/cluster_grd_work_queue.c index 1f317aaa96..041d97c64b 100644 --- a/src/backend/cluster/cluster_grd_work_queue.c +++ b/src/backend/cluster/cluster_grd_work_queue.c @@ -25,6 +25,7 @@ #include "cluster/cluster_ges.h" /* GesRequestPayload (spec-5.8 D8 coupling assert) */ #include "cluster/cluster_grd_work_queue.h" +#include "cluster/cluster_lmon.h" /* PGRAC: spec-7.2 D1 enqueue wakeup */ #include "cluster/cluster_shmem.h" #include "miscadmin.h" /* IsBootstrapProcessingMode */ #include "storage/lwlock.h" @@ -119,6 +120,19 @@ cluster_grd_work_queue_enqueue(uint32 source_node_id, const void *payload, uint1 cluster_grd_work_queue_state->count++; LWLockRelease(cluster_grd_work_queue_lock); + + /* + * PGRAC: spec-7.2 D1 -- wake the drain consumer (LMON) on enqueue, + * mirroring the cluster_grd_outbound enqueue family. Without this a + * locally-mastered GES request sat in the queue until LMON woke for + * some other reason (worst case a full heartbeat interval, <= 1s), + * which made local-master lock requests slower than remote-master + * ones. Publish-before-signal: the slot is visible (released the + * LWLock above) before the wakeup fires. + * Spec: spec-7.2-ic-data-plane-decoupling.md + */ + cluster_lmon_wakeup(); + return true; } From ad764c5d08bb22b7fff98ba21ad01aaf452f47db Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 09:00:15 +0800 Subject: [PATCH 02/58] perf(cluster): lower GCS retransmit initial backoff default 100ms -> 10ms LAN-scale pacing for the fast-deny / timeout retry path: 10/20/40/80 ms (total 150 ms) instead of 100/200/400/800 ms (total 1500 ms). The per-attempt reply wait is unchanged (cluster.gcs_reply_timeout_ms), so slow-but-successful serves are unaffected; only the sleep after a fast denial (DENIED_DEDUP_FULL / DENIED_EPOCH_STALE) or reply timeout shrinks. Floor lowered 10 -> 1 ms for tuning headroom; raise the GUC on slow or congested interconnects. Spec: spec-7.2-ic-data-plane-decoupling.md (D1) --- src/backend/cluster/cluster_guc.c | 8 +++++--- src/include/cluster/cluster_guc.h | 8 +++++--- src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..6476258e0a 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -3860,9 +3860,11 @@ cluster_init_guc(void) gettext_noop("Initial backoff before retry 1 (subsequent retries double)."), gettext_noop("Exponential backoff base for GCS block-ship retransmit: " "retry 1 waits this much, retry 2 doubles, etc. Default " - "100 → 100/200/400/800 ms for N=4 retries (total 1500 ms). " - "HC97. PGC_SUSET."), - &cluster_gcs_block_retransmit_initial_backoff_ms, 100, 10, 5000, PGC_SUSET, 0, NULL, NULL, + "10 → 10/20/40/80 ms for N=4 retries (total 150 ms; LAN " + "RTT scale, spec-7.2 D1). Raise on slow or congested " + "interconnects. The per-attempt reply wait itself stays " + "cluster.gcs_reply_timeout_ms. HC97. PGC_SUSET."), + &cluster_gcs_block_retransmit_initial_backoff_ms, 10, 1, 5000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("cluster.gcs_block_dedup_max_entries", diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..70d09dc5ae 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -902,9 +902,11 @@ extern int cluster_gcs_reply_timeout_ms; * * cluster.gcs_block_retransmit_initial_backoff_ms * type: int context: PGC_SUSET - * default: 100 (min 10, max 5000) - * Backoff before retry 1. Subsequent retries double: 100 → 200 → - * 400 → 800 ms (with default max_retries=4, total backoff = 1500 ms). + * default: 10 (min 1, max 5000; spec-7.2 D1 lowered from 100/10) + * Backoff before retry 1. Subsequent retries double: 10 → 20 → + * 40 → 80 ms (with default max_retries=4, total backoff = 150 ms). + * The per-attempt reply wait stays cluster.gcs_reply_timeout_ms; + * this only paces the fast-deny / timeout retry cadence (HC96). * * cluster.gcs_block_dedup_max_entries * type: int context: PGC_POSTMASTER diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 6b489f8c64..49015e48cc 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -15,7 +15,7 @@ # L5 CLUSTER_WAIT_EVENTS_COUNT = 85 (was 83 spec-2.33) # L6 3 NEW GUC visible + defaults + contexts: # cluster.gcs_block_retransmit_max_retries PGC_SUSET 4 -# cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 100 +# cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 10 (spec-7.2 D1) # cluster.gcs_block_dedup_max_entries PGC_POSTMASTER 1024 # L7 single-shot ship workload — retransmit_attempt_count=0 # L8 inject `cluster-gcs-block-drop-reply-before-send:skip:1` → @@ -153,7 +153,7 @@ sub gcs_int # ============================================================ for my $row ( [ 'cluster.gcs_block_retransmit_max_retries', '4', 'superuser' ], - [ 'cluster.gcs_block_retransmit_initial_backoff_ms', '100', 'superuser' ], + [ 'cluster.gcs_block_retransmit_initial_backoff_ms', '10', 'superuser' ], [ 'cluster.gcs_block_dedup_max_entries', '1024', 'postmaster' ], ) { From dbd828076d15e9f4ead9f8da73d376e7ce697a03 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 09:05:38 +0800 Subject: [PATCH 03/58] perf(cluster): wake LMON when LMS publishes a READY serve slot Both READY publish sites (CR page construct + undo/verdict serve) left the finished result parked until LMON's next natural wakeup (typically 100-250ms under load, worst case one heartbeat interval). Kick the shipper right after the atomic READY store, mirroring the enqueue-side wakeup family. Publish-before-signal preserved by the existing pg_write_barrier + atomic store ordering. Spec: spec-7.2-ic-data-plane-decoupling.md (D1) --- src/backend/cluster/cluster_cr_server.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 7aafcf3e5a..2f9e1bb703 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -53,6 +53,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope */ #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" /* PGRAC: spec-7.2 D1 READY-publish wakeup */ #include "cluster/cluster_shmem.h" #include "cluster/cluster_tt_durable.h" /* resolve_by_xid (D-i4 complete scan) */ #include "cluster/cluster_tt_slot.h" /* max_recycle_horizon (D-i4 bound) */ @@ -540,6 +541,11 @@ cluster_lms_cr_drain(void) pg_write_barrier(); pg_atomic_write_u32(&slot->state, CLUSTER_LMS_CR_READY); + /* PGRAC: spec-7.2 D1 -- wake the shipper (LMON) right away; + * without this the READY result sat until LMON's next natural + * wakeup (typ. 100-250ms, worst case one heartbeat). + * Publish-before-signal: READY store above precedes the kick. */ + cluster_lmon_wakeup(); continue; } @@ -580,6 +586,9 @@ cluster_lms_cr_drain(void) pg_write_barrier(); pg_atomic_write_u32(&slot->state, CLUSTER_LMS_CR_READY); + /* PGRAC: spec-7.2 D1 -- wake the shipper (see the undo/verdict + * READY publish above for the rationale). */ + cluster_lmon_wakeup(); } } From d68d03ce042b39faab85736d8bdad149896c823e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 09:42:43 +0800 Subject: [PATCH 04/58] perf(cluster): on-demand gating for lazy-able LMON duty-chain families Classify every LMON main-loop duty (24 entries, ClusterLmonDuty) into never-lazy correctness families (fence / sweeps / clean-leave / node- remove / reconfig / GRD recovery / BOC / sinval RESET / PI-discard -- order and cadence verbatim, I47-class order writs untouched) and 8 lazy-able queue-consumption families (GES work queue / ship-ready / grd outbound / sinval out x2 / tt-hint / dedup TTL / backup) gated behind a producer-set dirty bitmask plus a >= 1 Hz floor. - producers mark_dirty at enqueue (inside the shared push helpers where one exists); dedup TTL is time-based and rides the floor; the backup protocol self-re-marks while an operation is in flight (its advance steps are fence-driven, not producer-driven) - cluster.ic_duty_lazy (bool, SIGHUP, default on) is the escape hatch; off restores run-every-iteration behavior - classification is a no-default switch (-Wswitch guards new entries) and the unit truth table pins all 24 rows (157/157 unit green) Baseline: duty chain measured ~140us/iteration avg (max 923us) on a 2-node pair; this removes the fixed per-wakeup dilution tax, it is NOT the 0.5-1s ship-latency fix (that is the wakeup family + D2 WES). Spec: spec-7.2-ic-data-plane-decoupling.md (D1) --- src/backend/cluster/cluster_backup.c | 26 ++++ src/backend/cluster/cluster_cr_server.c | 2 + src/backend/cluster/cluster_grd_outbound.c | 5 + src/backend/cluster/cluster_grd_work_queue.c | 1 + src/backend/cluster/cluster_guc.c | 13 ++ src/backend/cluster/cluster_lmon.c | 155 ++++++++++++++++--- src/backend/cluster/cluster_sinval.c | 3 + src/backend/cluster/cluster_tt_status_hint.c | 3 + src/include/cluster/cluster_guc.h | 9 ++ src/include/cluster/cluster_lmon.h | 85 ++++++++++ src/test/cluster_unit/test_cluster_lmon.c | 50 +++++- 11 files changed, 333 insertions(+), 19 deletions(-) diff --git a/src/backend/cluster/cluster_backup.c b/src/backend/cluster/cluster_backup.c index 512632c5dc..8a28d52ca8 100644 --- a/src/backend/cluster/cluster_backup.c +++ b/src/backend/cluster/cluster_backup.c @@ -1099,6 +1099,7 @@ cluster_backup_coord_request(ClusterBackupWireOp op, const char *backup_id, } cluster_backup_state->coordinator_send_pending = true; LWLockRelease(&cluster_backup_state->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ cluster_lmon_wakeup(); started_at = GetCurrentTimestamp(); @@ -1278,6 +1279,7 @@ cluster_backup_lmon_queue_peer_reply(const ClusterBackupWireAck *reply, int32 de cluster_backup_state->peer_reply_dest = dest; cluster_backup_state->peer_reply_pending = true; LWLockRelease(&cluster_backup_state->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ } static bool @@ -1301,6 +1303,7 @@ cluster_backup_lmon_start_restore_point_prepare(const ClusterBackupWireRequest * cluster_backup_lmon_prepare_started_at = cluster_backup_restore_point_fence_start(); cluster_backup_lmon_restore_point_prepare_pending = true; cluster_backup_lmon_set_prepare_pending(true); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ return false; } @@ -1712,6 +1715,7 @@ cluster_backup_request_handler(const ClusterICEnvelope *env, const void *payload cluster_backup_state->peer_command_pending = true; } LWLockRelease(&cluster_backup_state->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } @@ -1896,6 +1900,8 @@ cluster_backup_lmon_process_peer_command(void) void cluster_backup_lmon_tick(void) { + bool in_flight; + if (cluster_backup_state == NULL || !cluster_enabled) return; @@ -1906,6 +1912,26 @@ cluster_backup_lmon_tick(void) cluster_backup_lmon_advance_restore_point_prepare(); cluster_backup_lmon_send_peer_reply(); cluster_backup_maybe_auto_restore_point(); + + /* + * PGRAC: spec-7.2 D1 -- the backup protocol is multi-phase and its + * advance steps are time-driven (fence quiesce), not producer-driven. + * While any operation is in flight, self-re-mark the duty so the + * lazy gate keeps the legacy every-iteration cadence until the + * protocol settles; steady state (no backup) reaps the lazy skip. + */ + if (cluster_backup_lmon_restore_point_prepare_pending) + in_flight = true; + else { + LWLockAcquire(&cluster_backup_state->lock.lock, LW_SHARED); + in_flight = cluster_backup_state->peer_reply_pending + || cluster_backup_state->coordinator_send_pending + || cluster_backup_state->peer_command_pending + || cluster_backup_state->peer_restore_point_prepare_pending; + LWLockRelease(&cluster_backup_state->lock.lock); + } + if (in_flight) + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); } static void diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 2f9e1bb703..eb431793dc 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -545,6 +545,7 @@ cluster_lms_cr_drain(void) * without this the READY result sat until LMON's next natural * wakeup (typ. 100-250ms, worst case one heartbeat). * Publish-before-signal: READY store above precedes the kick. */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SHIP_READY); cluster_lmon_wakeup(); continue; } @@ -588,6 +589,7 @@ cluster_lms_cr_drain(void) pg_atomic_write_u32(&slot->state, CLUSTER_LMS_CR_READY); /* PGRAC: spec-7.2 D1 -- wake the shipper (see the undo/verdict * READY publish above for the rationale). */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SHIP_READY); cluster_lmon_wakeup(); } } diff --git a/src/backend/cluster/cluster_grd_outbound.c b/src/backend/cluster/cluster_grd_outbound.c index dd3e99c9e4..917168658f 100644 --- a/src/backend/cluster/cluster_grd_outbound.c +++ b/src/backend/cluster/cluster_grd_outbound.c @@ -176,6 +176,9 @@ ring_push(uint8 msg_type, uint8 origin, uint32 dest_node_id, const void *payload cluster_grd_outbound_state->ring_head = (cluster_grd_outbound_state->ring_head + 1) % PGRAC_GES_OUTBOUND_RING_CAPACITY; cluster_grd_outbound_state->ring_count++; + /* PGRAC: spec-7.2 D1 — mark the drain family dirty inside the push + * helper so every producer (current and future) is covered. */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); return true; } @@ -207,6 +210,7 @@ reply_dirty_push(uint32 dest_node_id, const void *payload, uint16 payload_len) = (cluster_grd_outbound_state->reply_dirty_head + 1) % PGRAC_GES_REPLY_DIRTY_BUDGET; cluster_grd_outbound_state->reply_dirty_count++; cluster_grd_inc_ges_reply_deferred(); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); /* spec-7.2 D1 */ } static void @@ -239,6 +243,7 @@ cleanup_dirty_push(uint8 msg_type, uint8 origin, uint32 dest_node_id, const void = (cluster_grd_outbound_state->cleanup_dirty_head + 1) % PGRAC_GES_CLEANUP_DIRTY_BUDGET; cluster_grd_outbound_state->cleanup_dirty_count++; cluster_grd_inc_ges_cleanup_deferred(); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); /* spec-7.2 D1 */ } static void diff --git a/src/backend/cluster/cluster_grd_work_queue.c b/src/backend/cluster/cluster_grd_work_queue.c index 041d97c64b..b3700265ba 100644 --- a/src/backend/cluster/cluster_grd_work_queue.c +++ b/src/backend/cluster/cluster_grd_work_queue.c @@ -131,6 +131,7 @@ cluster_grd_work_queue_enqueue(uint32 source_node_id, const void *payload, uint1 * LWLock above) before the wakeup fires. * Spec: spec-7.2-ic-data-plane-decoupling.md */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GES_WORK_QUEUE); cluster_lmon_wakeup(); return true; diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 6476258e0a..99c7aa5bf7 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -720,6 +720,7 @@ bool cluster_gcs_block_local_cache = true; * remote row-lock conflict block until the holder completes; off reverts to * the spec-3.4d fail-closed (53R98) honest degradation. */ bool cluster_tx_enqueue_wait_enabled = true; +bool cluster_ic_duty_lazy = true; /* spec-7.2 D1 duty-chain on-demand gating */ int cluster_gcs_block_dedup_max_entries = 1024; /* @@ -3855,6 +3856,18 @@ cluster_init_guc(void) &cluster_tx_enqueue_wait_enabled, true, PGC_SUSET, 0, NULL, NULL, NULL); + DefineCustomBoolVariable( + "cluster.ic_duty_lazy", + gettext_noop("Run lazy-able LMON duty-chain drains on demand instead of every iteration."), + gettext_noop("When on (default), the queue-consumption duty families in the " + "LMON main loop (outbound drains / ship-ready / sinval out / " + "tt-hint / dedup TTL / backup / GES work queue) run only when " + "their producer marked them dirty or on the >= 1 Hz floor. " + "Correctness families (fence / sweeps / reconfig / recovery) " + "always run every iteration. Off restores the run-every-" + "iteration behavior (escape hatch). spec-7.2 D1. PGC_SIGHUP."), + &cluster_ic_duty_lazy, true, PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable( "cluster.gcs_block_retransmit_initial_backoff_ms", gettext_noop("Initial backoff before retry 1 (subsequent retries double)."), diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..0c98b23c66 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -183,6 +183,7 @@ cluster_lmon_shmem_init(void) memset(cluster_lmon_state, 0, sizeof(*cluster_lmon_state)); LWLockInitialize(&cluster_lmon_state->lwlock, LWTRANCHE_CLUSTER_LMON); cluster_lmon_state->status = CLUSTER_LMON_NOT_STARTED; + pg_atomic_init_u32(&cluster_lmon_state->lazy_duty_dirty, 0); } /* @@ -625,6 +626,76 @@ cluster_lmon_wakeup(void) (void)kill(pid, SIGUSR1); } +/* + * PGRAC: spec-7.2 D1 -- duty-chain on-demand gating (§3.7 classification). + * + * See the ClusterLmonDuty enum in cluster_lmon.h for the two-class + * table. The switch below has NO default case on purpose: adding an + * enum entry without classifying it here is a -Wswitch warning (CI + * warning budget catches it), and the unit truth table pins every row. + */ +bool +cluster_lmon_duty_is_lazy(ClusterLmonDuty duty) +{ + switch (duty) { + case CLUSTER_LMON_DUTY_LIVENESS: + case CLUSTER_LMON_DUTY_FENCE: + case CLUSTER_LMON_DUTY_GRD_DEAD_SWEEP: + case CLUSTER_LMON_DUTY_GRD_RECLAIM: + case CLUSTER_LMON_DUTY_GRD_STARVATION_SWEEP: + case CLUSTER_LMON_DUTY_DEADLOCK: + case CLUSTER_LMON_DUTY_GES_REPLY_WAIT_SWEEP: + case CLUSTER_LMON_DUTY_DIRECT_LAND_ABORTS: + case CLUSTER_LMON_DUTY_LMS_NATIVE_PROBE_RETRY: + case CLUSTER_LMON_DUTY_CLEAN_LEAVE: + case CLUSTER_LMON_DUTY_NODE_REMOVE: + case CLUSTER_LMON_DUTY_RECONFIG: + case CLUSTER_LMON_DUTY_GRD_RECOVERY: + case CLUSTER_LMON_DUTY_BOC_BROADCAST: + case CLUSTER_LMON_DUTY_SINVAL_RESET_ALL: + case CLUSTER_LMON_DUTY_PI_DISCARD: + return false; + case CLUSTER_LMON_DUTY_GES_WORK_QUEUE: + case CLUSTER_LMON_DUTY_SHIP_READY: + case CLUSTER_LMON_DUTY_GRD_OUTBOUND: + case CLUSTER_LMON_DUTY_SINVAL_OUT: + case CLUSTER_LMON_DUTY_SINVAL_ACK_OUT: + case CLUSTER_LMON_DUTY_TT_HINT: + case CLUSTER_LMON_DUTY_DEDUP_TTL: + case CLUSTER_LMON_DUTY_BACKUP: + return true; + case CLUSTER_LMON_DUTY_N: + break; + } + return false; +} + +void +cluster_lmon_duty_mark_dirty(ClusterLmonDuty duty) +{ + if (cluster_lmon_state == NULL) + return; /* pre-shmem: the >= 1 Hz floor covers it */ + if (!cluster_lmon_duty_is_lazy(duty)) + return; + pg_atomic_fetch_or_u32(&cluster_lmon_state->lazy_duty_dirty, + 1u << (duty - CLUSTER_LMON_DUTY_LAZY_FIRST)); +} + +bool +cluster_lmon_duty_should_run(ClusterLmonDuty duty, bool force_all) +{ + uint32 bit; + uint32 old; + + if (!cluster_lmon_duty_is_lazy(duty)) + return true; + if (!cluster_ic_duty_lazy || force_all || cluster_lmon_state == NULL) + return true; + bit = 1u << (duty - CLUSTER_LMON_DUTY_LAZY_FIRST); + old = pg_atomic_fetch_and_u32(&cluster_lmon_state->lazy_duty_dirty, ~bit); + return (old & bit) != 0; +} + TimestampTz cluster_lmon_spawned_at(void) { @@ -911,6 +982,7 @@ LmonMain(void) int rdma_cm_fd = -1; int rdma_cq_fd = -1; TimestampTz next_heartbeat_at; + TimestampTz next_duty_floor_at; /* PGRAC: spec-7.2 D1 lazy-duty floor */ int32 self_id = cluster_node_id; int32 pi; @@ -932,6 +1004,7 @@ LmonMain(void) } next_heartbeat_at = GetCurrentTimestamp() + HEARTBEAT_INTERVAL_MS * INT64CONST(1000); + next_duty_floor_at = 0; /* PGRAC: spec-7.2 D1 — first iteration forces all duties */ for (;;) { WaitEvent ev[2 * CLUSTER_MAX_NODES + 4]; @@ -939,6 +1012,7 @@ LmonMain(void) long wait_ms; TimestampTz now; int32 i; + bool force_all_duties; CHECK_FOR_INTERRUPTS(); @@ -950,6 +1024,18 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + /* + * PGRAC: spec-7.2 D1 -- >= 1 Hz floor for the lazy duty + * families (§3.5 backstop): every lazy-able drain runs at + * least once per heartbeat interval even if its producer + * never marked it dirty. Never-lazy duties ignore this and + * run every iteration, order verbatim. + */ + now = GetCurrentTimestamp(); + force_all_duties = (now >= next_duty_floor_at); + if (force_all_duties) + next_duty_floor_at = now + HEARTBEAT_INTERVAL_MS * INT64CONST(1000); + lmon_advance_liveness_tick(); /* @@ -1007,10 +1093,14 @@ LmonMain(void) * point but the read returns false in the skeleton path because * LMS never advances past STARTING for the ownership purpose. */ - cluster_ges_lmon_drain_work_queue(); + /* PGRAC: spec-7.2 D1 — lazy-gated (producer marks dirty at + * enqueue; >= 1 Hz floor backstops a missed mark). */ + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GES_WORK_QUEUE, force_all_duties)) + cluster_ges_lmon_drain_work_queue(); /* PGRAC: spec-6.12b — ship finished CR-server results (LMS * constructed them; only LMON owns the IC connections). */ - cluster_lms_cr_ship_ready(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) + cluster_lms_cr_ship_ready(); /* * spec-5.16 (orphan-grant, Rule 8.A) — reclaim abandoned reply-wait * tombstones whose bounded TTL has elapsed. This is the documented @@ -1022,7 +1112,8 @@ LmonMain(void) * wait table full" (cap = cluster.ges_reply_wait_max_entries). */ (void)cluster_ges_reply_wait_sweep_timeout(GetCurrentTimestamp()); - cluster_grd_outbound_lmon_drain_send(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GRD_OUTBOUND, force_all_duties)) + cluster_grd_outbound_lmon_drain_send(); (void)cluster_gcs_block_lmon_drain_direct_land_aborts(); cluster_lms_native_probe_retry_tick(); @@ -1053,14 +1144,21 @@ LmonMain(void) * in LMON because LMON owns the process-local IC fds. */ cluster_scn_lmon_drain_boc_broadcast(); - cluster_sinval_drain_outbound_and_broadcast(); - cluster_sinval_drain_ack_outbound_and_send(); + /* PGRAC: spec-7.2 D1 — sinval outbound drains are lazy-gated; + * the RESET-all sentinel below stays never-lazy (order- + * sensitive protocol, §3.6). */ + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_OUT, force_all_duties)) + cluster_sinval_drain_outbound_and_broadcast(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT, force_all_duties)) + cluster_sinval_drain_ack_outbound_and_send(); cluster_sinval_broadcast_reset_all(); /* spec-3.2 D6: LMON drain cross-node TT status hint outbound. * Fire-and-forget; L172 family — only LMON owns tier1 fds. */ - cluster_tt_status_hint_drain_outbound(); - cluster_backup_lmon_tick(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_TT_HINT, force_all_duties)) + cluster_tt_status_hint_drain_outbound(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_BACKUP, force_all_duties)) + cluster_backup_lmon_tick(); /* * spec-2.34 D6 (HC93 leg a): TTL sweep of the GCS block @@ -1068,9 +1166,11 @@ LmonMain(void) * 2 × max_retransmit_window age and in-flight entries * abandoned past the same threshold. Runs in LMON context * with raw LWLock + hash_seq (per L150; no buffer pin / no - * ResourceOwner). + * ResourceOwner). PGRAC: spec-7.2 D1 — TTL/time-based, so + * floor-driven (>= 1 Hz), no producer mark. */ - cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_DEDUP_TTL, force_all_duties)) + cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); /* spec-6.12h D-h2: drain checkpoint-confirmed PI-discard write * notes and route each to the block's master (fire-and-forget; @@ -1586,8 +1686,12 @@ LmonMain(void) cluster_ic_rdma_lmon_stop(); } else { /* Stub / mock / disabled mode -- preserve spec-1.11 simple loop. */ + TimestampTz next_duty_floor_at = 0; /* PGRAC: spec-7.2 D1 lazy-duty floor */ + for (;;) { int rc; + bool force_all_duties; + TimestampTz dnow; CHECK_FOR_INTERRUPTS(); @@ -1599,6 +1703,13 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + /* PGRAC: spec-7.2 D1 — >= 1 Hz floor (see the TIER_1 loop). */ + dnow = GetCurrentTimestamp(); + force_all_duties = (dnow >= next_duty_floor_at); + if (force_all_duties) + next_duty_floor_at + = dnow + (int64)cluster_interconnect_heartbeat_interval_ms * INT64CONST(1000); + lmon_advance_liveness_tick(); /* @@ -1654,23 +1765,31 @@ LmonMain(void) * so the outbound drain is intentionally NOT hoisted (a stub * single node has no peers to send to). */ - cluster_ges_lmon_drain_work_queue(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GES_WORK_QUEUE, force_all_duties)) + cluster_ges_lmon_drain_work_queue(); (void)cluster_gcs_block_lmon_drain_direct_land_aborts(); /* PGRAC: spec-6.12b — ship finished CR-server results (LMS * constructed them; only LMON owns the IC connections). */ - cluster_lms_cr_ship_ready(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) + cluster_lms_cr_ship_ready(); - cluster_sinval_drain_outbound_and_broadcast(); - cluster_sinval_drain_ack_outbound_and_send(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_OUT, force_all_duties)) + cluster_sinval_drain_outbound_and_broadcast(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT, force_all_duties)) + cluster_sinval_drain_ack_outbound_and_send(); cluster_sinval_broadcast_reset_all(); /* spec-3.2 D6: LMON drain cross-node TT status hint outbound. * Fire-and-forget; L172 family — only LMON owns tier1 fds. */ - cluster_tt_status_hint_drain_outbound(); - cluster_backup_lmon_tick(); - - /* spec-2.34 D6 (HC93 leg a): TTL sweep GCS block dedup HTAB. */ - cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_TT_HINT, force_all_duties)) + cluster_tt_status_hint_drain_outbound(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_BACKUP, force_all_duties)) + cluster_backup_lmon_tick(); + + /* spec-2.34 D6 (HC93 leg a): TTL sweep GCS block dedup HTAB. + * PGRAC: spec-7.2 D1 — TTL/time-based, floor-driven. */ + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_DEDUP_TTL, force_all_duties)) + cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); /* spec-6.12h D-h2: drain checkpoint-confirmed PI-discard notes. */ cluster_gcs_block_pi_discard_drain(); diff --git a/src/backend/cluster/cluster_sinval.c b/src/backend/cluster/cluster_sinval.c index f96daa8850..e8f6af3df0 100644 --- a/src/backend/cluster/cluster_sinval.c +++ b/src/backend/cluster/cluster_sinval.c @@ -325,6 +325,7 @@ cluster_sinval_enqueue_batch(const SharedInvalidationMessage *msgs, int n) * tier1 TCP fds. Wake LMON immediately; its main loop drains the * outbound queue via cluster_ic_send_envelope_fanout(). */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SINVAL_OUT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); return true; } @@ -365,6 +366,7 @@ cluster_sinval_enqueue_batch_with_ack_flag(const SharedInvalidationMessage *msgs pg_atomic_write_u32(&ClusterSinvalOutbound->tail, (tail + 1) % ClusterSinvalOutbound->capacity); LWLockRelease(&ClusterSinvalOutbound->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SINVAL_OUT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); return true; } @@ -1111,6 +1113,7 @@ cluster_sinval_ack_outbound_enqueue(uint64 batch_id, int32 sender_node, uint16 s pg_atomic_write_u32(&ClusterSinvalAckOutbound->tail, next_tail); LWLockRelease(&ClusterSinvalAckOutbound->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } diff --git a/src/backend/cluster/cluster_tt_status_hint.c b/src/backend/cluster/cluster_tt_status_hint.c index 1bb4e32fe3..71de287b9b 100644 --- a/src/backend/cluster/cluster_tt_status_hint.c +++ b/src/backend/cluster/cluster_tt_status_hint.c @@ -292,6 +292,7 @@ cluster_tt_status_hint_emit(const ClusterTTStatusKey *key, ClusterTTStatus statu pg_atomic_fetch_add_u64(&ClusterTTHintCounters->emit_count, 1); /* L174 idempotent latch wake — LMON drain promptly. */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_TT_HINT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } @@ -345,6 +346,7 @@ cluster_tt_status_hint_emit_subcommitted(const ClusterTTStatusKey *child_key, LWLockRelease(&ClusterTTHintOutbound->lock.lock); pg_atomic_fetch_add_u64(&ClusterTTHintCounters->emit_count, 1); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_TT_HINT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } @@ -409,6 +411,7 @@ cluster_tt_status_hint_emit_multixact_overlay(const ClusterMultiXactKey *key, ui LWLockRelease(&ClusterMultiXactHintOutbound->lock.lock); pg_atomic_fetch_add_u64(&ClusterTTHintCounters->emit_count, 1); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_TT_HINT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index 70d09dc5ae..22e50a3aef 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -925,6 +925,15 @@ extern int cluster_gcs_block_retransmit_initial_backoff_ms; * (default on). See cluster_guc.c for semantics. */ extern bool cluster_gcs_block_local_cache; extern bool cluster_tx_enqueue_wait_enabled; /* spec-5.2 D4 */ + +/* + * cluster.ic_duty_lazy + * type: bool context: PGC_SIGHUP default: on + * spec-7.2 D1 — gate the lazy-able (queue-consumption) LMON duty + * families behind producer dirty marks + a >= 1 Hz floor. Never-lazy + * correctness families are unaffected. Off = pre-7.2 behavior. + */ +extern bool cluster_ic_duty_lazy; extern int cluster_gcs_block_dedup_max_entries; /* PGRAC: spec-4.7 D1 — bounded wait (ms) on a RECOVERING block resource diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 63efc4e045..782b4b9eab 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -98,6 +98,7 @@ #define CLUSTER_LMON_H #include "datatype/timestamp.h" +#include "port/atomics.h" #include "storage/lwlock.h" @@ -138,8 +139,92 @@ typedef struct ClusterLmonSharedState { TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ bool shutdown_requested; /* postmaster sets; LMON main loop polls + exits */ + + /* + * PGRAC: spec-7.2 D1 duty-chain on-demand gating. Dirty bitmask for + * the lazy-able (queue-consumption) duty families: producers set their + * family bit at enqueue time (fetch-or, lock-free, NOT under lwlock); + * the LMON main loop test-and-clears it to decide whether the drain + * runs this iteration. A missed bit is latency-bounded, never a + * correctness issue: the heartbeat-interval floor forces every duty + * at >= 1 Hz (cluster.ic_duty_lazy = off restores run-every-iteration). + */ + pg_atomic_uint32 lazy_duty_dirty; } ClusterLmonSharedState; +/* + * ClusterLmonDuty -- spec-7.2 D1 classification table (§3.7). + * + * EVERY tick/drain call in the LMON main-loop duty chain is enumerated + * here and classified by cluster_lmon_duty_is_lazy(): + * + * never-lazy (correctness families): fence / sweeps / reconfig / + * recovery / epoch-quorum / order-writ members (I47: dead-sweep + * MUST run BEFORE reconfig; clean-leave MUST run BEFORE reconfig). + * These run every iteration, order and cadence verbatim. + * + * lazy-able (queue-consumption families): drains whose producers + * mark_dirty at enqueue time (or that are purely TTL/time-based and + * ride the >= 1 Hz floor). Skipping an iteration only delays the + * drain, bounded by the floor. + * + * A NEW duty added to the chain without an enum entry is never-lazy by + * construction (a plain unconditional call). To make it lazy it must + * be enumerated + classified here, and the unit truth table + * (test_cluster_lmon.c) re-pinned -- that is the anti-drift gate. + */ +typedef enum ClusterLmonDuty { + /* never-lazy (correctness families) */ + CLUSTER_LMON_DUTY_LIVENESS = 0, + CLUSTER_LMON_DUTY_FENCE, + CLUSTER_LMON_DUTY_GRD_DEAD_SWEEP, /* I47: MUST run BEFORE reconfig */ + CLUSTER_LMON_DUTY_GRD_RECLAIM, + CLUSTER_LMON_DUTY_GRD_STARVATION_SWEEP, + CLUSTER_LMON_DUTY_DEADLOCK, + CLUSTER_LMON_DUTY_GES_REPLY_WAIT_SWEEP, /* 5.16 rule-8.A backstop */ + CLUSTER_LMON_DUTY_DIRECT_LAND_ABORTS, + CLUSTER_LMON_DUTY_LMS_NATIVE_PROBE_RETRY, + CLUSTER_LMON_DUTY_CLEAN_LEAVE, /* CL-I13: MUST run BEFORE reconfig */ + CLUSTER_LMON_DUTY_NODE_REMOVE, /* INV-LF1: before reconfig */ + CLUSTER_LMON_DUTY_RECONFIG, + CLUSTER_LMON_DUTY_GRD_RECOVERY, /* AFTER reconfig tick (spec-4.6) */ + CLUSTER_LMON_DUTY_BOC_BROADCAST, + CLUSTER_LMON_DUTY_SINVAL_RESET_ALL, /* order-sensitive protocol */ + CLUSTER_LMON_DUTY_PI_DISCARD, + /* lazy-able (queue-consumption families) -- keep contiguous; the + * dirty bit is (duty - CLUSTER_LMON_DUTY_LAZY_FIRST) */ + CLUSTER_LMON_DUTY_GES_WORK_QUEUE, + CLUSTER_LMON_DUTY_SHIP_READY, + CLUSTER_LMON_DUTY_GRD_OUTBOUND, + CLUSTER_LMON_DUTY_SINVAL_OUT, + CLUSTER_LMON_DUTY_SINVAL_ACK_OUT, + CLUSTER_LMON_DUTY_TT_HINT, + CLUSTER_LMON_DUTY_DEDUP_TTL, /* TTL/time-based: floor-driven, no producer */ + CLUSTER_LMON_DUTY_BACKUP, + CLUSTER_LMON_DUTY_N +} ClusterLmonDuty; + +#define CLUSTER_LMON_DUTY_LAZY_FIRST CLUSTER_LMON_DUTY_GES_WORK_QUEUE + +StaticAssertDecl(CLUSTER_LMON_DUTY_N - CLUSTER_LMON_DUTY_LAZY_FIRST <= 32, + "lazy duty families must fit the uint32 dirty bitmask"); + +/* + * Duty-gating API (spec-7.2 D1). + * + * cluster_lmon_duty_is_lazy pure classification (unit-pinned). + * cluster_lmon_duty_mark_dirty producer side, any process; lock-free + * fetch-or; no-op for never-lazy duties + * and pre-shmem calls. Call BEFORE the + * cluster_lmon_wakeup() kick. + * cluster_lmon_duty_should_run LMON loop side; true for never-lazy, + * lazy=off, or force_all (>= 1 Hz floor); + * otherwise test-and-clears the bit. + */ +extern bool cluster_lmon_duty_is_lazy(ClusterLmonDuty duty); +extern void cluster_lmon_duty_mark_dirty(ClusterLmonDuty duty); +extern bool cluster_lmon_duty_should_run(ClusterLmonDuty duty, bool force_all); + /* * Public API. diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..a2f4a7de12 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -199,6 +199,9 @@ int cluster_interconnect_heartbeat_interval_ms = 1000; int cluster_interconnect_connect_timeout_ms = 5000; int cluster_interconnect_recv_timeout_ms = 30000; +/* spec-7.2 D1 GUC (cluster_lmon_duty_should_run references it). */ +bool cluster_ic_duty_lazy = true; + #include "cluster/cluster_ic_tier1.h" int @@ -807,6 +810,50 @@ UT_TEST(test_lmon_public_symbols_linkable) UT_ASSERT_NOT_NULL((void *)LmonMain); } +/* + * spec-7.2 D1 -- duty classification truth table (§3.7). + * + * Pins EVERY ClusterLmonDuty row to its never-lazy / lazy-able class. + * Re-classifying a duty (or adding one) MUST update this table + * deliberately -- that is the anti-drift gate against a future spec + * silently sliding a correctness family into the lazy set. + */ +UT_TEST(test_lmon_duty_lazy_truth_table) +{ + /* never-lazy (correctness families) */ + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_LIVENESS)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_FENCE)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_DEAD_SWEEP)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_RECLAIM)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_STARVATION_SWEEP)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_DEADLOCK)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GES_REPLY_WAIT_SWEEP)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_DIRECT_LAND_ABORTS)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_LMS_NATIVE_PROBE_RETRY)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_CLEAN_LEAVE)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_NODE_REMOVE)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_RECONFIG)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_RECOVERY)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_BOC_BROADCAST)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SINVAL_RESET_ALL)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_PI_DISCARD)); + /* lazy-able (queue-consumption families) */ + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GES_WORK_QUEUE)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SHIP_READY)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_OUTBOUND)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SINVAL_OUT)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_TT_HINT)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_DEDUP_TTL)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_BACKUP)); + /* enum shape: lazy block is contiguous + fits the uint32 bitmask */ + UT_ASSERT_EQ(CLUSTER_LMON_DUTY_N - CLUSTER_LMON_DUTY_LAZY_FIRST, 8); + /* pre-shmem safety: mark is a no-op, should_run fails open (true) */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); + UT_ASSERT(cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GRD_OUTBOUND, false)); + UT_ASSERT(cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_FENCE, false)); +} + /* ============================================================ * Test runner @@ -815,12 +862,13 @@ UT_TEST(test_lmon_public_symbols_linkable) int main(void) { - UT_PLAN(5); + UT_PLAN(6); UT_RUN(test_lmon_status_enum_values_frozen); UT_RUN(test_lmon_shared_state_size_under_4kb); UT_RUN(test_lmon_status_to_string_lookup); UT_RUN(test_lmon_status_unknown_returns_unknown); UT_RUN(test_lmon_public_symbols_linkable); + UT_RUN(test_lmon_duty_lazy_truth_table); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 127b6ebc7e87685650d193e55a8851d949dde451 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 09:54:53 +0800 Subject: [PATCH 05/58] feat(cluster): requester-side block-ship latency histogram (16 us buckets) Sixteen log-scale buckets (500us .. 30s + inf) in ClusterGcsBlockShared, recorded at the single normal-exit funnel of the block request path (GRANTED / STORAGE_FALLBACK / READ_IMAGE completions; terminal-ereport exits lose the sample, mirroring the xp scopes). Always on -- unlike the GUC-gated xnode-profile scopes -- because this is the ruler for the ship-latency value gate (p99 < 20ms, p50 < 5ms) and the later wait- closure acceptance legs. Exposed as dump category 'gcs' keys ship_hist_us_le_ / ship_hist_us_inf. Spec: spec-7.2-ic-data-plane-decoupling.md (D6 slice) --- src/backend/cluster/cluster_debug.c | 17 ++++++ src/backend/cluster/cluster_gcs_block.c | 65 ++++++++++++++++++++++ src/include/cluster/cluster_gcs_block.h | 17 ++++++ src/test/cluster_unit/test_cluster_debug.c | 12 ++++ 4 files changed, 111 insertions(+) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..6ce21bee45 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1831,6 +1831,23 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_block_wal_flush_before_ship_count())); emit_row(rsinfo, "gcs", "block_ship_bytes_total", fmt_int64((int64)cluster_gcs_get_block_ship_bytes_total())); + /* PGRAC: spec-7.2 D6 — requester ship-latency histogram (16 rows, + * keys ship_hist_us_le_ + ship_hist_us_inf). */ + { + int hb; + + for (hb = 0; hb < CLUSTER_GCS_SHIP_HIST_BUCKETS; hb++) { + char histkey[48]; + uint64 bound = cluster_gcs_block_ship_hist_bound_us(hb); + + if (bound == UINT64_MAX) + snprintf(histkey, sizeof(histkey), "ship_hist_us_inf"); + else + snprintf(histkey, sizeof(histkey), "ship_hist_us_le_" UINT64_FORMAT, bound); + emit_row(rsinfo, "gcs", histkey, + fmt_int64((int64)cluster_gcs_block_ship_hist_count(hb))); + } + } /* spec-6.13 D8: RDMA tier3/direct-land copy path observability. */ emit_row(rsinfo, "gcs", "scratch_copy_count", fmt_int64((int64)cluster_gcs_get_scratch_copy_count())); diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..9638cf8915 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -281,10 +281,39 @@ typedef struct ClusterGcsBlockShared { * comparable version unit (per-thread WAL makes * cross-node LSN comparison meaningless) */ } pi_note_ring[CLUSTER_GCS_PI_NOTE_RING_SIZE]; + + /* PGRAC: spec-7.2 D6 — requester-side block-ship latency histogram. + * Bucketed at the single normal-exit funnel of + * cluster_gcs_send_block_request_and_wait (GRANTED / STORAGE_FALLBACK / + * READ_IMAGE completions only; ereport exits lose the sample, mirroring + * the xp scopes). This is the ruler for the spec-7.2 value gate + * (ship p99 < 20ms, p50 < 5ms) and the 7.7/7.8 wait-closure legs. */ + pg_atomic_uint64 ship_latency_hist[CLUSTER_GCS_SHIP_HIST_BUCKETS]; } ClusterGcsBlockShared; static ClusterGcsBlockShared *ClusterGcsBlock = NULL; + +/* PGRAC: spec-7.2 D6 — ship-latency histogram bucket upper bounds (us). + * 15 bounds -> 16 buckets; the last bucket is the +inf overflow. */ +static const uint64 gcs_ship_hist_bounds_us[CLUSTER_GCS_SHIP_HIST_BUCKETS - 1] + = { 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, + 200000, 500000, 1000000, 2000000, 5000000, 10000000, 30000000 }; + +/* Record one completed ship into the histogram (requester context). */ +static void +gcs_block_ship_hist_record(TimestampTz started_at) +{ + uint64 elapsed_us; + int b = 0; + + if (ClusterGcsBlock == NULL) + return; + elapsed_us = (uint64)(GetCurrentTimestamp() - started_at); + while (b < CLUSTER_GCS_SHIP_HIST_BUCKETS - 1 && elapsed_us > gcs_ship_hist_bounds_us[b]) + b++; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->ship_latency_hist[b], 1); +} static ClusterGcsBlockBackendBlock *gcs_block_backend_blocks = NULL; @@ -381,6 +410,9 @@ cluster_gcs_block_shmem_init(void) if (!found) { memset(ClusterGcsBlock, 0, sizeof(*ClusterGcsBlock)); + /* PGRAC: spec-7.2 D6 — ship-latency histogram buckets. */ + for (i = 0; i < CLUSTER_GCS_SHIP_HIST_BUCKETS; i++) + pg_atomic_init_u64(&ClusterGcsBlock->ship_latency_hist[i], 0); pg_atomic_init_u64(&ClusterGcsBlock->block_request_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->block_reply_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->block_timeout_count, 0); @@ -1330,6 +1362,8 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans ClusterXpScope xp_recv; bool xp_is_read; bool xp_is_index; + /* PGRAC: spec-7.2 D6 — ship-latency histogram start stamp. */ + TimestampTz ship_started_at; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), @@ -1381,6 +1415,11 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans else xp_idx.active = false; + /* PGRAC: spec-7.2 D6 — ship-latency histogram start stamp (always on, + * unlike the GUC-gated xp scopes above; the histogram is the value- + * gate ruler so it must not depend on profiling being enabled). */ + ship_started_at = GetCurrentTimestamp(); + cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_reserve_slot(tag, (uint8)transition_id, current_master, &request_id); @@ -1970,6 +2009,13 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans cluster_xp_end(&xp_idx); cluster_xp_end(&xp_req); + /* PGRAC: spec-7.2 D6 — record the completed ship into the latency + * histogram (GRANTED / STORAGE_FALLBACK / READ_IMAGE all delivered a + * usable page; the terminal-denied tail below ereports and loses the + * sample, mirroring the xp scopes). */ + if (granted || granted_storage_fallback || read_image) + gcs_block_ship_hist_record(ship_started_at); + /* spec-5.2 D2: GRANTED / STORAGE_FALLBACK record durable ownership (the * caller mirrors PCM state); READ_IMAGE is a one-shot non-durable read so * the caller must leave buf->pcm_state == N. */ @@ -6321,6 +6367,25 @@ cluster_gcs_get_block_ship_bytes_total(void) return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->block_ship_bytes_total) : 0; } +/* PGRAC: spec-7.2 D6 — ship-latency histogram accessors (dump + tests). */ +uint64 +cluster_gcs_block_ship_hist_bound_us(int bucket) +{ + if (bucket < 0 || bucket >= CLUSTER_GCS_SHIP_HIST_BUCKETS) + return 0; + if (bucket == CLUSTER_GCS_SHIP_HIST_BUCKETS - 1) + return UINT64_MAX; /* +inf overflow bucket */ + return gcs_ship_hist_bounds_us[bucket]; +} + +uint64 +cluster_gcs_block_ship_hist_count(int bucket) +{ + if (ClusterGcsBlock == NULL || bucket < 0 || bucket >= CLUSTER_GCS_SHIP_HIST_BUCKETS) + return 0; + return pg_atomic_read_u64(&ClusterGcsBlock->ship_latency_hist[bucket]); +} + /* PGRAC: spec-6.13 D8 — RDMA tier3/direct-land copy observability. */ uint64 cluster_gcs_get_scratch_copy_count(void) diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 06662971b9..97af1bdae1 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1788,6 +1788,23 @@ extern uint64 cluster_gcs_get_block_storage_fallback_count(void); extern uint64 cluster_gcs_get_block_master_not_holder_count(void); extern uint64 cluster_gcs_get_block_wal_flush_before_ship_count(void); extern uint64 cluster_gcs_get_block_ship_bytes_total(void); + +/* + * PGRAC: spec-7.2 D6 — requester-side block-ship latency histogram. + * + * 16 log-scale buckets in microseconds; bucket b counts completions + * with elapsed <= bound(b), last bucket is the +inf overflow. Samples + * are recorded at the single normal-exit funnel of + * cluster_gcs_send_block_request_and_wait (GRANTED / STORAGE_FALLBACK / + * READ_IMAGE); terminal-ereport exits lose the sample. This is the + * ruler for the spec-7.2 value gate (p99 < 20ms, p50 < 5ms) and the + * 7.7/7.8 wait-closure legs. dump category 'gcs', keys + * ship_hist_us_le_ + ship_hist_us_inf. + */ +#define CLUSTER_GCS_SHIP_HIST_BUCKETS 16 +extern uint64 cluster_gcs_block_ship_hist_bound_us(int bucket); +extern uint64 cluster_gcs_block_ship_hist_count(int bucket); + /* PGRAC: spec-6.13 D8 — RDMA tier3/direct-land copy observability. */ extern uint64 cluster_gcs_get_scratch_copy_count(void); extern uint64 cluster_gcs_get_live_sge_send_count(void); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..6d3537272f 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -940,6 +940,18 @@ cluster_gcs_get_block_request_count(void) { return 0; } +/* spec-7.2 D6 stubs: ship-latency histogram accessors. */ +uint64 +cluster_gcs_block_ship_hist_bound_us(int bucket) +{ + return (bucket == 15) ? UINT64_MAX : (uint64)(bucket + 1) * 1000; +} +uint64 +cluster_gcs_block_ship_hist_count(int bucket) +{ + (void)bucket; + return 0; +} uint64 cluster_gcs_get_block_reply_count(void) { From 9dcbc4f778398be9141268d58f1bd0a66597885b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 09:58:34 +0800 Subject: [PATCH 06/58] fix(cluster): sync backoff C-var initializer with lowered boot_val check_GUC_init asserts the C variable initializer matches the GUC boot_val; the D1 default lowering changed only the registration. Caught by initdb (cassert) on the first post-change cluster boot. Spec: spec-7.2-ic-data-plane-decoupling.md (D1) --- src/backend/cluster/cluster_guc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 99c7aa5bf7..2eafe04848 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -707,7 +707,7 @@ int cluster_gcs_reply_timeout_ms = 5000; * HC92 + HC97 — see cluster_guc.h for semantics. */ int cluster_gcs_block_retransmit_max_retries = 4; -int cluster_gcs_block_retransmit_initial_backoff_ms = 100; +int cluster_gcs_block_retransmit_initial_backoff_ms = 10; /* spec-7.2 D1 */ /* PGRAC: spec-4.7a D2/Q8 — hold-until-revoked node-level PCM cache kill-switch. * ON (default): the bufmgr acquire gate skips the remote master round-trip when From 7b9b713646f97226f7c0e2597da689bada7149cc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 10:05:55 +0800 Subject: [PATCH 07/58] test(cluster): gcs dump-key count baselines 67 -> 83 (ship histogram rows) The 16 ship_hist_us_* rows added by the D6 histogram slice land in dump category 'gcs'; t/110 L1 and t/112 L3 pin the category cardinality. Spec: spec-7.2-ic-data-plane-decoupling.md (D6 slice) --- src/test/cluster_tap/t/110_gcs_loopback.pl | 8 ++++---- .../cluster_tap/t/112_gcs_block_retransmit_2node.pl | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index 84faab2ede..de017655ed 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -7,7 +7,7 @@ # any wire send (HC72), so wire path coverage is effectively limited # to SQL-visible surface invariants: # -# L1 fresh cluster startup: pg_cluster_state.gcs has 67 keys +# L1 fresh cluster startup: pg_cluster_state.gcs has 83 keys # L2 api_state = "active" after postmaster phase 1 init # L3 WAIT_EVENT_GCS_REPLY_WAIT registered in pg_stat_cluster_wait_events # L4 CLUSTER_WAIT_EVENTS_COUNT == 118 (cumulative through spec-6.13) @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 67 keys. +# L1 — pg_cluster_state.gcs surface has 83 keys (spec-7.2 D6 hist). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L1 pg_cluster_state.gcs category has 67 keys (spec-2.37 D12)'); + '83', + 'L1 pg_cluster_state.gcs category has 83 keys (spec-7.2 D6)'); # L2 — api_state = "active" after postmaster phase 1 init. diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 49015e48cc..3e0545d6b2 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a) +# L3 pg_cluster_state.gcs has 83 keys (cumulative through spec-7.2 D6 hist) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) # L5 CLUSTER_WAIT_EVENTS_COUNT = 85 (was 83 spec-2.33) @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs category has 83 keys (cumulative through spec-7.2 D6 hist). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '83', + 'L3 node0 pg_cluster_state.gcs category has 83 keys'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '83', + 'L3 node1 pg_cluster_state.gcs category has 83 keys'); # ============================================================ From 784418166ff162320eeff7aaffc627f54d75e4d7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 10:48:01 +0800 Subject: [PATCH 08/58] feat(cluster): pgrac.conf per-node data_addr field (DATA-plane listener) Optional [node.N] key, host:port, validated like public_addr. Empty or absent means the node offers no DATA plane; never derived from interconnect_addr by offset (adjacent-port collisions on single-host deployments make an implicit default unsound). The consumer lands with the LMS data-plane loop; startup enforcement for multi-node clusters arrives with the plane flip so intermediate trees keep running without conf changes. ClusterNodeInfo grows by the appended field (layout-append rule), which pushes the 128-slot ClusterConf container past its old 64 KiB anchor -- budget raised to 128 KiB (shmem-only anchor, not a wire contract). Spec: spec-7.2-ic-data-plane-decoupling.md (D2a) --- src/backend/cluster/cluster_conf.c | 17 ++++++++++- src/include/cluster/cluster_conf.h | 17 +++++++++-- src/test/cluster_tap/t/013_conf.pl | 36 +++++++++++++++++++++++ src/test/cluster_unit/test_cluster_conf.c | 24 +++++++++++---- 4 files changed, 86 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_conf.c b/src/backend/cluster/cluster_conf.c index 28a9f4e27d..81d1b94999 100644 --- a/src/backend/cluster/cluster_conf.c +++ b/src/backend/cluster/cluster_conf.c @@ -78,7 +78,11 @@ PG_FUNCTION_INFO_V1(cluster_get_nodes); StaticAssertDecl(sizeof(ClusterNodeRole) == sizeof(int32), "ClusterNodeRole must be int32-sized for shmem ABI stability"); -StaticAssertDecl(sizeof(ClusterConf) <= 65536, "ClusterConf must fit in 64 KiB"); +/* PGRAC: spec-7.2 D2 — budget raised 64 -> 128 KiB when data_addr joined + * the per-node entry (128 slots x 128B pushed the old bound; the region + * is shmem-only, so the budget is an anchor against accidental bloat, + * not a wire/disk contract). cluster-conf-design.md §3.2 amended. */ +StaticAssertDecl(sizeof(ClusterConf) <= 131072, "ClusterConf must fit in 128 KiB"); /* ============================================================ @@ -478,6 +482,17 @@ apply_node_field(ClusterNodeInfo *n, const char *key, const char *value, const c strlcpy(n->public_addr, value, sizeof(n->public_addr)); return true; } + /* PGRAC: spec-7.2 D2 — optional DATA-plane listener address. Empty + * (or absent) = data plane off for this node; never derived from + * interconnect_addr (r1-F2: no port guessing). */ + if (strcmp(key, "data_addr") == 0) { + if (*value != '\0' && !validate_addr_format(value)) { + *out_err = "data_addr must be empty or in host:port form"; + return false; + } + strlcpy(n->data_addr, value, sizeof(n->data_addr)); + return true; + } if (strcmp(key, "role") == 0) { ClusterNodeRole role; diff --git a/src/include/cluster/cluster_conf.h b/src/include/cluster/cluster_conf.h index c37661392f..2a6ae754db 100644 --- a/src/include/cluster/cluster_conf.h +++ b/src/include/cluster/cluster_conf.h @@ -111,12 +111,25 @@ typedef struct ClusterNodeInfo { uint16 rdma_port; uint16 rdma_pkey; uint32 rdma_qkey; + + /* + * PGRAC: spec-7.2 D2 — DATA-plane (LMS<->LMS) listener address, + * host:port. Optional: empty means this node offers no DATA plane + * (the CONTROL plane on interconnect_addr is unaffected). Never + * derived from interconnect_addr by offset — adjacent-port + * collisions on single-host deployments make an implicit default + * unsound (spec-7.2 r1-F2), so it is declared explicitly or the + * data plane stays off (fail-closed once the GCS block family + * flips to the DATA plane). Appended per the layout-append rule. + */ + char data_addr[CLUSTER_NODE_ADDR_LEN]; } ClusterNodeInfo; /* - * Top-level container in shmem. Sized to ~64 KiB (accepted overhead - * given the simplicity it buys; see docs/cluster-conf-design.md §3.2). + * Top-level container in shmem. Sized to ~128 KiB (accepted overhead + * given the simplicity it buys; see docs/cluster-conf-design.md §3.2 — + * budget raised from 64 KiB when spec-7.2 D2 appended data_addr). */ typedef struct ClusterConf { uint32 magic; diff --git a/src/test/cluster_tap/t/013_conf.pl b/src/test/cluster_tap/t/013_conf.pl index a9200432d6..ae2c97f442 100644 --- a/src/test/cluster_tap/t/013_conf.pl +++ b/src/test/cluster_tap/t/013_conf.pl @@ -272,6 +272,42 @@ 'startup failure log includes line number context'); +# ---------- +# spec-7.2 D2: data_addr node key — valid host:port parses and the +# server starts (the DATA plane consumer lands with the LMS loop; here +# we only pin parse acceptance); malformed value is a startup FATAL. +# ---------- +unlink $conf_path; +PostgreSQL::Test::Utils::append_to_file($conf_path, <<'EOC'); +[node.0] +interconnect_addr = 10.0.0.10:6432 +data_addr = 10.0.0.10:6532 +EOC + +$node->adjust_conf('postgresql.conf', 'cluster.node_id', '0'); +unlink $node->logfile; +$node->start; +is($node->safe_psql('postgres', 'SELECT 1'), '1', + 'spec-7.2 data_addr host:port accepted at startup'); +$node->stop; + +unlink $conf_path; +PostgreSQL::Test::Utils::append_to_file($conf_path, <<'EOC'); +[node.0] +interconnect_addr = 10.0.0.10:6432 +data_addr = not-an-addr +EOC + +unlink $node->logfile; +$start_failed = !$node->start(fail_ok => 1); +ok($start_failed, + 'spec-7.2 malformed data_addr refuses startup'); + +$log = PostgreSQL::Test::Utils::slurp_file($node->logfile); +like($log, qr/data_addr must be empty or in host:port form/, + 'spec-7.2 malformed data_addr log names the key'); + + # ---------- # Recovery: remove pgrac.conf, restore cluster.node_id default, server # should come back up on the single-node fallback path. diff --git a/src/test/cluster_unit/test_cluster_conf.c b/src/test/cluster_unit/test_cluster_conf.c index a202eff583..6e9343cd42 100644 --- a/src/test/cluster_unit/test_cluster_conf.c +++ b/src/test/cluster_unit/test_cluster_conf.c @@ -247,12 +247,25 @@ UT_TEST(test_rdma_node_field_budgets) { UT_ASSERT_EQ(CLUSTER_NODE_RDMA_ADDR_LEN, 64); UT_ASSERT_EQ(CLUSTER_NODE_RDMA_GID_LEN, 40); - UT_ASSERT(sizeof(ClusterConf) <= 65536); + UT_ASSERT(sizeof(ClusterConf) <= 131072); } -UT_TEST(test_conf_size_under_64k) +/* spec-7.2 D2: budget 64 -> 128 KiB when data_addr joined the node entry + * (shmem-only anchor against accidental bloat, not a wire contract). */ +UT_TEST(test_conf_size_under_128k) { - UT_ASSERT(sizeof(ClusterConf) <= 65536); + UT_ASSERT(sizeof(ClusterConf) <= 131072); +} + +/* spec-7.2 D2: data_addr appended to ClusterNodeInfo — budget + the + * append-only layout rule (alignment preserved). */ +UT_TEST(test_data_addr_field_budget) +{ + ClusterNodeInfo n; + + UT_ASSERT_EQ(sizeof(n.data_addr), CLUSTER_NODE_ADDR_LEN); + UT_ASSERT_EQ(sizeof(ClusterNodeInfo) % 8, 0); + UT_ASSERT(sizeof(ClusterConf) <= 131072); } @@ -388,13 +401,14 @@ UT_TEST(test_cluster_enabled_linkable) int main(void) { - UT_PLAN(20); + UT_PLAN(21); UT_RUN(test_max_nodes_constant); UT_RUN(test_conf_magic_constant); UT_RUN(test_node_role_int32_sized); UT_RUN(test_node_info_alignment); UT_RUN(test_rdma_node_field_budgets); - UT_RUN(test_conf_size_under_64k); + UT_RUN(test_conf_size_under_128k); + UT_RUN(test_data_addr_field_budget); UT_RUN(test_role_to_string_primary); UT_RUN(test_role_to_string_standby_arbiter); UT_RUN(test_role_from_string_valid); From 6d6370e6e7e8510d2999d865dd6d71496f927966 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:02:29 +0800 Subject: [PATCH 09/58] feat(cluster): tier1 per-plane substrate + HELLO plane/conn_epoch bytes Introduce ClusterICPlane (CONTROL = 0, DATA = 1) and carve the single tier1 shmem region into one instance per plane. Process-local tier1 state (fd arrays, buffers, HELLO machines) is naturally per-process, so a process's plane is implied: the working alias Tier1Shmem follows tier1_my_plane (CONTROL by default; the LMS loop re-aims it via cluster_ic_tier1_set_my_plane). Every existing tier1 function -- the CONNECTED send gate included -- lands on its own plane's peers[] and listener metadata unchanged; LMON, backends and SQL observers stay on the CONTROL instance and contract. HELLO rides plane + conn_epoch in the documented-zero pad (capabilities precedent, V1 stays 64B): plane byte at offset 40, conn_epoch LE at 44-51, spec-7.3 worker bytes (41/42) reserved zero. A CONTROL HELLO with epoch 0 stays byte-identical to pre-7.2 (unit compat pin); verify paths reject cross-plane HELLOs, and a DATA HELLO with epoch 0 is refused fail-closed (INV-7.2-CONN-EPOCH substrate). Plane-selected addresses: the DATA plane binds/dials data_addr only -- no data_addr means no DATA plane (LOG + listener off; never falls back to the CONTROL address). RDMA lanes stay CONTROL-owned (DATA is TCP-only in 7.2). Spec: spec-7.2-ic-data-plane-decoupling.md (D2b) --- src/backend/cluster/cluster_ic.c | 29 +++- src/backend/cluster/cluster_ic_rdma.c | 7 +- src/backend/cluster/cluster_ic_tier1.c | 197 +++++++++++++++++++----- src/include/cluster/cluster_ic.h | 57 ++++++- src/include/cluster/cluster_ic_tier1.h | 9 ++ src/test/cluster_unit/test_cluster_ic.c | 56 +++++-- 6 files changed, 303 insertions(+), 52 deletions(-) diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index edec2444c7..e7e96e7911 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -533,7 +533,13 @@ cluster_ic_recv_exact(int32 *out_sender_node_id, void *buf, size_t bufsize, * 6 2 envelope_version (PGRAC_IC_ENVELOPE_VERSION_V1 = 1) * 8 4 source_node_id (signed int32) * 12 24 cluster_name (NUL-terminated; truncated to 23 + NUL) - * 36 28 _pad (always zero on send;ignored on receive) + * 36 4 capabilities bitmask (smart-fusion; zero = none) + * 40 1 plane (spec-7.2: 0 = CONTROL, 1 = DATA) + * 41 2 worker_id / n_workers (spec-7.3 reserved, zero in 7.2) + * 43 1 reserved (zero) + * 44 8 conn_epoch (spec-7.2: cluster epoch at connect; + * zero from pre-7.2 senders / unenforced on CONTROL) + * 52 12 _pad tail (always zero on send;ignored on receive) * ------ ----- * total 64 * @@ -574,9 +580,20 @@ ic_le_read_uint32(const uint8 *buf) | ((uint32)buf[3] << 24); } +/* PGRAC: spec-7.2 D2 — 64-bit LE helper (HELLO conn_epoch). */ +static inline void +ic_le_write_uint64(uint8 *buf, uint64 v) +{ + int i; + + for (i = 0; i < 8; i++) + buf[i] = (uint8)((v >> (8 * i)) & 0xFF); +} + void cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version, - uint16 envelope_version, int32 source_node_id, const char *cluster_name) + uint16 envelope_version, int32 source_node_id, const char *cluster_name, + ClusterICPlane plane, uint64 conn_epoch) { size_t name_len; uint32 capabilities = 0; @@ -608,6 +625,14 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version capabilities |= PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); + + /* + * PGRAC: spec-7.2 D2 — plane byte + connection epoch ride the + * documented-zero pad (capabilities precedent; V1 stays 64B). + * Offsets 41/42 stay zero (spec-7.3 worker fields). + */ + out_buf[PGRAC_IC_HELLO_PLANE_OFFSET] = (uint8)plane; + ic_le_write_uint64(out_buf + PGRAC_IC_HELLO_CONN_EPOCH_OFFSET, conn_epoch); } bool diff --git a/src/backend/cluster/cluster_ic_rdma.c b/src/backend/cluster/cluster_ic_rdma.c index 8b2fc19ff5..787351791f 100644 --- a/src/backend/cluster/cluster_ic_rdma.c +++ b/src/backend/cluster/cluster_ic_rdma.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "cluster/cluster_guc.h" #include "cluster/cluster_conf.h" +#include "cluster/cluster_epoch.h" /* PGRAC: spec-7.2 D2 HELLO conn_epoch */ #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_rdma.h" @@ -600,8 +601,12 @@ rdma_build_private_data(ClusterICRdmaPrivateData *private_data, uint8 lane_type) } private_data->lane_type = lane_type; private_data->capabilities = CLUSTER_IC_RDMA_PRIVATE_CAP_BLOCK_REPLY; + /* PGRAC: spec-7.2 D2 — RDMA lanes are CONTROL-plane owned (LMON); + * the DATA plane is TCP-only in 7.2 (RDMA data plane = spec-6.13 + * forward). conn_epoch rides along but is unenforced on CONTROL. */ cluster_ic_build_hello(private_data->hello, PGRAC_IC_HELLO_VERSION_V1, - PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, cluster_name); + PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, cluster_name, + CLUSTER_IC_PLANE_CONTROL, cluster_epoch_get_current()); } static bool diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 2ab1516045..c55f14c9ed 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -73,6 +73,7 @@ #include "cluster/cluster_conf.h" #include "cluster/cluster_elog.h" +#include "cluster/cluster_epoch.h" /* PGRAC: spec-7.2 D2 HELLO conn_epoch */ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_chunk.h" /* cluster_ic_chunk_reset_peer (spec-2.4) */ #include "cluster/cluster_ic.h" @@ -117,6 +118,22 @@ typedef struct ClusterICTier1Shmem { #define PGRAC_IC_TIER1_SHMEM_MAGIC ((uint32)0x54494331U) /* "TIC1" */ +/* + * PGRAC: spec-7.2 D2 — per-plane shmem instances inside the single + * "pgrac cluster_ic_tier1" region ([0] = CONTROL, [1] = DATA). + * + * Tier1Shmem stays the working alias and points at THIS process's + * plane instance: process-local tier1 state (fd arrays, buffers, + * HELLO state machines) is naturally per-process, so the plane a + * process owns is implied by the process — LMON = CONTROL (the + * default), LMS = DATA (set via cluster_ic_tier1_set_my_plane at + * LmsMain entry). Every existing tier1 function keeps reading + * Tier1Shmem unchanged and lands on its own plane's peers[] / + * listener metadata / CONNECTED gate. Cross-plane observers (SQL + * views) address Tier1ShmemByPlane[] explicitly. + */ +static ClusterICTier1Shmem *Tier1ShmemByPlane[CLUSTER_IC_PLANE_N] = { NULL, NULL }; +static ClusterICPlane tier1_my_plane = CLUSTER_IC_PLANE_CONTROL; static ClusterICTier1Shmem *Tier1Shmem = NULL; /* @@ -364,6 +381,7 @@ static const char * peer_addr(int32 peer_id) { const ClusterNodeInfo *n; + const char *addr; if (peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) return NULL; @@ -372,12 +390,24 @@ peer_addr(int32 peer_id) if (n == NULL) return NULL; + /* + * PGRAC: spec-7.2 D2 — plane-selected peer address. The DATA plane + * connects to the peer's declared data_addr; a peer without one + * offers no DATA plane and is unreachable on it (NULL, no connect + * attempts — never fall back to the CONTROL address: r1-F2 no + * port guessing, and mixing planes on one socket breaks the + * per-plane single-stream ordering). + */ + addr = (tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? n->data_addr : n->interconnect_addr; + if (addr[0] == '\0') + return NULL; + if (Tier1Shmem != NULL && Tier1Shmem->peers[peer_id].interconnect_addr[0] == '\0') { - strlcpy(Tier1Shmem->peers[peer_id].interconnect_addr, n->interconnect_addr, + strlcpy(Tier1Shmem->peers[peer_id].interconnect_addr, addr, sizeof(Tier1Shmem->peers[peer_id].interconnect_addr)); Tier1Shmem->peers[peer_id].node_id = peer_id; } - return n->interconnect_addr; + return addr; } @@ -388,53 +418,83 @@ peer_addr(int32 peer_id) static Size tier1_shmem_size(void) { - return MAXALIGN(sizeof(ClusterICTier1Shmem)); + /* PGRAC: spec-7.2 D2 — one instance per plane (CONTROL + DATA). */ + return CLUSTER_IC_PLANE_N * MAXALIGN(sizeof(ClusterICTier1Shmem)); } static void tier1_shmem_init(void) { bool found; + char *base; + int plane; - Tier1Shmem = (ClusterICTier1Shmem *)ShmemInitStruct("pgrac cluster_ic_tier1", - tier1_shmem_size(), &found); + base = (char *)ShmemInitStruct("pgrac cluster_ic_tier1", tier1_shmem_size(), &found); - if (!found) { - int i; + /* PGRAC: spec-7.2 D2 — carve one instance per plane; the working + * alias follows this process's plane (CONTROL unless + * cluster_ic_tier1_set_my_plane re-aims it, i.e. in LMS). */ + for (plane = 0; plane < CLUSTER_IC_PLANE_N; plane++) + Tier1ShmemByPlane[plane] + = (ClusterICTier1Shmem *)(base + plane * MAXALIGN(sizeof(ClusterICTier1Shmem))); + Tier1Shmem = Tier1ShmemByPlane[tier1_my_plane]; - memset(Tier1Shmem, 0, tier1_shmem_size()); - Tier1Shmem->magic = PGRAC_IC_TIER1_SHMEM_MAGIC; - Tier1Shmem->listener_port = -1; - Tier1Shmem->listener_pid = 0; - Tier1Shmem->listener_incarnation = 0; - - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - Tier1Shmem->peers[i].node_id = -1; - Tier1Shmem->peers[i].state = (int32)CLUSTER_IC_PEER_DOWN; - Tier1Shmem->peers[i].last_errno = 0; - Tier1Shmem->peers[i].last_connect_at = 0; - pg_atomic_init_u64(&Tier1Shmem->peers[i].heartbeat_send_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].heartbeat_recv_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].msg_send_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].msg_recv_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].bytes_send, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].bytes_recv, 0); - /* spec-2.4 D10 NEW counters */ - pg_atomic_init_u64(&Tier1Shmem->peers[i].stale_epoch_drop_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].lamport_observe_advance_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].chunk_reassembly_timeout_count, 0); - pg_atomic_init_u32(&Tier1Shmem->peers[i].chunk_reassembly_active, 0); - /* spec-2.29 D20 + D18b: hostile-spoof + epoch piggyback counters. */ - pg_atomic_init_u64(&Tier1Shmem->peers[i].unreasonable_epoch_jump_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].epoch_observe_advance_count, 0); - /* spec-2.4 v1.0.1 F3: LMON-mediated close request flag. */ - pg_atomic_init_u32(&Tier1Shmem->peers[i].close_requested, 0); + if (!found) { + memset(base, 0, tier1_shmem_size()); + + for (plane = 0; plane < CLUSTER_IC_PLANE_N; plane++) { + ClusterICTier1Shmem *s = Tier1ShmemByPlane[plane]; + int i; + + s->magic = PGRAC_IC_TIER1_SHMEM_MAGIC; + s->listener_port = -1; + s->listener_pid = 0; + s->listener_incarnation = 0; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + s->peers[i].node_id = -1; + s->peers[i].state = (int32)CLUSTER_IC_PEER_DOWN; + s->peers[i].last_errno = 0; + s->peers[i].last_connect_at = 0; + pg_atomic_init_u64(&s->peers[i].heartbeat_send_count, 0); + pg_atomic_init_u64(&s->peers[i].heartbeat_recv_count, 0); + pg_atomic_init_u64(&s->peers[i].msg_send_count, 0); + pg_atomic_init_u64(&s->peers[i].msg_recv_count, 0); + pg_atomic_init_u64(&s->peers[i].bytes_send, 0); + pg_atomic_init_u64(&s->peers[i].bytes_recv, 0); + /* spec-2.4 D10 NEW counters */ + pg_atomic_init_u64(&s->peers[i].stale_epoch_drop_count, 0); + pg_atomic_init_u64(&s->peers[i].lamport_observe_advance_count, 0); + pg_atomic_init_u64(&s->peers[i].chunk_reassembly_timeout_count, 0); + pg_atomic_init_u32(&s->peers[i].chunk_reassembly_active, 0); + /* spec-2.29 D20 + D18b: hostile-spoof + epoch piggyback counters. */ + pg_atomic_init_u64(&s->peers[i].unreasonable_epoch_jump_count, 0); + pg_atomic_init_u64(&s->peers[i].epoch_observe_advance_count, 0); + /* spec-2.4 v1.0.1 F3: LMON-mediated close request flag. */ + pg_atomic_init_u32(&s->peers[i].close_requested, 0); + } } } Assert(Tier1Shmem->magic == PGRAC_IC_TIER1_SHMEM_MAGIC); } +/* + * PGRAC: spec-7.2 D2 — re-aim the working alias at this process's plane. + * + * Called once at aux-process entry BEFORE any tier1 use (LmsMain → + * DATA). Processes that never call it stay on CONTROL, which keeps + * LMON, backends and every SQL observer on the historic instance. + */ +void +cluster_ic_tier1_set_my_plane(ClusterICPlane plane) +{ + Assert(plane >= 0 && plane < CLUSTER_IC_PLANE_N); + tier1_my_plane = plane; + if (Tier1ShmemByPlane[plane] != NULL) + Tier1Shmem = Tier1ShmemByPlane[plane]; +} + static const ClusterShmemRegion cluster_ic_tier1_region = { .name = "pgrac cluster_ic_tier1", .size_fn = tier1_shmem_size, @@ -990,10 +1050,31 @@ cluster_ic_tier1_listener_bind(void) "interconnect_addr.", cluster_node_id))); - if (!parse_host_port(self->interconnect_addr, self_host, sizeof(self_host), &self_port)) - ereport(FATAL, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("cluster_ic tier1: cannot parse interconnect_addr \"%s\"", - self->interconnect_addr))); + /* + * PGRAC: spec-7.2 D2 — plane-selected listener address. The DATA + * plane binds the node's declared data_addr; no declaration means + * this node offers no DATA plane (LOG + no listener — the caller + * treats -1 as plane-off; fail-closed enforcement arrives with + * the GCS-block plane flip). Never derived from interconnect_addr + * (r1-F2: adjacent-port collisions make offset defaults unsound). + */ + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA && self->data_addr[0] == '\0') { + ereport(LOG, (errmsg("cluster_ic tier1: no data_addr declared for node %d; " + "DATA plane disabled", + cluster_node_id))); + return -1; + } + + if (!parse_host_port((tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? self->data_addr + : self->interconnect_addr, + self_host, sizeof(self_host), &self_port)) + ereport( + FATAL, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("cluster_ic tier1: cannot parse %s \"%s\"", + (tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? "data_addr" : "interconnect_addr", + (tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? self->data_addr + : self->interconnect_addr))); fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (fd < 0) @@ -1221,8 +1302,11 @@ cluster_ic_tier1_finish_connect(int32 peer_id, int peer_fd) * already-sent prefix. */ self_cluster_name = (ClusterConfShmem != NULL) ? ClusterConfShmem->cluster_name : ""; + /* PGRAC: spec-7.2 D2 — HELLO carries this process's plane + the + * cluster epoch at connect time (INV-7.2-CONN-EPOCH substrate). */ cluster_ic_build_hello(tier1_hello_send_buf[peer_id], PGRAC_IC_HELLO_VERSION_V1, - PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, self_cluster_name); + PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, self_cluster_name, + tier1_my_plane, cluster_epoch_get_current()); tier1_hello_send_remaining[peer_id] = PGRAC_IC_HELLO_BYTES; return cluster_ic_tier1_continue_hello_send(peer_id, peer_fd); @@ -1338,6 +1422,29 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) return false; } + /* + * PGRAC: spec-7.2 D2 — plane match: a HELLO arriving on this + * process's listener/connection must claim the same plane (a + * CONTROL peer dialing the DATA port, or vice versa, is a wiring + * error; pre-7.2 senders read as plane 0 = CONTROL and are only + * acceptable on the CONTROL plane). DATA additionally requires a + * nonzero conn_epoch (INV-7.2-CONN-EPOCH: zero would defeat the + * connection-generation fence; fail-closed). + */ + if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { + peer_record_error(peer_id, 0, "08P01", "HELLO plane mismatch (peer=%d mine=%d)", + (int)cluster_ic_hello_plane(&msg), (int)tier1_my_plane); + cluster_ic_tier1_close_peer(peer_id, "HELLO plane mismatch"); + Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; + return false; + } + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA && cluster_ic_hello_conn_epoch(&msg) == 0) { + peer_record_error(peer_id, 0, "08P01", "DATA HELLO missing conn_epoch"); + cluster_ic_tier1_close_peer(peer_id, "DATA HELLO missing conn_epoch"); + Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; + return false; + } + peer_info = cluster_conf_lookup_node(msg.source_node_id); if (peer_info == NULL) { peer_record_error(peer_id, 0, "08P01", "HELLO unknown source_node_id %d", @@ -1512,6 +1619,18 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear return false; } + /* PGRAC: spec-7.2 D2 — plane match + DATA conn_epoch (see the + * named-peer verify path for the rationale). */ + if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { + ereport(LOG, (errmsg("cluster_ic tier1 HELLO plane mismatch (peer=%d mine=%d)", + (int)cluster_ic_hello_plane(&msg), (int)tier1_my_plane))); + return false; + } + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA && cluster_ic_hello_conn_epoch(&msg) == 0) { + ereport(LOG, (errmsg("cluster_ic tier1 DATA HELLO missing conn_epoch"))); + return false; + } + peer_info = cluster_conf_lookup_node(msg.source_node_id); if (peer_info == NULL) { ereport(LOG, diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 981734a896..c8b0b859f3 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -218,6 +218,21 @@ extern const ClusterICOps *ClusterICOps_Active; */ #define PGRAC_IC_HELLO_MAGIC ((uint32)0x4F4C4C48) /* "HLLO" LE */ #define PGRAC_IC_HELLO_VERSION_V1 ((uint16)1) + +/* + * PGRAC: spec-7.2 D2 — interconnect plane. CONTROL is the historic + * LMON-owned connection (heartbeat / membership / GES / everything + * registered before spec-7.2); DATA is the LMS-owned connection the + * GCS block family migrates to. Zero MUST stay CONTROL: a pre-7.2 + * HELLO carries all-zero pad bytes and must keep parsing as a + * CONTROL-plane peer. + */ +typedef enum ClusterICPlane { + CLUSTER_IC_PLANE_CONTROL = 0, + CLUSTER_IC_PLANE_DATA = 1 +} ClusterICPlane; + +#define CLUSTER_IC_PLANE_N 2 /* * spec-2.3 D3: PGRAC_IC_ENVELOPE_VERSION_V1 is now defined in * cluster_ic_envelope.h as ((uint8)1) — the authoritative @@ -228,6 +243,17 @@ extern const ClusterICOps *ClusterICOps_Active; #define PGRAC_IC_CLUSTER_NAME_MAX 24 #define PGRAC_IC_HELLO_CAPABILITIES_OFFSET 36 #define PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 ((uint32)0x00000001U) +/* + * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero + * pad region (capabilities precedent: occupy pad bytes, do not resize V1). + * A pre-7.2 sender leaves them zero => plane CONTROL, conn_epoch 0. + * Offsets 41/42 are reserved for spec-7.3 (worker_id / n_workers) and + * stay zero in 7.2; offset 43 pads the group; 52-63 remain reserved. + */ +#define PGRAC_IC_HELLO_PLANE_OFFSET 40 +#define PGRAC_IC_HELLO_WORKER_ID_OFFSET 41 /* spec-7.3 reserved, zero in 7.2 */ +#define PGRAC_IC_HELLO_N_WORKERS_OFFSET 42 /* spec-7.3 reserved, zero in 7.2 */ +#define PGRAC_IC_HELLO_CONN_EPOCH_OFFSET 44 typedef struct ClusterICHelloMsg { uint32 magic; /* PGRAC_IC_HELLO_MAGIC */ @@ -249,6 +275,34 @@ cluster_ic_hello_capabilities(const ClusterICHelloMsg *msg) return ((uint32)p[0]) | ((uint32)p[1] << 8) | ((uint32)p[2] << 16) | ((uint32)p[3] << 24); } +/* PGRAC: spec-7.2 D2 — plane byte accessor (_pad offset 40 - 36 = 4). */ +static inline ClusterICPlane +cluster_ic_hello_plane(const ClusterICHelloMsg *msg) +{ + if (msg == NULL) + return CLUSTER_IC_PLANE_CONTROL; + return (ClusterICPlane) + msg->_pad[PGRAC_IC_HELLO_PLANE_OFFSET - PGRAC_IC_HELLO_CAPABILITIES_OFFSET]; +} + +/* PGRAC: spec-7.2 D2 — connection epoch accessor (_pad offset 44 - 36 = 8; + * LE uint64). Zero = pre-7.2 sender (or CONTROL plane, which does not + * enforce it); the DATA-plane verify path rejects zero fail-closed. */ +static inline uint64 +cluster_ic_hello_conn_epoch(const ClusterICHelloMsg *msg) +{ + const uint8 *p; + uint64 v = 0; + int i; + + if (msg == NULL) + return 0; + p = msg->_pad + (PGRAC_IC_HELLO_CONN_EPOCH_OFFSET - PGRAC_IC_HELLO_CAPABILITIES_OFFSET); + for (i = 7; i >= 0; i--) + v = (v << 8) | (uint64)p[i]; + return v; +} + /* * spec-2.2 D2 (post-codex review) -- HELLO wire encode/decode helpers. @@ -273,7 +327,8 @@ cluster_ic_hello_capabilities(const ClusterICHelloMsg *msg) */ extern void cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version, uint16 envelope_version, int32 source_node_id, - const char *cluster_name); + const char *cluster_name, ClusterICPlane plane, + uint64 conn_epoch); extern bool cluster_ic_parse_hello(const uint8 in_buf[PGRAC_IC_HELLO_BYTES], ClusterICHelloMsg *out_msg); diff --git a/src/include/cluster/cluster_ic_tier1.h b/src/include/cluster/cluster_ic_tier1.h index 0bab9a7a23..49fa1b5260 100644 --- a/src/include/cluster/cluster_ic_tier1.h +++ b/src/include/cluster/cluster_ic_tier1.h @@ -133,6 +133,15 @@ extern void cluster_ic_tier1_shmem_register(void); */ extern int cluster_ic_tier1_listener_bind(void); +/* + * PGRAC: spec-7.2 D2 — re-aim this process's tier1 working state at a + * plane ([0] = CONTROL, [1] = DATA inside the single tier1 region). + * Call once at aux-process entry BEFORE any tier1 use (LmsMain → DATA); + * processes that never call it stay on CONTROL (LMON, backends, SQL + * observers — the historic instance and contract). + */ +extern void cluster_ic_tier1_set_my_plane(ClusterICPlane plane); + /* * Accept exactly one pending connection on the listener fd. Returns * true if accepted (sets *out_peer_fd and *out_peer_id from HELLO once diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 6a08b4104f..76d9a357d2 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -519,7 +519,7 @@ UT_TEST(test_hello_wire_roundtrip) bool ok; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 42, - "pgrac-test"); + "pgrac-test", CLUSTER_IC_PLANE_CONTROL, 0); ok = cluster_ic_parse_hello(wire, &parsed); UT_ASSERT(ok); @@ -554,7 +554,7 @@ UT_TEST(test_hello_wire_reference_bytes) * unintended endian flips, and uninitialized memory leakage. */ cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, - 0x01020304, "AB"); + 0x01020304, "AB", CLUSTER_IC_PLANE_CONTROL, 0); /* magic */ UT_ASSERT_EQ(wire[0], 0x48); @@ -577,11 +577,47 @@ UT_TEST(test_hello_wire_reference_bytes) UT_ASSERT_EQ(wire[13], 'B'); for (i = 14; i < 36; i++) UT_ASSERT_EQ(wire[i], 0); - /* _pad must be all zero */ + /* _pad must be all zero: a CONTROL-plane HELLO with conn_epoch 0 is + * byte-identical to the pre-7.2 wire (spec-7.2 D2 compat pin). */ for (i = 36; i < PGRAC_IC_HELLO_BYTES; i++) UT_ASSERT_EQ(wire[i], 0); } +/* + * spec-7.2 D2 — DATA-plane HELLO reference bytes: plane byte at offset + * 40, conn_epoch LE at 44-51, spec-7.3 worker bytes (41/42) still zero, + * tail 52-63 still zero. Plus accessor roundtrip. + */ +UT_TEST(test_hello_wire_data_plane_bytes) +{ + uint8 wire[PGRAC_IC_HELLO_BYTES]; + ClusterICHelloMsg parsed; + int i; + + cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 3, "AB", + CLUSTER_IC_PLANE_DATA, UINT64CONST(0x1122334455667788)); + + UT_ASSERT_EQ(wire[PGRAC_IC_HELLO_PLANE_OFFSET], 1); + UT_ASSERT_EQ(wire[PGRAC_IC_HELLO_WORKER_ID_OFFSET], 0); + UT_ASSERT_EQ(wire[PGRAC_IC_HELLO_N_WORKERS_OFFSET], 0); + UT_ASSERT_EQ(wire[43], 0); + /* conn_epoch = 0x1122334455667788 little-endian */ + UT_ASSERT_EQ(wire[44], 0x88); + UT_ASSERT_EQ(wire[45], 0x77); + UT_ASSERT_EQ(wire[46], 0x66); + UT_ASSERT_EQ(wire[47], 0x55); + UT_ASSERT_EQ(wire[48], 0x44); + UT_ASSERT_EQ(wire[49], 0x33); + UT_ASSERT_EQ(wire[50], 0x22); + UT_ASSERT_EQ(wire[51], 0x11); + for (i = 52; i < PGRAC_IC_HELLO_BYTES; i++) + UT_ASSERT_EQ(wire[i], 0); + + UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); + UT_ASSERT_EQ(cluster_ic_hello_plane(&parsed), CLUSTER_IC_PLANE_DATA); + UT_ASSERT(cluster_ic_hello_conn_epoch(&parsed) == UINT64CONST(0x1122334455667788)); +} + UT_TEST(test_hello_smart_fusion_capability_gate) { uint8 wire[PGRAC_IC_HELLO_BYTES]; @@ -591,7 +627,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, - "sf-off"); + "sf-off", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), 0); @@ -599,7 +635,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_interconnect_tier = CLUSTER_IC_TIER_2; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, - "sf-tier-mismatch"); + "sf-tier-mismatch", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), 0); @@ -607,7 +643,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, - "sf-tier-match"); + "sf-tier-match", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2); @@ -621,7 +657,8 @@ UT_TEST(test_hello_parse_rejects_bad_magic) uint8 wire[PGRAC_IC_HELLO_BYTES]; ClusterICHelloMsg parsed; - cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "x"); + cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "x", + CLUSTER_IC_PLANE_CONTROL, 0); /* Corrupt magic */ wire[0] = 0xDE; wire[1] = 0xAD; @@ -638,7 +675,7 @@ UT_TEST(test_hello_build_truncates_long_name) const char *long_name = "this-cluster-name-is-way-longer-than-the-fixed-cap-on-purpose"; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 7, - long_name); + long_name, CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); /* @@ -654,7 +691,7 @@ UT_TEST(test_hello_build_truncates_long_name) int main(void) { - UT_PLAN(20); /* spec-2.3 D3: 6 ClusterMsgHeader/msg_send/recv tests deleted */ + UT_PLAN(21); /* spec-2.3 D3: 6 ClusterMsgHeader/msg_send/recv tests deleted */ UT_RUN(test_ic_send_bytes_linkable); UT_RUN(test_ic_recv_bytes_linkable); UT_RUN(test_ic_init_linkable); @@ -674,6 +711,7 @@ main(void) /* HELLO wire encode/decode + reference bytes (post-codex review) */ UT_RUN(test_hello_wire_roundtrip); UT_RUN(test_hello_wire_reference_bytes); + UT_RUN(test_hello_wire_data_plane_bytes); /* spec-7.2 D2 */ UT_RUN(test_hello_smart_fusion_capability_gate); UT_RUN(test_hello_parse_rejects_bad_magic); UT_RUN(test_hello_build_truncates_long_name); From ca7341f419984904f657805ca4ec6e63d3539026 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:21:22 +0800 Subject: [PATCH 10/58] fix(cluster): DATA HELLO conn_epoch check is equality vs current epoch The D2b nonzero-ness check false-positived on a fresh cluster: the initial epoch is legitimately 0 on both sides, so every initial DATA HELLO was refused in a reconnect storm (caught by the first two-node smoke). The sound INV-7.2-CONN-EPOCH receiver check is equality with this node's current cluster epoch -- a dialer on a stale epoch is refused and reconnects after observing the bump (fail-closed, self-healing); cross-version senders are already excluded by the HELLO version gate. Spec: spec-7.2-ic-data-plane-decoupling.md (D2b fix) --- src/backend/cluster/cluster_ic_tier1.c | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index c55f14c9ed..4f22ec7c88 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -1427,9 +1427,14 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) * process's listener/connection must claim the same plane (a * CONTROL peer dialing the DATA port, or vice versa, is a wiring * error; pre-7.2 senders read as plane 0 = CONTROL and are only - * acceptable on the CONTROL plane). DATA additionally requires a - * nonzero conn_epoch (INV-7.2-CONN-EPOCH: zero would defeat the - * connection-generation fence; fail-closed). + * acceptable on the CONTROL plane). On DATA the conn_epoch must + * EQUAL this node's current cluster epoch (INV-7.2-CONN-EPOCH: + * the connection is bound to the epoch it was established in — a + * dialer still on a stale epoch is refused and reconnects after + * observing the bump; fail-closed, self-healing. The initial + * epoch is legitimately 0 on both sides, so equality — not + * nonzero-ness — is the sound check; cross-version senders are + * already excluded by the version gate above). */ if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { peer_record_error(peer_id, 0, "08P01", "HELLO plane mismatch (peer=%d mine=%d)", @@ -1438,9 +1443,13 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; return false; } - if (tier1_my_plane == CLUSTER_IC_PLANE_DATA && cluster_ic_hello_conn_epoch(&msg) == 0) { - peer_record_error(peer_id, 0, "08P01", "DATA HELLO missing conn_epoch"); - cluster_ic_tier1_close_peer(peer_id, "DATA HELLO missing conn_epoch"); + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA + && cluster_ic_hello_conn_epoch(&msg) != cluster_epoch_get_current()) { + peer_record_error(peer_id, 0, "08P01", + "DATA HELLO conn_epoch mismatch (peer=" UINT64_FORMAT + " mine=" UINT64_FORMAT ")", + cluster_ic_hello_conn_epoch(&msg), cluster_epoch_get_current()); + cluster_ic_tier1_close_peer(peer_id, "DATA HELLO conn_epoch mismatch"); Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; return false; } @@ -1619,15 +1628,18 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear return false; } - /* PGRAC: spec-7.2 D2 — plane match + DATA conn_epoch (see the - * named-peer verify path for the rationale). */ + /* PGRAC: spec-7.2 D2 — plane match + DATA conn_epoch equality (see + * the named-peer verify path for the rationale). */ if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { ereport(LOG, (errmsg("cluster_ic tier1 HELLO plane mismatch (peer=%d mine=%d)", (int)cluster_ic_hello_plane(&msg), (int)tier1_my_plane))); return false; } - if (tier1_my_plane == CLUSTER_IC_PLANE_DATA && cluster_ic_hello_conn_epoch(&msg) == 0) { - ereport(LOG, (errmsg("cluster_ic tier1 DATA HELLO missing conn_epoch"))); + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA + && cluster_ic_hello_conn_epoch(&msg) != cluster_epoch_get_current()) { + ereport(LOG, (errmsg("cluster_ic tier1 DATA HELLO conn_epoch mismatch " + "(peer=" UINT64_FORMAT " mine=" UINT64_FORMAT ")", + cluster_ic_hello_conn_epoch(&msg), cluster_epoch_get_current()))); return false; } From 391a40f62e5aab9c5e79fdbb40810fe77db3c076 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:21:22 +0800 Subject: [PATCH 11/58] feat(cluster): LMS-owned DATA-plane interconnect loop New cluster_lms_data_plane.c: the LMS aux process binds the node's declared data_addr, dials peers by the same mesh role as the CONTROL plane, runs the HELLO state machines over its own fds, and waits on a WaitEventSet (DATA sockets + MyLatch) in place of the historic plain WaitLatch -- the latch-driven park-serve drains are untouched. No heartbeats and no liveness scan on this plane: CONTROL (CSSD/LMON) is the single liveness authority (Q-D2-1); connect-establishment timeouts, TCP-error close + backoff reconnect, and requester timeouts cover the rest. A node without data_addr runs no DATA plane and LMS keeps the historic latch-only loop. No message routes over the plane yet: the GCS block family flips in the D3/D4 atomic switch; until then the pump only carries HELLO. Spec: spec-7.2-ic-data-plane-decoupling.md (D2c) --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_lms.c | 22 +- src/backend/cluster/cluster_lms_data_plane.c | 410 +++++++++++++++++++ src/include/cluster/cluster_lms.h | 13 + 4 files changed, 443 insertions(+), 3 deletions(-) create mode 100644 src/backend/cluster/cluster_lms_data_plane.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 5ea41237f7..7e8ca0fd45 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -143,6 +143,7 @@ OBJS = \ cluster_lmd_wait_state.o \ cluster_lmon.o \ cluster_lms.o \ + cluster_lms_data_plane.o \ cluster_lns.o \ cluster_membership.o \ cluster_mrp.o \ diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index c3b9a39c24..d3d03e7678 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -695,6 +695,11 @@ LmsMain(void) cluster_lms_bump_restart_generation_at_main_entry(); (void)cluster_ges_dedup_drop_stale_entries(); + /* PGRAC: spec-7.2 D2 — bring up the LMS-owned DATA-plane listener + + * mesh (false = plane off for this node; the loop below then keeps + * the historic latch-only wait and park-serve keeps working). */ + (void)cluster_lms_data_plane_startup(); + /* Transition to READY. */ LWLockAcquire(&cluster_lms_state->lwlock, LW_EXCLUSIVE); cluster_lms_state->ready_at = GetCurrentTimestamp(); @@ -734,11 +739,22 @@ LmsMain(void) * failure becomes a DENIED result; LMS never exits over a serve). */ cluster_lms_cr_drain(); - (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, - LMS_IDLE_TIMEOUT_MS, WAIT_EVENT_PG_SLEEP); - ResetLatch(MyLatch); + /* PGRAC: spec-7.2 D2 — with a live DATA plane the wait moves into + * the data-plane tick (WaitEventSet: DATA sockets + MyLatch, latch + * reset inside); the historic plain WaitLatch stays the fallback + * when the plane is off. */ + if (cluster_lms_data_plane_enabled()) + cluster_lms_data_plane_tick(LMS_IDLE_TIMEOUT_MS); + else { + (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + LMS_IDLE_TIMEOUT_MS, WAIT_EVENT_PG_SLEEP); + ResetLatch(MyLatch); + } } + /* PGRAC: spec-7.2 D2 — close DATA-plane fds before teardown. */ + cluster_lms_data_plane_shutdown(); + /* PGRAC: spec-6.12b — retract the wake latch before teardown. */ cluster_cr_server_publish_lms_latch(NULL); diff --git a/src/backend/cluster/cluster_lms_data_plane.c b/src/backend/cluster/cluster_lms_data_plane.c new file mode 100644 index 0000000000..dc60bb6447 --- /dev/null +++ b/src/backend/cluster/cluster_lms_data_plane.c @@ -0,0 +1,410 @@ +/*------------------------------------------------------------------------- + * + * cluster_lms_data_plane.c + * pgrac DATA-plane (LMS<->LMS) interconnect loop — spec-7.2 D2. + * + * The LMS aux process owns the DATA plane: its own tier1 listener + * (bound to the node's declared data_addr), its own active-role + * connects, its own HELLO state machines, and — after the spec-7.2 + * D3/D4 plane flip — the GCS block family dispatch and replies. + * This mirrors the LMON tier1 mesh loop (cluster_lmon.c) with the + * control-plane duties removed: + * + * - NO heartbeat send and NO heartbeat-liveness scan: the CONTROL + * plane (CSSD/LMON) is the single liveness authority (spec-7.2 + * Q-D2-1). A dead peer is discovered via CONTROL membership + * events (spec-7.2 D5 wires the forced reset), via TCP errors + * on use, or via requester timeouts (fail-closed, 53R90 family). + * - Connect-establishment timeouts still apply (a peer that never + * completes HELLO must not pin a pending slot forever). + * - Reconnect backoff mirrors the LMON cadence (one heartbeat + * interval between attempts). + * + * Process-plane discipline: LmsMain aims this process's tier1 state + * at the DATA plane (cluster_ic_tier1_set_my_plane) before any tier1 + * use, so every tier1 helper called below lands on the DATA-plane + * shmem instance and this process's private fd/buffer statics. The + * CONTROL plane in LMON is untouched. + * + * A node without a declared data_addr runs no DATA plane: startup + * returns false and LmsMain keeps its historic latch-only loop + * (park-serve construction keeps working; the plane flip in D3/D4 + * is what makes a missing DATA plane a hard error for block ships). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_lms_data_plane.c + * + * NOTES + * This is a pgrac-original file. Compiled only in --enable-cluster + * builds. Spec: spec-7.2-ic-data-plane-decoupling.md (D2c). + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "cluster/cluster_conf.h" +#include "cluster/cluster_elog.h" +#include "cluster/cluster_guc.h" +#include "cluster/cluster_ic.h" +#include "cluster/cluster_ic_tier1.h" +#include "cluster/cluster_lms.h" +#include "miscadmin.h" +#include "storage/latch.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +#ifdef USE_PGRAC_CLUSTER + +/* Per-peer DATA-plane connection substate (mirrors the LMON tracker). */ +typedef enum LmsDpSubstate { + LMS_DP_DOWN = 0, + LMS_DP_CONNECT_PEND, + LMS_DP_HELLO_SENDING, + LMS_DP_HELLO_WAIT, + LMS_DP_CONNECTED +} LmsDpSubstate; + +typedef struct LmsDpPeerTrack { + int fd; + LmsDpSubstate substate; + bool is_active; /* mesh role: we dial (true) or they dial us */ + TimestampTz next_attempt_at; + TimestampTz connect_started_at; +} LmsDpPeerTrack; + +static LmsDpPeerTrack dp_track[CLUSTER_MAX_NODES]; +static int dp_pending_fds[CLUSTER_MAX_NODES]; +static int dp_listener_fd = -1; +static WaitEventSet *dp_wes = NULL; +static bool dp_wes_dirty = true; +static bool dp_enabled = false; + +/* + * cluster_lms_data_plane_startup — bind the DATA listener + init the mesh. + * + * Returns true when the DATA plane is live (listener bound); false = + * plane off for this node (no data_addr declared, cluster disabled, or + * a stub/mock interconnect tier). Callable once from LmsMain before + * the READY transition. + */ +bool +cluster_lms_data_plane_startup(void) +{ + int32 self_id = cluster_node_id; + int32 pi; + + Assert(MyBackendType == B_LMS); + + if (!cluster_enabled) + return false; + if (cluster_interconnect_tier != CLUSTER_IC_TIER_1 + && cluster_interconnect_tier != CLUSTER_IC_TIER_2 + && cluster_interconnect_tier != CLUSTER_IC_TIER_3) + return false; + + /* Aim this process's tier1 state at the DATA plane BEFORE any use. */ + cluster_ic_tier1_set_my_plane(CLUSTER_IC_PLANE_DATA); + + dp_listener_fd = cluster_ic_tier1_listener_bind(); + if (dp_listener_fd < 0) + return false; /* no data_addr — plane off (LOG emitted inside) */ + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + dp_track[pi].is_active = false; + dp_track[pi].next_attempt_at = 0; + dp_track[pi].connect_started_at = 0; + dp_pending_fds[pi] = -1; + + if (pi == self_id) + continue; + if (cluster_conf_lookup_node(pi) == NULL) + continue; + dp_track[pi].is_active + = (cluster_ic_mesh_role_for_pair(self_id, pi) == CLUSTER_IC_MESH_ACTIVE); + } + + dp_wes_dirty = true; + dp_enabled = true; + ereport(LOG, (errmsg("cluster_lms: DATA-plane listener bound (node %d)", self_id))); + return true; +} + +bool +cluster_lms_data_plane_enabled(void) +{ + return dp_enabled; +} + +/* Rebuild the WaitEventSet when the fd set changed. */ +static void +dp_rebuild_wes(void) +{ + int32 pi; + + if (dp_wes != NULL) { + FreeWaitEventSet(dp_wes); + dp_wes = NULL; + } + + dp_wes = CreateWaitEventSet(CurrentMemoryContext, 3 + 2 * CLUSTER_MAX_NODES); + AddWaitEventToSet(dp_wes, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); + AddWaitEventToSet(dp_wes, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, NULL, NULL); + if (dp_listener_fd >= 0) + AddWaitEventToSet(dp_wes, WL_SOCKET_READABLE, dp_listener_fd, NULL, (void *)(intptr_t)-1); + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + int events; + + if (dp_track[pi].fd < 0) + continue; + + switch (dp_track[pi].substate) { + case LMS_DP_CONNECT_PEND: + case LMS_DP_HELLO_SENDING: + events = WL_SOCKET_WRITEABLE; + break; + case LMS_DP_HELLO_WAIT: + events = WL_SOCKET_READABLE; + break; + case LMS_DP_CONNECTED: + events = WL_SOCKET_READABLE; + if (cluster_ic_tier1_pending_outbound(pi)) + events |= WL_SOCKET_WRITEABLE; + break; + default: + continue; + } + + AddWaitEventToSet(dp_wes, events, dp_track[pi].fd, NULL, (void *)(intptr_t)pi); + } + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_pending_fds[pi] < 0) + continue; + AddWaitEventToSet(dp_wes, WL_SOCKET_READABLE, dp_pending_fds[pi], NULL, + (void *)(intptr_t)(CLUSTER_MAX_NODES + pi)); + } + + dp_wes_dirty = false; +} + +/* + * cluster_lms_data_plane_tick — one management + wait + dispatch round. + * + * Called from the LmsMain loop in place of the historic plain + * WaitLatch: the wait now also wakes on DATA sockets. MyLatch is + * reset here, so the caller must run its latch-driven drains (CR + * park-serve etc.) BEFORE calling in, mirroring the LMON loop shape. + */ +void +cluster_lms_data_plane_tick(long timeout_ms) +{ + WaitEvent ev[2 * CLUSTER_MAX_NODES + 3]; + TimestampTz now; + int n_events; + int32 pi; + int i; + + Assert(dp_enabled); + + now = GetCurrentTimestamp(); + + /* Active-role reconnect for DOWN peers whose backoff elapsed. */ + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + int new_fd = -1; + + if (pi == cluster_node_id || !dp_track[pi].is_active) + continue; + if (dp_track[pi].substate != LMS_DP_DOWN) + continue; + if (dp_track[pi].next_attempt_at > now) + continue; + + dp_track[pi].next_attempt_at + = now + (int64)cluster_interconnect_heartbeat_interval_ms * INT64CONST(1000); + + /* A peer without data_addr is unreachable on this plane — + * connect_one fails fast via the NULL peer_addr; skip quietly + * (the conf may declare it later; SIGHUP reload re-checks). */ + if (cluster_ic_tier1_connect_one(pi, &new_fd) && new_fd >= 0) { + dp_track[pi].fd = new_fd; + dp_track[pi].substate = LMS_DP_CONNECT_PEND; + dp_track[pi].connect_started_at = now; + dp_wes_dirty = true; + } + } + + /* Connect-establishment timeout scan (no liveness scan on DATA: + * CONTROL is the liveness authority, Q-D2-1). */ + { + int64 connect_to_us = (int64)cluster_interconnect_connect_timeout_ms * INT64CONST(1000); + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd < 0) + continue; + if ((dp_track[pi].substate == LMS_DP_CONNECT_PEND + || dp_track[pi].substate == LMS_DP_HELLO_SENDING + || dp_track[pi].substate == LMS_DP_HELLO_WAIT) + && dp_track[pi].connect_started_at > 0 + && now > dp_track[pi].connect_started_at + connect_to_us) { + cluster_ic_tier1_close_peer(pi, "data-plane connect timeout"); + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + dp_track[pi].connect_started_at = 0; + dp_wes_dirty = true; + } + } + } + + if (dp_wes_dirty) + dp_rebuild_wes(); + + if (timeout_ms < 0) + timeout_ms = 0; + + n_events = WaitEventSetWait(dp_wes, timeout_ms, ev, lengthof(ev), WAIT_EVENT_PG_SLEEP); + + for (i = 0; i < n_events; i++) { + intptr_t tag = (intptr_t)ev[i].user_data; + + if (ev[i].events & WL_LATCH_SET) { + ResetLatch(MyLatch); + continue; + } + + if (tag == -1) { + /* Listener: drain all pending accepts into anon slots. */ + for (;;) { + int new_fd = -1; + int32 dummy_peer_id = -1; + int slot; + + if (!cluster_ic_tier1_accept_one(&new_fd, &dummy_peer_id)) + break; + if (new_fd < 0) + break; + + for (slot = 0; slot < CLUSTER_MAX_NODES; slot++) + if (dp_pending_fds[slot] < 0) + break; + if (slot >= CLUSTER_MAX_NODES) { + (void)close(new_fd); + continue; + } + dp_pending_fds[slot] = new_fd; + dp_wes_dirty = true; + } + } else if (tag >= 0 && tag < CLUSTER_MAX_NODES) { + int32 peer = (int32)tag; + int peer_fd = dp_track[peer].fd; + + if (peer_fd < 0) + continue; + + if (dp_track[peer].substate == LMS_DP_CONNECT_PEND + && (ev[i].events & WL_SOCKET_WRITEABLE)) { + if (cluster_ic_tier1_finish_connect(peer, peer_fd)) { + if (cluster_ic_tier1_hello_send_remaining(peer) == 0) + dp_track[peer].substate = LMS_DP_CONNECTED; + else + dp_track[peer].substate = LMS_DP_HELLO_SENDING; + dp_wes_dirty = true; + } else { + dp_track[peer].fd = -1; + dp_track[peer].substate = LMS_DP_DOWN; + dp_wes_dirty = true; + } + } else if (dp_track[peer].substate == LMS_DP_HELLO_SENDING + && (ev[i].events & WL_SOCKET_WRITEABLE)) { + if (cluster_ic_tier1_continue_hello_send(peer, peer_fd)) { + if (cluster_ic_tier1_hello_send_remaining(peer) == 0) { + dp_track[peer].substate = LMS_DP_CONNECTED; + dp_wes_dirty = true; + } + } else { + dp_track[peer].fd = -1; + dp_track[peer].substate = LMS_DP_DOWN; + dp_wes_dirty = true; + } + } else if (dp_track[peer].substate == LMS_DP_CONNECTED + && (ev[i].events & WL_SOCKET_READABLE)) { + /* Generic envelope pump: recv + verify + dispatch. No + * DATA msg_type is registered before the D3/D4 flip, so + * pre-flip traffic is limited to HELLO/errors; post- + * flip this is the block-family dispatch entry. */ + if (!cluster_ic_tier1_recv_heartbeat_drain(peer, peer_fd)) { + cluster_ic_tier1_close_peer(peer, "data-plane recv failed"); + dp_track[peer].fd = -1; + dp_track[peer].substate = LMS_DP_DOWN; + dp_wes_dirty = true; + } + } + } else if (tag >= CLUSTER_MAX_NODES && tag < 2 * CLUSTER_MAX_NODES) { + /* Anonymous accepted fd: accumulate + verify HELLO. */ + int slot = (int)(tag - CLUSTER_MAX_NODES); + int pend_fd = dp_pending_fds[slot]; + int32 learned = -1; + + if (pend_fd < 0) + continue; + + if (cluster_ic_tier1_continue_hello_recv(slot, pend_fd, &learned)) { + if (learned >= 0) { + /* HELLO complete: bind to the learned peer. */ + if (dp_track[learned].fd >= 0 && dp_track[learned].fd != pend_fd) + cluster_ic_tier1_close_peer(learned, "data-plane duplicate connection"); + dp_track[learned].fd = pend_fd; + dp_track[learned].substate = LMS_DP_CONNECTED; + dp_pending_fds[slot] = -1; + cluster_ic_tier1_anon_hello_reset(slot); + dp_wes_dirty = true; + } + /* else: still accumulating; stay registered READABLE */ + } else { + (void)close(pend_fd); + dp_pending_fds[slot] = -1; + cluster_ic_tier1_anon_hello_reset(slot); + dp_wes_dirty = true; + } + } + } +} + +/* Close everything we own (LmsMain teardown path). */ +void +cluster_lms_data_plane_shutdown(void) +{ + int32 pi; + + if (!dp_enabled) + return; + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd >= 0) { + cluster_ic_tier1_close_peer(pi, "lms shutdown"); + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + } + if (dp_pending_fds[pi] >= 0) { + (void)close(dp_pending_fds[pi]); + dp_pending_fds[pi] = -1; + } + } + if (dp_wes != NULL) { + FreeWaitEventSet(dp_wes); + dp_wes = NULL; + } + dp_enabled = false; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index 99c0e47f55..3feb43ee5a 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -294,6 +294,19 @@ extern void cluster_lms_request_shutdown(void); */ extern void LmsMain(void) pg_attribute_noreturn(); +/* + * PGRAC: spec-7.2 D2 — LMS-owned DATA-plane interconnect loop + * (cluster_lms_data_plane.c). startup binds the data_addr listener and + * aims this process's tier1 state at the DATA plane (false = plane off: + * no data_addr / cluster off / stub tier); tick runs one management + + * WaitEventSet round (sockets + MyLatch — replaces the historic plain + * WaitLatch when the plane is live); shutdown closes owned fds. + */ +extern bool cluster_lms_data_plane_startup(void); +extern bool cluster_lms_data_plane_enabled(void); +extern void cluster_lms_data_plane_tick(long timeout_ms); +extern void cluster_lms_data_plane_shutdown(void); + /* * Read-only accessors for SQL view + diagnostics. */ From 9ddb5ff6373d8092491566c773ad92d0c6bf1bed Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:21:22 +0800 Subject: [PATCH 12/58] test(cluster): harnesses declare per-node data_addr ClusterPair/Triple/Quad allocate one extra free port per node and declare it as data_addr in the generated pgrac.conf (allocator- provided, never offset-derived -- adjacent-port collisions made offset defaults unsound). data_ports recorded on the harness object. Spec: spec-7.2-ic-data-plane-decoupling.md (D2d) --- src/test/perl/PostgreSQL/Test/ClusterPair.pm | 11 ++++++++++- src/test/perl/PostgreSQL/Test/ClusterQuad.pm | 15 ++++++++++++--- src/test/perl/PostgreSQL/Test/ClusterTriple.pm | 13 +++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/test/perl/PostgreSQL/Test/ClusterPair.pm b/src/test/perl/PostgreSQL/Test/ClusterPair.pm index 74c1e0c34d..1f84f527f5 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterPair.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterPair.pm @@ -75,12 +75,16 @@ sub new_pair { my ($class, $cluster_name, %opts) = @_; - # Allocate 4 distinct free ports. get_free_port() reserves the port + # Allocate 6 distinct free ports. get_free_port() reserves the port # until the test exits, so successive calls return distinct values. + # spec-7.2 D2: every node also gets an explicit DATA-plane port + # (data_addr) -- allocator-provided, never offset-derived (r1-F2). my $pg_port_0 = PostgreSQL::Test::Cluster::get_free_port(); my $pg_port_1 = PostgreSQL::Test::Cluster::get_free_port(); my $ic_port_0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic_port_1 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port_0 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port_1 = PostgreSQL::Test::Cluster::get_free_port(); my $node0 = PostgreSQL::Test::Cluster->new("${cluster_name}_node0", port => $pg_port_0); @@ -241,9 +245,11 @@ name = $cluster_name [node.0] interconnect_addr = 127.0.0.1:$ic_port_0 +data_addr = 127.0.0.1:$data_port_0 [node.1] interconnect_addr = 127.0.0.1:$ic_port_1 +data_addr = 127.0.0.1:$data_port_1 EOC # node1's pgrac.conf -- same except optionally a different cluster_name @@ -255,9 +261,11 @@ name = $alt_name [node.0] interconnect_addr = 127.0.0.1:$ic_port_0 +data_addr = 127.0.0.1:$data_port_0 [node.1] interconnect_addr = 127.0.0.1:$ic_port_1 +data_addr = 127.0.0.1:$data_port_1 EOC PostgreSQL::Test::Utils::append_to_file( @@ -271,6 +279,7 @@ EOC cluster_name => $cluster_name, pg_ports => [ $pg_port_0, $pg_port_1 ], ic_ports => [ $ic_port_0, $ic_port_1 ], + data_ports => [ $data_port_0, $data_port_1 ], voting_disk_paths => \@voting_disk_paths, wal_threads_root => $wal_threads_root, shared_data_root => $shared_data_root, diff --git a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm index 3b3a2bb29f..9cf7c45510 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm @@ -71,9 +71,12 @@ sub new_quad { my ($class, $cluster_name, %opts) = @_; - # Allocate 8 distinct free ports (4 PG + 4 IC). + # Allocate 12 distinct free ports (4 PG + 4 IC + 4 DATA). + # spec-7.2 D2: DATA-plane ports are allocator-provided, never + # offset-derived (r1-F2). my @pg_ports; my @ic_ports; + my @data_ports; for (0 .. $NODES - 1) { push @pg_ports, PostgreSQL::Test::Cluster::get_free_port(); @@ -82,6 +85,10 @@ sub new_quad { push @ic_ports, PostgreSQL::Test::Cluster::get_free_port(); } + for (0 .. $NODES - 1) + { + push @data_ports, PostgreSQL::Test::Cluster::get_free_port(); + } my @nodes; for my $i (0 .. $NODES - 1) @@ -194,8 +201,9 @@ sub new_quad my $peers_block = ""; for my $i (0 .. $NODES - 1) { - $peers_block .= - "[node.$i]\ninterconnect_addr = 127.0.0.1:$ic_ports[$i]\n\n"; + $peers_block .= "[node.$i]\n" + . "interconnect_addr = 127.0.0.1:$ic_ports[$i]\n" + . "data_addr = 127.0.0.1:$data_ports[$i]\n\n"; } my $pgrac_conf_body = < $cluster_name, pg_ports => \@pg_ports, ic_ports => \@ic_ports, + data_ports => \@data_ports, voting_disk_paths => \@voting_disk_paths, wal_threads_root => $wal_threads_root, shared_data_root => $shared_data_root, diff --git a/src/test/perl/PostgreSQL/Test/ClusterTriple.pm b/src/test/perl/PostgreSQL/Test/ClusterTriple.pm index 1268e1f524..4b625e1792 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterTriple.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterTriple.pm @@ -57,6 +57,13 @@ sub new_triple PostgreSQL::Test::Cluster::get_free_port(), PostgreSQL::Test::Cluster::get_free_port(), ); + # spec-7.2 D2: explicit DATA-plane ports (allocator-provided, never + # offset-derived -- r1-F2). + my @data_ports = ( + PostgreSQL::Test::Cluster::get_free_port(), + PostgreSQL::Test::Cluster::get_free_port(), + PostgreSQL::Test::Cluster::get_free_port(), + ); my @nodes; for my $i (0 .. 2) @@ -176,8 +183,9 @@ sub new_triple my $peers_block = ""; for my $i (0 .. 2) { - $peers_block .= - "[node.$i]\ninterconnect_addr = 127.0.0.1:$ic_ports[$i]\n\n"; + $peers_block .= "[node.$i]\n" + . "interconnect_addr = 127.0.0.1:$ic_ports[$i]\n" + . "data_addr = 127.0.0.1:$data_ports[$i]\n\n"; } my $pgrac_conf_body = < $cluster_name, pg_ports => \@pg_ports, ic_ports => \@ic_ports, + data_ports => \@data_ports, voting_disk_paths => \@voting_disk_paths, wal_threads_root => $wal_threads_root, shared_data_root => $shared_data_root, From 8722f79e7c9278e2fa52c56b0a01af9b90ec1d8a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:38:12 +0800 Subject: [PATCH 13/58] feat(cluster): msg_type plane field + router plane gates ClusterICMsgTypeInfo gains an owning-plane byte (zero = CONTROL, so every existing registration is CONTROL without edits) plus the LMS producer macros (_LMS / _LMS_WORKER / _LMS_DATA; the combined mask pre-includes the spec-7.3 worker bit). Two gates enforce the plane: - dispatch: a frame whose msg_type belongs to the other plane is dropped fail-closed and counted (plane_misroute_reject, per-plane tier1 instance); the connection stays up (unlike the peer-level unregistered/forged-broadcast rejections). - physical send (after the dest=self short-circuit, which stays open to every mask-allowed logical producer): cross-plane send ereports -- an LMS-context emitter of a CONTROL message must stage it into the CONTROL outbound ring for LMON (staging rule). No msg_type is DATA yet: the five GCS block types flip in one commit at the end of the D3+D4 sequence (no half-migrated window). Spec: spec-7.2-ic-data-plane-decoupling.md (D3) --- src/backend/cluster/cluster_ic_router.c | 38 +++++++++++++++++++ src/backend/cluster/cluster_ic_tier1.c | 28 ++++++++++++++ src/include/cluster/cluster_ic_router.h | 25 ++++++++++++ src/include/cluster/cluster_ic_tier1.h | 5 +++ .../cluster_unit/test_cluster_ic_router.c | 16 ++++++++ 5 files changed, 112 insertions(+) diff --git a/src/backend/cluster/cluster_ic_router.c b/src/backend/cluster/cluster_ic_router.c index d8c7eddbaf..0c7f6900ad 100644 --- a/src/backend/cluster/cluster_ic_router.c +++ b/src/backend/cluster/cluster_ic_router.c @@ -204,6 +204,25 @@ cluster_ic_send_envelope(uint8 msg_type, int32 dest_node_id, const void *payload if (dest_node_id == cluster_node_id) return CLUSTER_IC_SEND_DONE; + /* + * PGRAC: spec-7.2 D3 — (3b) plane gate on the PHYSICAL send leg + * (after the self short-circuit, which stays open to every mask- + * allowed logical producer). A wire send uses this process's fds, + * so the msg_type's plane must be the plane this process owns: + * LMS emitting a CONTROL message must stage it into the CONTROL + * outbound ring for LMON instead (r4 ruling), and nothing but LMS + * may put a DATA frame on the wire. ereport — a cross-plane send + * is a coding defect, not a runtime condition to tolerate. + */ + if ((ClusterICPlane)info->plane != cluster_ic_tier1_my_plane()) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cluster_ic msg_type %u (\"%s\") is plane-%d; cannot send from " + "plane-%d owner process", + msg_type, info->name, (int)info->plane, (int)cluster_ic_tier1_my_plane()), + errhint("Stage the message into the owning plane's outbound ring instead " + "(spec-7.2 staging rule)."))); + /* (4) broadcast destination check (chunk wrap inherits broadcast_ok=true * implicit -- chunked send is always unicast or broadcast at caller level). */ @@ -317,6 +336,25 @@ cluster_ic_dispatch_envelope(const ClusterICEnvelope *env, const void *payload, if (env->dest_node_id == PGRAC_IC_BROADCAST && !info->broadcast_ok) return false; + /* + * PGRAC: spec-7.2 D3 — plane misroute gate. A msg_type is + * dispatched ONLY in its owning plane's process: a DATA frame + * arriving in LMON (or a CONTROL frame in LMS) is a wiring or + * routing defect, never "close enough" — drop the frame fail-closed + * and count it (plane_misroute_reject on this plane's tier1 + * instance). Return true: the peer connection itself is healthy + * (unlike the unregistered/forged-broadcast rejections above, which + * are peer-level failures). + */ + if ((ClusterICPlane)info->plane != cluster_ic_tier1_my_plane()) { + cluster_ic_tier1_bump_plane_misroute_reject(); + ereport(LOG, (errmsg("cluster_ic: dropping msg_type %u (\"%s\") — plane %d frame " + "dispatched in plane-%d owner process", + env->msg_type, info->name, (int)info->plane, + (int)cluster_ic_tier1_my_plane()))); + return true; + } + /* * spec-2.3 §3.5 + Q14 + R3 防御层: PG_TRY/PG_CATCH wrap. Catches * elevel == ERROR (handler violated §3.5 by raising ERROR) -- LOG diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 4f22ec7c88..30761a600d 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -113,6 +113,9 @@ typedef struct ClusterICTier1Shmem { pid_t listener_pid; /* current LMON pid */ uint64 listener_incarnation; /* ++ on each LMON respawn */ uint32 magic; /* sanity check */ + /* PGRAC: spec-7.2 D3 — frames dropped by the dispatch plane gate + * (a msg_type dispatched in the wrong plane's owner process). */ + pg_atomic_uint64 plane_misroute_reject; ClusterICPeerStateShmem peers[CLUSTER_MAX_NODES]; } ClusterICTier1Shmem; @@ -450,6 +453,7 @@ tier1_shmem_init(void) s->listener_port = -1; s->listener_pid = 0; s->listener_incarnation = 0; + pg_atomic_init_u64(&s->plane_misroute_reject, 0); for (i = 0; i < CLUSTER_MAX_NODES; i++) { s->peers[i].node_id = -1; @@ -495,6 +499,30 @@ cluster_ic_tier1_set_my_plane(ClusterICPlane plane) Tier1Shmem = Tier1ShmemByPlane[plane]; } +/* PGRAC: spec-7.2 D3 — this process's plane (router plane gates). */ +ClusterICPlane +cluster_ic_tier1_my_plane(void) +{ + return tier1_my_plane; +} + +/* PGRAC: spec-7.2 D3 — dispatch plane-gate drop counter (this plane's + * instance; pre-shmem no-op keeps the router callable in unit stubs). */ +void +cluster_ic_tier1_bump_plane_misroute_reject(void) +{ + if (Tier1Shmem != NULL) + pg_atomic_fetch_add_u64(&Tier1Shmem->plane_misroute_reject, 1); +} + +uint64 +cluster_ic_tier1_get_plane_misroute_reject(ClusterICPlane plane) +{ + if (plane < 0 || plane >= CLUSTER_IC_PLANE_N || Tier1ShmemByPlane[plane] == NULL) + return 0; + return pg_atomic_read_u64(&Tier1ShmemByPlane[plane]->plane_misroute_reject); +} + static const ClusterShmemRegion cluster_ic_tier1_region = { .name = "pgrac cluster_ic_tier1", .size_fn = tier1_shmem_size, diff --git a/src/include/cluster/cluster_ic_router.h b/src/include/cluster/cluster_ic_router.h index 5cb23bc0bd..945647ae28 100644 --- a/src/include/cluster/cluster_ic_router.h +++ b/src/include/cluster/cluster_ic_router.h @@ -98,6 +98,19 @@ | CLUSTER_IC_PRODUCER_BGWRITER | CLUSTER_IC_PRODUCER_CHECKPOINTER) /* spec-2.X may add more as new BackendType slots get assigned */ +/* + * PGRAC: spec-7.2 D3 (r3/r4 amend) — LMS producer bits for the DATA-plane + * msg_type family. _LMS_DATA pre-includes the 7.3 worker bit so the + * five GCS block registrations widen exactly once. REDECLARE stays on + * the CONTROL mask untouched (r4 ruling), and CONTROL-only msg_types + * never gain these bits — a migrated handler that must emit a CONTROL + * message stages it into the CONTROL outbound ring for LMON to send + * (direct send from LMS = producer-gate ERROR by construction). + */ +#define CLUSTER_IC_PRODUCER_LMS ((uint32)(1u << B_LMS)) +#define CLUSTER_IC_PRODUCER_LMS_WORKER ((uint32)(1u << B_LMS_WORKER)) +#define CLUSTER_IC_PRODUCER_LMS_DATA (CLUSTER_IC_PRODUCER_LMS | CLUSTER_IC_PRODUCER_LMS_WORKER) + /* ============================================================ * ClusterICMsgTypeInfo -- registration record for one msg_type. @@ -114,6 +127,18 @@ typedef struct ClusterICMsgTypeInfo { uint32 allowed_producer_mask; /* OR of CLUSTER_IC_PRODUCER_* */ bool broadcast_ok; /* dest = PGRAC_IC_BROADCAST allowed? */ void (*handler)(const ClusterICEnvelope *env, const void *payload); + + /* + * PGRAC: spec-7.2 D3 — owning plane (ClusterICPlane; uint8 keeps the + * const-initializer shape). Zero = CONTROL, so every existing + * registration is CONTROL without edits. A msg_type is sent and + * dispatched ONLY in its plane's owner process: the physical-send + * layer ereports on a cross-plane send, dispatch drops the frame + * fail-closed (plane_misroute_reject). Flip discipline (H-5 + user + * ruling): the five GCS block types turn DATA in ONE commit at the + * end of the D3+D4 sequence — no half-migrated window. + */ + uint8 plane; } ClusterICMsgTypeInfo; diff --git a/src/include/cluster/cluster_ic_tier1.h b/src/include/cluster/cluster_ic_tier1.h index 49fa1b5260..10aaf0ac41 100644 --- a/src/include/cluster/cluster_ic_tier1.h +++ b/src/include/cluster/cluster_ic_tier1.h @@ -141,6 +141,11 @@ extern int cluster_ic_tier1_listener_bind(void); * observers — the historic instance and contract). */ extern void cluster_ic_tier1_set_my_plane(ClusterICPlane plane); +extern ClusterICPlane cluster_ic_tier1_my_plane(void); + +/* PGRAC: spec-7.2 D3 — dispatch plane-gate drop counter (per plane). */ +extern void cluster_ic_tier1_bump_plane_misroute_reject(void); +extern uint64 cluster_ic_tier1_get_plane_misroute_reject(ClusterICPlane plane); /* * Accept exactly one pending connection on the listener fd. Returns diff --git a/src/test/cluster_unit/test_cluster_ic_router.c b/src/test/cluster_unit/test_cluster_ic_router.c index 4edca9b0a7..32ddbd7680 100644 --- a/src/test/cluster_unit/test_cluster_ic_router.c +++ b/src/test/cluster_unit/test_cluster_ic_router.c @@ -314,6 +314,22 @@ void cluster_ic_tier1_bump_epoch_observe_advance(int32 peer_id pg_attribute_unused()) {} +/* spec-7.2 D3 stubs: plane gates (this harness models the CONTROL-plane + * owner; the plane-gate truth table is pinned in the test body). */ +static ClusterICPlane router_test_my_plane = CLUSTER_IC_PLANE_CONTROL; +static uint64 router_test_misroute_count = 0; + +ClusterICPlane +cluster_ic_tier1_my_plane(void) +{ + return router_test_my_plane; +} +void +cluster_ic_tier1_bump_plane_misroute_reject(void) +{ + router_test_misroute_count++; +} + /* MyBackendType -- writable from tests to flip producer scope */ extern BackendType MyBackendType; BackendType MyBackendType = B_INVALID; From 5af21a0a9c8e984be4dbca3a578129ff10a8c456 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:39:53 +0800 Subject: [PATCH 14/58] feat(cluster): route the X->S downgrade notify through the staging helper cluster_gcs_send_transition_nowait sent GCS_REQUEST raw; post-flip its callers (the remote X->S downgrade inside the FORWARD handler and the BAST-nudge branch) execute in LMS, where a direct CONTROL-plane send trips the router plane gate by design. Route through gcs_send_envelope_or_loopback, whose existing != B_LMON branch stages the frame into the GRD outbound ring for LMON to send (the D0 staging case; fire-and-forget semantics unchanged, reply still HC74-drops). Spec: spec-7.2-ic-data-plane-decoupling.md (D3) --- src/backend/cluster/cluster_gcs.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/backend/cluster/cluster_gcs.c b/src/backend/cluster/cluster_gcs.c index df04a899c2..052eec4a01 100644 --- a/src/backend/cluster/cluster_gcs.c +++ b/src/backend/cluster/cluster_gcs.c @@ -499,8 +499,14 @@ gcs_send_envelope_or_loopback(uint8 msg_type, int32 dest_node, const void *paylo /* * GCS requests can be produced from bufmgr/content-lock backend paths. - * Tier1 sockets are owned by LMON, so non-LMON request producers enqueue - * to the existing GRD outbound ring and wait on the normal reply slot. + * CONTROL-plane sockets are owned by LMON, so non-LMON request + * producers enqueue to the existing GRD outbound ring and wait on the + * normal reply slot. PGRAC: spec-7.2 D3 — this branch is also the + * staging rule for LMS: a migrated DATA-plane handler that must emit + * GCS_REQUEST (a CONTROL message; the ㉕ X->S downgrade notify and + * the BAST-nudge branch) lands here by the same != B_LMON test and + * goes through the ring for LMON to send — a direct send would trip + * the router's plane gate by design. */ if (msg_type == PGRAC_IC_MSG_GCS_REQUEST && MyBackendType != B_LMON) return cluster_grd_outbound_enqueue_backend_msg(msg_type, (uint32)dest_node, payload, @@ -725,8 +731,15 @@ cluster_gcs_send_transition_nowait(BufferTag tag, PcmLockTransition transition_i pg_atomic_fetch_add_u64(&ClusterGcs->encode_payload_bytes, sizeof(payload)); pg_atomic_fetch_add_u64(&ClusterGcs->send_request_count, 1); - return cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_REQUEST, master_node, &payload, - sizeof(payload)) + /* + * PGRAC: spec-7.2 D3 — route through the loopback/ring-aware helper + * instead of a raw send: post-flip this fire-and-forget notify is + * emitted from the LMS-context FORWARD/BAST paths, where GCS_REQUEST + * (CONTROL plane) must stage into the outbound ring for LMON (the + * D0-①b staging case). LMON callers keep the direct send. + */ + return gcs_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_REQUEST, master_node, &payload, + sizeof(payload)) == CLUSTER_IC_SEND_DONE; } From fe2da00005ed243781eefc533e676e6d547153f5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:51:57 +0800 Subject: [PATCH 15/58] feat(cluster): DATA-plane outbound ring + registry-probed twin plumbing The dormant half of the D3/D4 atomic flip: - cluster_lms_outbound.c: DATA twin of the GRD outbound ring (Q6-B; single-tail FIFO, LMS is the only consumer, WOULD_BLOCK requeues at the HEAD so per-peer byte streams never reorder -- INV-7.2-DATA-FIFO). The GCS block REQUEST pre-send hook (direct-land arm) migrates with the DATA consumer. New shmem region + t/020 baselines. - cluster_lms_wakeup(): SIGUSR1 -> latch (was SIG_IGN in the LMS skeleton); ring enqueue publishes before the kick. - backend staging entry (enqueue_backend_msg) routes by the msg_type's registered plane: DATA -> LMS ring, everything else unchanged. - cluster_gcs_block_family_on_data_plane(): registry probe (REPLY stands in for all five). The LMON ship_ready / pi_discard tick sites skip when the family is DATA; the LMS loop runs both plus the DATA ring drain when it is. Every branch reads CONTROL today -- the flip commit only edits the six registration structs and the whole pipeline pivots at once (no half-migrated window). Spec: spec-7.2-ic-data-plane-decoupling.md (D4) --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_gcs_block.c | 15 ++ src/backend/cluster/cluster_grd_outbound.c | 15 ++ src/backend/cluster/cluster_lmon.c | 12 +- src/backend/cluster/cluster_lms.c | 45 +++- src/backend/cluster/cluster_lms_outbound.c | 230 +++++++++++++++++++ src/backend/cluster/cluster_shmem.c | 3 + src/include/cluster/cluster_gcs_block.h | 3 + src/include/cluster/cluster_lms.h | 12 + src/test/cluster_tap/t/020_shmem_registry.pl | 4 +- src/test/cluster_unit/test_cluster_lmon.c | 7 + src/test/cluster_unit/test_cluster_shmem.c | 4 + 12 files changed, 341 insertions(+), 10 deletions(-) create mode 100644 src/backend/cluster/cluster_lms_outbound.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 7e8ca0fd45..83c829c27c 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -144,6 +144,7 @@ OBJS = \ cluster_lmon.o \ cluster_lms.o \ cluster_lms_data_plane.o \ + cluster_lms_outbound.o \ cluster_lns.o \ cluster_membership.o \ cluster_mrp.o \ diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 9638cf8915..3de18692b8 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -300,6 +300,21 @@ static const uint64 gcs_ship_hist_bounds_us[CLUSTER_GCS_SHIP_HIST_BUCKETS - 1] = { 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000, 2000000, 5000000, 10000000, 30000000 }; +/* + * PGRAC: spec-7.2 D3/D4 — registry probe: is the GCS block family on + * the DATA plane? REPLY stands in for all five (they flip atomically, + * H-5). Both LMON tick sites (ship_ready / pi_discard) and the LMS + * data-plane loop consult this so the flip commit only edits the six + * registration structs and everything pivots at once. + */ +bool +cluster_gcs_block_family_on_data_plane(void) +{ + const ClusterICMsgTypeInfo *info = cluster_ic_get_msg_type_info(PGRAC_IC_MSG_GCS_BLOCK_REPLY); + + return info != NULL && (ClusterICPlane)info->plane == CLUSTER_IC_PLANE_DATA; +} + /* Record one completed ship into the histogram (requester context). */ static void gcs_block_ship_hist_record(TimestampTz started_at) diff --git a/src/backend/cluster/cluster_grd_outbound.c b/src/backend/cluster/cluster_grd_outbound.c index 917168658f..83a453f197 100644 --- a/src/backend/cluster/cluster_grd_outbound.c +++ b/src/backend/cluster/cluster_grd_outbound.c @@ -40,6 +40,7 @@ #include "cluster/cluster_grd_outbound.h" #include "cluster/cluster_ic_rdma.h" #include "cluster/cluster_ic_router.h" +#include "cluster/cluster_lms.h" /* PGRAC: spec-7.2 D4 DATA-ring routing */ #include "cluster/cluster_ic_tier1.h" #include "cluster/cluster_lmon.h" #include "cluster/cluster_shmem.h" @@ -289,6 +290,20 @@ cluster_grd_outbound_enqueue_backend_msg(uint8 msg_type, uint32 dest_node_id, co Assert(cluster_grd_outbound_state != NULL); + /* + * PGRAC: spec-7.2 D4 — plane routing at the single backend staging + * entry: a msg_type registered on the DATA plane goes to the DATA + * twin ring (LMS drains + sends); everything else stays on this + * CONTROL ring (LMON). Before the plane flip every type reads + * CONTROL here, so this branch is dormant until the flip commit. + */ + { + const ClusterICMsgTypeInfo *pinfo = cluster_ic_get_msg_type_info(msg_type); + + if (pinfo != NULL && (ClusterICPlane)pinfo->plane == CLUSTER_IC_PLANE_DATA) + return cluster_lms_outbound_enqueue(msg_type, dest_node_id, payload, payload_len); + } + LWLockAcquire(cluster_grd_outbound_lock, LW_EXCLUSIVE); /* Reserved pool: BACKEND_REQUEST may consume ring slots only up to diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 0c98b23c66..24a51bed42 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -1099,7 +1099,8 @@ LmonMain(void) cluster_ges_lmon_drain_work_queue(); /* PGRAC: spec-6.12b — ship finished CR-server results (LMS * constructed them; only LMON owns the IC connections). */ - if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) + if (!cluster_gcs_block_family_on_data_plane() + && cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) cluster_lms_cr_ship_ready(); /* * spec-5.16 (orphan-grant, Rule 8.A) — reclaim abandoned reply-wait @@ -1175,7 +1176,8 @@ LmonMain(void) /* spec-6.12h D-h2: drain checkpoint-confirmed PI-discard write * notes and route each to the block's master (fire-and-forget; * only LMON owns the tier1 fds, L172 family). */ - cluster_gcs_block_pi_discard_drain(); + if (!cluster_gcs_block_family_on_data_plane()) + cluster_gcs_block_pi_discard_drain(); CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); @@ -1770,7 +1772,8 @@ LmonMain(void) (void)cluster_gcs_block_lmon_drain_direct_land_aborts(); /* PGRAC: spec-6.12b — ship finished CR-server results (LMS * constructed them; only LMON owns the IC connections). */ - if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) + if (!cluster_gcs_block_family_on_data_plane() + && cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) cluster_lms_cr_ship_ready(); if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_OUT, force_all_duties)) @@ -1792,7 +1795,8 @@ LmonMain(void) cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); /* spec-6.12h D-h2: drain checkpoint-confirmed PI-discard notes. */ - cluster_gcs_block_pi_discard_drain(); + if (!cluster_gcs_block_family_on_data_plane()) + cluster_gcs_block_pi_discard_drain(); CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index d3d03e7678..ddf65bbd72 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,7 +59,8 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" -#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" #include "cluster/cluster_grd_outbound.h" @@ -381,6 +382,22 @@ cluster_lms_get_pid(void) return pid; } +/* + * PGRAC: spec-7.2 D4 — wake the LMS data-plane loop (mirror of + * cluster_lmon_wakeup): SIGUSR1 → lms_sigusr1_handler → SetLatch. + * Safe from any context; self-kick and pre-spawn calls no-op. + */ +void +cluster_lms_wakeup(void) +{ + pid_t pid = cluster_lms_get_pid(); + + if (pid <= 0 || pid == MyProcPid) + return; + + (void)kill(pid, SIGUSR1); +} + TimestampTz cluster_lms_get_spawned_at(void) { @@ -651,6 +668,14 @@ cluster_lms_wake_drain(void) * latch / ShutdownRequestPending) * ============================================================ */ +/* PGRAC: spec-7.2 D4 — SIGUSR1 wakeup handler (latch only; rule 16: + * async-signal-safe operations exclusively). */ +static void +lms_sigusr1_handler(SIGNAL_ARGS) +{ + SetLatch(MyLatch); +} + void LmsMain(void) { @@ -667,7 +692,9 @@ LmsMain(void) pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); /* No ProcSignal slot in the skeleton; see auxprocess.c early setup. */ - pqsignal(SIGUSR1, SIG_IGN); + /* PGRAC: spec-7.2 D4 — SIGUSR1 = data-plane wakeup (was SIG_IGN in + * the skeleton). Handler only sets the latch (async-signal-safe). */ + pqsignal(SIGUSR1, lms_sigusr1_handler); pqsignal(SIGUSR2, SIG_IGN); pqsignal(SIGCHLD, SIG_DFL); @@ -743,9 +770,19 @@ LmsMain(void) * the data-plane tick (WaitEventSet: DATA sockets + MyLatch, latch * reset inside); the historic plain WaitLatch stays the fallback * when the plane is off. */ - if (cluster_lms_data_plane_enabled()) + if (cluster_lms_data_plane_enabled()) { + /* PGRAC: spec-7.2 D4 — once the GCS block family flips to + * the DATA plane, LMS ships its own READY results and + * drives the PI-discard notes (the LMON tick twins go + * quiet via the same registry probe); the DATA outbound + * ring drains here either way (empty before the flip). */ + if (cluster_gcs_block_family_on_data_plane()) { + cluster_lms_cr_ship_ready(); + cluster_gcs_block_pi_discard_drain(); + } + (void)cluster_lms_outbound_drain_send(); cluster_lms_data_plane_tick(LMS_IDLE_TIMEOUT_MS); - else { + } else { (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, LMS_IDLE_TIMEOUT_MS, WAIT_EVENT_PG_SLEEP); ResetLatch(MyLatch); diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c new file mode 100644 index 0000000000..d1d5b4a5fc --- /dev/null +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -0,0 +1,230 @@ +/*------------------------------------------------------------------------- + * + * cluster_lms_outbound.c + * pgrac DATA-plane outbound ring — spec-7.2 D4 (Q6-B twin ring). + * + * Backend-context producers of DATA-plane messages (GCS block + * REQUEST / INVALIDATE) stage frames here; the LMS data-plane loop + * drains and sends them over its own fds. This is the DATA twin of + * the CONTROL-plane cluster_grd_outbound ring, kept to the same + * minimal single-tail-FIFO semantics (Q6-B: one consumer, one lock, + * no claim/compaction machinery — the r1-F3 argument against a + * shared dual-reader ring). Enqueue marks no LMON duty and wakes + * LMS, not LMON. + * + * Ordering note (INV-7.2-DATA-FIFO): per-peer DATA frames keep a + * single ordered stream because (a) this ring is FIFO, (b) LMS is + * its only consumer, and (c) LMS sends on per-peer sockets with the + * tier1 partial-frame buffer (WOULD_BLOCK requeues at the head + * before anything newer is sent to that peer). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_lms_outbound.c + * + * NOTES + * This is a pgrac-original file. Compiled only in --enable-cluster + * builds. Spec: spec-7.2-ic-data-plane-decoupling.md (D4). + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_gcs_block.h" /* GcsBlockRequestPayload (pre-send hook) */ +#include "cluster/cluster_ic.h" +#include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_ic_router.h" +#include "cluster/cluster_lms.h" +#include "cluster/cluster_shmem.h" +#include "miscadmin.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/elog.h" + +#ifdef USE_PGRAC_CLUSTER + +#define PGRAC_LMS_OUTBOUND_CAPACITY 256 +#define PGRAC_LMS_OUTBOUND_PAYLOAD_MAX 128 + +typedef struct ClusterLmsOutboundSlot { + uint32 dest_node_id; + uint8 msg_type; + uint16 payload_len; + uint8 payload[PGRAC_LMS_OUTBOUND_PAYLOAD_MAX]; +} ClusterLmsOutboundSlot; + +typedef struct ClusterLmsOutboundState { + uint32 head; /* next slot to fill */ + uint32 tail; /* next slot to drain */ + uint32 count; + ClusterLmsOutboundSlot ring[PGRAC_LMS_OUTBOUND_CAPACITY]; +} ClusterLmsOutboundState; + +static ClusterLmsOutboundState *cluster_lms_outbound_state = NULL; +static LWLock *cluster_lms_outbound_lock = NULL; + +static Size +cluster_lms_outbound_shmem_size(void) +{ + return MAXALIGN(sizeof(ClusterLmsOutboundState)); +} + +static void +cluster_lms_outbound_shmem_init(void) +{ + bool found; + + cluster_lms_outbound_state = (ClusterLmsOutboundState *)ShmemInitStruct( + "pgrac cluster lms data outbound", cluster_lms_outbound_shmem_size(), &found); + if (!found) + memset(cluster_lms_outbound_state, 0, sizeof(*cluster_lms_outbound_state)); + + if (!IsBootstrapProcessingMode()) + cluster_lms_outbound_lock = &(GetNamedLWLockTranche("ClusterLmsDataOutbound"))[0].lock; +} + +static const ClusterShmemRegion cluster_lms_outbound_region = { + .name = "pgrac cluster lms data outbound", + .size_fn = cluster_lms_outbound_shmem_size, + .init_fn = cluster_lms_outbound_shmem_init, + .lwlock_count = 1, + .owner_subsys = "cluster_lms_outbound", + .reserved_flags = 0, +}; + +void +cluster_lms_outbound_shmem_register(void) +{ + cluster_shmem_register_region(&cluster_lms_outbound_region); +} + +/* + * cluster_lms_outbound_enqueue — stage one DATA-plane frame for LMS. + * + * Returns false on full ring / oversized payload / pre-shmem call; + * callers treat false as WOULD_BLOCK (retry via the normal request + * retry machinery). Publish-before-signal: the slot is visible + * before the LMS wakeup fires. + */ +bool +cluster_lms_outbound_enqueue(uint8 msg_type, uint32 dest_node_id, const void *payload, + uint16 payload_len) +{ + ClusterLmsOutboundSlot *slot; + + if (cluster_lms_outbound_state == NULL || cluster_lms_outbound_lock == NULL) + return false; + if (payload_len > PGRAC_LMS_OUTBOUND_PAYLOAD_MAX) + return false; + + LWLockAcquire(cluster_lms_outbound_lock, LW_EXCLUSIVE); + if (cluster_lms_outbound_state->count >= PGRAC_LMS_OUTBOUND_CAPACITY) { + LWLockRelease(cluster_lms_outbound_lock); + return false; + } + slot = &cluster_lms_outbound_state->ring[cluster_lms_outbound_state->head]; + slot->dest_node_id = dest_node_id; + slot->msg_type = msg_type; + slot->payload_len = payload_len; + if (payload_len > 0) + memcpy(slot->payload, payload, payload_len); + cluster_lms_outbound_state->head + = (cluster_lms_outbound_state->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; + cluster_lms_outbound_state->count++; + LWLockRelease(cluster_lms_outbound_lock); + + cluster_lms_wakeup(); + return true; +} + +/* Head-requeue for WOULD_BLOCK (preserves per-peer FIFO). */ +static void +lms_outbound_requeue_head(const ClusterLmsOutboundSlot *slot) +{ + LWLockAcquire(cluster_lms_outbound_lock, LW_EXCLUSIVE); + if (cluster_lms_outbound_state->count < PGRAC_LMS_OUTBOUND_CAPACITY) { + cluster_lms_outbound_state->tail + = (cluster_lms_outbound_state->tail + PGRAC_LMS_OUTBOUND_CAPACITY - 1) + % PGRAC_LMS_OUTBOUND_CAPACITY; + cluster_lms_outbound_state->ring[cluster_lms_outbound_state->tail] = *slot; + cluster_lms_outbound_state->count++; + } + /* full → drop; fire-and-forget layers self-heal via retransmit */ + LWLockRelease(cluster_lms_outbound_lock); +} + +/* + * cluster_lms_outbound_drain_send — LMS data-plane loop: drain + send. + * + * Bounded batch per call. WOULD_BLOCK requeues at the HEAD so the + * per-peer byte stream is never reordered (INV-7.2-DATA-FIFO). The + * GCS block REQUEST pre-send hook (direct-land arm) migrates here + * with the DATA consumer (D0-② coupling item ①). + */ +int +cluster_lms_outbound_drain_send(void) +{ + int sent = 0; + + if (cluster_lms_outbound_state == NULL || cluster_lms_outbound_lock == NULL) + return 0; + + Assert(MyBackendType == B_LMS); + + while (sent < 64) { + ClusterLmsOutboundSlot slot; + ClusterICSendResult rc; + bool got = false; + + LWLockAcquire(cluster_lms_outbound_lock, LW_EXCLUSIVE); + if (cluster_lms_outbound_state->count > 0) { + slot = cluster_lms_outbound_state->ring[cluster_lms_outbound_state->tail]; + cluster_lms_outbound_state->tail + = (cluster_lms_outbound_state->tail + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; + cluster_lms_outbound_state->count--; + got = true; + } + LWLockRelease(cluster_lms_outbound_lock); + if (!got) + break; + + if (slot.msg_type == PGRAC_IC_MSG_GCS_BLOCK_REQUEST + && slot.payload_len == sizeof(GcsBlockRequestPayload)) + cluster_gcs_block_lmon_prepare_outbound_request((GcsBlockRequestPayload *)slot.payload, + (int32)slot.dest_node_id); + + rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, + slot.payload_len > 0 ? slot.payload : NULL, slot.payload_len); + if (rc == CLUSTER_IC_SEND_DONE) { + sent++; + continue; + } + if (rc == CLUSTER_IC_SEND_WOULD_BLOCK) { + lms_outbound_requeue_head(&slot); + break; + } + /* HARD_ERROR: peer down — drop; requesters retry fail-closed. */ + } + + return sent; +} + +uint32 +cluster_lms_outbound_depth(void) +{ + uint32 depth; + + if (cluster_lms_outbound_state == NULL || cluster_lms_outbound_lock == NULL) + return 0; + LWLockAcquire(cluster_lms_outbound_lock, LW_SHARED); + depth = cluster_lms_outbound_state->count; + LWLockRelease(cluster_lms_outbound_lock); + return depth; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index fa4b2db725..eabf487b01 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -681,6 +681,9 @@ cluster_init_shmem_module(void) cluster_grd_outbound_shmem_register(); if (cluster_shmem_lookup_region("pgrac cluster grd work queue") == NULL) cluster_grd_work_queue_shmem_register(); + /* PGRAC: spec-7.2 D4 — DATA-plane outbound ring (Q6-B twin). */ + if (cluster_shmem_lookup_region("pgrac cluster lms data outbound") == NULL) + cluster_lms_outbound_shmem_register(); /* * spec-2.4 D2: register cluster_epoch shmem region. 64-byte cache- diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 97af1bdae1..2f8417a522 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1805,6 +1805,9 @@ extern uint64 cluster_gcs_get_block_ship_bytes_total(void); extern uint64 cluster_gcs_block_ship_hist_bound_us(int bucket); extern uint64 cluster_gcs_block_ship_hist_count(int bucket); +/* PGRAC: spec-7.2 D3/D4 — registry probe for the atomic plane flip. */ +extern bool cluster_gcs_block_family_on_data_plane(void); + /* PGRAC: spec-6.13 D8 — RDMA tier3/direct-land copy observability. */ extern uint64 cluster_gcs_get_scratch_copy_count(void); extern uint64 cluster_gcs_get_live_sge_send_count(void); diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index 3feb43ee5a..9601b2bb7d 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -307,6 +307,18 @@ extern bool cluster_lms_data_plane_enabled(void); extern void cluster_lms_data_plane_tick(long timeout_ms); extern void cluster_lms_data_plane_shutdown(void); +/* + * PGRAC: spec-7.2 D4 — LMS wakeup (mirror of cluster_lmon_wakeup) + the + * DATA-plane outbound ring (cluster_lms_outbound.c; Q6-B twin ring: + * backends stage DATA frames, LMS drains and sends on its own fds). + */ +extern void cluster_lms_wakeup(void); +extern void cluster_lms_outbound_shmem_register(void); +extern bool cluster_lms_outbound_enqueue(uint8 msg_type, uint32 dest_node_id, const void *payload, + uint16 payload_len); +extern int cluster_lms_outbound_drain_send(void); +extern uint32 cluster_lms_outbound_depth(void); + /* * Read-only accessors for SQL view + diagnostics. */ diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..4af39b3e6f 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -109,9 +109,9 @@ # between "pgrac cluster cr relgen" and "pgrac cluster cr tuple stats"). # spec-6.12h D-h3a: +1 "pgrac cluster pi shadow" (PI ship-SCN shadow table; # sorts between "pgrac cluster oid lease,pgrac cluster pcm grd" and "pgrac cluster qvotec"). -my $expected_region_count = $has_visibility_inject ? '79' : '78'; +my $expected_region_count = $has_visibility_inject ? '80' : '79'; my $expected_regions = - 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo record cursor'; + 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lms data outbound,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo record cursor'; $expected_regions .= ',pgrac cluster visibility inject' if $has_visibility_inject; # spec-4.12 D7: cooperative write-fence region; always registered. Sorts after diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index a2f4a7de12..c7d7824a56 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -202,6 +202,13 @@ int cluster_interconnect_recv_timeout_ms = 30000; /* spec-7.2 D1 GUC (cluster_lmon_duty_should_run references it). */ bool cluster_ic_duty_lazy = true; +/* spec-7.2 D4 stub: plane-flip registry probe (skeleton = CONTROL). */ +bool +cluster_gcs_block_family_on_data_plane(void) +{ + return false; +} + #include "cluster/cluster_ic_tier1.h" int diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index d5bfd4ca76..89bed4f2b3 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -772,6 +772,10 @@ cluster_grd_outbound_shmem_register(void) void cluster_grd_work_queue_shmem_register(void) {} +/* spec-7.2 D4 stub: DATA-plane outbound ring registration. */ +void +cluster_lms_outbound_shmem_register(void) +{} /* spec-2.16 cluster_shmem.c also calls RequestNamedLWLockTranche for the * 2 NEW Step 2 tranches. PG built-in stub already declared in this From 52e087fbc097769d35baa2376e143764f20cdb8d Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 11:53:07 +0800 Subject: [PATCH 16/58] fix(cluster): request the DATA-ring named LWLock tranche GetNamedLWLockTranche requires a matching RequestNamedLWLockTranche in the process_shmem_requests window (I15 pattern); without it the first tranche lookup FATALs at startup. Also stub the register hook for the shmem unit harness. Spec: spec-7.2-ic-data-plane-decoupling.md (D4) --- src/backend/cluster/cluster_lms_outbound.c | 7 +++++++ src/backend/cluster/cluster_shmem.c | 1 + src/include/cluster/cluster_lms.h | 1 + src/test/cluster_unit/test_cluster_shmem.c | 5 ++++- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c index d1d5b4a5fc..e7f4b9a5ec 100644 --- a/src/backend/cluster/cluster_lms_outbound.c +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -103,6 +103,13 @@ cluster_lms_outbound_shmem_register(void) cluster_shmem_register_region(&cluster_lms_outbound_region); } +/* Named-tranche request (process_shmem_requests window; I15 pattern). */ +void +cluster_lms_outbound_request_lwlocks(void) +{ + RequestNamedLWLockTranche("ClusterLmsDataOutbound", 1); +} + /* * cluster_lms_outbound_enqueue — stage one DATA-plane frame for LMS. * diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index eabf487b01..6021ccd2ca 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -918,6 +918,7 @@ cluster_request_shmem(void) cluster_cr_pool_request_lwlocks(); /* spec-5.51: CR pool named LWLock tranche */ cluster_resolver_cache_request_lwlocks(); /* spec-5.55: resolver cache LWLock tranche */ cluster_ges_dedup_shmem_request(); + cluster_lms_outbound_request_lwlocks(); /* PGRAC: spec-7.2 D4 DATA ring */ /* * spec-4.5a G5: pg_xact_remote SLRU manages its own shmem (SimpleLru), diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index 9601b2bb7d..31b8b6a999 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -314,6 +314,7 @@ extern void cluster_lms_data_plane_shutdown(void); */ extern void cluster_lms_wakeup(void); extern void cluster_lms_outbound_shmem_register(void); +extern void cluster_lms_outbound_request_lwlocks(void); extern bool cluster_lms_outbound_enqueue(uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len); extern int cluster_lms_outbound_drain_send(void); diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index 89bed4f2b3..a5f87916c9 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -772,10 +772,13 @@ cluster_grd_outbound_shmem_register(void) void cluster_grd_work_queue_shmem_register(void) {} -/* spec-7.2 D4 stub: DATA-plane outbound ring registration. */ +/* spec-7.2 D4 stubs: DATA-plane outbound ring registration + tranche. */ void cluster_lms_outbound_shmem_register(void) {} +void +cluster_lms_outbound_request_lwlocks(void) +{} /* spec-2.16 cluster_shmem.c also calls RequestNamedLWLockTranche for the * 2 NEW Step 2 tranches. PG built-in stub already declared in this From 08a8f28fc66b213ae9dfc245412bc12dfb4649a6 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 12:57:57 +0800 Subject: [PATCH 17/58] feat(cluster): INV-7.2-CONN-EPOCH substrate + DATA-plane fence linkage Connection-generation fence for the DATA plane (spec-7.2 D5): - conn_epoch recorded per peer at every CONNECTED transition (dialer stamps its HELLO epoch; accept sides record the peer's), giving the connection an epoch identity. - sender gate in tier1_send_bytes: a DATA frame never goes out on a connection bound to a different epoch (HARD_ERROR -> close + re-HELLO rebinds at the current epoch). Cross-epoch interleave on one byte stream would defeat the reconfig-rebuild ordering argument (8.A). CONTROL is exempt -- its single LMON stream carries the epoch events themselves. - epoch watch in the LMS tick: a bump proactively force-closes the whole DATA mesh (reconnect immediately); the sender gate covers the window before the tick notices. - write-fence linkage: while the fence is enforcing and writes are disallowed, LMS sends nothing on the DATA plane (ship_ready / pi-discard / ring drain all held; thaw wakes the latch and the held frames drain). Block images must not leave a fenced node. Spec: spec-7.2-ic-data-plane-decoupling.md (D5) --- src/backend/cluster/cluster_ic_tier1.c | 22 ++++++++++++++ src/backend/cluster/cluster_lms.c | 22 +++++++++----- src/backend/cluster/cluster_lms_data_plane.c | 32 ++++++++++++++++++++ src/include/cluster/cluster_ic_tier1.h | 12 ++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 30761a600d..f82e483e8b 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -476,6 +476,7 @@ tier1_shmem_init(void) pg_atomic_init_u64(&s->peers[i].epoch_observe_advance_count, 0); /* spec-2.4 v1.0.1 F3: LMON-mediated close request flag. */ pg_atomic_init_u32(&s->peers[i].close_requested, 0); + s->peers[i].conn_epoch = 0; } } } @@ -599,6 +600,20 @@ tier1_send_bytes(int32 target_node_id, const void *buf, size_t len) || Tier1Shmem->peers[target_node_id].state != (int32)CLUSTER_IC_PEER_CONNECTED) return CLUSTER_IC_SEND_WOULD_BLOCK; /* HELLO pending; caller retries */ + /* + * PGRAC: spec-7.2 D5 (INV-7.2-CONN-EPOCH sender gate) — never put a + * new epoch's DATA frame on a connection established under an older + * epoch: cross-epoch interleave on one byte stream would defeat the + * reconfig-rebuild ordering argument (8.A face). HARD_ERROR makes + * the data-plane loop close the peer; reconnect + re-HELLO rebinds + * at the current epoch (the epoch-watch in the LMS tick also force- + * closes proactively on a bump). CONTROL is exempt — its single + * LMON stream is the epoch-event carrier itself. + */ + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA + && Tier1Shmem->peers[target_node_id].conn_epoch != cluster_epoch_get_current()) + return CLUSTER_IC_SEND_HARD_ERROR; + /* * Hardening v1.0.1 F1 (spec-2.2 v1.0.1 + spec-2.3 v1.0.1 L68): * per-peer outbound buffer for partial writes. If a previous send @@ -1396,6 +1411,9 @@ cluster_ic_tier1_continue_hello_send(int32 peer_id, int peer_fd) */ if (Tier1Shmem != NULL) { Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_CONNECTED; + /* PGRAC: spec-7.2 D5 — bind the connection to its epoch (dialer + * side stamped this epoch into its HELLO). */ + Tier1Shmem->peers[peer_id].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); } ereport(LOG, @@ -1506,6 +1524,8 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) } Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_CONNECTED; + /* PGRAC: spec-7.2 D5 — bind the connection to the peer's HELLO epoch. */ + Tier1Shmem->peers[peer_id].conn_epoch = cluster_ic_hello_conn_epoch(&msg); Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(peer_id); /* cache addr in shmem for view */ cluster_sf_note_peer_hello_capabilities(peer_id, cluster_ic_hello_capabilities(&msg)); @@ -1684,6 +1704,8 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear if (Tier1Shmem != NULL) { peer_record_error(learned, 0, "", ""); /* clear any prior */ Tier1Shmem->peers[learned].state = (int32)CLUSTER_IC_PEER_CONNECTED; + /* PGRAC: spec-7.2 D5 — bind the connection to the peer's HELLO epoch. */ + Tier1Shmem->peers[learned].conn_epoch = cluster_ic_hello_conn_epoch(&msg); Tier1Shmem->peers[learned].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(learned); cluster_sf_note_peer_hello_capabilities(learned, cluster_ic_hello_capabilities(&msg)); diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index ddf65bbd72..d8a7f228a9 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,8 +59,9 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" -#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ -#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_write_fence.h" /* PGRAC: spec-7.2 D5 fence linkage */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" #include "cluster/cluster_grd_outbound.h" @@ -775,12 +776,19 @@ LmsMain(void) * the DATA plane, LMS ships its own READY results and * drives the PI-discard notes (the LMON tick twins go * quiet via the same registry probe); the DATA outbound - * ring drains here either way (empty before the flip). */ - if (cluster_gcs_block_family_on_data_plane()) { - cluster_lms_cr_ship_ready(); - cluster_gcs_block_pi_discard_drain(); + * ring drains here either way (empty before the flip). + * D5 fence linkage: while the write-fence is enforcing and + * writes are disallowed, the DATA plane sends nothing — + * block images must not leave a fenced node. Thaw wakes + * the latch and the held frames drain (pure-read probe, + * no-throw, per the write_fence_allowed contract). */ + if (!(cluster_write_fence_enforcing() && !cluster_write_fence_allowed())) { + if (cluster_gcs_block_family_on_data_plane()) { + cluster_lms_cr_ship_ready(); + cluster_gcs_block_pi_discard_drain(); + } + (void)cluster_lms_outbound_drain_send(); } - (void)cluster_lms_outbound_drain_send(); cluster_lms_data_plane_tick(LMS_IDLE_TIMEOUT_MS); } else { (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, diff --git a/src/backend/cluster/cluster_lms_data_plane.c b/src/backend/cluster/cluster_lms_data_plane.c index dc60bb6447..e6c2b7f34d 100644 --- a/src/backend/cluster/cluster_lms_data_plane.c +++ b/src/backend/cluster/cluster_lms_data_plane.c @@ -52,6 +52,7 @@ #include "cluster/cluster_conf.h" #include "cluster/cluster_elog.h" +#include "cluster/cluster_epoch.h" /* PGRAC: spec-7.2 D5 epoch watch */ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic.h" #include "cluster/cluster_ic_tier1.h" @@ -219,6 +220,37 @@ cluster_lms_data_plane_tick(long timeout_ms) now = GetCurrentTimestamp(); + /* + * PGRAC: spec-7.2 D5 (INV-7.2-CONN-EPOCH ③) — proactive connection + * reset on an epoch bump: every DATA connection is bound to the + * epoch it was established under, so a bump force-closes the mesh + * and the reconnect below re-HELLOs at the current epoch. The + * sender gate in tier1_send_bytes is the structural backstop for + * the window between the bump and this tick. + */ + { + static uint64 dp_last_epoch = 0; + static bool dp_epoch_seen = false; + uint64 cur_epoch = cluster_epoch_get_current(); + + if (!dp_epoch_seen) { + dp_last_epoch = cur_epoch; + dp_epoch_seen = true; + } else if (cur_epoch != dp_last_epoch) { + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd >= 0) { + cluster_ic_tier1_close_peer(pi, "data-plane epoch bump reset"); + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + dp_track[pi].connect_started_at = 0; + dp_track[pi].next_attempt_at = 0; /* reconnect immediately */ + dp_wes_dirty = true; + } + } + dp_last_epoch = cur_epoch; + } + } + /* Active-role reconnect for DOWN peers whose backoff elapsed. */ for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { int new_fd = -1; diff --git a/src/include/cluster/cluster_ic_tier1.h b/src/include/cluster/cluster_ic_tier1.h index 10aaf0ac41..e6a3ce349d 100644 --- a/src/include/cluster/cluster_ic_tier1.h +++ b/src/include/cluster/cluster_ic_tier1.h @@ -99,6 +99,18 @@ typedef struct ClusterICPeerStateShmem { * close. */ pg_atomic_uint32 close_requested; /* 0 = idle, 1 = pending close */ + + /* + * PGRAC: spec-7.2 D5 (INV-7.2-CONN-EPOCH) — the cluster epoch this + * connection was established under (recorded at the CONNECTED + * transition; DATA plane only, CONTROL leaves it 0/unenforced). + * The DATA physical-send gate refuses to put bytes on a connection + * whose conn_epoch != current epoch — an epoch bump structurally + * invalidates the old stream and forces close + re-HELLO, so a new + * epoch's frames can never interleave into an old epoch's byte + * stream. Written and read by the owning plane's process only. + */ + uint64 conn_epoch; } ClusterICPeerStateShmem; /* From f494f504a2ce6f9360f2539d34f6f4b77144088e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 13:07:36 +0800 Subject: [PATCH 18/58] feat(cluster): LMS data-plane wait events + injection points Two wait events (F12 four-point symmetry: enum / wait_event.c name map / views table / count): ClusterLmsDataRecv is the data-plane WaitEventSet wait; ClusterLmsDataSend is the same wait while any peer holds a backpressured partial frame (waiting for WRITEABLE drainage). CLUSTER_WAIT_EVENTS_COUNT 118 -> 120; every count pin across the TAP + unit surfaces follows. Two injection points: cluster-lms-data-dispatch fires before the inbound envelope pump of a readable DATA peer (hit surface for the flip e2e legs); cluster-lms-conn-reset SKIP forces a one-shot close of the whole DATA mesh (the reset/epoch legs' trigger) and shares the epoch-watch reset path. Registry 158 -> 160 rows (t/015). Spec: spec-7.2-ic-data-plane-decoupling.md (D6) --- src/backend/cluster/cluster_inject.c | 14 +++++++++ src/backend/cluster/cluster_lms_data_plane.c | 22 ++++++++++++-- src/backend/cluster/cluster_views.c | 4 ++- src/backend/utils/activity/wait_event.c | 30 +++++++++++-------- src/include/cluster/cluster_views.h | 2 +- src/include/utils/wait_event.h | 3 ++ src/test/cluster_tap/t/010_views.pl | 4 +-- src/test/cluster_tap/t/011_gviews.pl | 6 ++-- src/test/cluster_tap/t/012_ic.pl | 8 ++--- src/test/cluster_tap/t/013_conf.pl | 8 ++--- src/test/cluster_tap/t/014_ic_mock.pl | 4 +-- src/test/cluster_tap/t/015_inject.pl | 8 ++--- src/test/cluster_tap/t/016_perfmon.pl | 4 +-- src/test/cluster_tap/t/017_debug.pl | 4 +-- src/test/cluster_tap/t/020_shmem_registry.pl | 4 +-- src/test/cluster_tap/t/021_block_format.pl | 4 +-- src/test/cluster_tap/t/022_itl_slot.pl | 4 +-- .../cluster_tap/t/023_buffer_descriptor.pl | 4 +-- src/test/cluster_tap/t/030_acceptance.pl | 6 ++-- .../cluster_tap/t/108_pcm_state_machine.pl | 2 +- .../cluster_tap/t/111_gcs_block_ship_2node.pl | 2 +- .../cluster_tap/t/113_gcs_block_2way_2node.pl | 4 +-- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 2 +- .../cluster_tap/t/115_gcs_block_3way_3node.pl | 2 +- .../t/116_gcs_block_lost_write_2node.pl | 4 +-- .../t/117_sinval_broadcast_2node.pl | 2 +- .../t/118_sinval_ddl_propagation_2node.pl | 4 +-- .../t/203_cluster_tt_status_foundation.pl | 4 +-- .../test_cluster_gcs_block_retransmit.c | 6 ++-- .../test_cluster_stage2_acceptance.c | 6 ++-- src/test/cluster_unit/test_cluster_views.c | 8 ++--- 31 files changed, 116 insertions(+), 73 deletions(-) diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..81035e2ba9 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -358,6 +358,20 @@ static ClusterInjectPoint cluster_injection_points[] = { * that would otherwise serve FULL/PARTIAL. */ { .name = "cluster-lms-cr-construct" }, + /* + * spec-7.2 D6 — LMS data-plane observability injections. + * + * cluster-lms-data-dispatch: + * Fires in the LMS data-plane tick just before the inbound + * envelope pump for a readable DATA peer (hit-count surface for + * the flip e2e legs). + * cluster-lms-conn-reset: + * Fires in the tick's reset scan; SKIP forces a one-shot close + * of every DATA connection (reconnect + re-HELLO converge), the + * L4-family reset/epoch legs' trigger. + */ + { .name = "cluster-lms-data-dispatch" }, + { .name = "cluster-lms-conn-reset" }, /* * spec-6.12i D-i1 — undo-TT fetch serve refusal injection. * diff --git a/src/backend/cluster/cluster_lms_data_plane.c b/src/backend/cluster/cluster_lms_data_plane.c index e6c2b7f34d..14736b1263 100644 --- a/src/backend/cluster/cluster_lms_data_plane.c +++ b/src/backend/cluster/cluster_lms_data_plane.c @@ -56,6 +56,7 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic.h" #include "cluster/cluster_ic_tier1.h" +#include "cluster/cluster_inject.h" /* PGRAC: spec-7.2 D6 injection points */ #include "cluster/cluster_lms.h" #include "miscadmin.h" #include "storage/latch.h" @@ -228,6 +229,7 @@ cluster_lms_data_plane_tick(long timeout_ms) * sender gate in tier1_send_bytes is the structural backstop for * the window between the bump and this tick. */ + CLUSTER_INJECTION_POINT("cluster-lms-conn-reset"); { static uint64 dp_last_epoch = 0; static bool dp_epoch_seen = false; @@ -236,7 +238,8 @@ cluster_lms_data_plane_tick(long timeout_ms) if (!dp_epoch_seen) { dp_last_epoch = cur_epoch; dp_epoch_seen = true; - } else if (cur_epoch != dp_last_epoch) { + } else if (cur_epoch != dp_last_epoch + || cluster_injection_should_skip("cluster-lms-conn-reset")) { for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { if (dp_track[pi].fd >= 0) { cluster_ic_tier1_close_peer(pi, "data-plane epoch bump reset"); @@ -304,7 +307,21 @@ cluster_lms_data_plane_tick(long timeout_ms) if (timeout_ms < 0) timeout_ms = 0; - n_events = WaitEventSetWait(dp_wes, timeout_ms, ev, lengthof(ev), WAIT_EVENT_PG_SLEEP); + { + /* PGRAC: spec-7.2 D6 — wait identity: SEND while any peer has a + * backpressured partial frame (we are waiting for WRITEABLE + * drainage), RECV otherwise. */ + uint32 wait_kind = WAIT_EVENT_CLUSTER_LMS_DATA_RECV; + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd >= 0 && dp_track[pi].substate == LMS_DP_CONNECTED + && cluster_ic_tier1_pending_outbound(pi)) { + wait_kind = WAIT_EVENT_CLUSTER_LMS_DATA_SEND; + break; + } + } + n_events = WaitEventSetWait(dp_wes, timeout_ms, ev, lengthof(ev), wait_kind); + } for (i = 0; i < n_events; i++) { intptr_t tag = (intptr_t)ev[i].user_data; @@ -370,6 +387,7 @@ cluster_lms_data_plane_tick(long timeout_ms) } } else if (dp_track[peer].substate == LMS_DP_CONNECTED && (ev[i].events & WL_SOCKET_READABLE)) { + CLUSTER_INJECTION_POINT("cluster-lms-data-dispatch"); /* Generic envelope pump: recv + verify + dispatch. No * DATA msg_type is registered before the D3/D4 flip, so * pre-flip traffic is limited to HELLO/errors; post- diff --git a/src/backend/cluster/cluster_views.c b/src/backend/cluster/cluster_views.c index a1554dbbb2..1c85378157 100644 --- a/src/backend/cluster/cluster_views.c +++ b/src/backend/cluster/cluster_views.c @@ -276,7 +276,7 @@ static const uint32 cluster_wait_event_infos[CLUSTER_WAIT_EVENTS_COUNT] = { WAIT_EVENT_SINVAL_ACK_SEND, WAIT_EVENT_SINVAL_ACK_RECEIVE, - /* Cluster: Interconnect (9 = 7 + spec-6.13 busypoll + inline send) */ + /* Cluster: Interconnect (11 = 9 + spec-7.2 LMS data-plane recv/send) */ WAIT_EVENT_INTERCONNECT_RDMA_SEND, WAIT_EVENT_INTERCONNECT_RDMA_RECV, WAIT_EVENT_CLUSTER_IC_RDMA_POLL, @@ -286,6 +286,8 @@ static const uint32 cluster_wait_event_infos[CLUSTER_WAIT_EVENTS_COUNT] = { WAIT_EVENT_INTERCONNECT_TCP_FALLBACK, WAIT_EVENT_INTERCONNECT_TIER_SWITCH, WAIT_EVENT_INTERCONNECT_CONNECT_RETRY, + WAIT_EVENT_CLUSTER_LMS_DATA_RECV, /* PGRAC spec-7.2 D6 */ + WAIT_EVENT_CLUSTER_LMS_DATA_SEND, /* PGRAC spec-7.2 D6 */ /* Cluster: Undo (4) */ WAIT_EVENT_UNDO_REMOTE_READ, diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index a1844aac56..677e4e7df2 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -1225,18 +1225,18 @@ pgstat_get_wait_cluster_interconnect(WaitEventCluster w) case WAIT_EVENT_INTERCONNECT_RDMA_RECV: event_name = "InterconnectRdmaRecv"; break; - case WAIT_EVENT_CLUSTER_IC_RDMA_POLL: - event_name = "ClusterICRdmaPoll"; - break; - case WAIT_EVENT_INTERCONNECT_RDMA_BUSYPOLL: - event_name = "InterconnectRdmaBusypoll"; - break; - case WAIT_EVENT_INTERCONNECT_RDMA_INLINE_SEND: - event_name = "InterconnectRdmaInlineSend"; - break; - case WAIT_EVENT_CLUSTER_IC_RDMA_CONNECT: - event_name = "ClusterICRdmaConnect"; - break; + case WAIT_EVENT_CLUSTER_IC_RDMA_POLL: + event_name = "ClusterICRdmaPoll"; + break; + case WAIT_EVENT_INTERCONNECT_RDMA_BUSYPOLL: + event_name = "InterconnectRdmaBusypoll"; + break; + case WAIT_EVENT_INTERCONNECT_RDMA_INLINE_SEND: + event_name = "InterconnectRdmaInlineSend"; + break; + case WAIT_EVENT_CLUSTER_IC_RDMA_CONNECT: + event_name = "ClusterICRdmaConnect"; + break; case WAIT_EVENT_INTERCONNECT_TCP_FALLBACK: event_name = "ClusterICRdmaFallback"; break; @@ -1265,6 +1265,12 @@ pgstat_get_wait_cluster_interconnect(WaitEventCluster w) case WAIT_EVENT_CLUSTER_IC_RECONNECT: event_name = "ClusterICReconnect"; break; + case WAIT_EVENT_CLUSTER_LMS_DATA_RECV: + event_name = "ClusterLmsDataRecv"; + break; + case WAIT_EVENT_CLUSTER_LMS_DATA_SEND: + event_name = "ClusterLmsDataSend"; + break; default: break; } diff --git a/src/include/cluster/cluster_views.h b/src/include/cluster/cluster_views.h index 46b702f4df..49f8b1f1f9 100644 --- a/src/include/cluster/cluster_views.h +++ b/src/include/cluster/cluster_views.h @@ -51,7 +51,7 @@ * internal table in cluster_views.c stays in sync with the enum. */ #define CLUSTER_WAIT_EVENTS_COUNT \ - 118 /* spec-6.13 D8: RDMA busypoll + inline wait events included */ + 120 /* spec-7.2 D6: +2 LMS data-plane; spec-6.13 D8 RDMA included */ /* diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index c67502ae2c..1897b3af0b 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -468,6 +468,9 @@ typedef enum { WAIT_EVENT_CLUSTER_IC_TCP_SEND, WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT, WAIT_EVENT_CLUSTER_IC_RECONNECT, + /* PGRAC: spec-7.2 D6 -- LMS data-plane loop waits. */ + WAIT_EVENT_CLUSTER_LMS_DATA_RECV, + WAIT_EVENT_CLUSTER_LMS_DATA_SEND, /* Cluster: Undo (8 events) -- AD-010; spec-3.9 adds CR_CONSTRUCT; * spec-3.11 adds TT_DURABLE_IO; spec-3.18 D7 adds BUF_FLUSH + EXTENT_CLAIM */ diff --git a/src/test/cluster_tap/t/010_views.pl b/src/test/cluster_tap/t/010_views.pl index 524beb3e71..2f174371ea 100644 --- a/src/test/cluster_tap/t/010_views.pl +++ b/src/test/cluster_tap/t/010_views.pl @@ -50,8 +50,8 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/011_gviews.pl b/src/test/cluster_tap/t/011_gviews.pl index 61dda90bdc..c9c6d7c20b 100644 --- a/src/test/cluster_tap/t/011_gviews.pl +++ b/src/test/cluster_tap/t/011_gviews.pl @@ -15,7 +15,7 @@ # # What this test verifies: # - The global view exists and is queryable. -# - It returns exactly 118 rows (1 node x 118 cluster wait events). +# - It returns exactly 120 rows (1 node x 118 cluster wait events). # - It exposes exactly 1 distinct node_id at 0.17 (placeholder). # - The single node_id matches the cluster.node_id GUC. # - Per-class row counts match docs/wait-events-design.md §2.1. @@ -62,8 +62,8 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_gcluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/012_ic.pl b/src/test/cluster_tap/t/012_ic.pl index f894be28ad..dae334be93 100644 --- a/src/test/cluster_tap/t/012_ic.pl +++ b/src/test/cluster_tap/t/012_ic.pl @@ -102,13 +102,13 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_gcluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); # ---------- diff --git a/src/test/cluster_tap/t/013_conf.pl b/src/test/cluster_tap/t/013_conf.pl index ae2c97f442..1e2001a5f5 100644 --- a/src/test/cluster_tap/t/013_conf.pl +++ b/src/test/cluster_tap/t/013_conf.pl @@ -113,13 +113,13 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_gcluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', q{SHOW "cluster.interconnect_tier"}), 'stub', diff --git a/src/test/cluster_tap/t/014_ic_mock.pl b/src/test/cluster_tap/t/014_ic_mock.pl index 08f3cfbdda..dc848c4545 100644 --- a/src/test/cluster_tap/t/014_ic_mock.pl +++ b/src/test/cluster_tap/t/014_ic_mock.pl @@ -171,8 +171,8 @@ is( $node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..92e78027fa 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '160', + 'pg_stat_cluster_injections returns 160 rows (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- @@ -189,8 +189,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); # ---------- # Test 11 (Hardening v1.0.1 / codex review P2-2): SQL SRF rejects diff --git a/src/test/cluster_tap/t/016_perfmon.pl b/src/test/cluster_tap/t/016_perfmon.pl index cf0c4bbfb1..a7e3a98abe 100644 --- a/src/test/cluster_tap/t/016_perfmon.pl +++ b/src/test/cluster_tap/t/016_perfmon.pl @@ -154,8 +154,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..d37ca8a874 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -157,8 +157,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index 4af39b3e6f..bd7864b7c5 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -309,8 +309,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L17 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L17 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..6bd02693d5 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -199,8 +199,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L12 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L12 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..71889182cf 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -210,8 +210,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L12b pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L12b pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..9f1cbaaee5 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -137,8 +137,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L9 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..516f9c9b19 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -146,8 +146,8 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'E1 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'E1 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); ok($node->safe_psql('postgres', q{SELECT count(*) > 0 FROM pg_stat_cluster_wait_events WHERE type='Cluster: GES'}) @@ -159,7 +159,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', 'E4 pg_stat_gcluster_wait_events returns 118 rows (single-node, spec-6.13 RDMA wait surface)'); + '120', 'E4 pg_stat_gcluster_wait_events returns 120 rows (single-node, spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index cebe221c82..d4a4beb22a 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -81,7 +81,7 @@ # L6 — wait event count baseline through spec-6.1. my $wait_event_count = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_stat_cluster_wait_events"); -is($wait_event_count, '118', +is($wait_event_count, '120', 'L6 wait event baseline 118 (spec-6.13 RDMA wait surface)'); # L7 — no PCM wire opcode smoke (no SQL-visible PCM wire opcode enum surface) diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 9b5015e4e9..a24ed4796f 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -152,7 +152,7 @@ sub gcs_int_value { is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', + '120', 'L5 total cluster wait event count = 118 (spec-6.13 RDMA wait surface)'); diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index 085bf47bd0..20ea537952 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -211,8 +211,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L9 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); done_testing(); diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 2a479ae004..0debc65cbb 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -132,7 +132,7 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', + '120', 'L4 wait event count == 118 (spec-6.13 RDMA wait surface)'); diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index 8f0e94d343..ecbfa89b8b 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -125,7 +125,7 @@ sub gcs_int is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', + '120', "L4 node$i wait event count == 118 (spec-6.13 RDMA wait surface)"); is($node->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 5c3f5504ed..37b42c2730 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -101,8 +101,8 @@ sub gcs_int is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L2 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L2 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($pair->node0->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl index aea5668348..409dcebfb6 100644 --- a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl +++ b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl @@ -93,7 +93,7 @@ sub sinval_int # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', + '120', 'L3 wait event count == 118 (spec-6.13 RDMA wait surface)'); diff --git a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl index 89e2273a7f..70cd07c3da 100644 --- a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl +++ b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl @@ -81,8 +81,8 @@ # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L4 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L4 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ============================================================ # L5: 3 NEW ack wait events visible. diff --git a/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl b/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl index 5b033f5f43..8f8186b321 100644 --- a/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl +++ b/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl @@ -215,8 +215,8 @@ sub tt_int # additions to sinval surface). is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L9 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c index 095c16b2de..74a3b54fdb 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c @@ -196,7 +196,7 @@ UT_TEST(test_new_wait_events_distinct) } -UT_TEST(test_cluster_wait_events_count_118) +UT_TEST(test_cluster_wait_events_count_120) { /* spec-2.34 D7: 83 → 85 (+ 2 reliability wait events). * spec-2.36 D8: 85 → 88 (+ 3 CF 3-way wait events). @@ -207,7 +207,7 @@ UT_TEST(test_cluster_wait_events_count_118) * spec-4.7 D1: 98 → 99 (+ 1 GCS block RECOVERING short-wait). * spec-4.11 D5: 99 → 100 (+ 1 online thread recovery short-wait). * spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -313,7 +313,7 @@ main(void) UT_RUN(test_retry_total_backoff_default_1500ms); UT_RUN(test_lwtranche_distinct); UT_RUN(test_new_wait_events_distinct); - UT_RUN(test_cluster_wait_events_count_118); + UT_RUN(test_cluster_wait_events_count_120); UT_RUN(test_dedup_full_status_distinct_from_master_not_holder); UT_RUN(test_block_data_size_equals_blcksz); UT_RUN(test_dedup_entry_collision_field_layout); diff --git a/src/test/cluster_unit/test_cluster_stage2_acceptance.c b/src/test/cluster_unit/test_cluster_stage2_acceptance.c index 8a6390e67f..71a34d3593 100644 --- a/src/test/cluster_unit/test_cluster_stage2_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage2_acceptance.c @@ -210,14 +210,14 @@ UT_TEST(test_stage2_fault_inject_point_names) /* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 118 ===== */ -UT_TEST(test_stage2_wait_events_count_snapshot_118) +UT_TEST(test_stage2_wait_events_count_snapshot_120) { /* spec-2.39 D13 ship value. Future spec adding wait events MUST * update this snapshot (update-required contract per spec v0.2 F5 * — current state, not "==93 forever"). spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering). spec-4.11 D5: 99 → 100 * (+ ClusterThreadRecovery). spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -277,7 +277,7 @@ main(void) UT_RUN(test_stage2_msg_types_cumulative_registration); UT_RUN(test_stage2_capability_counter_symbols_linkable); UT_RUN(test_stage2_fault_inject_point_names); - UT_RUN(test_stage2_wait_events_count_snapshot_118); + UT_RUN(test_stage2_wait_events_count_snapshot_120); UT_RUN(test_stage2_sqlstate_53r60_through_95_encodable); UT_RUN(test_stage2_guc_enum_snapshot); UT_RUN(test_stage2_ic_msg_reserved_0_sentinel); diff --git a/src/test/cluster_unit/test_cluster_views.c b/src/test/cluster_unit/test_cluster_views.c index 00027881db..60eee45139 100644 --- a/src/test/cluster_unit/test_cluster_views.c +++ b/src/test/cluster_unit/test_cluster_views.c @@ -207,7 +207,7 @@ cluster_shmem_iter_regions(int *idx pg_attribute_unused(), UT_DEFINE_GLOBALS(); -UT_TEST(test_cluster_wait_events_count_is_118) +UT_TEST(test_cluster_wait_events_count_is_120) { /* * Cumulative registration roster: 61 prior + 3 added by spec-2.6 D11 @@ -239,8 +239,8 @@ UT_TEST(test_cluster_wait_events_count_is_118) * enum in wait_event.h and CLUSTER_WAIT_EVENTS_COUNT must move * together, and this test number must be bumped in lockstep. */ - /* spec-6.13 D8: RDMA tier3 wait events included -> 118. */ - UT_ASSERT_EQ(CLUSTER_WAIT_EVENTS_COUNT, 118); + /* spec-7.2 D6: +2 LMS data-plane -> 120. */ + UT_ASSERT_EQ(CLUSTER_WAIT_EVENTS_COUNT, 120); } @@ -285,7 +285,7 @@ int main(void) { UT_PLAN(5); - UT_RUN(test_cluster_wait_events_count_is_118); + UT_RUN(test_cluster_wait_events_count_is_120); UT_RUN(test_srf_symbol_linkable); UT_RUN(test_adg_srf_symbol_linkable); UT_RUN(test_first_event_is_ges_enqueue_acquire); From d91cb9181cb0c1915248becc543d55348da6b309 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 19:52:13 +0800 Subject: [PATCH 19/58] test(cluster): sync injection-registry and wait-event pins to spec-7.2 D6 The D6 observability commit added two injection points (cluster-lms-data-dispatch + cluster-lms-conn-reset, registry 158 -> 160) and two wait events (ClusterLmsDataRecv/Send, 118 -> 120) but refreshed only part of the pinned assertion surface. Left behind and failing on the branch tip: - t/015 test 3: the full name-list string_agg lacked the two new injection names (the row-count leg alone was updated) - t/020 L15: total registry size still pinned 158 - cluster_regress cluster_smoke: both wait-event count pins (regular + gcluster view) still expected 118 Spec: spec-7.2-ic-data-plane-decoupling.md --- src/test/cluster_regress/expected/cluster_smoke.out | 6 +++--- src/test/cluster_regress/sql/cluster_smoke.sql | 2 +- src/test/cluster_tap/t/015_inject.pl | 4 ++-- src/test/cluster_tap/t/020_shmem_registry.pl | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/cluster_regress/expected/cluster_smoke.out b/src/test/cluster_regress/expected/cluster_smoke.out index 062304fd1c..2d7ea5527d 100644 --- a/src/test/cluster_regress/expected/cluster_smoke.out +++ b/src/test/cluster_regress/expected/cluster_smoke.out @@ -76,14 +76,14 @@ SELECT attname, format_type(atttypid, atttypmod) (7 rows) -- ---------- --- 3. Cluster wait events: 118 rows (anchored by +-- 3. Cluster wait events: 120 rows (anchored by -- CLUSTER_WAIT_EVENTS_COUNT, spec-0.11 + StaticAssertDecl -- in cluster_views.c; spec-6.2 D10 +4 authority waits). -- ---------- SELECT count(*) FROM pg_stat_cluster_wait_events; count ------- - 118 + 120 (1 row) -- ---------- @@ -107,7 +107,7 @@ SELECT count(DISTINCT type) FROM pg_stat_cluster_wait_events; SELECT count(*) FROM pg_stat_gcluster_wait_events; count ------- - 118 + 120 (1 row) -- ---------- diff --git a/src/test/cluster_regress/sql/cluster_smoke.sql b/src/test/cluster_regress/sql/cluster_smoke.sql index 04fd94fa1c..866c81b441 100644 --- a/src/test/cluster_regress/sql/cluster_smoke.sql +++ b/src/test/cluster_regress/sql/cluster_smoke.sql @@ -44,7 +44,7 @@ SELECT attname, format_type(atttypid, atttypmod) -- ---------- --- 3. Cluster wait events: 118 rows (anchored by +-- 3. Cluster wait events: 120 rows (anchored by -- CLUSTER_WAIT_EVENTS_COUNT, spec-0.11 + StaticAssertDecl -- in cluster_views.c; spec-6.2 D10 +4 authority waits). -- ---------- diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 92e78027fa..0fd66564aa 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '160 injection point names match the full registry (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index bd7864b7c5..916ba76eb5 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '160', + 'L15 total injection registry size is 160 (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); # ---------- From 81b61b91651c8509c2f2f93b165965bc500217bd Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 19:52:38 +0800 Subject: [PATCH 20/58] fix(cluster): spawn LMS on hot-standby postmaster state An ADG standby without an LMS has no DATA-plane listener, so with the GCS block family on the LMS-owned DATA plane every cross-node block request from or to it retransmits to exhaustion. Mirror the MRP/RFS standby spawn: the LMS spawn gate accepts PM_HOT_STANDBY alongside PM_RUN. Grant ownership stays with LMON (HC4) on every node. Spec: spec-7.2-ic-data-plane-decoupling.md --- src/backend/postmaster/postmaster.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index b8afb18bc5..1a0f4ddccb 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -2017,8 +2017,17 @@ ServerLoop(void) * external child termination paths. cluster.lms_enabled = off * (PGC_POSTMASTER) keeps LMS un-forked and leaves the startup-time * PG-native fallback path active. + * + * PGRAC: spec-7.2 — LMS must also run in HOT STANDBY: with the + * GCS block family on the LMS-owned DATA plane, an ADG standby + * without an LMS has no DATA listener and every cross-node block + * request from or to it retransmits to exhaustion (t/335). The + * standby LMS serves the same roles as on a primary (DATA plane + * + CR/undo park-serve); grant ownership stays with LMON (HC4) + * on every node. */ - if (cluster_enabled && cluster_lms_enabled && LmsPID == 0 && pmState == PM_RUN) + if (cluster_enabled && cluster_lms_enabled && LmsPID == 0 + && (pmState == PM_RUN || pmState == PM_HOT_STANDBY)) LmsPID = StartLms(); /* From 82598e0e5c03227de9a564a2f2dbf8035360ff07 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 19:52:38 +0800 Subject: [PATCH 21/58] fix(cluster): bind DATA conn_epoch to the local current epoch Two-node cold restart deadlocked the DATA mesh: the HELLO verify layer rejected peers whose declared conn_epoch differed from ours, but cross-node epochs legitimately differ during the cold-form / reconfig window, and connection establishment is the very channel the envelope epoch observation converges through. Remove the HELLO-layer conn_epoch rejection (the plane-mismatch rejection stays) and record the LOCAL current epoch at all three CONNECTED sites, so the sender-side epoch gate compares a connection against the same node's view. Stale frames remain dropped per message by the envelope HC100 check -- receiver-side stale-epoch protection lives at the envelope layer, not the HELLO layer. Spec: spec-7.2-ic-data-plane-decoupling.md --- src/backend/cluster/cluster_ic_tier1.c | 37 ++++++++++---------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index f82e483e8b..fc3113d743 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -1489,16 +1489,6 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; return false; } - if (tier1_my_plane == CLUSTER_IC_PLANE_DATA - && cluster_ic_hello_conn_epoch(&msg) != cluster_epoch_get_current()) { - peer_record_error(peer_id, 0, "08P01", - "DATA HELLO conn_epoch mismatch (peer=" UINT64_FORMAT - " mine=" UINT64_FORMAT ")", - cluster_ic_hello_conn_epoch(&msg), cluster_epoch_get_current()); - cluster_ic_tier1_close_peer(peer_id, "DATA HELLO conn_epoch mismatch"); - Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; - return false; - } peer_info = cluster_conf_lookup_node(msg.source_node_id); if (peer_info == NULL) { @@ -1524,8 +1514,12 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) } Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_CONNECTED; - /* PGRAC: spec-7.2 D5 — bind the connection to the peer's HELLO epoch. */ - Tier1Shmem->peers[peer_id].conn_epoch = cluster_ic_hello_conn_epoch(&msg); + /* PGRAC: spec-7.2 D5 — bind the connection to THIS node's current + * epoch (our own view), so the sender gate compares apples to + * apples; the peer's HELLO epoch may differ transiently during a + * cold-form / reconfig window and per-message envelope HC100 (not + * the HELLO) is the receiver-side stale-epoch guard. */ + Tier1Shmem->peers[peer_id].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(peer_id); /* cache addr in shmem for view */ cluster_sf_note_peer_hello_capabilities(peer_id, cluster_ic_hello_capabilities(&msg)); @@ -1676,20 +1670,16 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear return false; } - /* PGRAC: spec-7.2 D2 — plane match + DATA conn_epoch equality (see - * the named-peer verify path for the rationale). */ + /* PGRAC: spec-7.2 D2 — plane match (a CONTROL peer dialing the DATA + * port, or vice versa, is a wiring error). No conn_epoch reject at + * HELLO: cross-node epoch may differ transiently during cold-form / + * reconfig, and per-message envelope HC100 is the stale-epoch guard + * (D5 §3.2 ④). */ if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { ereport(LOG, (errmsg("cluster_ic tier1 HELLO plane mismatch (peer=%d mine=%d)", (int)cluster_ic_hello_plane(&msg), (int)tier1_my_plane))); return false; } - if (tier1_my_plane == CLUSTER_IC_PLANE_DATA - && cluster_ic_hello_conn_epoch(&msg) != cluster_epoch_get_current()) { - ereport(LOG, (errmsg("cluster_ic tier1 DATA HELLO conn_epoch mismatch " - "(peer=" UINT64_FORMAT " mine=" UINT64_FORMAT ")", - cluster_ic_hello_conn_epoch(&msg), cluster_epoch_get_current()))); - return false; - } peer_info = cluster_conf_lookup_node(msg.source_node_id); if (peer_info == NULL) { @@ -1704,8 +1694,9 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear if (Tier1Shmem != NULL) { peer_record_error(learned, 0, "", ""); /* clear any prior */ Tier1Shmem->peers[learned].state = (int32)CLUSTER_IC_PEER_CONNECTED; - /* PGRAC: spec-7.2 D5 — bind the connection to the peer's HELLO epoch. */ - Tier1Shmem->peers[learned].conn_epoch = cluster_ic_hello_conn_epoch(&msg); + /* PGRAC: spec-7.2 D5 — bind to THIS node's current epoch (our own + * view; see the named-peer path for the rationale). */ + Tier1Shmem->peers[learned].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[learned].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(learned); cluster_sf_note_peer_hello_capabilities(learned, cluster_ic_hello_capabilities(&msg)); From 0ff0f1c6ae1dd15ce49612a5e836c441c714340a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 19:52:38 +0800 Subject: [PATCH 22/58] fix(cluster): fence-gate only the image-bearing LMS data-plane legs The D5 write-fence linkage paused the WHOLE LMS data-plane send side while the fence is enforcing and writes are disallowed. That turns the write fence into a read fence on the fenced node: the outbound ring carries only backend solicitations (REQUEST / FORWARD / CR + undo fetches -- the master's image REPLY is sent from the dispatch context and never staged), and those are exactly the frames a fenced node's own catalog reads and post-crash rejoin depend on. A full-cluster cold restart wedged behind its own reads: ring full, every connection died with a failed-to-enqueue FATAL, and the starved heartbeats fed a fail-stop reconfig storm. The ungated CONTROL ring never had this problem pre-flip. Keep the fence pause on the image-bearing legs (READY serve ship + PI-discard notes): block images must not leave a fenced node. Drain the solicitation ring unconditionally. Spec: spec-7.2-ic-data-plane-decoupling.md --- src/backend/cluster/cluster_lms.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index d8a7f228a9..ab0e7a227f 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -778,17 +778,24 @@ LmsMain(void) * quiet via the same registry probe); the DATA outbound * ring drains here either way (empty before the flip). * D5 fence linkage: while the write-fence is enforcing and - * writes are disallowed, the DATA plane sends nothing — - * block images must not leave a fenced node. Thaw wakes - * the latch and the held frames drain (pure-read probe, - * no-throw, per the write_fence_allowed contract). */ - if (!(cluster_write_fence_enforcing() && !cluster_write_fence_allowed())) { - if (cluster_gcs_block_family_on_data_plane()) { - cluster_lms_cr_ship_ready(); - cluster_gcs_block_pi_discard_drain(); - } - (void)cluster_lms_outbound_drain_send(); + * writes are disallowed, the IMAGE-BEARING legs (READY + * ship + PI notes) send nothing — block images must not + * leave a fenced node. The outbound ring is NOT fence + * gated: it carries only backend solicitations (REQUEST / + * FORWARD / CR + undo fetches — the master's image REPLY + * is sent from the dispatch context, never staged here), + * and a fenced node's reads must keep flowing exactly as + * they do on the ungated CONTROL ring pre-flip; gating + * them wedges post-crash rejoin behind its own catalog + * reads. Thaw wakes the latch and the held image legs + * resume (pure-read probe, no-throw, per the + * write_fence_allowed contract). */ + if (cluster_gcs_block_family_on_data_plane() + && !(cluster_write_fence_enforcing() && !cluster_write_fence_allowed())) { + cluster_lms_cr_ship_ready(); + cluster_gcs_block_pi_discard_drain(); } + (void)cluster_lms_outbound_drain_send(); cluster_lms_data_plane_tick(LMS_IDLE_TIMEOUT_MS); } else { (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, From c18c1b6375fee33b143a6f58bacc4ce03f0b4323 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 19:53:06 +0800 Subject: [PATCH 23/58] feat(cluster): flip the GCS block family onto the LMS DATA plane The atomic plane flip (H-5, no half-migrated window): the five block msg_types (REQUEST / REPLY / FORWARD / INVALIDATE / INVALIDATE_ACK) register plane = DATA with the producer mask extended by the LMS family. REDECLARE alone stays on the CONTROL plane: recovery re-declare must survive a DATA-mesh teardown mid-episode, and none of the five migrated handlers emits it. The registry probe pivots the LMON tick sites, the LMS loop and the outbound-ring routing off these six structs, so this one edit moves dispatch, physical sends and drains onto the LMS-owned tier1 instance. Riding the flip, as one commit must: - master-side fail-stop episode fence at the REQUEST wire handler entry (phase_for_tag == RECOVERING -> DENIED_RESOURCE_RECOVERING, 53R9L retry-safe): a stale-view remote requester routes straight to the master, so the requester-side acquire gate alone cannot close the phantom-holder overtake window - pgrac.conf post_validate hard gate: multi-node + cluster enabled + a real interconnect tier requires data_addr on every node (FATAL at startup instead of the first cross-node block request hanging) - pg_cluster_state gcs keys block_family_plane + plane_misroute_reject (83 -> 85) with unit stubs; TAP gcs-key / wait-event baselines and stale label texts refreshed (t/110-118) - per-node data_addr for every handwritten multi-node conf in the TAP fleet (t/078, 288, 289, 334, 335, 336, 337, 339, 346) - t/358 data-plane flip e2e: flip fact + DATA listeners on both nodes, bidirectional full-table X transfer with wire-request growth asserted after the swap (single-block request asserts are master-placement dependent: HC72 master==self short-circuits the wire REQUEST and rides FORWARD), staging survival via zero plane misroutes and zero plane-gate log errors, cross-held read-back; online-join write-gate polling and OID burn-align per the t/347 idiom Spec: spec-7.2-ic-data-plane-decoupling.md --- src/backend/cluster/cluster_conf.c | 24 ++ src/backend/cluster/cluster_debug.c | 9 + src/backend/cluster/cluster_gcs_block.c | 56 ++++- .../t/078_ic_tier1_partial_send_recv.pl | 4 + src/test/cluster_tap/t/110_gcs_loopback.pl | 16 +- .../cluster_tap/t/111_gcs_block_ship_2node.pl | 12 +- .../t/112_gcs_block_retransmit_2node.pl | 18 +- .../cluster_tap/t/113_gcs_block_2way_2node.pl | 12 +- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 16 +- .../cluster_tap/t/115_gcs_block_3way_3node.pl | 6 +- .../t/116_gcs_block_lost_write_2node.pl | 8 +- .../t/117_sinval_broadcast_2node.pl | 2 +- .../t/118_sinval_ddl_propagation_2node.pl | 6 +- .../t/288_cf_enqueue_concurrent.pl | 4 + src/test/cluster_tap/t/289_cf_bootstrap.pl | 8 + .../cluster_tap/t/334_ic_rdma_soft_roce.pl | 4 + .../t/335_adg_two_thread_rfs_apply.pl | 4 + .../cluster_tap/t/336_cluster_backup_pitr.pl | 4 + .../t/337_shared_catalog_ddl_2node.pl | 4 + .../t/339_shared_catalog_faults_2node.pl | 4 + .../346_shared_catalog_relmap_crash_2node.pl | 4 + .../cluster_tap/t/358_data_plane_flip_e2e.pl | 218 ++++++++++++++++++ src/test/cluster_unit/test_cluster_conf.c | 4 + src/test/cluster_unit/test_cluster_debug.c | 12 + 24 files changed, 406 insertions(+), 53 deletions(-) create mode 100644 src/test/cluster_tap/t/358_data_plane_flip_e2e.pl diff --git a/src/backend/cluster/cluster_conf.c b/src/backend/cluster/cluster_conf.c index 81d1b94999..e7028eb59f 100644 --- a/src/backend/cluster/cluster_conf.c +++ b/src/backend/cluster/cluster_conf.c @@ -58,6 +58,7 @@ #ifdef USE_PGRAC_CLUSTER #include "cluster/cluster_elog.h" /* CLUSTER_LOG */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_config_file */ +#include "cluster/cluster_ic.h" /* CLUSTER_IC_TIER_* (spec-7.2 data_addr gate) */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (stage 0.27) */ #endif @@ -611,6 +612,29 @@ post_validate(const char *path) "cluster_conf: node %d in \"%s\" is missing required field interconnect_addr", n->node_id, path))); } + + /* + * PGRAC: spec-7.2 flip — with the GCS block family on the DATA + * plane, a multi-node cluster on a real interconnect tier + * cannot serve block transfers without a per-node data_addr. + * Fail-closed at startup (r1-F2: no port guessing, no silent + * CONTROL fallback) instead of hanging the first cross-node + * block request. Single-node fallback and stub/mock tiers are + * exempt (no remote block traffic exists there). + */ + if (ClusterConfShmem->node_count > 1 && cluster_enabled + && (cluster_interconnect_tier == CLUSTER_IC_TIER_1 + || cluster_interconnect_tier == CLUSTER_IC_TIER_2 + || cluster_interconnect_tier == CLUSTER_IC_TIER_3) + && n->data_addr[0] == '\0') { + ereport(FATAL, (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("cluster_conf: node %d in \"%s\" is missing required field " + "data_addr (multi-node DATA plane, spec-7.2)", + n->node_id, path), + errhint("Declare data_addr = host:port for every [node.N] section; " + "the GCS block family ships over the LMS-owned DATA plane."))); + } + if (n->rdma_port == 0) n->rdma_port = 1; if (n->node_id == cluster_node_id) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 6ce21bee45..f3a774fcb8 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1831,6 +1831,15 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_block_wal_flush_before_ship_count())); emit_row(rsinfo, "gcs", "block_ship_bytes_total", fmt_int64((int64)cluster_gcs_get_block_ship_bytes_total())); + /* PGRAC: spec-7.2 flip — plane facts: which plane owns the block + * family (0 = CONTROL pre-flip, 1 = DATA) and the total frames the + * dispatch plane gate dropped (both tier1 instances). */ + emit_row(rsinfo, "gcs", "block_family_plane", + fmt_int64(cluster_gcs_block_family_on_data_plane() ? 1 : 0)); + emit_row( + rsinfo, "gcs", "plane_misroute_reject", + fmt_int64((int64)(cluster_ic_tier1_get_plane_misroute_reject(CLUSTER_IC_PLANE_CONTROL) + + cluster_ic_tier1_get_plane_misroute_reject(CLUSTER_IC_PLANE_DATA)))); /* PGRAC: spec-7.2 D6 — requester ship-latency histogram (16 rows, * keys ship_hist_us_le_ + ship_hist_us_inf). */ { diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 3de18692b8..1333e948b6 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -3397,6 +3397,24 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo return; } + /* + * PGRAC: spec-7.2 D5 — master-side fail-stop episode fence, symmetric + * with the requester-side acquire gate (cluster_pcm_lock.c). While a + * dead static master's block resources are mid-episode (survivor + * re-declare not yet complete, or merged replay not yet materialized), + * serving a request here could grant a block whose surviving holder + * has not re-declared yet (phantom-holder overtake window). The + * requester-side gate cannot close this alone: a remote requester with + * a stale view routes here directly. Fail-closed before any dedup / + * state change; DENIED_RESOURCE_RECOVERING -> sender maps to 53R9L + * (retry-safe). phase_for_tag counts the hit itself. + */ + if (cluster_gcs_block_phase_for_tag(req->tag) == GCS_BLOCK_RECOVERING) { + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_RESOURCE_RECOVERING, + InvalidXLogRecPtr, NULL); + return; + } + /* HC75 range guard — out of range never enters dedup HTAB. */ if (req->transition_id < PCM_TRANS_N_TO_S || req->transition_id > PCM_TRANS_S_TO_X_CLEANOUT) { gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, @@ -5466,31 +5484,48 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo /* ============================================================ * Dispatch table registration. + * + * PGRAC: spec-7.2 flip — the five block-family msg_types (REQUEST / + * REPLY / FORWARD / INVALIDATE / INVALIDATE_ACK) are registered on + * the DATA plane: the LMS-owned tier1 instance carries their frames, + * the LMS loop dispatches them, and the producer mask admits the LMS + * family for the drain-and-send leg. All five flip in this one edit + * (H-5: no half-migrated window; the registry probe above pivots the + * LMON tick sites and the LMS loop automatically). REDECLARE alone + * stays on the CONTROL plane (r4): recovery re-declare must survive a + * DATA-mesh teardown mid-episode, and the REDECLARE -> REDECLARE_DONE + * pair may not be split across planes. * ============================================================ */ static const ClusterICMsgTypeInfo gcs_block_request_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REQUEST, .name = "gcs_block_request", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_request_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; static const ClusterICMsgTypeInfo gcs_block_reply_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REPLY, .name = "gcs_block_reply", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_reply_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; /* PGRAC: spec-2.35 D8 — holder-side forward handler msg_type registration. */ static const ClusterICMsgTypeInfo gcs_block_forward_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_FORWARD, .name = "gcs_block_forward", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_forward_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; @@ -6207,20 +6242,25 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c } +/* PGRAC: spec-7.2 flip — DATA plane (see the dispatch-table comment). */ static const ClusterICMsgTypeInfo gcs_block_invalidate_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE, .name = "gcs_block_invalidate", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_invalidate_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; static const ClusterICMsgTypeInfo gcs_block_invalidate_ack_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK, .name = "gcs_block_invalidate_ack", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_invalidate_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; @@ -6307,12 +6347,18 @@ cluster_gcs_handle_block_redeclare_envelope(const ClusterICEnvelope *env, const } +/* PGRAC: spec-7.2 flip (r4) — REDECLARE stays on the CONTROL plane: + * survivor re-declare and its DONE barrier belong to the LMON-owned + * recovery episode and must not depend on the DATA mesh being up. + * None of the five migrated handlers emits REDECLARE (D0-①b audit), + * so no cross-plane staging leg is required here. */ static const ClusterICMsgTypeInfo gcs_block_redeclare_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REDECLARE, .name = "gcs_block_redeclare", .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, .broadcast_ok = false, .handler = cluster_gcs_handle_block_redeclare_envelope, + .plane = CLUSTER_IC_PLANE_CONTROL, }; diff --git a/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl b/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl index 997da11f84..823cbd5e76 100644 --- a/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl +++ b/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl @@ -66,6 +66,8 @@ sub build_hello # Declare 2 nodes so we can spoof "peer 1" connecting to "peer 0" (us). my $self_ic_port = PostgreSQL::Test::Cluster::get_free_port(); my $spoof_ic_port = PostgreSQL::Test::Cluster::get_free_port(); +my $self_data_port = PostgreSQL::Test::Cluster::get_free_port(); +my $spoof_data_port = PostgreSQL::Test::Cluster::get_free_port(); my $cluster_name = 'pgrac-078'; my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index de017655ed..9f508ee72c 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -7,10 +7,10 @@ # any wire send (HC72), so wire path coverage is effectively limited # to SQL-visible surface invariants: # -# L1 fresh cluster startup: pg_cluster_state.gcs has 83 keys +# L1 fresh cluster startup: pg_cluster_state.gcs has 85 keys # L2 api_state = "active" after postmaster phase 1 init # L3 WAIT_EVENT_GCS_REPLY_WAIT registered in pg_stat_cluster_wait_events -# L4 CLUSTER_WAIT_EVENTS_COUNT == 118 (cumulative through spec-6.13) +# L4 CLUSTER_WAIT_EVENTS_COUNT == 120 (cumulative through spec-7.2) # L5 msg_type registry surface visible: pg_cluster_ic_msg_types has # gcs_request + gcs_reply rows # L6 workload (SELECT/UPDATE/VACUUM) does NOT inc send_request_count @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 83 keys (spec-7.2 D6 hist). +# L1 — pg_cluster_state.gcs surface has 85 keys (spec-7.2 D6 hist). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '83', - 'L1 pg_cluster_state.gcs category has 83 keys (spec-7.2 D6)'); + '85', + 'L1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6)'); # L2 — api_state = "active" after postmaster phase 1 init. @@ -87,11 +87,11 @@ sub gcs_value { 'L3 ClusterGcsReplyWait wait event registered (spec-2.32 D7)'); -# L4 — CLUSTER_WAIT_EVENTS_COUNT == 118 (spec-6.13). +# L4 — CLUSTER_WAIT_EVENTS_COUNT == 120 (spec-7.2 D6). my $total_wait_events = $node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'); -is($total_wait_events, '118', - 'L4 wait_events count 118 (spec-6.13 RDMA wait surface)'); +is($total_wait_events, '120', + 'L4 wait_events count 120 (spec-7.2 D6 LMS data-plane wait surface)'); # L6 — Production workload does NOT trigger wire path (HC72 short-circuit). diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index a24ed4796f..323f16e364 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -8,7 +8,7 @@ # # L1 ClusterPair startup — both postmasters healthy + tier1 connected # L2 fresh baseline gcs counters on both nodes (block_* counters = 0) -# L3 pg_cluster_state.gcs category has 67 keys (cumulative through +# L3 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through # spec-6.13 direct-land observability) # L4 4 NEW wait events registered in pg_stat_cluster_wait_events: # ClusterGCSBlockShipWait, ClusterGCSBlockRequestDispatch, @@ -113,19 +113,19 @@ sub gcs_int_value { # ============================================================ -# L3: pg_cluster_state.gcs category has 67 keys +# L3: pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) # (cumulative GCS surface through spec-6.13 direct-land observability). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 3e0545d6b2..95ad463b15 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,10 +9,10 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 83 keys (cumulative through spec-7.2 D6 hist) +# L3 pg_cluster_state.gcs has 85 keys (cumulative through spec-7.2 D6 hist) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) -# L5 CLUSTER_WAIT_EVENTS_COUNT = 85 (was 83 spec-2.33) +# L5 CLUSTER_WAIT_EVENTS_COUNT = 120 (spec-7.2 +2) # L6 3 NEW GUC visible + defaults + contexts: # cluster.gcs_block_retransmit_max_retries PGC_SUSET 4 # cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 10 (spec-7.2 D1) @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 83 keys (cumulative through spec-7.2 D6 hist). +# L3: pg_cluster_state.gcs category has 85 keys (cumulative through spec-7.2 D6 hist). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '83', - 'L3 node0 pg_cluster_state.gcs category has 83 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '83', - 'L3 node1 pg_cluster_state.gcs category has 83 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys'); # ============================================================ @@ -144,8 +144,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L5 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L5 pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); # ============================================================ diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index 20ea537952..c04dc7dcb2 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 7 NEW counters all 0 + catversion >= 202605420 -# L3 pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.13) +# L3 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) # L4 cross-node forward path: node A read first → master_holder = A; # force same tag on node B via test-only injection → master # chooses forward path → A direct-ships to B → block_forward_sent @@ -108,18 +108,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 0debc65cbb..9319413f6a 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -12,8 +12,8 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 -# L3 pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a) -# L4 catversion lower-bound >= 202605430; wait event count == 118 +# L3 pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) +# L4 catversion lower-bound >= 202605430; wait event count == 120 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader # sees starvation_denied_pending_x_count tick @@ -106,18 +106,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); # ============================================================ @@ -133,7 +133,7 @@ sub gcs_int 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), '120', - 'L4 wait event count == 118 (spec-6.13 RDMA wait surface)'); + 'L4 wait event count == 120 (spec-7.2 D6 LMS data-plane wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index ecbfa89b8b..64f60a9614 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -126,12 +126,12 @@ sub gcs_int is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), '120', - "L4 node$i wait event count == 118 (spec-6.13 RDMA wait surface)"); + "L4 node$i wait event count == 120 (spec-7.2 D6 LMS data-plane wait surface)"); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - "L4 node$i pg_cluster_state.gcs has 67 keys"); + '85', + "L4 node$i pg_cluster_state.gcs has 85 keys"); } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 37b42c2730..b3fbdf32e3 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -20,7 +20,7 @@ # L7 SQLSTATE 53R93 ERRCODE_CLUSTER_LOST_WRITE_DETECTED literal- # encodable in PG SQL (catalog 形式 verification) # L8 GUC switch back to 'error' SHOW returns 'error' -# L9 pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.14a) +# L9 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) # L10 Reply status enum value 12 (DENIED_LOST_WRITE) is新增的 # 最大 value (baseline workload must not trigger lost-write) # L11 spec-2.41 D / P1-C — behavioral lost-write inject: a @@ -107,8 +107,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L2 pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.14a)'); + '85', + 'L2 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); # ============================================================ @@ -198,7 +198,7 @@ sub gcs_int is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', + '85', 'L9 node1 pg_cluster_state.gcs has 67 keys (cross-node parity)'); diff --git a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl index 409dcebfb6..81639615bd 100644 --- a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl +++ b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl @@ -94,7 +94,7 @@ sub sinval_int is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), '120', - 'L3 wait event count == 118 (spec-6.13 RDMA wait surface)'); + 'L3 wait event count == 120 (spec-7.2 D6 LMS data-plane wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl index 70cd07c3da..978a8259c2 100644 --- a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl +++ b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl @@ -13,7 +13,7 @@ # L1 catversion bump 202605450 → 202605460 # L2 pg_cluster_state sinval category has 16 keys (D8 +3 fanout + D9 +3 ack + D1 applied) # L3 3 NEW GUC defaults (sinval_ack_mode=peer_enqueued / timeout 5000 / slots 256) -# L4 pg_stat_cluster_wait_events count == 118 (cumulative through spec-6.13) +# L4 pg_stat_cluster_wait_events count == 120 (cumulative through spec-7.2) # L5 ClusterSinvalAckWait / ClusterSinvalAckSend / ClusterSinvalAckReceive visible # L6 53R95 ERRCODE_CLUSTER_SINVAL_ACK_TIMEOUT encodable # L7 node0 CREATE TABLE → node1 SELECT visible within 5s ack timeout @@ -77,12 +77,12 @@ 'L3c cluster.sinval_ack_wait_slots default 256'); # ============================================================ -# L4: pg_stat_cluster_wait_events count == 118. +# L4: pg_stat_cluster_wait_events count == 120. # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), '120', - 'L4 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); + 'L4 pg_stat_cluster_wait_events returns 120 rows (spec-7.2 D6 LMS data-plane wait surface)'); # ============================================================ # L5: 3 NEW ack wait events visible. diff --git a/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl b/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl index f695cd636b..a4f14e62b3 100644 --- a/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl +++ b/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl @@ -79,6 +79,8 @@ sub cf_counter my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---- node0 init -> backup -> node1 init_from_backup (shared sysid) ---- my $node0 = PostgreSQL::Test::Cluster->new('cf_cc_node0'); @@ -131,9 +133,11 @@ sub cf_counter [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/289_cf_bootstrap.pl b/src/test/cluster_tap/t/289_cf_bootstrap.pl index 43924f15eb..632c0c93ab 100644 --- a/src/test/cluster_tap/t/289_cf_bootstrap.pl +++ b/src/test/cluster_tap/t/289_cf_bootstrap.pl @@ -103,6 +103,8 @@ # ---- IC ports ---- my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (shared sysid). @@ -168,9 +170,11 @@ [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); @@ -446,6 +450,8 @@ sub poison_authority_from_node0 mkdir "$neg_shared/global" or die "mkdir neg global: $!"; my $neg_ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $neg_ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $neg_data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $neg_data_port1 = PostgreSQL::Test::Cluster::get_free_port(); my $neg = PostgreSQL::Test::Cluster->new('cf_boot_neg'); $neg->init; @@ -469,9 +475,11 @@ sub poison_authority_from_node0 [node.0] interconnect_addr = 127.0.0.1:$neg_ic0 +data_addr = 127.0.0.1:$neg_data_port0 [node.1] interconnect_addr = 127.0.0.1:$neg_ic1 +data_addr = 127.0.0.1:$neg_data_port1 EOC my $nret = $neg->start(fail_ok => 1); diff --git a/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl b/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl index 79e31609f1..504ac321d5 100644 --- a/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl +++ b/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl @@ -83,6 +83,8 @@ sub command_exists my $rdma_port0 = PostgreSQL::Test::Cluster::get_free_port(); my $rdma_port1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); for my $node ($pair->node0, $pair->node1) { @@ -100,6 +102,7 @@ sub write_rdma_pgrac_conf [node.0] interconnect_addr = 127.0.0.1:@{[$pair->ic_port(0)]} +data_addr = 127.0.0.1:$data_port0 rdma_addr = $rdma_host:$rdma_port0 rdma_gid = rxe rdma_port = 1 @@ -108,6 +111,7 @@ sub write_rdma_pgrac_conf [node.1] interconnect_addr = 127.0.0.1:@{[$pair->ic_port(1)]} +data_addr = 127.0.0.1:$data_port1 rdma_addr = $rdma_host:$rdma_port1 rdma_gid = rxe rdma_port = 1 diff --git a/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl b/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl index 56240b21f5..2e1c4a0a89 100644 --- a/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl +++ b/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl @@ -98,15 +98,19 @@ sub write_pair_conf my ($node0, $node1, $cluster_name) = @_; my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); my $conf = <data_dir . '/pgrac.conf', $conf); diff --git a/src/test/cluster_tap/t/336_cluster_backup_pitr.pl b/src/test/cluster_tap/t/336_cluster_backup_pitr.pl index e48af1e510..660ddb5f0f 100644 --- a/src/test/cluster_tap/t/336_cluster_backup_pitr.pl +++ b/src/test/cluster_tap/t/336_cluster_backup_pitr.pl @@ -440,6 +440,8 @@ sub configure_pair_restore my ($restore_node, $shared_dir, $target_scn) = @_; my $restore_ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $restore_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $restore_data_port0 = PostgreSQL::Test::Cluster::get_free_port(); + my $restore_data_port1 = PostgreSQL::Test::Cluster::get_free_port(); my $target_conf = defined($target_scn) ? "cluster.recovery_target_scn = '$target_scn'\n" : ""; @@ -452,9 +454,11 @@ sub configure_pair_restore [node.0] interconnect_addr = 127.0.0.1:$restore_ic0 +data_addr = 127.0.0.1:$restore_data_port0 [node.1] interconnect_addr = 127.0.0.1:$restore_ic1 +data_addr = 127.0.0.1:$restore_data_port1 EOC $restore_node->append_conf('postgresql.conf', "port = " . $restore_node->port . "\n" diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..1e1e038460 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -109,6 +109,8 @@ my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). @@ -183,9 +185,11 @@ [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl index 301827c967..ecd1a51bb0 100644 --- a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl +++ b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl @@ -143,6 +143,8 @@ my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). @@ -234,9 +236,11 @@ [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl b/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl index 2391157fc1..72db2d246a 100644 --- a/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl +++ b/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl @@ -101,6 +101,8 @@ my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid), @@ -188,9 +190,11 @@ [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl b/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl new file mode 100644 index 0000000000..6b4bbb8b69 --- /dev/null +++ b/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl @@ -0,0 +1,218 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 358_data_plane_flip_e2e.pl +# spec-7.2 D3+D4 atomic plane flip — bidirectional e2e + staging +# survival (the flip-commit acceptance legs, user-ruled hard +# constraints on the flip): +# +# L1 flip fact: pg_cluster_state gcs/block_family_plane = 1 on +# both nodes; DATA-plane listeners bound (log evidence) +# L2 forward transfer: node0 seeds + X-holds a shared table; +# node1's UPDATE ships the block over the LMS DATA plane +# (statement succeeds, requester block_request_count grows, +# plane_misroute_reject stays 0 on both nodes) +# L3 reverse transfer: node1 now holds; node0's UPDATE ships +# back (bidirectional evidence) +# L4 staging survival (D0-①b case): node1 READS an X-held block +# -- the holder-side LMS FORWARD path downgrades X->S and its +# GCS_REQUEST notify must reach LMON via the CONTROL ring, not +# a cross-plane direct send; no plane-gate ERROR may appear in +# either node's log +# L5 hygiene: zero plane misroutes + zero "cannot send from +# plane" / "plane mismatch" strings across the whole run +# +# Spec: spec-7.2-ic-data-plane-decoupling.md (flip commit) +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Time::HiRes qw(usleep); +use Test::More; + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'dpflip', + quorum_voting_disks => 3, + shared_data => 1, + storage_backend => 'block_device', + extra_conf => [ + 'autovacuum = off', + 'fsync = off', + 'shared_buffers = 64MB', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.online_join = on', + 'cluster.xid_striping = on', + 'cluster.crossnode_runtime_visibility = on', + 'cluster.crossnode_cr_data_plane = on', + 'cluster.block_self_contained = on', + ]); +$pair->start_pair; +usleep(2_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'CONTROL peers up 0->1'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'CONTROL peers up 1->0'); +my ($node0, $node1) = ($pair->node0, $pair->node1); + +sub gcs_int +{ + my ($node, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='gcs' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + +# Poll until a write statement succeeds on $node — the online-join +# admission gate rejects writable transactions until the join settles +# (t/347 pattern). +sub poll_write_ok +{ + my ($node, $sql, $timeout_s, $label) = @_; + $timeout_s //= 60; + my $deadline = time + $timeout_s; + while (time < $deadline) + { + my $ok = eval { $node->safe_psql('postgres', $sql); 1 } // 0; + return 1 if $ok; + usleep(500_000); + } + diag("poll_write_ok timeout ($label)"); + return 0; +} + +# ============================================================ +# L1 — flip fact + DATA listeners. +# ============================================================ +is(gcs_int($node0, 'block_family_plane'), 1, 'L1 node0 block family on DATA plane'); +is(gcs_int($node1, 'block_family_plane'), 1, 'L1 node1 block family on DATA plane'); + +my $log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +my $log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($log0, qr/DATA-plane listener bound/, 'L1 node0 DATA listener bound'); +like($log1, qr/DATA-plane listener bound/, 'L1 node1 DATA listener bound'); + +# ============================================================ +# shared table (relfilenode coincidence on the shared device). +# ============================================================ +ok(poll_write_ok($node0, 'CREATE TABLE flip_gate0 (x int)', 90, 'node0 write gate'), + 'node0 write gate open (join admitted)'); +ok(poll_write_ok($node1, 'CREATE TABLE flip_gate1 (x int)', 90, 'node1 write gate'), + 'node1 write gate open (join admitted)'); + +# The write-gate polling burned an uneven number of OIDs per node. Align +# the two OID counters (t/347 pattern): measure the relfilenode delta and +# burn single OIDs on the lagging node until an identical CREATE lands on +# the same relfilenode on both. +my ($p0, $p1) = ('', ''); +for my $attempt (1 .. 8) +{ + $node0->safe_psql('postgres', + 'CREATE TABLE flip_t (aid int, bal int) WITH (fillfactor = 50)'); + $node1->safe_psql('postgres', + 'CREATE TABLE flip_t (aid int, bal int) WITH (fillfactor = 50)'); + $p0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('flip_t')}); + $p1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('flip_t')}); + last if $p0 eq $p1; + my ($n0) = $p0 =~ /(\d+)$/; + my ($n1) = $p1 =~ /(\d+)$/; + my ($lag, $burn) = $n0 < $n1 ? ($node0, $n1 - $n0) : ($node1, $n0 - $n1); + $lag->safe_psql('postgres', + "SELECT lo_unlink(lo_create(0)) FROM generate_series(1, $burn)"); + $node0->safe_psql('postgres', 'DROP TABLE flip_t'); + $node1->safe_psql('postgres', 'DROP TABLE flip_t'); +} +is($p0, $p1, 'shared-table coincidence'); +$node0->safe_psql('postgres', + 'INSERT INTO flip_t SELECT g, 0 FROM generate_series(1, 200) g'); +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node1->safe_psql('postgres', 'CHECKPOINT'); + +sub timed_update_retry +{ + my ($node, $sql) = @_; + for my $attempt (1 .. 20) + { + return 1 if eval { $node->safe_psql('postgres', $sql); 1 }; + usleep(500_000); + } + return 0; +} + +# ============================================================ +# L2 — forward transfer (node0 holds, node1 ships in). +# +# Wire-request counters are asserted AFTER the L3 swap below: a single +# row's block may be mastered by the requester itself (HC72 master==self +# short-circuits the wire REQUEST and the transfer rides FORWARD), so a +# per-leg counter assert is master-placement dependent. The full-table +# X swaps guarantee every block is acquired by both nodes, so each node +# must send wire REQUESTs for its remote-mastered half. +# ============================================================ +my $req1_before = gcs_int($node1, 'block_request_count'); +my $req0_before = gcs_int($node0, 'block_request_count'); + +ok(timed_update_retry($node0, 'UPDATE flip_t SET bal = bal + 1'), + 'L2 setup: node0 X-holds the table'); +usleep(500_000); + +ok(timed_update_retry($node1, 'UPDATE flip_t SET bal = bal + 1 WHERE aid = 7'), + 'L2 node1 write of node0-held block succeeds over the DATA plane'); +is(gcs_int($node0, 'plane_misroute_reject'), 0, 'L2 node0 zero plane misroutes'); +is(gcs_int($node1, 'plane_misroute_reject'), 0, 'L2 node1 zero plane misroutes'); + +# ============================================================ +# L3 — reverse transfer (node1 now holds, node0 ships back). +# ============================================================ +ok(timed_update_retry($node1, 'UPDATE flip_t SET bal = bal + 1'), + 'L3 setup: node1 takes the table X (reverse ship)'); +usleep(500_000); + +ok(timed_update_retry($node0, 'UPDATE flip_t SET bal = bal + 1 WHERE aid = 11'), + 'L3 node0 write of node1-held block succeeds (bidirectional)'); +cmp_ok(gcs_int($node1, 'block_request_count'), '>', $req1_before, + 'L3 node1 issued wire block requests across the swap'); +cmp_ok(gcs_int($node0, 'block_request_count'), '>', $req0_before, + 'L3 node0 issued wire block requests across the swap'); + +# ============================================================ +# L4 — staging survival read-back: node0 X-holds the aid=11 block from +# L3; node1 scans the whole table across it (remote X-held blocks are +# served read-image, spec-5.2). The D0-①b staging case itself (holder- +# side FORWARD/BAST chain emitting GCS_REQUEST from LMS context via the +# CONTROL ring) was exercised by every X transfer in the L2/L3 swap — +# its survival proof is the transfers succeeding with zero plane +# misroutes (L2/L3) and zero plane-gate errors in either log (L5). +# +# No full-table re-hold here: the L2/L3 swap leaves foreign recycled- +# slot ITL residue behind, and resolving THAT is the #119 co-sample +# lane (a full-table write on node0 walls on the fail-closed retryable +# verdict until then; the read-back needs no new X). +# ============================================================ +my $sum = ''; +for my $attempt (1 .. 10) +{ + $sum = eval { $node1->safe_psql('postgres', 'SELECT count(*) FROM flip_t') }; + last if defined($sum) && $sum eq '200'; + usleep(300_000); +} +is($sum, '200', 'L4 node1 reads the whole table across X-held blocks'); + +# ============================================================ +# L5 — hygiene: no plane-gate errors anywhere in the run. +# ============================================================ +$pair->stop_pair; +$log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +$log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +unlike($log0, qr/cannot send from plane|plane mismatch|dropping msg_type/, + 'L5 node0 log free of plane-gate errors'); +unlike($log1, qr/cannot send from plane|plane mismatch|dropping msg_type/, + 'L5 node1 log free of plane-gate errors'); + +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_conf.c b/src/test/cluster_unit/test_cluster_conf.c index 6e9343cd42..9397d008be 100644 --- a/src/test/cluster_unit/test_cluster_conf.c +++ b/src/test/cluster_unit/test_cluster_conf.c @@ -80,6 +80,10 @@ bool cluster_allow_single_node = true; /* spec-2.1 D1; storage stub matches defa */ bool cluster_enabled = true; +/* spec-7.2 flip: post_validate consults the interconnect tier for the + * multi-node data_addr gate; stub = stub tier (gate exempt). */ +int cluster_interconnect_tier = 0; + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 6d3537272f..546ca62ce6 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -952,6 +952,18 @@ cluster_gcs_block_ship_hist_count(int bucket) (void)bucket; return 0; } +/* spec-7.2 flip stubs: plane facts. */ +bool +cluster_gcs_block_family_on_data_plane(void) +{ + return false; +} +uint64 +cluster_ic_tier1_get_plane_misroute_reject(ClusterICPlane plane) +{ + (void)plane; + return 0; +} uint64 cluster_gcs_get_block_reply_count(void) { From 88ab0078d2b8e8a0510297a92b05d31df98e379c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 21:25:01 +0800 Subject: [PATCH 24/58] test(cluster): sync stale count baselines to spec-7.2 D6 additions D6 added 2 injection points (cluster-lms-data-dispatch, cluster-lms-conn-reset), 2 Cluster: Interconnect wait events (ClusterLmsDataRecv/Send) and 1 shmem region (lms data outbound), but the D6 baseline patch only synced t/015 / t/020 / cluster_smoke and missed the same counts asserted elsewhere (L462 grep-all-assertion-surfaces). Sync the remaining stale baselines: - injection registry 158 -> 160 (+ 316 -> 320 sub-keys) t/017, t/018, t/021, t/022, t/023, t/024, t/030 - Cluster: Interconnect wait events 9 -> 11 t/010, t/011 - shmem regions 79 -> 80 (with visibility inject) t/021, t/022, t/023 Test-only; no source change. The runtime values were already correct; only the hardcoded expected counts were stale. All 9 files green. Spec: spec-7.2-ic-data-plane-decoupling.md --- src/test/cluster_tap/t/010_views.pl | 2 +- src/test/cluster_tap/t/011_gviews.pl | 2 +- src/test/cluster_tap/t/017_debug.pl | 8 ++++---- src/test/cluster_tap/t/018_shared_fs.pl | 4 ++-- src/test/cluster_tap/t/021_block_format.pl | 6 +++--- src/test/cluster_tap/t/022_itl_slot.pl | 6 +++--- src/test/cluster_tap/t/023_buffer_descriptor.pl | 6 +++--- src/test/cluster_tap/t/024_pcm_lock.pl | 4 ++-- src/test/cluster_tap/t/030_acceptance.pl | 6 +++--- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/test/cluster_tap/t/010_views.pl b/src/test/cluster_tap/t/010_views.pl index 2f174371ea..81182cab24 100644 --- a/src/test/cluster_tap/t/010_views.pl +++ b/src/test/cluster_tap/t/010_views.pl @@ -88,7 +88,7 @@ 'Cluster: Reconfig' => 8, # spec-5.18 D12: +ReconfigNodeRemoveCleanupWait 'Cluster: Recovery' => 7, # spec-4.12 D6: +ClusterWriteFenceVerify 'Cluster: Sinval' => 6, - 'Cluster: Interconnect' => 9, # spec-6.13: +busypoll + inline send waits + 'Cluster: Interconnect' => 11, # spec-6.13: +busypoll + inline send waits; spec-7.2 D6: +LmsDataRecv/Send 'Cluster: Undo' => 4, 'Cluster: ADG' => 4, ); diff --git a/src/test/cluster_tap/t/011_gviews.pl b/src/test/cluster_tap/t/011_gviews.pl index c9c6d7c20b..e7cbbdbb3d 100644 --- a/src/test/cluster_tap/t/011_gviews.pl +++ b/src/test/cluster_tap/t/011_gviews.pl @@ -102,7 +102,7 @@ 'Cluster: Reconfig' => 8, # spec-5.18 D12: +ReconfigNodeRemoveCleanupWait 'Cluster: Recovery' => 7, # spec-4.12 D6: +ClusterWriteFenceVerify 'Cluster: Sinval' => 6, - 'Cluster: Interconnect' => 9, # spec-6.13: +busypoll + inline send waits + 'Cluster: Interconnect' => 11, # spec-6.13: +busypoll + inline send waits; spec-7.2 D6: +LmsDataRecv/Send 'Cluster: Undo' => 4, 'Cluster: ADG' => 4, ); diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index d37ca8a874..10355ee2d8 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '158', - 'all 158 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '160', + 'all 160 injection points have a .fault_type entry under inject category (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '158', - 'all 158 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '160', + 'all 160 injection points have a .hits entry under inject category (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index 676dfc9e1a..bd5427f7ec 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L9 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '160', + 'L9 total injection registry size is 160 (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index 6bd02693d5..fac70dfd0b 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -56,7 +56,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound # ---------- @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '160', + 'L11 pg_stat_cluster_injections is 160 (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 71889182cf..ea0a8b7a2c 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -71,7 +71,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound # ---------- @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '160', + 'L12a pg_stat_cluster_injections is 160 (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index 9f1cbaaee5..741bc9dccc 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -61,7 +61,7 @@ # admission reason counters; +1 "pgrac cluster clean_leave" (spec-5.13); +1 # "pgrac cluster cr relgen" (spec-5.56 D4); +1 "pgrac cluster cr tuple stats" # (spec-5.54 D5); full list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound # ---------- @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '160', + 'L11 pg_stat_cluster_injections is 160 (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..7bb3245820 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -162,8 +162,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L6a pg_stat_cluster_injections has 158 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '160', + 'L6a pg_stat_cluster_injections has 160 entries (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 516f9c9b19..161ad271f8 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', 'M1 158 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '160', 'M1 160 injection points (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '316', - 'M5 inject category has 158×2 = 316 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + ) eq '320', + 'M5 inject category has 160×2 = 320 sub-keys (.fault_type + .hits; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); From 38236adf49737b2dfb5c45d71547dcd8530ea6f5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Tue, 7 Jul 2026 21:49:35 +0800 Subject: [PATCH 25/58] test(cluster): spec-7.2 D7 value gate + honest-SKIP fault legs (t/360) Adds t/360 covering the reachable D7 legs and honest-SKIPping the legs blocked on a pre-existing substrate wall (user ruling 2026-07-07; see Stage-7 substrate follow-up register): Reachable (real assertions): L1 flip fact: block_family_plane=1 both nodes, DATA listeners bound L2 value gate: ping-pong X, requester ship histogram p50<5ms, p99<20ms (measured p50/p99 <= 500us on loopback, ~40x headroom) L3 hygiene: zero plane misroutes, no plane-gate errors KNOWN-BLOCKED (SKIP with reason; mirrors t/339's pattern): L4 kill -9 LMS x3 -- F1-8 block_device crash-recovery raw_layout_lock needs GES, unavailable before the crash-restarting node re-joins (recovery-vs-membership substrate gap, orthogonal to the DATA plane) L5 conn-reset injection -- F6-1 persistent GUC storm crashes the passive LMS with kevent() EBADF; needs one-shot arm + WES fd hardening L6 data-dispatch injection -- F6-2 point sits off the actual block recv path (hits stay 0); needs repositioning 23 tests, 6 honest SKIPs. t/348 (3-node) regression green. Spec: spec-7.2-ic-data-plane-decoupling.md --- .../t/360_lms_data_plane_faults.pl | 282 ++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 src/test/cluster_tap/t/360_lms_data_plane_faults.pl diff --git a/src/test/cluster_tap/t/360_lms_data_plane_faults.pl b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl new file mode 100644 index 0000000000..2deb926bf4 --- /dev/null +++ b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl @@ -0,0 +1,282 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 360_lms_data_plane_faults.pl +# spec-7.2 D7 — LMS DATA-plane value gate + fault-leg coverage. +# Complements t/358 (flip core L1-L5) with the quantitative value +# gate and documents the fault legs that are KNOWN-BLOCKED on a +# pre-existing substrate wall. +# +# REACHABLE legs (real assertions): +# L1 flip fact: block_family_plane = 1 on both nodes; DATA-plane +# listeners bound (log evidence). +# L2 value gate: ping-pong the shared table X between the nodes, +# then the requester ship-latency histogram must show +# p50 < 5ms and p99 < 20ms (裁决④ loopback口径). Actual +# percentiles are diag'd. +# L3 hygiene: zero plane misroutes + no plane-gate errors. +# +# KNOWN-BLOCKED legs (honest SKIP; see docs/stage7-substrate- +# findings.md — Stage-7 substrate follow-ups, user ruling 2026-07-07): +# L4 kill -9 LMS x3 crash recovery — F1-8: a SIGKILL of the LMS +# triggers a node crash-restart, but block_device crash- +# recovery replays a relation create/extend that needs the +# raw_layout_lock via GES (cluster_lock_acquire_seven_step), +# and GES is unavailable before the node re-joins the cluster +# -> recovery FATAL "could not acquire raw layout lock". This +# is a recovery-vs-membership ordering gap in the spec-6.0a +# block_device backend, orthogonal to the spec-7.2 DATA plane. +# L5 conn-reset injection reset->reconnect — F6-1: the only arm +# path is the process-local injection GUC, which is persistent +# and storms a reset every LMS tick (not the one-shot epoch +# bump it models); the storm crashes the passive LMS with +# kevent() EBADF and then hits the same F1-8 recovery wall. +# The real single-bump reset path (cur_epoch != dp_last_epoch) +# is clean (rebuild-before-wait within the tick). +# L6 data-dispatch injection reachability — F6-2: the +# cluster-lms-data-dispatch point sits on the LMS async recv +# pump's CONNECTED&READABLE branch, which the actual block +# REPLY recv does not traverse (hits stay 0), so it needs +# repositioning before it can be armed as a reachability leg. +# +# Spec: spec-7.2-ic-data-plane-decoupling.md §4 / D7 +# +# IDENTIFICATION +# src/test/cluster_tap/t/360_lms_data_plane_faults.pl +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Time::HiRes qw(usleep); +use Test::More; + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'dpfault', + quorum_voting_disks => 3, + shared_data => 1, + storage_backend => 'block_device', + extra_conf => [ + 'autovacuum = off', + 'fsync = off', + 'shared_buffers = 64MB', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.online_join = on', + 'cluster.xid_striping = on', + 'cluster.crossnode_runtime_visibility = on', + 'cluster.crossnode_cr_data_plane = on', + 'cluster.block_self_contained = on', + ]); +$pair->start_pair; +usleep(2_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'CONTROL peers up 0->1'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'CONTROL peers up 1->0'); +my ($node0, $node1) = ($pair->node0, $pair->node1); + +sub gcs_int +{ + my ($node, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='gcs' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + +sub poll_write_ok +{ + my ($node, $sql, $timeout_s, $label) = @_; + $timeout_s //= 60; + my $deadline = time + $timeout_s; + while (time < $deadline) + { + return 1 if eval { $node->safe_psql('postgres', $sql); 1 }; + usleep(500_000); + } + diag("poll_write_ok timeout ($label)"); + return 0; +} + +sub timed_update_retry +{ + my ($node, $sql, $tries) = @_; + $tries //= 30; + for my $attempt (1 .. $tries) + { + return 1 if eval { $node->safe_psql('postgres', $sql); 1 }; + usleep(500_000); + } + return 0; +} + +# ============================================================ +# L1 — flip fact + DATA listeners (reachable). +# ============================================================ +is(gcs_int($node0, 'block_family_plane'), 1, 'L1 node0 block family on DATA plane'); +is(gcs_int($node1, 'block_family_plane'), 1, 'L1 node1 block family on DATA plane'); + +my $log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +my $log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($log0, qr/DATA-plane listener bound/, 'L1 node0 DATA listener bound'); +like($log1, qr/DATA-plane listener bound/, 'L1 node1 DATA listener bound'); + +# ============================================================ +# shared table (relfilenode coincidence, t/347 OID-align pattern). +# ============================================================ +ok(poll_write_ok($node0, 'CREATE TABLE fg0 (x int)', 90, 'node0 gate'), + 'node0 write gate open (join admitted)'); +ok(poll_write_ok($node1, 'CREATE TABLE fg1 (x int)', 90, 'node1 gate'), + 'node1 write gate open (join admitted)'); + +my ($p0, $p1) = ('', ''); +for my $attempt (1 .. 8) +{ + $node0->safe_psql('postgres', + 'CREATE TABLE fault_t (aid int, bal int) WITH (fillfactor = 50)'); + $node1->safe_psql('postgres', + 'CREATE TABLE fault_t (aid int, bal int) WITH (fillfactor = 50)'); + $p0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('fault_t')}); + $p1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('fault_t')}); + last if $p0 eq $p1; + my ($n0) = $p0 =~ /(\d+)$/; + my ($n1) = $p1 =~ /(\d+)$/; + my ($lag, $burn) = $n0 < $n1 ? ($node0, $n1 - $n0) : ($node1, $n0 - $n1); + $lag->safe_psql('postgres', + "SELECT lo_unlink(lo_create(0)) FROM generate_series(1, $burn)"); + $node0->safe_psql('postgres', 'DROP TABLE fault_t'); + $node1->safe_psql('postgres', 'DROP TABLE fault_t'); +} +is($p0, $p1, 'shared-table coincidence'); +$node0->safe_psql('postgres', + 'INSERT INTO fault_t SELECT g, 0 FROM generate_series(1, 200) g'); +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node1->safe_psql('postgres', 'CHECKPOINT'); + +# ============================================================ +# L2 — value gate: ping-pong the table X, then p50 < 5ms, p99 < 20ms. +# ============================================================ +my $swaps = 0; +for my $r (1 .. 20) +{ + last unless timed_update_retry($node0, 'UPDATE fault_t SET bal = bal + 1', 20); + usleep(100_000); + last unless timed_update_retry($node1, 'UPDATE fault_t SET bal = bal + 1', 20); + usleep(100_000); + $swaps++; +} +ok($swaps > 0, "L2 completed $swaps X ping-pong swaps over the DATA plane"); + +my @bounds_us = (500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, + 200000, 500000, 1000000, 2000000, 5000000, 10000000, 30000000); +my @counts; +my $total = 0; +for my $b (0 .. $#bounds_us) +{ + my $c = gcs_int($node0, "ship_hist_us_le_$bounds_us[$b]") + + gcs_int($node1, "ship_hist_us_le_$bounds_us[$b]"); + $counts[$b] = $c; + $total += $c; +} +my $inf = gcs_int($node0, 'ship_hist_us_inf') + gcs_int($node1, 'ship_hist_us_inf'); +$total += $inf; + +sub pct_bound +{ + my ($pct, $tot, $counts_ref, $bounds_ref) = @_; + my $target = $pct * $tot; + my $cum = 0; + for my $b (0 .. $#$bounds_ref) + { + $cum += $counts_ref->[$b]; + return $bounds_ref->[$b] if $cum >= $target; + } + return undef; # +inf overflow +} + +my $p50 = pct_bound(0.50, $total, \@counts, \@bounds_us); +my $p99 = pct_bound(0.99, $total, \@counts, \@bounds_us); +diag(sprintf('L2 value gate: ships=%d p50<=%s us p99<=%s us (+inf=%d)', + $total, defined($p50) ? $p50 : 'inf', defined($p99) ? $p99 : 'inf', $inf)); + +ok($total > 0, "L2 requester histogram recorded ship samples ($total)"); +ok(defined($p50) && $p50 <= 5000, + sprintf('L2 value gate p50 < 5ms (p50 <= %s us)', defined($p50) ? $p50 : 'inf')); +ok(defined($p99) && $p99 <= 20000, + sprintf('L2 value gate p99 < 20ms (p99 <= %s us)', defined($p99) ? $p99 : 'inf')); + +# ============================================================ +# L3 — hygiene: zero plane misroutes. +# ============================================================ +is(gcs_int($node0, 'plane_misroute_reject'), 0, 'L3 node0 zero plane misroutes'); +is(gcs_int($node1, 'plane_misroute_reject'), 0, 'L3 node1 zero plane misroutes'); + +# ============================================================ +# L4 — KNOWN-BLOCKED: kill -9 LMS x3 crash recovery. +# (F1-8: block_device crash-recovery raw_layout_lock needs GES, which is +# unavailable before the crash-restarting node re-joins -> recovery FATAL. +# Orthogonal to the spec-7.2 DATA plane; see docs/stage7-substrate- +# findings.md. The SIGKILL is NOT issued here -- it would crash the node +# into an unrecoverable state.) +# ============================================================ +SKIP: { + skip 'F1-8 KNOWN-BLOCKED: block_device crash-recovery raw_layout_lock ' + . 'depends on GES, unavailable before a crash-restarting node re-joins ' + . '(recovery FATAL "could not acquire raw layout lock"); recovery-vs-' + . 'membership substrate gap, orthogonal to the spec-7.2 DATA plane', 3; + ok(0, 'L4.1 LMS crash-restart recovers + DATA listener rebinds'); + ok(0, 'L4.2 CONTROL membership reconnects after LMS crash'); + ok(0, 'L4.3 cross-node block transfer recovers after LMS crash (x3)'); +} + +# ============================================================ +# L5 — KNOWN-BLOCKED: conn-reset injection reset -> reconnect. +# (F6-1: the injection GUC arm is persistent and storms a reset every LMS +# tick, crashing the passive LMS with kevent() EBADF, then hitting the F1-8 +# wall. The injection cannot be single-fired via the GUC; the real single- +# bump epoch reset path is clean. Not armed here.) +# ============================================================ +SKIP: { + skip 'F6-1 KNOWN-BLOCKED: cluster-lms-conn-reset GUC arm is persistent ' + . '(storms every tick, not the one-shot epoch bump it models) and ' + . 'crashes the passive LMS with kevent() EBADF, then hits F1-8; needs ' + . 'one-shot arm + WES fd-lifecycle hardening', 2; + ok(0, 'L5.1 conn-reset injection resets the DATA mesh once'); + ok(0, 'L5.2 mesh reconnects + block transfer converges after reset'); +} + +# ============================================================ +# L6 — KNOWN-BLOCKED: data-dispatch injection reachability. +# (F6-2: the cluster-lms-data-dispatch point is on the LMS async recv pump's +# CONNECTED&READABLE branch, which the actual block REPLY recv does not +# traverse -- hits stay 0 even armed at startup. Needs repositioning onto +# the real block recv path before it can be a reachability leg.) +# ============================================================ +SKIP: { + skip 'F6-2 KNOWN-BLOCKED: cluster-lms-data-dispatch sits off the actual ' + . 'block REPLY recv path (hits stay 0 even armed at startup); needs ' + . 'repositioning', 1; + ok(0, 'L6 cluster-lms-data-dispatch fires on the live block recv path'); +} + +# ============================================================ +# hygiene tail: no plane-gate errors anywhere in the run. +# ============================================================ +$pair->stop_pair; +$log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +$log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +unlike($log0, qr/cannot send from plane|plane mismatch/, + 'L3 node0 log free of plane-gate errors'); +unlike($log1, qr/cannot send from plane|plane mismatch/, + 'L3 node1 log free of plane-gate errors'); + +done_testing(); From 590b3e1c6877f8a77fee887119cd216d5ab3b7aa Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 15:53:12 +0800 Subject: [PATCH 26/58] fix(cluster): make qvotec markers async for lmon --- src/backend/cluster/cluster_clean_leave.c | 248 +++++-- src/backend/cluster/cluster_debug.c | 10 + src/backend/cluster/cluster_guc.c | 10 + src/backend/cluster/cluster_inject.c | 1 + src/backend/cluster/cluster_lmon.c | 119 ++++ src/backend/cluster/cluster_node_remove.c | 166 ++++- src/backend/cluster/cluster_qvotec.c | 5 + src/backend/cluster/cluster_reconfig.c | 641 +++++++++++++++--- src/backend/cluster/cluster_write_fence.c | 34 + src/include/cluster/cluster_clean_leave.h | 11 + src/include/cluster/cluster_guc.h | 1 + src/include/cluster/cluster_lmon.h | 9 + src/include/cluster/cluster_marker_async.h | 208 ++++++ src/include/cluster/cluster_node_remove.h | 11 + src/include/cluster/cluster_reconfig.h | 18 + src/include/cluster/cluster_write_fence.h | 11 + src/test/cluster_tap/t/015_inject.pl | 8 +- src/test/cluster_tap/t/017_debug.pl | 12 +- src/test/cluster_tap/t/018_shared_fs.pl | 4 +- src/test/cluster_tap/t/020_shmem_registry.pl | 4 +- src/test/cluster_tap/t/021_block_format.pl | 4 +- src/test/cluster_tap/t/022_itl_slot.pl | 4 +- .../cluster_tap/t/023_buffer_descriptor.pl | 10 +- src/test/cluster_tap/t/024_pcm_lock.pl | 10 +- src/test/cluster_tap/t/030_acceptance.pl | 10 +- ...358_cluster_2_29a_marker_async_liveness.pl | 176 +++++ src/test/cluster_unit/Makefile | 2 +- src/test/cluster_unit/test_cluster_debug.c | 25 + src/test/cluster_unit/test_cluster_lmon.c | 24 +- .../cluster_unit/test_cluster_marker_async.c | 236 +++++++ src/test/cluster_unit/test_cluster_qvotec.c | 8 + src/test/cluster_unit/test_cluster_reconfig.c | 20 + 32 files changed, 1864 insertions(+), 196 deletions(-) create mode 100644 src/include/cluster/cluster_marker_async.h create mode 100644 src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl create mode 100644 src/test/cluster_unit/test_cluster_marker_async.c diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index d6152a4e28..78a95163d9 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -59,6 +59,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ #include "cluster/cluster_reconfig.h" /* apply_clean_leave + record/is_clean_departed + join_in_progress */ @@ -745,6 +746,126 @@ cluster_clean_leave_ic_send_ack(int32 dest_node_id, int32 leaving_node_id, uint6 /* qvotec-side per-process handshake cursors (mirror the fence-marker ones). */ static uint64 cl_qvotec_last_processed_marker_seq = 0; static uint64 cl_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync cl_lmon_marker_async; +static ClusterLeaveIntentMarker cl_lmon_marker; +static int cl_lmon_marker_phase = 0; +static int32 cl_lmon_marker_leaving = -1; +static uint64 cl_lmon_marker_epoch = 0; +static bool cl_lmon_marker_submitted = false; + +static void cl_build_marker(ClusterLeaveIntentMarker *m, uint8 marker_phase, int32 leaving, + uint64 epoch); + +typedef enum ClusterLeaveAsyncMarkerResult { + CL_LEAVE_ASYNC_PENDING = 0, + CL_LEAVE_ASYNC_ACKED, + CL_LEAVE_ASYNC_FAILED +} ClusterLeaveAsyncMarkerResult; + +static ClusterMarkerAsyncKind +cl_marker_phase_kind(int phase) +{ + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED; + return CLUSTER_MARKER_KIND_UNKNOWN; +} + +static void +cl_release_lmon_marker_stage(void) +{ + cluster_marker_async_release_stage(&cl_lmon_marker_async); + memset(&cl_lmon_marker, 0, sizeof(cl_lmon_marker)); + cl_lmon_marker_phase = 0; + cl_lmon_marker_leaving = -1; + cl_lmon_marker_epoch = 0; + cl_lmon_marker_submitted = false; +} + +static bool +cl_start_lmon_marker_stage(const ClusterLeaveIntentMarker *m, int phase, int32 leaving, + uint64 epoch) +{ + if (cl_lmon_marker_async.has_staged_event) + return false; + cl_lmon_marker = *m; + cl_lmon_marker_phase = phase; + cl_lmon_marker_leaving = leaving; + cl_lmon_marker_epoch = epoch; + cl_lmon_marker_async.has_staged_event = true; + cl_lmon_marker_submitted = false; + return true; +} + +static ClusterLeaveAsyncMarkerResult +cl_poll_lmon_marker_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_LEAVE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + if (!cl_lmon_marker_async.has_staged_event) + return CL_LEAVE_ASYNC_FAILED; + + now = GetCurrentTimestamp(); + kind = cl_marker_phase_kind(cl_lmon_marker_phase); + if (!cl_lmon_marker_submitted) { + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, + kind, cl_lmon_marker_leaving, now)) + return CL_LEAVE_ASYNC_PENDING; + cl_lmon_marker_submitted = true; + return CL_LEAVE_ASYNC_PENDING; + } + + pr = cluster_clean_leave_poll_marker_async(&cl_lmon_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return CL_LEAVE_ASYNC_PENDING; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, cl_lmon_marker_leaving, elapsed_us); + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + + cluster_reconfig_note_marker_slow_ack(kind, cl_lmon_marker_leaving, elapsed_us); + if (result != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + return CL_LEAVE_ASYNC_ACKED; +} + +static void +cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) +{ + ClusterLeaveIntentMarker cm; + ClusterLeaveAsyncMarkerResult ar; + + if (cl_lmon_marker_async.has_staged_event) { + if (cl_lmon_marker_phase != CLUSTER_LEAVE_MARKER_PHASE_COMMITTED + || cl_lmon_marker_leaving != leaving || cl_lmon_marker_epoch != committed_epoch) + return; + } else { + cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); + (void)cl_start_lmon_marker_stage(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + committed_epoch); + } + + ar = cl_poll_lmon_marker_stage(); + if (ar == CL_LEAVE_ASYNC_ACKED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, committed_epoch); + cl_release_lmon_marker_stage(); + } else if (ar == CL_LEAVE_ASYNC_FAILED) { + ereport(LOG, + (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); + } +} ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) @@ -787,6 +908,38 @@ cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) } } +bool +cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (cl_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cl_state->marker_request_seq, + &cl_state->marker_completion_seq)) + return false; + + cl_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &cl_state->marker_request_seq, + &cl_state->marker_completion_seq, cl_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (cl_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cl_state->marker_completion_seq, &cl_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512) { @@ -818,6 +971,7 @@ cluster_clean_leave_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&cl_state->marker_completion_seq, cl_qvotec_inflight_marker_seq); cl_qvotec_last_processed_marker_seq = cl_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -1640,6 +1794,44 @@ cl_coordinator_commit(int32 leaving) uint64 new_epoch; ClusterLeaveIntentMarker m; + if (cl_lmon_marker_async.has_staged_event) { + ClusterLeaveAsyncMarkerResult ar = cl_poll_lmon_marker_stage(); + int phase = cl_lmon_marker_phase; + uint64 staged_epoch = cl_lmon_marker_epoch; + + if (ar == CL_LEAVE_ASYNC_PENDING) + return; + if (ar == CL_LEAVE_ASYNC_FAILED) + return; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { + cl_release_lmon_marker_stage(); + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); + if (new_epoch == 0) { + ereport(LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); + return; + } + cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + new_epoch); + (void)cl_poll_lmon_marker_stage(); + ereport(LOG, + (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", + leaving, (unsigned long long)new_epoch))); + return; + } + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, staged_epoch); + cl_release_lmon_marker_stage(); + return; + } + cl_release_lmon_marker_stage(); + return; + } + /* CL-I3 pre-check: refuse to commit a clean leave on a version a real death * already bumped — the leaving node will then observe a non-CLEAN_LEAVE event * and escalate. This is a cheap early-out; the authoritative guard is the @@ -1668,51 +1860,9 @@ cl_coordinator_commit(int32 leaving) /* (1) COMMITTING(E) marker (coordinator's own slot, before the bump; NOT a * trust basis). Not durable -> do not commit. */ cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, baseline_epoch + 1); - if (cluster_clean_leave_submit_marker(&m) != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster clean-leave: COMMITTING marker not durable for node %d; " - "not committing (the leaving node will retry/escalate)", - leaving))); - return; - } - - /* (2) the real commit point: guarded epoch bump (only if still baseline) + - * publish CLEAN_LEAVE + record clean-departed. Returns 0 if a concurrent - * reconfig moved the epoch off the baseline after the COMMITTING marker — - * fail-closed, do NOT write the COMMITTED marker (CL-I3). */ - new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); - if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); - return; /* CL-I3: stale baseline (or cluster disabled / bad id) */ - } - - /* - * (3) COMMITTED(E) marker (after the bump; the ONLY rebuild trust basis). - * Post-commit it MUST become majority-durable — the durable truth-source must - * exist BEFORE the leaving node departs (P1-V0.7 exit gate; §2.5). Try once - * here. If it reaches majority, mark durable + tell the leaving node it may - * exit (LEAVE_COMMITTED). If NOT, do not give up: committed_marker_durable - * stays 0, cl_survivor_tick retries the marker every tick, and the leaving - * node stays in BARRIER_WAIT (no LEAVE_COMMITTED) until it is durable — it - * never departs without a durable truth-source. The leave is already - * committed (epoch bumped); we never revert. - */ - cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); - if (cluster_clean_leave_submit_marker(&m) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - cl_send_committed(leaving, new_epoch); - } else { - ereport( - LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the COMMITTED " - "marker is not yet majority-durable; retrying each tick (leaving node waits)", - leaving, (unsigned long long)new_epoch))); - } - - ereport(LOG, (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", - leaving, (unsigned long long)new_epoch))); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, + baseline_epoch + 1); + (void)cl_poll_lmon_marker_stage(); } /* survivor (incl. coordinator) side of another node's leave. */ @@ -1773,13 +1923,9 @@ cl_survivor_tick(int32 leaving) if (coordinator == cluster_node_id && cluster_reconfig_is_clean_departed(leaving)) { uint64 committed_epoch = cluster_reconfig_get_clean_departed_epoch(leaving); - if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) { - ClusterLeaveIntentMarker cm; + if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) + cl_drive_committed_marker_stage(leaving, committed_epoch); - cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); - if (cluster_clean_leave_submit_marker(&cm) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - } /* Re-send LEAVE_COMMITTED every tick once durable until the leaver is gone * (best-effort IC): the leaving node will not depart until it receives one, * and step 3 holds the slot until it is CSSD-dead, so delivery is assured. */ diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..6ece9182d4 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -544,6 +544,12 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", + fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", + fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_slow_iter_count", + fmt_int64((int64)cluster_lmon_slow_iter_count())); } /* @@ -1475,6 +1481,10 @@ dump_reconfig_join(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_reconfig_get_join_timeout_count())); emit_row(rsinfo, "reconfig_join", "clean_departed_cleared_count", fmt_int64((int64)cluster_reconfig_get_clean_departed_cleared_count())); + emit_row(rsinfo, "reconfig", "marker_slow_ack_count", + fmt_int64((int64)cluster_reconfig_get_marker_slow_ack_count())); + emit_row(rsinfo, "reconfig", "marker_timeout_count", + fmt_int64((int64)cluster_reconfig_get_marker_timeout_count())); } /* diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..869e7c2866 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -307,6 +307,7 @@ int cluster_phase4_timeout = 30; * LMON main-loop tick / WaitLatch timeout in milliseconds. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; /* spec-1.12 Sprint B D8: cluster.lck_main_loop_interval (mirror). */ int cluster_lck_main_loop_interval = 1000; @@ -2573,6 +2574,15 @@ cluster_init_guc(void) &cluster_lmon_main_loop_interval, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.lmon_slow_iteration_warn_ms", + gettext_noop("LMON main-loop slow-iteration warning threshold in milliseconds."), + gettext_noop("A value greater than zero logs LMON main-loop iterations above this " + "duration; the lmon_slow_iter_count counter is maintained even when " + "logging is disabled with 0."), + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( "cluster.lck_main_loop_interval", gettext_noop("LCK main-loop tick interval in milliseconds."), diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..332a4e97ff 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -199,6 +199,7 @@ static ClusterInjectPoint cluster_injection_points[] = { /* spec-2.6 Sprint A Step 4 D14 — 5 qvotec / quorum-lite injects. */ { .name = "cluster-qvotec-poll-pre" }, { .name = "cluster-qvotec-poll-post" }, + { .name = "cluster-qvotec-marker-service-hold" }, { .name = "cluster-voting-disk-write-fail" }, { .name = "cluster-quorum-loss-broadcast" }, { .name = "cluster-collision-detect" }, diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..51884d0a80 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -681,6 +681,61 @@ cluster_lmon_main_loop_iters(void) return result; } +uint64 +cluster_lmon_last_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->last_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_max_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->max_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_slow_iter_count(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->slow_iter_count; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +void +cluster_lmon_marker_complete_wakeup(void) +{ + struct Latch *latch; + + if (cluster_lmon_state == NULL) + return; + + latch = cluster_lmon_state->lmon_latch; + if (latch != NULL) + SetLatch(latch); +} + /* ============================================================ * LMON main entry (AuxiliaryProcessMain dispatch target). @@ -720,12 +775,30 @@ lmon_publish_status(ClusterLmonStatus status) cluster_lmon_state->ready_at = 0; cluster_lmon_state->last_liveness_tick_at = 0; cluster_lmon_state->main_loop_iters = 0; + cluster_lmon_state->last_iter_us = 0; + cluster_lmon_state->max_iter_us = 0; + cluster_lmon_state->slow_iter_count = 0; + cluster_lmon_state->lmon_latch = NULL; } else if (status == CLUSTER_LMON_READY) { cluster_lmon_state->ready_at = now; + cluster_lmon_state->lmon_latch = MyLatch; + } else if (status == CLUSTER_LMON_EXITED) { + cluster_lmon_state->lmon_latch = NULL; } LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_clear_latch(int code, Datum arg) +{ + if (cluster_lmon_state == NULL) + return; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->lmon_latch = NULL; + LWLockRelease(&cluster_lmon_state->lwlock); +} + static bool lmon_shutdown_requested(void) @@ -760,6 +833,45 @@ lmon_advance_liveness_tick(void) LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_record_iteration(TimestampTz iter_started_at) +{ + TimestampTz now = GetCurrentTimestamp(); + uint64 elapsed_us; + bool slow; + bool should_log = false; + static TimestampTz last_slow_log_at = 0; + + if (now <= iter_started_at) + elapsed_us = 0; + else + elapsed_us = (uint64)(now - iter_started_at); + + slow = (cluster_lmon_slow_iteration_warn_ms >= 0) + && elapsed_us >= (uint64)cluster_lmon_slow_iteration_warn_ms * 1000ULL; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->last_iter_us = elapsed_us; + if (elapsed_us > cluster_lmon_state->max_iter_us) + cluster_lmon_state->max_iter_us = elapsed_us; + if (slow) + cluster_lmon_state->slow_iter_count++; + LWLockRelease(&cluster_lmon_state->lwlock); + + if (slow && cluster_lmon_slow_iteration_warn_ms > 0 + && (last_slow_log_at == 0 + || now - last_slow_log_at >= INT64CONST(60) * INT64CONST(1000000))) { + last_slow_log_at = now; + should_log = true; + } + + if (should_log) + ereport(LOG, + (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); +} + void LmonMain(void) @@ -811,6 +923,7 @@ LmonMain(void) /* Publish SPAWNING (records pid + spawned_at). */ lmon_publish_status(CLUSTER_LMON_SPAWNING); + on_shmem_exit(lmon_clear_latch, (Datum)0); /* Sprint B inject: ready-publish (test slow startup / phase 1 wait timeout). */ CLUSTER_INJECTION_POINT("cluster-lmon-ready-publish"); @@ -938,6 +1051,7 @@ LmonMain(void) int n_events; long wait_ms; TimestampTz now; + TimestampTz iter_started_at; int32 i; CHECK_FOR_INTERRUPTS(); @@ -950,6 +1064,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1414,6 +1529,7 @@ LmonMain(void) wait_ms = 1; } + lmon_record_iteration(iter_started_at); n_events = WaitEventSetWait(wes, wait_ms, ev, lengthof(ev), WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT); @@ -1588,6 +1704,7 @@ LmonMain(void) /* Stub / mock / disabled mode -- preserve spec-1.11 simple loop. */ for (;;) { int rc; + TimestampTz iter_started_at; CHECK_FOR_INTERRUPTS(); @@ -1599,6 +1716,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1677,6 +1795,7 @@ LmonMain(void) CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); + lmon_record_iteration(iter_started_at); rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, cluster_lmon_main_loop_interval, WAIT_EVENT_CLUSTER_BGPROC_LMON_MAIN_LOOP); diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 8825ac6828..830a7a8d66 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -57,6 +57,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" #include "cluster/cluster_membership.h" #include "cluster/cluster_node_remove.h" #include "cluster/cluster_pcm_lock.h" @@ -162,6 +163,14 @@ cluster_node_remove_get_state(ClusterNodeRemoveState *out) static uint64 nr_qvotec_last_processed_marker_seq = 0; static uint64 nr_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync nr_marker_async; +static ClusterRemovalMarker nr_marker_stage; +static int nr_marker_phase = 0; +static int32 nr_marker_node_id = -1; +static uint64 nr_marker_epoch = 0; +static uint64 nr_marker_removed_incarnation = 0; +static uint64 nr_marker_removal_event_id = 0; +static bool nr_marker_submitted = false; ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) @@ -195,6 +204,38 @@ cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) } } +bool +cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (nr_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&nr_state->marker_request_seq, + &nr_state->marker_completion_seq)) + return false; + + nr_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &nr_state->marker_request_seq, + &nr_state->marker_completion_seq, nr_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (nr_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, + &nr_state->marker_result, now, out_result, out_elapsed_us); +} + bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out) { @@ -223,6 +264,7 @@ cluster_node_remove_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&nr_state->marker_completion_seq, nr_qvotec_inflight_marker_seq); nr_qvotec_last_processed_marker_seq = nr_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -241,24 +283,124 @@ cluster_node_remove_publish_qvotec_latch(struct Latch *latch) on_shmem_exit(nr_clear_qvotec_latch, (Datum)0); } -/* build + submit one durable removal marker; returns true iff majority-durable. */ +static ClusterMarkerAsyncKind +nr_marker_phase_kind(int phase) +{ + switch (phase) { + case CLUSTER_REMOVAL_MARKER_REMOVING: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING; + case CLUSTER_REMOVAL_MARKER_SHRUNK: + return CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK; + case CLUSTER_REMOVAL_MARKER_REMOVED: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED; + default: + return CLUSTER_MARKER_KIND_UNKNOWN; + } +} + +static void +nr_release_marker_stage(void) +{ + cluster_marker_async_release_stage(&nr_marker_async); + memset(&nr_marker_stage, 0, sizeof(nr_marker_stage)); + nr_marker_phase = 0; + nr_marker_node_id = -1; + nr_marker_epoch = 0; + nr_marker_removed_incarnation = 0; + nr_marker_removal_event_id = 0; + nr_marker_submitted = false; +} + +static bool +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, + uint64 removed_incarnation, uint64 removal_event_id) +{ + return nr_marker_async.has_staged_event && nr_marker_phase == phase + && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch + && nr_marker_removed_incarnation == removed_incarnation + && nr_marker_removal_event_id == removal_event_id; +} + +/* build + poll one durable removal marker; returns true iff majority-durable. */ static bool nr_write_marker(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, uint64 removal_event_id) { ClusterRemovalMarker m; + TimestampTz now; + uint32 result = CLUSTER_REMOVAL_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + now = GetCurrentTimestamp(); + kind = nr_marker_phase_kind(phase); + + if (nr_marker_async.has_staged_event + && !nr_marker_stage_matches(phase, node_id, remove_epoch, removed_incarnation, + removal_event_id)) { + if (!nr_marker_submitted) { + if (cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, + nr_marker_phase_kind(nr_marker_phase), + nr_marker_node_id, now)) + nr_marker_submitted = true; + return false; + } + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } else if (pr == CLUSTER_MARKER_POLL_ACKED) { + cluster_reconfig_note_marker_slow_ack(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } + return false; + } + + if (!nr_marker_async.has_staged_event) { + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; + m.version = CLUSTER_REMOVAL_MARKER_VERSION; + m.phase = (uint16)phase; + m.removed_node_id = node_id; + m.remove_epoch = remove_epoch; + m.removed_incarnation = removed_incarnation; + m.removal_event_id = removal_event_id; + cluster_removal_marker_compute_crc(&m); + + nr_marker_stage = m; + nr_marker_phase = phase; + nr_marker_node_id = node_id; + nr_marker_epoch = remove_epoch; + nr_marker_removed_incarnation = removed_incarnation; + nr_marker_removal_event_id = removal_event_id; + nr_marker_async.has_staged_event = true; + nr_marker_async.staged_expect_epoch = remove_epoch; + nr_marker_submitted = false; + } + + if (!nr_marker_submitted) { + if (!cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, kind, + node_id, now)) + return false; + nr_marker_submitted = true; + return false; + } + + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return false; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return false; + } - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; - m.version = CLUSTER_REMOVAL_MARKER_VERSION; - m.phase = (uint16)phase; - m.removed_node_id = node_id; - m.remove_epoch = remove_epoch; - m.removed_incarnation = removed_incarnation; - m.removal_event_id = removal_event_id; - cluster_removal_marker_compute_crc(&m); - - return cluster_node_remove_submit_marker(&m) == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; + cluster_reconfig_note_marker_slow_ack(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return result == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; } diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 8eb25379a7..6fd071a17a 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -94,6 +94,7 @@ #include "cluster/cluster_elog.h" /* CLUSTER_LOG (best-effort logging) */ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ +#include "cluster/cluster_inject.h" #include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ #include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ @@ -883,6 +884,10 @@ qvotec_poll_once(void) * carried forward every poll like the fence marker (R12), and acked majority- * durable from the self-slot write tally. */ have_removal_submit = cluster_node_remove_qvotec_poll_pending(&removal_submit_marker); + if ((have_submit || have_leave_submit || have_join_submit || have_removal_submit) + && cluster_cr_injection_armed("cluster-qvotec-marker-service-hold", NULL)) + return; + memset(&apply_lease_request, 0, sizeof(apply_lease_request)); memset(&apply_lease_winner, 0, sizeof(apply_lease_winner)); apply_lease_winner.owner_node_id = -1; diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 1f79a6c9b9..931ea26b8d 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -77,6 +77,7 @@ #include "cluster/cluster_clean_leave.h" /* v1.0.4 — cluster_clean_leave_in_progress (serialize) */ #include "cluster/cluster_guc.h" /* cluster_enabled, cluster_online_join */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_voting_disk_io.h" /* spec-5.15 D4 — region-3 join-marker slot I/O */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D4 — durable fence marker submit */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum */ @@ -114,6 +115,38 @@ StaticAssertDecl(CLUSTER_RECONFIG_TOUCH_KIND_COUNT == CLUSTER_TOUCH_KIND_COUNT, */ static ClusterReconfigState *ReconfigShmem = NULL; +typedef struct ClusterReconfigFenceStage { + ClusterMarkerAsync async; + ReconfigEvent event; + ClusterFenceMarker marker; + int32 node_id; + uint64 last_incarnation; + uint64 removal_event_id; + bool submitted; +} ClusterReconfigFenceStage; + +typedef struct ClusterReconfigJoinPrepareStage { + ClusterMarkerAsync async; + ReconfigEvent event; + uint8 join_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 joiner_incarnations[CLUSTER_MAX_NODES]; + int next_node; + bool submitted; +} ClusterReconfigJoinPrepareStage; + +typedef struct ClusterReconfigJoinCommitStage { + ClusterMarkerAsync async; + ClusterJoinCommitMarker marker; + int32 node_id; + uint64 admitted_incarnation; + bool submitted; +} ClusterReconfigJoinCommitStage; + +static ClusterReconfigFenceStage failstop_fence_stage; +static ClusterReconfigFenceStage node_removed_fence_stage; +static ClusterReconfigJoinPrepareStage join_prepare_stage; +static ClusterReconfigJoinCommitStage join_commit_stage; + /* ============================================================ * Shmem region lifecycle. @@ -211,6 +244,8 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->join_reject_count, 0); pg_atomic_init_u64(&ReconfigShmem->join_timeout_count, 0); pg_atomic_init_u64(&ReconfigShmem->clean_departed_cleared_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_slow_ack_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_timeout_count, 0); } /* @@ -1032,6 +1067,44 @@ cluster_reconfig_get_clean_departed_cleared_count(void) : pg_atomic_read_u64(&ReconfigShmem->clean_departed_cleared_count); } +void +cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem == NULL || elapsed_us < 1000000ULL) + return; + + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); + ereport(LOG, + (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +void +cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem != NULL) + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); + ereport(LOG, + (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_slow_ack_count); +} + +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_timeout_count); +} + /* * cluster_reconfig_join_in_progress -- Hardening v1.0.4 (spec-5.13 clean-leave x * spec-5.15 online-join serialization, P1-1/P2): is a membership JOIN currently in @@ -1148,6 +1221,383 @@ cluster_reconfig_join_publish_proven(uint64 admitted_epoch) return advanced >= ((members / 2u) + 1u); } +static void +cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) +{ + cluster_marker_async_release_stage(&stage->async); + stage->submitted = false; + memset(&stage->event, 0, sizeof(stage->event)); + memset(&stage->marker, 0, sizeof(stage->marker)); + stage->node_id = -1; + stage->last_incarnation = 0; + stage->removal_event_id = 0; +} + +static bool +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (stage->submitted) + return true; + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, + target_node, now)) + return false; + stage->submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_failstop_fence_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!failstop_fence_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, now)) + return true; + + pr = cluster_write_fence_poll_marker_async(&failstop_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_broadcast_local_procsig(); + } else { + ereport(LOG, + (errmsg("cluster reconfig: fence marker did not reach a voting-disk majority " + "for epoch %llu; not publishing reconfig event (write-fenced, will retry)", + (unsigned long long)failstop_fence_stage.event.new_epoch))); + } + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; +} + +static uint64 +cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 removal_event_id, + uint64 last_incarnation) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!node_removed_fence_stage.async.has_staged_event) + return 0; + if (node_removed_fence_stage.node_id != removed_node_id + || node_removed_fence_stage.removal_event_id != removal_event_id) + return 0; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&node_removed_fence_stage, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, now)) + return 0; + + pr = cluster_write_fence_poll_marker_async(&node_removed_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return 0; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " + "voting-disk majority for epoch %llu; not publishing removal " + "(write-fenced, will retry)", + removed_node_id, + (unsigned long long)node_removed_fence_stage.event.new_epoch))); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); + cluster_reconfig_publish_event(&node_removed_fence_stage.event); + cluster_reconfig_record_removed(removed_node_id, node_removed_fence_stage.event.new_epoch, + false); + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_shrink_to_removed(removed_node_id, last_incarnation); + LWLockRelease(&ReconfigShmem->lock); + + { + uint64 new_epoch = node_removed_fence_stage.event.new_epoch; + + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return new_epoch; + } +} + +static void +cluster_reconfig_release_join_prepare_stage(void) +{ + cluster_marker_async_release_stage(&join_prepare_stage.async); + memset(&join_prepare_stage.event, 0, sizeof(join_prepare_stage.event)); + memset(join_prepare_stage.join_bitmap, 0, sizeof(join_prepare_stage.join_bitmap)); + memset(join_prepare_stage.joiner_incarnations, 0, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; +} + +static void +cluster_reconfig_abort_join_prepare_stage(void) +{ + int i; + + if (ReconfigShmem != NULL) { + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (!dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + continue; + ReconfigShmem->pending_join_bitmap[i / 8] &= (uint8) ~(1u << (i % 8)); + if (cluster_membership_get_state(i) == CLUSTER_MEMBER_JOINING) + cluster_membership_set_state(i, CLUSTER_MEMBER_DEAD); + } + LWLockRelease(&ReconfigShmem->lock); + } + cluster_reconfig_release_join_prepare_stage(); +} + +static bool +cluster_reconfig_submit_join_prepare_current(TimestampTz now) +{ + ClusterJoinCommitMarker m; + int i; + + for (i = join_prepare_stage.next_node; i < CLUSTER_MAX_NODES; i++) { + if (dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + break; + } + join_prepare_stage.next_node = i; + if (i >= CLUSTER_MAX_NODES) { + cluster_reconfig_publish_event(&join_prepare_stage.event); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + cluster_reconfig_broadcast_local_procsig(); + cluster_reconfig_release_join_prepare_stage(); + return true; + } + + if (join_prepare_stage.submitted) + return true; + + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_JCMK_MAGIC; + m.version = CLUSTER_JCMK_VERSION; + m.node_id = i; + m.phase = CLUSTER_JCMK_PHASE_PREPARE; + m.admitted_incarnation = join_prepare_stage.joiner_incarnations[i]; + m.generation = join_prepare_stage.joiner_incarnations[i]; + m.admitted_epoch = join_prepare_stage.event.new_epoch; + cluster_join_marker_compute_crc(&m); + + if (!cluster_reconfig_submit_join_marker_async(&join_prepare_stage.async, i, &m, + CLUSTER_MARKER_KIND_JOIN_PREPARE, now)) + return false; + join_prepare_stage.submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_join_prepare_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + int target; + + if (!join_prepare_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_prepare_stage.submitted) { + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; + } + + target = join_prepare_stage.next_node; + pr = cluster_reconfig_poll_join_marker_async(&join_prepare_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, + elapsed_us); + cluster_reconfig_abort_join_prepare_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, + elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach a " + "voting-disk majority; not publishing JOIN_PENDING (will retry)", + target))); + cluster_reconfig_abort_join_prepare_stage(); + return true; + } + + join_prepare_stage.submitted = false; + join_prepare_stage.next_node++; + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; +} + +static void +cluster_reconfig_release_join_commit_stage(void) +{ + cluster_marker_async_release_stage(&join_commit_stage.async); + memset(&join_commit_stage.marker, 0, sizeof(join_commit_stage.marker)); + join_commit_stage.node_id = -1; + join_commit_stage.admitted_incarnation = 0; + join_commit_stage.submitted = false; +} + +static bool +cluster_reconfig_publish_join_commit(int32 node_id, uint64 admitted_incarnation, + uint64 expected_epoch) +{ + uint64 old_epoch, new_epoch; + XLogRecPtr lsn; + ReconfigEvent evt; + uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint64 incs[CLUSTER_MAX_NODES]; + + if (cluster_epoch_get_current() + 1 != expected_epoch) + return false; + + CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + + cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); + if (new_epoch != expected_epoch) + return false; + lsn = GetXLogInsertRecPtr(); + cluster_epoch_set_changed_at_lsn((uint64)lsn); + cluster_gcs_block_on_epoch_advance(new_epoch); + cluster_sinval_reset_all_on_reconfig(); + cluster_tt_status_flush_all((uint32)new_epoch); + + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); + cluster_membership_record_admitted(node_id, admitted_incarnation); + ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); + LWLockRelease(&ReconfigShmem->lock); + + if (cluster_reconfig_is_clean_departed(node_id)) + pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); + cluster_reconfig_clear_clean_departed(node_id); + + dead_bitmap_set_bit(jb, node_id); + memset(incs, 0, sizeof(incs)); + incs[node_id] = admitted_incarnation; + + memset(&evt, 0, sizeof(evt)); + evt.event_id = cluster_reconfig_compute_event_id_v2( + RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); + evt.coordinator_node_id = cluster_node_id; + evt.old_epoch = old_epoch; + evt.new_epoch = new_epoch; + memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + evt.applied_at = GetCurrentTimestamp(); + evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); + evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; + cluster_reconfig_publish_event(&evt); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); + + return true; +} + +static bool +cluster_reconfig_poll_join_commit_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + uint64 admitted_incarnation = 0; + uint64 admitted_generation = 0; + ClusterMarkerPollResult pr; + + if (!join_commit_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_commit_stage.submitted) { + if (!cluster_reconfig_submit_join_marker_async( + &join_commit_stage.async, join_commit_stage.node_id, &join_commit_stage.marker, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, now)) + return true; + join_commit_stage.submitted = true; + return true; + } + + pr = cluster_reconfig_poll_join_marker_async(&join_commit_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " + "voting-disk majority; not committing (will retry)", + join_commit_stage.node_id))); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + if (!cluster_reconfig_get_observed_slot(join_commit_stage.node_id, &admitted_incarnation, + &admitted_generation) + || admitted_incarnation != join_commit_stage.admitted_incarnation + || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, + admitted_generation) + != CLUSTER_JOIN_ACCEPT + || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, + join_commit_stage.admitted_incarnation, + join_commit_stage.async.staged_expect_epoch)) { + pg_atomic_fetch_add_u64(&ReconfigShmem->join_reject_count, 1); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_release_join_commit_stage(); + return true; +} + /* * spec-5.15 D4 — coordinator-side join driver (called from the tick when * online_join is on and self is the min-MEMBER coordinator). Phase-1: fresh @@ -1166,6 +1616,10 @@ cluster_reconfig_drive_joins(int coordinator) if (state == NULL) return; + if (cluster_reconfig_poll_join_prepare_stage()) + return; + if (cluster_reconfig_poll_join_commit_stage()) + return; /* Phase-1 detection + a snapshot of the current pending set, under the lock * (compute_join_bitmap reads membership_state). */ @@ -1182,8 +1636,7 @@ cluster_reconfig_drive_joins(int coordinator) (void)cluster_reconfig_get_observed_slot(i, &joiner_incarnations[i], NULL); } cluster_reconfig_apply_join_as_coordinator(join_bitmap, coordinator, joiner_incarnations); - /* enter the JOIN_PENDING transition fail-closed on every local backend. */ - cluster_reconfig_broadcast_local_procsig(); + return; } /* Phase-2: commit pending joins that have converged. The just-added Phase-1 @@ -1646,6 +2099,7 @@ cluster_reconfig_lmon_tick(void) uint64 cssd_dead_generation; uint64 event_id; int i; + bool failstop_stage_handled = false; /* L20: runtime feature flag check first line. */ if (!cluster_enabled) @@ -1665,6 +2119,7 @@ cluster_reconfig_lmon_tick(void) self_id = cluster_node_id; if (self_id < 0 || self_id >= CLUSTER_MAX_NODES) return; /* defensive: bad self id, cannot participate */ + failstop_stage_handled = cluster_reconfig_poll_failstop_fence_stage(); /* * §3.1 + F11: build the raw CSSD DEAD bitmap, filtering out un-declared @@ -1886,7 +2341,7 @@ cluster_reconfig_lmon_tick(void) * stabilizing the survivor base), THEN the join edge. Each is an independent * ReconfigEvent; neither early-returns past the other. */ - if (!dead_bitmap_is_zero(dead_bitmap)) { + if (!failstop_stage_handled && !dead_bitmap_is_zero(dead_bitmap)) { CLUSTER_INJECTION_POINT("cluster-reconfig-decide-coordinator"); /* §3.2 P1.2: event_id from dead_bitmap + dead_generation snapshot. */ @@ -2123,13 +2578,14 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " - "majority for epoch %llu; not publishing reconfig event " - "(write-fenced, will retry)", - (unsigned long long)new_epoch))); - return; /* fail-closed: epoch bumped, event NOT published, recovery NOT started */ - } + failstop_fence_stage.event = evt; + failstop_fence_stage.marker = marker; + failstop_fence_stage.node_id = coordinator_node_id; + failstop_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + coordinator_node_id, GetCurrentTimestamp()); + return; /* fail-closed until the staged marker is majority-durable */ } cluster_reconfig_publish_event(&evt); @@ -2287,6 +2743,9 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 return 0; if (removed_node_id < 0 || removed_node_id >= CLUSTER_MAX_NODES) return 0; + if (node_removed_fence_stage.async.has_staged_event) + return cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); CLUSTER_INJECTION_POINT("cluster-node-remove-shrink-committing"); @@ -2340,13 +2799,25 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 * a concurrent dead set, if any, is carried by its own fail-stop fence. */ memcpy(marker.fenced_dead_bitmap, removed_with_n, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " - "voting-disk majority for epoch %llu; not publishing removal " - "(write-fenced, will retry)", - removed_node_id, (unsigned long long)new_epoch))); - return 0; /* fail-closed: epoch bumped, removal NOT published, driver retries */ - } + node_removed_fence_stage.event.event_id + = cluster_reconfig_compute_removal_event_id(removed_with_n, removal_event_id); + node_removed_fence_stage.event.coordinator_node_id = cluster_node_id; + node_removed_fence_stage.event.old_epoch = old_epoch; + node_removed_fence_stage.event.new_epoch = new_epoch; + memcpy(node_removed_fence_stage.event.dead_bitmap, empty_dead, + CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + node_removed_fence_stage.event.applied_at = GetCurrentTimestamp(); + node_removed_fence_stage.event.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + node_removed_fence_stage.event.cssd_dead_generation = cssd_dead_generation; + node_removed_fence_stage.event.reconfig_kind = RECONFIG_KIND_NODE_REMOVED; + node_removed_fence_stage.marker = marker; + node_removed_fence_stage.node_id = removed_node_id; + node_removed_fence_stage.last_incarnation = last_incarnation; + node_removed_fence_stage.removal_event_id = removal_event_id; + node_removed_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); + return 0; /* fail-closed until the staged fence marker ACKs */ } CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); @@ -2431,6 +2902,42 @@ cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMa } } +bool +cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now) +{ + int wait_ms; + + if (ReconfigShmem == NULL || m == NULL || a == NULL) + return false; + if (target_node < 0 || target_node >= CLUSTER_MAX_NODES) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&ReconfigShmem->join_marker_request_seq, + &ReconfigShmem->join_marker_completion_seq)) + return false; + + ReconfigShmem->join_marker_target_node_id = target_node; + ReconfigShmem->join_pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit( + a, &ReconfigShmem->join_marker_request_seq, &ReconfigShmem->join_marker_completion_seq, + ReconfigShmem->join_qvotec_latch, now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (ReconfigShmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &ReconfigShmem->join_marker_completion_seq, + &ReconfigShmem->join_marker_result, now, out_result, + out_elapsed_us); +} + bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512) { @@ -2464,6 +2971,7 @@ cluster_reconfig_join_qvotec_complete(bool acked) pg_atomic_write_u64(&ReconfigShmem->join_marker_completion_seq, join_qvotec_inflight_marker_seq); join_qvotec_last_processed_marker_seq = join_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -2614,27 +3122,6 @@ cluster_reconfig_apply_join_as_coordinator( } LWLockRelease(&ReconfigShmem->lock); - /* Durable PREPARE marker per joiner (records the presented incarnation; does - * NOT seed — only COMMITTED is a basis). Best-effort: PREPARE failure does - * not block the JOIN_PENDING publish (the COMMITTED marker in Phase-2 is the - * commit point that must be majority-durable). */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - ClusterJoinCommitMarker m; - - if (!dead_bitmap_test_bit(join_bitmap, i)) - continue; - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_JCMK_MAGIC; - m.version = CLUSTER_JCMK_VERSION; - m.node_id = i; - m.phase = CLUSTER_JCMK_PHASE_PREPARE; - m.admitted_incarnation = joiner_incarnations[i]; - m.generation = joiner_incarnations[i]; /* monotonic per node (read-newest intent) */ - m.admitted_epoch = new_epoch; - cluster_join_marker_compute_crc(&m); - (void)cluster_reconfig_submit_join_marker(i, &m); - } - memset(&evt, 0, sizeof(evt)); evt.event_id = cluster_reconfig_compute_event_id_v2(RECONFIG_KIND_JOIN_PENDING, empty_dead, join_bitmap, @@ -2647,20 +3134,21 @@ cluster_reconfig_apply_join_as_coordinator( evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; evt.cssd_dead_generation = cssd_dead_generation; evt.reconfig_kind = RECONFIG_KIND_JOIN_PENDING; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + + join_prepare_stage.event = evt; + memcpy(join_prepare_stage.join_bitmap, join_bitmap, sizeof(join_prepare_stage.join_bitmap)); + memcpy(join_prepare_stage.joiner_incarnations, joiner_incarnations, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; + join_prepare_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_join_prepare_stage(); } bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) { ClusterJoinCommitMarker m; - uint64 old_epoch, new_epoch; - XLogRecPtr lsn; - ReconfigEvent evt; - uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint64 incs[CLUSTER_MAX_NODES]; if (!cluster_enabled || ReconfigShmem == NULL) return false; @@ -2704,60 +3192,17 @@ cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) m.commit_nonce = ((uint64)cluster_node_id << 56) ^ (uint64)GetCurrentTimestamp(); cluster_join_marker_compute_crc(&m); - if (cluster_reconfig_submit_join_marker(node_id, &m) != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " - "voting-disk majority; not committing (will retry)", - node_id))); + if (join_commit_stage.async.has_staged_event) return false; - } - - CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); - - /* - * ② publish: bump JOIN_COMMITTED epoch + state MEMBER + last_admitted + clear - * pending + clear clean_departed[node] (INV-J10). Strict order — the publish - * (epoch + state MEMBER) precedes the joiner opening its write gate (D5). - */ - cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); - lsn = GetXLogInsertRecPtr(); - cluster_epoch_set_changed_at_lsn((uint64)lsn); - cluster_gcs_block_on_epoch_advance(new_epoch); - cluster_sinval_reset_all_on_reconfig(); - cluster_tt_status_flush_all((uint32)new_epoch); - - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); - cluster_membership_record_admitted(node_id, admitted_incarnation); - ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); - LWLockRelease(&ReconfigShmem->lock); + join_commit_stage.marker = m; + join_commit_stage.node_id = node_id; + join_commit_stage.admitted_incarnation = admitted_incarnation; + join_commit_stage.async.has_staged_event = true; + join_commit_stage.async.staged_expect_epoch = m.admitted_epoch; + join_commit_stage.submitted = false; + (void)cluster_reconfig_poll_join_commit_stage(); - /* INV-J10: clear the in-shmem clean_departed suppression so a clean-left node - * that just rejoined has its later real fail-stop honored again (the durable - * supersede across restart is resolved by the seed, RC-5). */ - if (cluster_reconfig_is_clean_departed(node_id)) - pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); - cluster_reconfig_clear_clean_departed(node_id); - - dead_bitmap_set_bit(jb, node_id); - memset(incs, 0, sizeof(incs)); - incs[node_id] = admitted_incarnation; - - memset(&evt, 0, sizeof(evt)); - evt.event_id = cluster_reconfig_compute_event_id_v2( - RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); - evt.coordinator_node_id = cluster_node_id; - evt.old_epoch = old_epoch; - evt.new_epoch = new_epoch; - memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - evt.applied_at = GetCurrentTimestamp(); - evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; - evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); - evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); - - return true; + return false; } diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index 2baebb13bd..d4988b042d 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -40,6 +40,7 @@ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_voting_disks */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_qvotec.h" /* ClusterVotingSlot (marker layout asserts) */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_voting_disk_io.h" /* D6 direct durable marker read */ @@ -769,6 +770,38 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m) } } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (cluster_write_fence_shmem == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq)) + return false; + + memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); + return cluster_marker_async_submit( + a, &cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq, + cluster_write_fence_shmem->qvotec_latch, now, + (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (cluster_write_fence_shmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cluster_write_fence_shmem->marker_completion_seq, + &cluster_write_fence_shmem->marker_result, now, out_result, + out_elapsed_us); +} + /* * cluster_write_fence_clear_qvotec_latch / _publish_qvotec_latch -- qvotec publishes * its MyLatch at startup so LMON can wake it; auto-cleared at proc_exit so a stale @@ -835,6 +868,7 @@ cluster_write_fence_qvotec_complete(bool acked) pg_atomic_write_u64(&cluster_write_fence_shmem->marker_completion_seq, qvotec_inflight_marker_seq); qvotec_last_processed_marker_seq = qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index e49ecfc194..2d75de8e2a 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -65,6 +65,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ #define CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES 16 /* Default drain deadline; aligns with the feature-082 30s barrier. */ @@ -408,6 +410,15 @@ extern void cluster_clean_leave_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); +extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, + const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, + int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512); extern void cluster_clean_leave_qvotec_complete(bool acked); extern void cluster_clean_leave_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..1b3a06016f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -469,6 +469,7 @@ extern int cluster_phase4_timeout; * range: [100, 60000] (millisecond) */ extern int cluster_lmon_main_loop_interval; +extern int cluster_lmon_slow_iteration_warn_ms; /* diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 63efc4e045..1b6a516f4b 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -100,6 +100,7 @@ #include "datatype/timestamp.h" #include "storage/lwlock.h" +struct Latch; /* * ClusterLmonStatus -- HC2 SSOT for LMON lifecycle state. @@ -137,6 +138,10 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 max_iter_us; /* high-water iteration wall time */ + uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ + struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ bool shutdown_requested; /* postmaster sets; LMON main loop polls + exits */ } ClusterLmonSharedState; @@ -188,6 +193,10 @@ extern TimestampTz cluster_lmon_spawned_at(void); extern TimestampTz cluster_lmon_ready_at(void); extern TimestampTz cluster_lmon_last_liveness_tick_at(void); extern int64 cluster_lmon_main_loop_iters(void); +extern uint64 cluster_lmon_last_iter_us(void); +extern uint64 cluster_lmon_max_iter_us(void); +extern uint64 cluster_lmon_slow_iter_count(void); +extern void cluster_lmon_marker_complete_wakeup(void); /* * Status enum -> canonical lowercase string ("not_started", "spawning", diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h new file mode 100644 index 0000000000..792e583b47 --- /dev/null +++ b/src/include/cluster/cluster_marker_async.h @@ -0,0 +1,208 @@ +/*------------------------------------------------------------------------- + * + * cluster_marker_async.h + * Small process-local async FSM for qvotec marker submit mailboxes. + * + * The mailbox remains the existing request_seq/completion_seq/result slot. + * This wrapper changes only the waiting shape: submit publishes a request and + * returns; the owning LMON tick polls completion without sleeping. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_MARKER_ASYNC_H +#define CLUSTER_MARKER_ASYNC_H + +#include "c.h" + +#include + +#include "datatype/timestamp.h" +#include "port/atomics.h" +#include "storage/latch.h" + +typedef enum ClusterMarkerAsyncState { + CLUSTER_MARKER_ASYNC_IDLE = 0, + CLUSTER_MARKER_ASYNC_SUBMITTED +} ClusterMarkerAsyncState; + +typedef enum ClusterMarkerPollResult { + CLUSTER_MARKER_POLL_IDLE = 0, + CLUSTER_MARKER_POLL_PENDING, + CLUSTER_MARKER_POLL_ACKED, + CLUSTER_MARKER_POLL_TIMEOUT +} ClusterMarkerPollResult; + +typedef enum ClusterMarkerAsyncKind { + CLUSTER_MARKER_KIND_UNKNOWN = 0, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + CLUSTER_MARKER_KIND_JOIN_PREPARE, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED +} ClusterMarkerAsyncKind; + +typedef struct ClusterMarkerAsync { + ClusterMarkerAsyncState state; + uint64 inflight_seq; + uint64 deadline_us; + TimestampTz submitted_at; + ClusterMarkerAsyncKind kind; + int32 target_node; + + /* + * Caller-owned staged publish/commit state. While true, the owner must not + * re-enter the pre-bump path; it either submits/polls this record, publishes + * after ACK, or releases it after failure/timeout. + */ + bool has_staged_event; + uint64 staged_expect_epoch; +} ClusterMarkerAsync; + +static inline const char * +cluster_marker_async_kind_name(ClusterMarkerAsyncKind kind) +{ + switch (kind) { + case CLUSTER_MARKER_KIND_FENCE_FAILSTOP: + return "fence_failstop"; + case CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED: + return "fence_node_removed"; + case CLUSTER_MARKER_KIND_JOIN_PREPARE: + return "join_prepare"; + case CLUSTER_MARKER_KIND_JOIN_COMMITTED: + return "join_committed"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING: + return "clean_leave_committing"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED: + return "clean_leave_committed"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING: + return "node_remove_removing"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK: + return "node_remove_shrunk"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED: + return "node_remove_removed"; + case CLUSTER_MARKER_KIND_UNKNOWN: + default: + return "unknown"; + } +} + +static inline void +cluster_marker_async_init(ClusterMarkerAsync *a) +{ + memset(a, 0, sizeof(*a)); + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->target_node = -1; +} + +static inline bool +cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) +{ + return a != NULL && a->state == CLUSTER_MARKER_ASYNC_SUBMITTED; +} + +static inline bool +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq) +{ + return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); +} + +static inline bool +cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, + TimestampTz now, uint64 timeout_us, + ClusterMarkerAsyncKind kind, int32 target_node) +{ + uint64 seq; + + Assert(a != NULL); + Assert(request_seq != NULL); + Assert(completion_seq != NULL); + + if (a->state == CLUSTER_MARKER_ASYNC_SUBMITTED) + return true; + if (cluster_marker_async_mailbox_busy(request_seq, completion_seq)) + return false; + + pg_write_barrier(); + seq = pg_atomic_add_fetch_u64(request_seq, 1); + if (qvotec_latch != NULL) + SetLatch(qvotec_latch); + + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->inflight_seq = seq; + a->deadline_us = (uint64)now + timeout_us; + a->submitted_at = now; + a->kind = kind; + a->target_node = target_node; + return true; +} + +static inline ClusterMarkerPollResult +cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + uint64 elapsed; + + Assert(a != NULL); + Assert(completion_seq != NULL); + Assert(result_slot != NULL); + + if (out_result != NULL) + *out_result = 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (a->state != CLUSTER_MARKER_ASYNC_SUBMITTED) + return CLUSTER_MARKER_POLL_IDLE; + + if (pg_atomic_read_u64(completion_seq) == a->inflight_seq) { + pg_read_barrier(); + if (out_result != NULL) + *out_result = pg_atomic_read_u32(result_slot); + elapsed = ((uint64)now > (uint64)a->submitted_at) + ? ((uint64)now - (uint64)a->submitted_at) + : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_ACKED; + } + + if ((uint64)now >= a->deadline_us) { + elapsed = ((uint64)now > (uint64)a->submitted_at) + ? ((uint64)now - (uint64)a->submitted_at) + : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_TIMEOUT; + } + + return CLUSTER_MARKER_POLL_PENDING; +} + +static inline void +cluster_marker_async_release_stage(ClusterMarkerAsync *a) +{ + if (a == NULL) + return; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->inflight_seq = 0; + a->deadline_us = 0; + a->submitted_at = 0; + a->kind = CLUSTER_MARKER_KIND_UNKNOWN; + a->target_node = -1; + a->has_staged_event = false; + a->staged_expect_epoch = 0; +} + +#endif /* CLUSTER_MARKER_ASYNC_H */ diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index ce6bbd7b19..1df3d8c868 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -74,6 +74,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + #include "cluster/cluster_write_fence.h" /* CLUSTER_FENCE_MARKER_BYTES (marker offset) */ /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ @@ -415,6 +417,15 @@ extern void cluster_node_remove_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); +extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, + const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, + int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out); extern void cluster_node_remove_qvotec_complete(bool acked); extern void cluster_node_remove_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index e34c3a1942..3e6abadc16 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -57,6 +57,7 @@ #include "storage/lwlock.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ struct Latch; /* spec-5.15 D4 — join-marker qvotec mailbox latch (pointer only) */ @@ -316,6 +317,8 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 join_reject_count; pg_atomic_uint64 join_timeout_count; pg_atomic_uint64 clean_departed_cleared_count; + pg_atomic_uint64 marker_slow_ack_count; + pg_atomic_uint64 marker_timeout_count; /* * spec-5.15 D1 — per declared node, the freshest voting-slot incarnation + @@ -580,10 +583,25 @@ extern bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarn */ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); +extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, + TimestampTz now); +extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512); extern void cluster_reconfig_join_qvotec_complete(bool acked); extern void cluster_reconfig_publish_join_qvotec_latch(struct Latch *latch); +extern void cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern void cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern uint64 cluster_reconfig_get_marker_slow_ack_count(void); +extern uint64 cluster_reconfig_get_marker_timeout_count(void); + /* ============================================================ * spec-5.15 D5 — joiner-side write gate + admission (INV-J9 / §2.4 / §2.7). * ============================================================ diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index 7543664e72..dad13ff00d 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -39,6 +39,8 @@ #ifndef CLUSTER_WRITE_FENCE_H #define CLUSTER_WRITE_FENCE_H +#include "cluster/cluster_marker_async.h" + /* * cluster_write_fence_decide -- the PURE cooperative write-fence judge (spec-4.12 * D3). A shared-storage write is allowed iff enforcement is on AND the local @@ -517,6 +519,15 @@ extern void cluster_write_fence_note_baseline_stale(void); */ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); +extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, + int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern void cluster_write_fence_publish_qvotec_latch(struct Latch *latch); extern bool cluster_write_fence_qvotec_poll_pending(ClusterFenceMarker *out); extern void cluster_write_fence_qvotec_complete(bool acked); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..2035b40072 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'pg_stat_cluster_injections returns 159 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '159 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..6d53bad1df 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -63,8 +63,8 @@ 'postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'all 55 categories appear (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'all 56 categories appear (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); # ---------- @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '158', - 'all 158 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '158', - 'all 158 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index 676dfc9e1a..42f8817b91 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L9 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L9 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..eab706fd4c 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '159', + 'L15 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..57a321a3a6 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..f2cac828a8 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L12a pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..2c41642a72 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- @@ -168,8 +168,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12 pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12 pg_cluster_state has 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -232,7 +232,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..62f66a1aa8 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -162,8 +162,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L6a pg_stat_cluster_injections has 158 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L6a pg_stat_cluster_injections has 159 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', @@ -189,8 +189,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L7b pg_cluster_state has 55 distinct categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L7b pg_cluster_state has 56 distinct categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -209,7 +209,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..3672d99bce 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', 'M1 158 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', 'M1 159 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '316', - 'M5 inject category has 158×2 = 316 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + ) eq '318', + 'M5 inject category has 159×2 = 318 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); @@ -395,8 +395,8 @@ is($node->safe_psql('postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'O2 pg_cluster_state has all 55 categories (spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'O2 pg_cluster_state has all 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE value IS NULL}), diff --git a/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl new file mode 100644 index 0000000000..e2f19d6454 --- /dev/null +++ b/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl @@ -0,0 +1,176 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 358_cluster_2_29a_marker_async_liveness.pl +# BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. +# +# The hold injection freezes qvotec marker completion until released. Before +# BUG-C1, LMON synchronously waited inside marker submit, stopped sending HELLOs, +# and peers could false-DEAD it during a long marker write. This TAP holds the +# marker service for >=90s while an online join wants a PREPARE marker and checks: +# L1 hold injection is registered. +# L2 a true leave/restart creates a join marker backlog. +# L3 during a 95s hold, LMON keeps iterating and both live nodes keep each +# other CSSD-alive (no false-DEAD). +# L4 marker timeout/backlog is observable and does not publish admission. +# L5 releasing the hold lets the join converge. +# L6 an actual stopped node still becomes DEAD (no true-DEAD regression). +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterPair; +use Test::More; +use Time::HiRes qw(usleep); + +sub poll_until +{ + my ($node, $query, $expected, $timeout_s, $label) = @_; + $expected //= 't'; + $timeout_s //= 30; + my $deadline = time + $timeout_s; + my $last = ''; + while (time < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) } // ''; + return 1 if defined $last && $last eq $expected; + usleep(200_000); + } + diag("poll_until timeout ($label): last='$last' expected='$expected'"); + return 0; +} + +sub state_int +{ + my ($node, $category, $key) = @_; + my $v = $node->safe_psql('postgres', + "SELECT COALESCE((SELECT value::bigint FROM pg_cluster_state " + . "WHERE category = '$category' AND key = '$key'), 0)"); + return $v + 0; +} + +sub set_injection +{ + my ($node, $value) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$value'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(700_000); + return; +} + +my @conf = ( + 'autovacuum = off', + 'jit = off', + 'cluster.online_join = on', + 'cluster.join_convergence_timeout_ms = 180000', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 8', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.lmon_slow_iteration_warn_ms = 0', +); + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'marker_async_liveness', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [@conf]); +$pair->start_pair; +usleep(3_000_000); + +my $node0 = $pair->node0; +my $node1 = $pair->node1; + +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_cluster_injections + WHERE name = 'cluster-qvotec-marker-service-hold'}), + '1', + 'L1 hold injection point is registered'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 node1 sees node0 connected'); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 30, 'node1 initially member'), + 'L1 node1 initially MEMBER'); + +my $epoch0 = $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->stop; +ok(poll_until($node0, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead'), + 'L2 node0 detects the stopped node1 as dead'); +ok(poll_until($node0, + q{SELECT state <> 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'node1 demoted from member'), + 'L2 node1 is demoted from MEMBER before rejoin'); + +set_injection($node0, 'cluster-qvotec-marker-service-hold'); +my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); + +$node1->start; +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 CSSD alive after restart'), + 'L2 node1 is live again while marker service is held'); + +sleep 95; + +my $lmon_iters_after = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_after = state_int($node0, 'reconfig', 'marker_timeout_count'); + +cmp_ok($lmon_iters_after, '>', $lmon_iters_before + 20, + 'L3 node0 LMON keeps iterating during >=90s qvotec marker hold'); +ok(poll_until($node1, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 5, 'node1 still sees node0 alive after hold'), + 'L3 node1 still sees node0 CSSD-alive after the hold (no false-DEAD)'); +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 5, 'node0 still sees node1 alive after hold'), + 'L3 node0 still sees node1 CSSD-alive after the hold'); + +cmp_ok($timeouts_after, '>', $timeouts_before, + 'L4 marker timeout/backlog counter advances while qvotec marker service is held'); +isnt($node0->safe_psql('postgres', + q{SELECT state FROM pg_cluster_membership WHERE node_id = 1}), + 'member', + 'L4 held/timeout marker does not publish node1 admission'); + +set_injection($node0, ''); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 joins after hold release'), + 'L5 node0 admits node1 after hold release'); +ok(poll_until($node1, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 self member after hold release'), + 'L5 node1 self-state reaches MEMBER after hold release'); +cmp_ok($node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0, + '>', $epoch0, + 'L5 membership epoch advances after rejoin'); + +$node1->stop; +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead after final stop'), + 'L6 true DEAD detection still works after async marker liveness fix'); +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'membership dead after final stop'), + 'L6 membership fail-stop path still demotes a truly stopped peer'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..0d753ce2b6 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -41,7 +41,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ - test_cluster_epoch test_cluster_fence test_cluster_reconfig \ + test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory \ test_cluster_tt_slot_allocator test_cluster_itl_reader_real_triple \ test_cluster_itl_cleanout test_cluster_visibility_inject test_cluster_itl_cleanout_perf \ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..b0a4fe5746 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -2638,6 +2638,21 @@ cluster_lmon_main_loop_iters(void) { return 0; } +uint64 +cluster_lmon_last_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_max_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_slow_iter_count(void) +{ + return 0; +} int cluster_lmon_status(void) { @@ -3377,6 +3392,16 @@ cluster_reconfig_get_clean_departed_cleared_count(void) { return 0; } +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return 0; +} +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return 0; +} void cluster_touched_peers_self_hex(char *buf, Size buflen) { diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..15bc7894b1 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -178,6 +178,7 @@ cluster_postmaster_start_lmon(void) * MyLatch + cluster_inject framework. Stubs cover them all -- * runtime LmonMain is not exercised. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; #include "cluster/cluster_inject.h" int cluster_injection_armed_count = 0; @@ -590,6 +591,9 @@ pg_usleep(long microsec pg_attribute_unused()) /* Sprint B: Latch / WaitLatch / ResetLatch stubs (LmonMain runtime * is not invoked at unit-test level). */ struct Latch *MyLatch = NULL; +void +SetLatch(struct Latch *latch pg_attribute_unused()) +{} int WaitLatch(struct Latch *latch pg_attribute_unused(), int wakeEvents pg_attribute_unused(), long timeout pg_attribute_unused(), uint32 wait_event_info pg_attribute_unused()) @@ -600,6 +604,11 @@ void ResetLatch(struct Latch *latch pg_attribute_unused()) {} +#include "storage/ipc.h" +void +on_shmem_exit(pg_on_exit_callback function pg_attribute_unused(), Datum arg pg_attribute_unused()) +{} + /* cluster_lmon.c references MyBackendType (set by LmonMain). */ #include "miscadmin.h" BackendType MyBackendType = B_INVALID; @@ -804,9 +813,21 @@ UT_TEST(test_lmon_public_symbols_linkable) UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_size); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_init); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_register); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_last_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_max_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_slow_iter_count); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_marker_complete_wakeup); UT_ASSERT_NOT_NULL((void *)LmonMain); } +UT_TEST(test_lmon_iteration_counters_null_safe) +{ + UT_ASSERT_EQ((unsigned long long)cluster_lmon_last_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_max_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_slow_iter_count(), 0ULL); + cluster_lmon_marker_complete_wakeup(); +} + /* ============================================================ * Test runner @@ -815,12 +836,13 @@ UT_TEST(test_lmon_public_symbols_linkable) int main(void) { - UT_PLAN(5); + UT_PLAN(6); UT_RUN(test_lmon_status_enum_values_frozen); UT_RUN(test_lmon_shared_state_size_under_4kb); UT_RUN(test_lmon_status_to_string_lookup); UT_RUN(test_lmon_status_unknown_returns_unknown); UT_RUN(test_lmon_public_symbols_linkable); + UT_RUN(test_lmon_iteration_counters_null_safe); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c new file mode 100644 index 0000000000..1618c8ea2d --- /dev/null +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -0,0 +1,236 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_marker_async.c + * Unit tests for the process-local qvotec marker async FSM. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_marker_async.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_marker_async.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +static int ut_set_latch_count = 0; + +void +SetLatch(Latch *latch pg_attribute_unused()) +{ + ut_set_latch_count++; +} + +static void +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, pg_atomic_uint32 *result_slot) +{ + cluster_marker_async_init(a); + pg_atomic_init_u64(request_seq, 0); + pg_atomic_init_u64(completion_seq, 0); + pg_atomic_init_u32(result_slot, 0); + ut_set_latch_count = 0; +} + +UT_TEST(test_init_defaults_idle) +{ + ClusterMarkerAsync a; + + cluster_marker_async_init(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); +} + +UT_TEST(test_submit_publishes_one_request) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 7); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(pg_atomic_read_u64(&completion_seq), 0); + UT_ASSERT_EQ(ut_set_latch_count, 1); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.deadline_us, 6000); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_FAILSTOP); + UT_ASSERT_EQ(a.target_node, 7); +} + +UT_TEST(test_poll_pending_before_completion) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 99; + uint64 elapsed = 99; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_PREPARE, 2)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 1500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_PENDING); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(result, 0); + UT_ASSERT_EQ(elapsed, 0); +} + +UT_TEST(test_poll_ack_returns_result_and_elapsed) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 0; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, 3)); + pg_atomic_write_u32(&result_slot, 42); + pg_atomic_write_u64(&completion_seq, 1); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_ACKED); + UT_ASSERT_EQ(result, 42); + UT_ASSERT_EQ(elapsed, 1500); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_poll_timeout_releases_submit_state) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 1000, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, 4)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2000, NULL, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_TIMEOUT); + UT_ASSERT_EQ(elapsed, 1000); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_mailbox_busy_does_not_publish) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + pg_atomic_write_u64(&request_seq, 2); + pg_atomic_write_u64(&completion_seq, 1); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, 5); + + UT_ASSERT(!ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 2); + UT_ASSERT_EQ(ut_set_latch_count, 0); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_reentrant_submit_does_not_bump_again) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, 6)); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1200, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 99); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED); + UT_ASSERT_EQ(a.target_node, 6); +} + +UT_TEST(test_release_stage_clears_staged_record) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + a.has_staged_event = true; + a.staged_expect_epoch = 123; + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED, 8)); + + cluster_marker_async_release_stage(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); + UT_ASSERT_EQ(a.staged_expect_epoch, 0); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_init_defaults_idle); + UT_RUN(test_submit_publishes_one_request); + UT_RUN(test_poll_pending_before_completion); + UT_RUN(test_poll_ack_returns_result_and_elapsed); + UT_RUN(test_poll_timeout_releases_submit_state); + UT_RUN(test_mailbox_busy_does_not_publish); + UT_RUN(test_reentrant_submit_does_not_bump_again); + UT_RUN(test_release_stage_clears_staged_record); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_qvotec.c b/src/test/cluster_unit/test_cluster_qvotec.c index 62b2099932..3400d68962 100644 --- a/src/test/cluster_unit/test_cluster_qvotec.c +++ b/src/test/cluster_unit/test_cluster_qvotec.c @@ -212,6 +212,14 @@ void cluster_elog_init(void) {} +#include "cluster/cluster_inject.h" +bool +cluster_cr_injection_armed(const char *name pg_attribute_unused(), + uint64 *out_param pg_attribute_unused()) +{ + return false; +} + /* Step 3 D7 stubs: signal/ps_display/procsignal symbols not linked * here (cluster_qvotec.c references them for ClusterQvotecMain; * unit test never invokes Main, just address-takes for T-6). */ diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 573406761c..5860c19831 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -357,6 +357,26 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), + const ClusterFenceMarker *m pg_attribute_unused(), + ClusterMarkerAsyncKind kind pg_attribute_unused(), + int32 target_node pg_attribute_unused(), + TimestampTz now pg_attribute_unused()) +{ + return false; +} +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), + TimestampTz now pg_attribute_unused(), + uint32 *out_result pg_attribute_unused(), + uint64 *out_elapsed_us pg_attribute_unused()) +{ + return CLUSTER_MARKER_POLL_IDLE; +} +void +cluster_lmon_marker_complete_wakeup(void) +{} /* spec-3.1 D7 stub: cluster_reconfig_apply_epoch_bump_as_coordinator * calls cluster_tt_status_flush_all. Fixture has no TT overlay shmem; From 3eeeea1db625bf585d9e5ae7d1882f3f749bdea3 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:01:32 +0800 Subject: [PATCH 27/58] ci: include marker async TAP in nightly --- .github/workflows/nightly.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..307a387f2e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,12 +176,13 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and + # t/358 spec-2.29a marker-async LMON liveness. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-358", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 From c7a2c9fc25884e97f1ceaf12478f78958f342dd0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:10:03 +0800 Subject: [PATCH 28/58] test: stub assertions in marker async unit --- src/test/cluster_unit/test_cluster_marker_async.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c index 1618c8ea2d..9823fd69bf 100644 --- a/src/test/cluster_unit/test_cluster_marker_async.c +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -33,6 +33,14 @@ UT_DEFINE_GLOBALS(); static int ut_set_latch_count = 0; +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + void SetLatch(Latch *latch pg_attribute_unused()) { From 8924f6e732966db954e39bc105cf9a4a27a5f086 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 15:54:48 +0800 Subject: [PATCH 29/58] fix(cluster): shared catalog native-era XID authority Publish and seal native-era XID authority after seed shutdown, adopt prehistory from per-node anchors, and max-merge startup nextXid before ShmemVariableCache seeding. Spec: spec-6.15b-xid-authority-native-era.md --- .github/workflows/nightly.yml | 4 +- docs/reference/system-views.md | 3 + docs/user-guide/configuration.md | 47 ++ src/backend/access/transam/xlog.c | 57 ++ src/backend/access/transam/xlogrecovery.c | 10 +- src/backend/cluster/Makefile | 2 + .../cluster/cluster_catalog_bootstrap.c | 180 ++++++- src/backend/cluster/cluster_catalog_migrate.c | 5 +- src/backend/cluster/cluster_debug.c | 14 + src/backend/cluster/cluster_shmem.c | 16 + src/backend/cluster/cluster_xid_authority.c | 340 ++++++++++++ src/backend/cluster/cluster_xid_prehistory.c | 450 ++++++++++++++++ src/backend/utils/errcodes.txt | 11 + src/include/cluster/cluster_catalog_migrate.h | 8 +- src/include/cluster/cluster_xid_authority.h | 194 +++++++ .../t/337_shared_catalog_ddl_2node.pl | 2 + .../t/339_shared_catalog_faults_2node.pl | 26 +- .../346_shared_catalog_relmap_crash_2node.pl | 14 +- .../t/361_xid_authority_native_era_2node.pl | 410 +++++++++++++++ src/test/cluster_unit/Makefile | 18 +- src/test/cluster_unit/test_cluster_debug.c | 16 + .../cluster_unit/test_cluster_pi_shadow.c | 2 +- .../cluster_unit/test_cluster_xid_authority.c | 488 ++++++++++++++++++ 23 files changed, 2290 insertions(+), 27 deletions(-) create mode 100644 src/backend/cluster/cluster_xid_authority.c create mode 100644 src/backend/cluster/cluster_xid_prehistory.c create mode 100644 src/include/cluster/cluster_xid_authority.h create mode 100644 src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl create mode 100644 src/test/cluster_unit/test_cluster_xid_authority.c diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..8422820ee2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -177,11 +177,11 @@ jobs: - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and t/361 XID authority native-era adoption. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docs/reference/system-views.md b/docs/reference/system-views.md index 649aa954c8..9ceab9b723 100644 --- a/docs/reference/system-views.md +++ b/docs/reference/system-views.md @@ -210,6 +210,9 @@ must not be read as evidence that early transfer is active. | `smart_fusion` | `origin_suspect_count` | Pending dependencies associated with an origin suspected dead or unavailable. | | `smart_fusion` | `dep_lost_failclosed_count` | Missing or malformed dependency evidence rejected fail-closed. | | `smart_fusion` | `retry_failclosed_count` | Retryable Smart Fusion fail-closed outcomes, primarily commit-brake timeout. | +| `catalog` | `xid_authority_native_hw` | Sealed native-era xid high-water for shared-catalog formation; `0` when the authority is inactive or unreadable. | +| `catalog` | `xid_authority_sealed` | `t` when the seed completed a clean native-era shutdown and published adoptable XID prehistory. | +| `catalog` | `xid_prehistory_adopted` | Process-local marker showing whether this postmaster boot adopted native-era pg_xact prehistory. | Example: diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 007c2cb90e..76d6cc8f73 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -776,6 +776,9 @@ Requirements and behaviour: * `cluster.node_id` must be between 0 and 15; startup is refused otherwise. Striped clusters are therefore limited to **16 nodes**. * `cluster.online_join` must be `on`; startup is refused otherwise. +* `cluster.shared_catalog = on` with more than one declared node requires + `cluster.xid_striping = on`; startup fails closed with SQLSTATE `53RB5` + if a multi-node shared-catalog cluster is formed without striping. * All nodes must run with the same `cluster.xid_striping` setting. The cluster records its striping state durably on the voting disk; a node whose configuration disagrees with that recorded state (or @@ -791,6 +794,50 @@ cluster.xid_striping = on cluster.online_join = on ``` +### Shared-catalog native-era XID authority + +A shared-catalog cluster that is formed from a single seed node has a +short native era: the seed runs with `cluster.enabled = off` while it +creates initial catalog rows, roles, schemas, and seed data. Those +transactions consume the seed node's local xids before other nodes have +joined. During the seed node's clean shutdown, pgrac publishes two +formation files under `cluster.shared_data_dir/global`: +`pgrac_xid_authority` (the sealed native xid high-water) and +`pgrac_xid_prehistory` (the seed node's pg_xact truth for those xids). + +Provisioning sequence: + +1. Create or clone joiner data directories before the seed load if they + must share the seed node's system identifier. +2. Start the seed with `cluster.shared_catalog = on`, + `cluster.controlfile_shared_authority = on`, `cluster.enabled = off`, + and a fixed `cluster.node_id`; perform the seed load. +3. Stop the seed cleanly. Do not use immediate shutdown for the final + native-era stop; joiners refuse an unsealed authority with SQLSTATE + `53RB5`. +4. Start all nodes with `cluster.enabled = on`, `cluster.online_join = on`, + and `cluster.xid_striping = on`. Joiners adopt the prehistory before + WAL startup seeds `nextXid`, so seed tuples and roles are judged from + first-hand pg_xact truth rather than hint bits. + +Do not delete or re-create the authority files to recover a failed join. +A missing, unsealed, or corrupt authority/prehistory is intentionally +fail-closed (`53RB5`); restore the shared tree files from backup or +re-provision the cluster. After the first `cluster.enabled = on` boot, +the native era is permanently closed and a later `cluster.enabled = off` +boot against the same shared tree is refused. + +Observe formation state with: + +```sql +SELECT key, value + FROM pg_cluster_state + WHERE category = 'catalog' + AND key IN ('xid_authority_native_hw', + 'xid_authority_sealed', + 'xid_prehistory_adopted'); +``` + ### `cluster.xid_herding_slack` | | | diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670985611f..03d5719b89 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -201,6 +201,7 @@ #include "cluster/cluster_guc.h" /* PGRAC: spec-5.6 cluster_controlfile_shared_authority */ #include "cluster/cluster_hw_snapshot.h" /* PGRAC: spec-5.7 D3 HW authority checkpoint snapshot */ #include "cluster/cluster_xid_stripe_xlog.h" /* PGRAC: spec-6.15 D5d checkpoint re-emit */ +#include "cluster/cluster_xid_authority.h" /* PGRAC: spec-6.15b native-era XID authority */ #include "cluster/cluster_recovery_anchor.h" /* PGRAC: spec-5.6a per-node recovery anchor */ #include "cluster/cluster_lms.h" /* PGRAC: spec-5.6 GES-ready boundary for CF X */ #endif @@ -5593,6 +5594,35 @@ StartupXLOG(void) */ if (cluster_recovery_anchor_active() && !InRecovery) checkPoint = cluster_recovery_anchor_get()->checkPointCopy; + + /* + * PGRAC (spec-6.15b D5): shared_catalog nodes must reconcile their + * transaction horizon with the sealed native-era XID authority before + * ShmemVariableCache is seeded and before StartupCLOG computes its latest + * page. This deliberately consumes a dedicated never-lowered authority, + * not the shared pg_control CheckPoint copy (spec-5.6a keeps that + * per-writer). + */ + if (cluster_shared_catalog && cluster_enabled) + { + ClusterXidAuthorityHeader auth; + + if (!cluster_xid_authority_read(&auth) || + (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable during WAL startup"), + errdetail("StartupXLOG cannot seed nextXid without a sealed native-era high-water."), + errhint("Complete a clean native-era seed shutdown before starting shared_catalog nodes."))); + + if (U64FromFullTransactionId(checkPoint.nextXid) < auth.native_hw_full) + { + uint64 own_next = U64FromFullTransactionId(checkPoint.nextXid); + + checkPoint.nextXid = FullTransactionIdFromU64(auth.native_hw_full); + elog(LOG, "cluster shared_catalog: merged nextXid with XID authority native high-water %llu (own was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } + } #endif /* initialize shared memory variables from the checkpoint record */ @@ -7549,6 +7579,33 @@ CreateCheckPoint(int flags) */ END_CRIT_SECTION(); +#ifdef USE_PGRAC_CLUSTER + /* + * PGRAC (spec-6.15b D3): after the shutdown checkpoint record, + * control-file update, CLOG flush, and per-node recovery anchor publish + * are complete, publish the native-era pg_xact prehistory and seal the + * shared XID authority. This does durable file I/O and may fail closed, + * so it must stay outside the checkpoint critical section. + */ + if (shutdown && cluster_shared_catalog && !cluster_enabled) + { + uint64 native_hw = U64FromFullTransactionId(checkPoint.nextXid); + + if (checkPoint.nextMulti > FirstMultiXactId) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native-era seed loads that create MultiXacts are not supported"), + errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", + checkPoint.nextMulti, FirstMultiXactId), + errhint("Recreate the seed without MultiXact-producing operations, or move the load in-protocol."))); + + cluster_xid_prehistory_publish(DataDir, native_hw); + cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); + elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", + (unsigned long long)native_hw); + } + +#endif + /* * Let smgr do post-checkpoint cleanup (eg, deleting old files). */ diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 7c3daf86a5..c0b7a009c7 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3273,8 +3273,16 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl /* * ShmemVariableCache->nextXid must be beyond record's xid. + * + * PGRAC: a merged-recovery FOREIGN record belongs to a peer's xid + * authority, not this node's allocator. Advancing the local nextXid here + * would make StartupCLOG/TrimCLOG expect local pg_xact pages for peer + * striped xids, while the peer outcome truth lives in pg_xact_remote. */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); +#ifdef USE_PGRAC_CLUSTER + if (!(cluster_recmerge_window_active && cluster_recmerge_apply_foreign)) +#endif + AdvanceNextFullTransactionIdPastXid(record->xl_xid); /* * Before replaying this record, check if this record causes the current diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 5ea41237f7..1a21622e4d 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -220,6 +220,8 @@ OBJS = \ cluster_views.o \ cluster_wal_state.o \ cluster_wal_thread.o \ + cluster_xid_authority.o \ + cluster_xid_prehistory.o \ cluster_xid_stripe.o \ cluster_xid_stripe_boot.o \ cluster_xid_stripe_xlog.o \ diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 9fee3e481c..96dd7912c6 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -35,6 +35,10 @@ */ #include "postgres.h" +#include +#include + +#include "access/transam.h" #include "access/xlog.h" #include "catalog/pg_control.h" #include "cluster/cluster_catalog_bootstrap.h" @@ -44,13 +48,184 @@ #include "cluster/cluster_inject.h" #include "cluster/cluster_mode.h" #include "cluster/cluster_oid_lease.h" +#include "cluster/cluster_recovery_anchor.h" #include "cluster/cluster_startup_phase.h" +#include "cluster/cluster_xid_authority.h" #include "miscadmin.h" +#include "storage/fd.h" + +static bool +cluster_catalog_backup_label_present(void) +{ + char label_path[MAXPGPATH]; + struct stat st; + + snprintf(label_path, sizeof(label_path), "%s/%s", DataDir, BACKUP_LABEL_FILE); + return stat(label_path, &st) == 0; +} + +static void +cluster_catalog_xid_authority_corrupt_fatal(const char *detail) +{ + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable at catalog bootstrap"), + errdetail("%s", detail), + errhint("Restore \"%s/%s\" (or its .bak) from a backup of the shared tree; " + "do not delete or re-seed it from a stale xid high-water.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH))); +} + + +/* + * Early D6 topology sniff: cluster_conf_load() runs later when shmem exists, + * but shared_catalog bootstrap must reject multi-node xid_striping=off before + * it seeds/adopts any authority. This deliberately counts only [node.N] + * section headers; the real parser remains the startup SSOT and will still + * validate syntax, required fields, and node_id consistency later. + */ +static int +cluster_catalog_declared_node_count_early(void) +{ + const char *path = (cluster_config_file != NULL && cluster_config_file[0] != '\0') + ? cluster_config_file + : "pgrac.conf"; + FILE *f; + char line[1024]; + int nodes = 0; + + f = AllocateFile(path, "r"); + if (f == NULL) + return 1; + + while (fgets(line, sizeof(line), f) != NULL) { + const char *p = line; + + while (*p != '\0' && isspace((unsigned char)*p)) + p++; + if (strncmp(p, "[node.", 6) == 0) + nodes++; + } + FreeFile(f); + + return nodes > 0 ? nodes : 1; +} + +static void +cluster_catalog_vet_xid_striping_for_shared_catalog(void) +{ + int nodes; + + if (!cluster_enabled || cluster_xid_striping) + return; + + nodes = cluster_catalog_declared_node_count_early(); + if (nodes > 1) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", nodes), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); +} + +static void +cluster_catalog_prepare_xid_authority(const ControlFileData *cf, + ClusterCatalogMigrateResult migrate_result) +{ + ClusterXidAuthorityHeader auth; + bool have_auth; + + have_auth = cluster_xid_authority_read(&auth); + if (!have_auth && cluster_xid_authority_present()) + cluster_catalog_xid_authority_corrupt_fatal( + "Neither the primary shared XID authority nor its .bak fallback passes validation."); + + if (!have_auth) { + if (migrate_result != CLUSTER_CATALOG_MIGRATE_SEEDED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is absent for an existing shared catalog tree"), + errdetail("The shared catalog marker exists, but \"%s/%s\" does not.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH), + errhint("Re-seed the shared tree with a build carrying spec-6.15b."))); + + if (cluster_xid_authority_seed_if_absent( + U64FromFullTransactionId(cf->checkPointCopy.nextXid))) + elog(LOG, "cluster shared_catalog: seeded XID authority native high-water at %llu", + (unsigned long long)U64FromFullTransactionId(cf->checkPointCopy.nextXid)); + + if (!cluster_xid_authority_read(&auth)) + cluster_catalog_xid_authority_corrupt_fatal( + "The shared XID authority was just seeded but cannot be read back."); + } + + if (!cluster_enabled) { + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"), + errdetail("The shared XID authority is already marked cluster-era started."), + errhint( + "Keep cluster.enabled=on for this shared tree, or destroy and re-form it."))); + return; + } + + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is not sealed"), + errdetail("The seed node has not completed a clean native-era shutdown."), + errhint( + "Start the seed with cluster.enabled=off, stop it cleanly, then start joiners."))); + + if (cluster_catalog_backup_label_present()) + elog(LOG, "cluster shared_catalog: skipped XID prehistory adopt on backup_label boot"); + else { + ClusterRecoveryAnchor ra; + uint64 own_next; + + if (!cluster_recovery_anchor_read(cf->system_identifier, &ra, NULL)) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("per-node recovery anchor is unavailable for XID prehistory adopt"), + errdetail("The joiner cannot determine its own pre-adopt nextXid without \"%s\".", + cluster_recovery_anchor_path() != NULL ? cluster_recovery_anchor_path() + : "(unset)"), + errhint( + "Re-provision this node or verify cluster.shared_data_dir before startup."))); + + own_next = U64FromFullTransactionId(ra.checkPointCopy.nextXid); + if (own_next < auth.native_hw_full) { + cluster_xid_prehistory_adopt(DataDir, auth.native_hw_full); + elog(LOG, + "cluster shared_catalog: adopted XID prehistory through native high-water %llu " + "(own nextXid was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } else + elog(LOG, + "cluster shared_catalog: skipped XID prehistory adopt; own nextXid %llu >= native " + "high-water %llu", + (unsigned long long)own_next, (unsigned long long)auth.native_hw_full); + } + + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) { + cluster_xid_authority_mark_cluster_era(); + elog(LOG, + "cluster shared_catalog: marked XID authority cluster era started at native " + "high-water %llu", + (unsigned long long)auth.native_hw_full); + } +} void cluster_catalog_startup_prepare(void) { ControlFileData cf; + ClusterCatalogMigrateResult migrate_result; /* Postmaster-once: only the postmaster seeds; forked backends inherit. */ if (IsUnderPostmaster) @@ -66,6 +241,8 @@ cluster_catalog_startup_prepare(void) return; /* off: stock per-node catalog */ } + cluster_catalog_vet_xid_striping_for_shared_catalog(); + /* * shared_catalog=on requires the shared pg_control authority (D1 vet), so * it is the single source of the cluster-wide next-OID high-water. Reading @@ -113,7 +290,8 @@ cluster_catalog_startup_prepare(void) * the (D3-flipped) shared smgr route. system_identifier is the shared * pg_control's cluster-wide value both seed and join agree on. */ - cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + migrate_result = cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + cluster_catalog_prepare_xid_authority(&cf, migrate_result); } /* diff --git a/src/backend/cluster/cluster_catalog_migrate.c b/src/backend/cluster/cluster_catalog_migrate.c index 961179b008..bb17838229 100644 --- a/src/backend/cluster/cluster_catalog_migrate.c +++ b/src/backend/cluster/cluster_catalog_migrate.c @@ -588,7 +588,7 @@ vet_seed_no_tablespaces(const char *local_pgdata) FreeDir(dir); } -void +ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) { ClusterCatalogAuthorityMarker existing; @@ -624,7 +624,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) elog(LOG, "cluster shared_catalog: adopted shared catalog authority (sysid " UINT64_FORMAT ")", system_identifier); - return; + return CLUSTER_CATALOG_MIGRATE_ADOPTED; } /* @@ -645,6 +645,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) "cluster shared_catalog: seeded shared catalog authority under \"%s\" " "(sysid " UINT64_FORMAT ")", cluster_shared_data_dir, system_identifier); + return CLUSTER_CATALOG_MIGRATE_SEEDED; } /* diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..187ce5f79e 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -130,6 +130,7 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_catalog_stats.h" /* catalog counters (spec-6.14 D10b) */ #include "cluster/cluster_oid_lease.h" /* catalog category (spec-6.14 D10) */ #include "cluster/cluster_relmap_authority.h" /* relmap authority state (spec-6.14 D5) */ +#include "cluster/cluster_xid_authority.h" /* XID authority state (spec-6.15b D7) */ #include "cluster/cluster_remote_xact.h" /* remote outcome counters (spec-4.5a D11) */ #include "cluster/cluster_ic.h" /* ClusterICOps_Active, ClusterICTier */ #include "cluster/cluster_ic_tier1.h" /* listener metadata accessors (Hardening v1.0.1 F3) */ @@ -3035,6 +3036,19 @@ dump_catalog(ReturnSetInfo *rsinfo) emit_row(rsinfo, "catalog", "relmap_shared_pending_owner_node", fmt_int64(rmok ? (int64)rmhdr.owner_node : -1)); + /* spec-6.15b D7: native-era XID authority and this-boot adopt marker. */ + { + ClusterXidAuthorityHeader xahdr; + bool xaok = cluster_shared_catalog && cluster_xid_authority_read(&xahdr); + + emit_row(rsinfo, "catalog", "xid_authority_native_hw", + fmt_int64(xaok ? (int64)xahdr.native_hw_full : 0)); + emit_row(rsinfo, "catalog", "xid_authority_sealed", + fmt_bool(xaok && (xahdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) != 0)); + emit_row(rsinfo, "catalog", "xid_prehistory_adopted", + fmt_bool(cluster_xid_prehistory_was_adopted())); + } + /* * D10b shared-catalog data-plane counters: cluster-path visibility * resolutions of catalog-page tuples, their fail-closed (53R97-family) diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index fa4b2db725..5444528540 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -1077,6 +1077,22 @@ cluster_init_shmem(void) errhint("Enable cluster.online_join on every node, or disable " "cluster.xid_striping."))); + /* + * spec-6.15b D6: shared_catalog puts catalog heap pages on one shared + * storage image, so a multi-node cluster must not allocate dense native + * xids from every node. Without striping, two nodes can reuse the seed + * native-era xid range and poison shared catalog visibility. + */ + if (cluster_enabled && cluster_shared_catalog && cluster_conf_node_count() > 1 + && !cluster_xid_striping) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", cluster_conf_node_count()), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); + /* * Stage 0.18: bind the cluster_ic vtable for the configured tier. * Stub mode allocates no shmem; tier1+ (Stage 2+) will piggyback diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c new file mode 100644 index 0000000000..0d73ad4552 --- /dev/null +++ b/src/backend/cluster/cluster_xid_authority.c @@ -0,0 +1,340 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.c + * Shared XID authority: never-lowered native-era xid high-water plus + * one-way lifecycle flags, as a torn-safe file under + * cluster.shared_data_dir (spec-6.15b D1). + * + * The authority records the first xid NOT consumed by the native era + * (the pre-formation cluster.enabled=off single-writer window of the + * seed node) and two one-way flags: SEALED (a clean native-era + * shutdown published a complete high-water together with the pg_xact + * prehistory blob, so joiners may adopt) and CLUSTER_ERA (a + * cluster.enabled=on boot closed the native era forever). Writers are + * single by construction: the seed node's postmaster/checkpointer + * during the native era, and the catalog bootstrap window afterwards. + * + * File discipline mirrors cluster_oid_lease.c: temp + fsync + .bak + * roll + durable_rename on write; primary-then-.bak fail-closed read; + * ENOENT-only-absent presence probe. The pure classify layer is + * standalone-linkable for cluster_unit. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1, §4 U1-U3/U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* On-disk image size: the fixed header, exactly. */ +#define CLUSTER_XID_AUTHORITY_FILE_SIZE ((int)sizeof(ClusterXidAuthorityHeader)) + +/* ============================================================ + * Pure layer (no elog/shmem/fd; standalone-linkable for cluster_unit). + * ============================================================ */ + +/* + * cluster_xid_authority_classify -- validate an authority image buffer. + */ +ClusterXidAuthorityValidity +cluster_xid_authority_classify(const char *buf, size_t len) +{ + ClusterXidAuthorityHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidAuthorityHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_AUTHORITY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * build_path -- join cluster_shared_data_dir + relpath into dst. Returns + * false when the shared root is not configured. + */ +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * write_durable -- write buf into tmp, fsync, then durable_rename over final. + * PANICs on any I/O failure (mirrors cluster_oid_lease write_durable). + */ +static void +write_durable(const char *tmp, const char *final, const char *buf) +{ + int fd; + + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + + errno = 0; + if (write(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE) != CLUSTER_XID_AUTHORITY_FILE_SIZE) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + if (durable_rename(tmp, final, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, final))); +} + +/* + * roll_primary_to_bak -- preserve an existing primary into .bak before a new + * primary write. A missing/short primary (first write) is skipped. + */ +static void +roll_primary_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char buf[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + int fd; + int r; + + fd = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (fd < 0) + return; /* first write: no prior primary */ + + r = read(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return; /* short/odd primary: don't manufacture a .bak */ + + write_durable(baktmp, bak, buf); +} + +/* + * read_image -- read a candidate authority file and classify it. Missing or + * short is INVALID_SHORT. + */ +static ClusterXidAuthorityValidity +read_image(const char *path, char *image) +{ + int fd; + int r; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + r = read(fd, image, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + return cluster_xid_authority_classify(image, CLUSTER_XID_AUTHORITY_FILE_SIZE); +} + +/* + * cluster_xid_authority_read -- fail-closed read of the shared authority. + * Tries primary then .bak; returns false when neither is trustworthy. Never + * ereports (safe on the bootstrap early-read path). + */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (read_image(primary_path, image) != CLUSTER_XID_AUTHORITY_VALID) { + if (read_image(bak_path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; /* fail-closed: neither trustworthy */ + } + + memcpy(out, image, sizeof(*out)); + return true; +} + +/* + * cluster_xid_authority_present -- does any authority image (primary or .bak) + * exist on disk at all, trustworthy or not? Lets callers distinguish + * "absent: genuine first seed" from "present but corrupt: fail-closed" -- a + * corrupt authority must never be silently re-seeded (a re-seed from a + * checkpointed value could put already-consumed xids back "in the future"). + * Never ereports. + * + * Fail-closed errno discipline: only ENOENT proves absence (mirrors + * cluster_oid_authority_present; EIO/EACCES/ESTALE report present). + */ +bool +cluster_xid_authority_present(void) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + struct stat st; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (stat(primary_path, &st) == 0 || errno != ENOENT) + return true; + if (stat(bak_path, &st) == 0 || errno != ENOENT) + return true; + return false; +} + +/* + * write_header -- CRC-stamp and atomically install an authority header. + */ +static void +write_header(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + roll_primary_to_bak(primary, bak, baktmp); + write_durable(tmp, primary, buffer); +} + +/* + * cluster_xid_authority_seed_if_absent -- read-then-seed: if the authority is + * already present (any trustworthy image) do nothing; otherwise create the + * unsealed initial image. Returns true when this call created it. + */ +bool +cluster_xid_authority_seed_if_absent(uint64 initial_native_hw) +{ + ClusterXidAuthorityHeader hdr; + + if (cluster_xid_authority_read(&hdr)) + return false; /* already seeded (join node / prior seed) */ + if (cluster_xid_authority_present()) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at seed"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "do not re-seed from a stale xid high-water.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + + memset(&hdr, 0, sizeof(hdr)); + hdr.flags = 0; + hdr.native_hw_full = initial_native_hw; + hdr.next_multi = 0; + write_header(&hdr); + return true; +} + +/* + * cluster_xid_authority_publish_native -- monotone raise of the native-era + * high-water; seal=true additionally stamps SEALED. Flags are never + * cleared and the high-water is never lowered (spec §3.1). An absent + * authority is created outright (crash between bootstrap-seed and the + * first publish); a present-but-corrupt one PANICs rather than re-seed. + */ +void +cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, bool seal) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) { + if (cluster_xid_authority_present()) + ereport(PANIC, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at publish"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "re-seeding from a stale high-water could re-issue consumed xids.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + memset(&hdr, 0, sizeof(hdr)); + } + + if (native_hw_full > hdr.native_hw_full) + hdr.native_hw_full = native_hw_full; + if (next_multi > hdr.next_multi) + hdr.next_multi = next_multi; + if (seal) + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_SEALED; + + write_header(&hdr); +} + +/* + * cluster_xid_authority_mark_cluster_era -- one-way CLUSTER_ERA stamp (first + * cluster.enabled=on boot; spec §3.1). No-op when already stamped. A + * missing/corrupt authority PANICs: the catalog bootstrap seeds it before + * any caller can reach this point, so absence here is real damage. + */ +void +cluster_xid_authority_mark_cluster_era(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at cluster-era stamp"))); + + if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + return; + + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; + write_header(&hdr); +} diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c new file mode 100644 index 0000000000..9333c1eaba --- /dev/null +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -0,0 +1,450 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_prehistory.c + * Native-era pg_xact prehistory: publish the seed node's CLOG page run + * [0, native_hw) into a CRC-guarded blob under cluster.shared_data_dir, + * and adopt it into a joiner's local pg_xact (spec-6.15b D2). + * + * The native era (pre-formation, cluster.enabled=off) is a single- + * writer window: the seed node's local pg_xact is the complete commit- + * status truth for every xid below the native high-water, including + * aborted ones. Publishing that page run with the shared tree lets a + * pre-seed-lineage joiner adopt first-hand truth instead of trusting + * hint bits (false-invisible / poison-stamp hazard; spec §0). This is + * a one-shot formation-time carry of pre-cluster history -- the same + * bytes a post-seed clone would have carried -- NOT a runtime mirror of + * cluster-era foreign commit bits (the rejected CLOG-overlay design; + * spec §1.4.1 boundary note). + * + * Both publish and adopt stream page-at-a-time (no large allocations): + * a CRC pass validates or stamps the whole image, then a second pass + * moves the bytes. Their callers run in single-writer windows (the + * seed's shutdown checkpoint; the joiner's postmaster-once bootstrap, + * strictly before StartupCLOG), so the two passes see stable bytes. + * Adopt fsyncs every touched segment and the pg_xact directory before + * returning (spec P2). + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_prehistory.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D2, §4 U1/U4) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* + * CLOG geometry, restated locally so the module stays standalone-linkable + * for cluster_unit: 2 status bits per xact -> 4 xacts per byte -> BLCKSZ*4 + * xacts per page (clog.c CLOG_XACTS_PER_PAGE, private to clog.c), and 32 + * pages per SLRU segment file (slru.h SLRU_PAGES_PER_SEGMENT). The unit + * truth table locks this arithmetic against drift. + */ +#define PREHISTORY_XACTS_PER_PAGE ((uint32)(BLCKSZ * 4)) +#define PREHISTORY_PAGES_PER_SEGMENT 32 + +static bool prehistory_adopted_this_boot = false; + +/* ============================================================ + * Pure layer. + * ============================================================ */ + +/* + * cluster_xid_prehistory_payload_bytes -- whole-page payload size covering + * xids [0, native_hw_full); 0 for an empty or over-cap native era (callers + * treat 0 as "nothing to carry" / refuse respectively). + */ +uint32 +cluster_xid_prehistory_payload_bytes(uint64 native_hw_full) +{ + uint64 pages; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + return 0; + + pages = (native_hw_full + PREHISTORY_XACTS_PER_PAGE - 1) / PREHISTORY_XACTS_PER_PAGE; + return (uint32)(pages * BLCKSZ); +} + +/* + * cluster_xid_prehistory_classify -- validate a full in-memory blob image + * (header + payload). Used by the unit truth table; the streaming adopt + * path performs the same checks incrementally. + */ +ClusterXidAuthorityValidity +cluster_xid_prehistory_classify(const char *buf, size_t len) +{ + ClusterXidPrehistoryHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidPrehistoryHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + if (len != sizeof(hdr) + hdr.payload_len + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(hdr.native_hw_full)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidPrehistoryHeader, crc)); + COMP_CRC32C(crc, buf + sizeof(hdr), hdr.payload_len); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Streaming file plumbing. + * ============================================================ */ + +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * read_clog_page -- read local pg_xact page `pageno` into buf. Fail-closed: + * a missing or short segment is FATAL -- the caller's window (post + * CheckPointGuts / clean shutdown) guarantees every allocated page is on + * disk, so a hole would mean fabricating "aborted" truth for real xids. + */ +static void +read_clog_page(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not open pg_xact segment \"%s\" for prehistory publish: %m", seg), + errhint("The native-era CLOG must be complete on disk; publish runs only " + "after a CLOG flush."))); + if (lseek(fd, offset, SEEK_SET) != offset || read(fd, buf, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of pg_xact segment \"%s\" page %u for prehistory publish", seg, + pageno), + errhint("The native-era CLOG must be complete on disk; a hole would fabricate " + "commit-status truth."))); + CloseTransientFile(fd); +} + +/* + * roll_blob_to_bak -- preserve an existing primary blob into .bak (streamed + * in page-sized chunks; variable length). Missing primary is skipped. + */ +static void +roll_blob_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char chunk[BLCKSZ]; + int src; + int dst; + int r; + + src = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (src < 0) + return; /* first write: no prior primary */ + + dst = OpenTransientFile(baktmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (dst < 0) { + CloseTransientFile(src); + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", baktmp))); + } + while ((r = read(src, chunk, sizeof(chunk))) > 0) { + if (write(dst, chunk, r) != r) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", baktmp))); + } + CloseTransientFile(src); + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", baktmp))); + CloseTransientFile(dst); + if (durable_rename(baktmp, bak, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", baktmp, bak))); +} + +/* ============================================================ + * Publish (seed side). + * ============================================================ */ + +/* + * cluster_xid_prehistory_publish -- snapshot local pg_xact [0, native_hw) + * into the shared blob, torn-safe. Two passes over the source pages: one + * to compute the image CRC, one to write; the caller's single-writer + * window (seed shutdown checkpoint / catalog-seed StartupXLOG) keeps the + * bytes stable between the passes. + */ +void +cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + char page[BLCKSZ]; + uint32 payload_len; + uint32 pages; + uint32 p; + int fd; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native-era xid high-water " UINT64_FORMAT + " is outside the prehistory publish range", + native_hw_full), + errhint("Native-era seed loads are capped at " UINT64_FORMAT + " xids; split the seed load or move it in-protocol.", + CLUSTER_XID_PREHISTORY_MAX_XID))); + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_PREHISTORY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + payload_len = cluster_xid_prehistory_payload_bytes(native_hw_full); + pages = payload_len / BLCKSZ; + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_PREHISTORY_MAGIC; + hdr.version = CLUSTER_XID_PREHISTORY_VERSION; + hdr.native_hw_full = native_hw_full; + hdr.payload_len = payload_len; + hdr.reserved = 0; + + /* pass 1: CRC over header fields + source pages */ + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + COMP_CRC32C(hdr.crc, page, BLCKSZ); + } + FIN_CRC32C(hdr.crc); + + /* pass 2: stream header + pages into tmp, then install torn-safe */ + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + errno = 0; + if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + errno = 0; + if (write(fd, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + } + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + roll_blob_to_bak(primary, bak, baktmp); + if (durable_rename(tmp, primary, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, primary))); +} + +/* ============================================================ + * Adopt (joiner side). + * ============================================================ */ + +/* + * verify_blob -- streaming validation of one candidate blob file against + * the sealed authority's native_hw. Returns true (and leaves *out_pages + * set) only when magic/version/hw/length/CRC all hold. + */ +static bool +verify_blob(const char *path, uint64 native_hw_full, uint32 *out_pages) +{ + ClusterXidPrehistoryHeader hdr; + char page[BLCKSZ]; + pg_crc32c crc; + uint32 pages; + uint32 p; + int fd; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return false; + if (read(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + CloseTransientFile(fd); + return false; + } + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC || hdr.version != CLUSTER_XID_PREHISTORY_VERSION + || hdr.native_hw_full != native_hw_full + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(native_hw_full)) { + CloseTransientFile(fd); + return false; + } + + pages = hdr.payload_len / BLCKSZ; + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + if (read(fd, page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(fd); + return false; + } + COMP_CRC32C(crc, page, BLCKSZ); + } + FIN_CRC32C(crc); + CloseTransientFile(fd); + + if (!EQ_CRC32C(crc, hdr.crc)) + return false; + + *out_pages = pages; + return true; +} + +/* + * cluster_xid_prehistory_adopt -- decode the shared blob into local pg_xact + * page files. Validates the whole image (primary, then .bak) BEFORE the + * first local write -- a joiner must never take half a truth. Writes are + * whole-page, sequential from page 0, fsynced per touched segment, then the + * pg_xact directory is fsynced (spec P2: complete and durable strictly + * before StartupCLOG). Idempotent: re-running overwrites the same bytes. + */ +void +cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char seg[MAXPGPATH]; + char dir[MAXPGPATH]; + char page[BLCKSZ]; + const char *src_path = NULL; + uint32 pages = 0; + uint32 p; + int src; + int dst = -1; + int cur_segno = -1; + int fd; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is missing, corrupt, or does not match the " + "sealed authority high-water " UINT64_FORMAT, + native_hw_full), + errhint("The seed node must complete a clean native-era shutdown before " + "joiners can adopt; restore \"%s\" from the shared tree's backup " + "if it was damaged.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + + /* pass 2: stream the validated pages into local pg_xact segments */ + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not re-open shared XID prehistory \"%s\": %m", src_path))); + + for (p = 0; p < pages; p++) { + int segno = (int)(p / PREHISTORY_PAGES_PER_SEGMENT); + off_t offset = (off_t)(p % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + if (read(src, page, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of shared XID prehistory \"%s\" page %u", src_path, p))); + + if (segno != cur_segno) { + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, segno); + dst = OpenTransientFile(seg, O_RDWR | O_CREAT | PG_BINARY); + if (dst < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact segment \"%s\": %m", seg))); + cur_segno = segno; + } + errno = 0; + if (lseek(dst, offset, SEEK_SET) != offset || write(dst, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not write pg_xact segment \"%s\": %m", seg))); + } + } + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + CloseTransientFile(src); + + /* P2: directory fsync makes the adopted truth durable before StartupCLOG */ + snprintf(dir, sizeof(dir), "%s/pg_xact", local_pgdata); + fd = OpenTransientFile(dir, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact directory \"%s\": %m", dir))); + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact directory \"%s\": %m", dir))); + CloseTransientFile(fd); + prehistory_adopted_this_boot = true; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return prehistory_adopted_this_boot; +} diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be8af8e758..ac8fe1c784 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -875,6 +875,17 @@ Section: Class 53 - Insufficient Resources (pgrac extension) # the first own checkpoint). 53RB3 E ERRCODE_CLUSTER_RECOVERY_ANCHOR_UNAVAILABLE cluster_recovery_anchor_unavailable +# spec-6.15b: shared XID authority / native-era prehistory unavailable. +# Under cluster.shared_catalog=on a joiner must reconcile its transaction +# horizon against the sealed shared XID authority and adopt the native-era +# prehistory (pg_xact truth for pre-formation seed xids); a missing, +# unsealed or corrupt authority/prehistory fails closed rather than judge +# seed rows through a stale horizon or a truthless local CLOG. Also raised +# when a cluster.enabled=off boot tries to re-enter the native seed era on +# a formed cluster's shared tree. (53RB4 is claimed by the spec-7.1 lane's +# ERRCODE_CLUSTER_MXID_HALFSPACE_LIMIT; do not reuse.) +53RB5 E ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE cluster_xid_authority_unavailable + Section: Class 54 - Program Limit Exceeded # this is for wired-in limits, not resource exhaustion problems (class borrowed from DB2) diff --git a/src/include/cluster/cluster_catalog_migrate.h b/src/include/cluster/cluster_catalog_migrate.h index aa17cae1cb..18e3c640ec 100644 --- a/src/include/cluster/cluster_catalog_migrate.h +++ b/src/include/cluster/cluster_catalog_migrate.h @@ -61,6 +61,11 @@ typedef struct ClusterCatalogAuthorityMarker { pg_crc32c crc; /* over [0, offsetof(crc)) */ } ClusterCatalogAuthorityMarker; +typedef enum ClusterCatalogMigrateResult { + CLUSTER_CATALOG_MIGRATE_SEEDED = 0, + CLUSTER_CATALOG_MIGRATE_ADOPTED +} ClusterCatalogMigrateResult; + /* * cluster_catalog_migrate_tree -- establish the shared catalog relation tree * for this node. Postmaster-once, only meaningful when @@ -74,7 +79,8 @@ typedef struct ClusterCatalogAuthorityMarker { * Fail-closed: FATAL 53RB0 on a foreign/mismatched or unreadable authority. * system_identifier is the cluster-wide value from the shared pg_control. */ -extern void cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier); +extern ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, + uint64 system_identifier); /* * cluster_catalog_vet_no_unlogged -- spec-6.14 Q12 enable-time vet: FATAL diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h new file mode 100644 index 0000000000..ed9471907a --- /dev/null +++ b/src/include/cluster/cluster_xid_authority.h @@ -0,0 +1,194 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.h + * Shared XID authority + native-era prehistory (spec-6.15b D1/D2). + * + * Under cluster.shared_catalog the pre-formation ("native era", + * cluster.enabled=off single-writer) xid consumption of the seed node is + * invisible to every other node: a joiner's per-node recovery anchor was + * seeded from its pre-seed clone-era control file, so its transaction + * horizon (nextXid) stays below the seed's committed xids and MVCC + * silently judges seed rows "in the future" (false-invisible), while its + * local pg_xact carries no commit bits for them (poison-hint hazard on + * the shared catalog pages). + * + * Two small durable files under cluster.shared_data_dir close this: + * + * global/pgrac_xid_authority -- never-lowered native-era xid + * high-water (first xid NOT consumed by the native era) plus two + * one-way lifecycle flags: SEALED (a clean native-era shutdown + * published a complete high-water + prehistory; joiners may adopt) + * and CLUSTER_ERA (a cluster.enabled=on boot happened; the native + * era is closed forever and may not be re-entered). + * + * global/pgrac_xid_prehistory -- the raw local pg_xact page run + * covering [0, native_hw), CRC-guarded. A joiner whose own + * nextXid proves it is a pre-seed lineage adopts these bytes into + * its local pg_xact before StartupCLOG, giving it first-hand + * commit-status truth for every native-era xid (no hint-bit + * trust, no CLOG overlay of cluster-era foreign bits -- see the + * spec's no-clog-overlay boundary note). + * + * Torn-safe file discipline mirrors cluster_oid_lease.c: temp + fsync + + * .bak roll + durable_rename on write; primary-then-.bak fail-closed + * read; ENOENT-only-absent presence probe. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_xid_authority.h + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_XID_AUTHORITY_H +#define CLUSTER_XID_AUTHORITY_H + +#include "port/pg_crc32c.h" + +/* ============================================================ + * On-disk authority image (spec-6.15b §2). + * ============================================================ */ + +#define CLUSTER_XID_AUTHORITY_MAGIC 0x0141D617 +#define CLUSTER_XID_AUTHORITY_VERSION 1 + +/* A clean native-era shutdown published a complete hw + prehistory. */ +#define CLUSTER_XID_AUTHORITY_FLAG_SEALED 0x0001 +/* A cluster.enabled=on boot closed the native era forever (one-way). */ +#define CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA 0x0002 + +typedef struct ClusterXidAuthorityHeader { + uint32 magic; + uint32 version; + uint32 flags; + uint32 reserved; /* zero */ + uint64 native_hw_full; /* FullTransactionId value: first xid NOT + * consumed by the native era; never lowered */ + uint64 next_multi; /* native-era nextMulti at publish; vet-only + * (spec Q5: >FirstMultiXactId refuses seed) */ + pg_crc32c crc; /* over all preceding bytes */ +} ClusterXidAuthorityHeader; + +/* Relative paths under cluster.shared_data_dir. */ +#define CLUSTER_XID_AUTHORITY_REL_PATH "global/pgrac_xid_authority" +#define CLUSTER_XID_AUTHORITY_BAK_REL_PATH "global/pgrac_xid_authority.bak" +#define CLUSTER_XID_AUTHORITY_TMP_REL_PATH "global/pgrac_xid_authority.tmp" +#define CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH "global/pgrac_xid_authority.bak.tmp" + +typedef enum ClusterXidAuthorityValidity { + CLUSTER_XID_AUTHORITY_VALID = 0, + CLUSTER_XID_AUTHORITY_INVALID_SHORT, + CLUSTER_XID_AUTHORITY_INVALID_MAGIC, + CLUSTER_XID_AUTHORITY_INVALID_CRC, +} ClusterXidAuthorityValidity; + +/* ============================================================ + * On-disk prehistory image (spec-6.15b D2). + * ============================================================ */ + +#define CLUSTER_XID_PREHISTORY_MAGIC 0x0142D617 +#define CLUSTER_XID_PREHISTORY_VERSION 1 + +/* + * Refusal cap on the native era (spec §3.4): 8M xids = 256 CLOG pages = + * 2MB of payload. A seed load that consumes more must be split or moved + * in-protocol; the cap keeps the blob bounded and the adopt O(small). + */ +#define CLUSTER_XID_PREHISTORY_MAX_XID UINT64CONST(8388608) + +typedef struct ClusterXidPrehistoryHeader { + uint32 magic; + uint32 version; + uint64 native_hw_full; /* payload covers xids [0, native_hw_full) */ + uint32 payload_len; /* raw pg_xact bytes following this header */ + uint32 reserved; /* zero */ + pg_crc32c crc; /* over header fields above + payload */ +} ClusterXidPrehistoryHeader; + +#define CLUSTER_XID_PREHISTORY_REL_PATH "global/pgrac_xid_prehistory" +#define CLUSTER_XID_PREHISTORY_BAK_REL_PATH "global/pgrac_xid_prehistory.bak" +#define CLUSTER_XID_PREHISTORY_TMP_REL_PATH "global/pgrac_xid_prehistory.tmp" +#define CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH "global/pgrac_xid_prehistory.bak.tmp" + +/* ============================================================ + * Pure layer (standalone-linkable; exercised by cluster_unit). + * ============================================================ */ + +extern ClusterXidAuthorityValidity cluster_xid_authority_classify(const char *buf, size_t len); + +/* + * cluster_xid_prehistory_payload_bytes -- whole-CLOG-page payload size + * covering xids [0, native_hw_full). 0 when native_hw_full is 0 (nothing + * to carry); the per-page constant mirrors CLOG_XACTS_PER_PAGE without + * dragging clog.c internals into the unit build. + */ +extern uint32 cluster_xid_prehistory_payload_bytes(uint64 native_hw_full); + +extern ClusterXidAuthorityValidity cluster_xid_prehistory_classify(const char *buf, size_t len); + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * Fail-closed read: primary then .bak; false when neither validates. + * Never ereports (safe on the bootstrap early-read path). + */ +extern bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out); + +/* + * ENOENT-only-absent presence probe (spec-6.14 §3.6 posture): any stat() + * failure other than ENOENT reports present, so a transiently unreadable + * authority is never re-seeded over. + */ +extern bool cluster_xid_authority_present(void); + +/* + * Seed the authority (unsealed) when neither image exists. Returns true + * when this call created it. A trustworthy existing image is a no-op. + */ +extern bool cluster_xid_authority_seed_if_absent(uint64 initial_native_hw); + +/* + * Monotone raise of native_hw_full + set SEALED (never lowers, never + * clears flags). seal=false publishes a crash-safe interim high-water + * during the native era without opening adoption. PANICs on I/O error. + */ +extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, + bool seal); + +/* One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). */ +extern void cluster_xid_authority_mark_cluster_era(void); + +/* ============================================================ + * Prehistory publish / adopt (spec-6.15b D2; P2: adopt is complete and + * fsynced strictly before StartupCLOG -- both run under the postmaster / + * shutdown-checkpoint single-writer windows, file-level only). + * ============================================================ */ + +/* + * Seed side: snapshot local pg_xact pages [0, native_hw_full) into the + * shared prehistory file (torn-safe). PANICs on I/O error; FATALs when + * the native era exceeds CLUSTER_XID_PREHISTORY_MAX_XID. + */ +extern void cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full); + +/* + * Joiner side: decode the shared prehistory into local pg_xact page files, + * pg_fsync each touched segment and the pg_xact directory before returning. + * Idempotent (same-byte overwrite). FATALs (53RB5) on a missing/corrupt + * blob or a native_hw mismatch against the sealed authority. + */ +extern void cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full); +extern bool cluster_xid_prehistory_was_adopted(void); + +#endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..1d45da20c0 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -161,6 +161,8 @@ # ---------- my $cluster_conf = <safe_psql('postgres', 'SELECT 1'), '1', 'R0: node0 is up'); +# Immediately after online_join + striping activation, a backend's bootstrap +# catalog read can transiently hit a peer whose block view is still rebuilding. +# Poll both nodes with the same tolerant shape used by t/339. +my $n0_up = 0; +for (1 .. 60) +{ + my ($rc) = $node0->psql('postgres', 'SELECT 1'); + if (defined $rc && $rc == 0) { $n0_up = 1; last; } + usleep(500_000); +} +is($n0_up, 1, 'R0: node0 is up'); is($n1_up, 1, 'R0: node1 is up'); $node0->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl new file mode 100644 index 0000000000..bacd49ac69 --- /dev/null +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -0,0 +1,410 @@ +#------------------------------------------------------------------------- +# +# 361_xid_authority_native_era_2node.pl +# spec-6.15b BUG-A1/A2 -- shared_catalog native-era XID authority. +# +# node1 is a cold pre-seed clone of node0: it has the same sysid and a +# pre-seed pg_control/pg_xact horizon, but it does not contain node0's +# cluster.enabled=off seed transactions. node0 then seeds shared_catalog, +# consumes native xids, cleanly shuts down, and publishes the sealed XID +# authority plus pg_xact prehistory. On first cluster boot node1 must use +# its per-node recovery anchor's pre-seed nextXid as the adoption proof, +# copy the native-era prehistory into local pg_xact before StartupCLOG, and +# max-merge nextXid with the sealed authority before ShmemVariableCache is +# seeded. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared catalog requires --enable-cluster'; +} + +sub make_shared_root +{ + my $root = PostgreSQL::Test::Utils::tempdir(); + mkdir "$root/global" or die "mkdir $root/global: $!"; + return $root; +} + +sub make_voting_disks +{ + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); + my @disks; + for my $i (0 .. 2) + { + my $p = "$disk_dir/disk$i"; + open(my $fh, '>', $p) or die "open $p: $!"; + binmode $fh; + print $fh ("\0" x (128 * 512)); + close $fh; + push @disks, $p; + } + return join(',', @disks); +} + +sub cold_clone_data_dir +{ + my ($src, $dst) = @_; + + PostgreSQL::Test::RecursiveCopy::copypath( + $src->data_dir, + $dst->data_dir, + filterfn => sub { + my $rel = shift; + return ($rel ne 'log' && $rel ne 'postmaster.pid'); + }); + chmod(0700, $dst->data_dir) or die "chmod clone data dir: $!"; + + # The cold copy carries node0's port; append node1's port so the last + # setting wins, matching init_from_backup's post-copy rewrite. + $dst->append_conf('postgresql.conf', 'port = ' . $dst->port . "\n"); +} + +sub append_pgrac_conf +{ + my ($node, $name, $ic0, $ic1) = @_; + my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); +} + +sub start_background +{ + my ($node) = @_; + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $node->data_dir, + '-l', $node->logfile, '-o', '--cluster-name=' . $node->name, 'start'); +} + +sub wait_sql_eq +{ + my ($node, $sql, $want, $name, $attempts) = @_; + $attempts //= 80; + my $got = ''; + + for (1 .. $attempts) + { + my ($rc, $out) = $node->psql('postgres', $sql); + if (defined $rc && $rc == 0 && defined $out) + { + $got = $out; + last if $got eq $want; + } + usleep(250_000); + } + is($got, $want, $name); +} + +sub read_xid_authority +{ + my ($shared_root) = @_; + my $path = "$shared_root/global/pgrac_xid_authority"; + open(my $fh, '<:raw', $path) or die "open $path: $!"; + read($fh, my $buf, 40) or die "read $path: $!"; + close $fh; + my ($magic, $version, $flags, $reserved, $native_hw, $next_multi, $crc) = + unpack('L< L< L< L< Q< Q< L<', $buf); + return { + magic => $magic, + version => $version, + flags => $flags, + native_hw => $native_hw, + next_multi => $next_multi, + crc => $crc, + }; +} + +sub corrupt_file +{ + my ($path) = @_; + open(my $fh, '>:raw', $path) or die "open $path: $!"; + print $fh "not-a-valid-xid-authority"; + close $fh; +} + +sub append_common_shared_catalog_conf +{ + my ($node, $shared_root) = @_; + $node->append_conf('postgresql.conf', <append_conf('postgresql.conf', <new('sxid_no_stripe'); + + $n->init; + append_common_shared_catalog_conf($n, $shared_root); + $n->append_conf('postgresql.conf', <start(fail_ok => 1); + ok($start_failed, 'D6: shared_catalog multi-node refuses cluster.xid_striping=off'); + like(PostgreSQL::Test::Utils::slurp_file($n->logfile), + qr/requires cluster\.xid_striping/, + 'D6: startup log names the xid_striping requirement'); +} + +# ------------------------------------------------------------------------- +# Main green path: pre-seed clone adopts node0 native-era pg_xact truth. +# ------------------------------------------------------------------------- +my $shared_root = make_shared_root(); +my $disks_csv = make_voting_disks(); +my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); +my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); + +my $node0 = PostgreSQL::Test::Cluster->new('sxid_node0'); +$node0->init(allows_streaming => 1); +$node0->start; +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node0->stop; + +my $node1 = PostgreSQL::Test::Cluster->new('sxid_node1'); +cold_clone_data_dir($node0, $node1); + +append_common_shared_catalog_conf($node0, $shared_root); +$node0->append_conf('postgresql.conf', <start; +ok(-e "$shared_root/global/pgrac_catalog_authority", + 'G1: node0 seeded the shared catalog authority marker'); +ok(-e "$shared_root/global/pg_control", + 'G1: node0 seeded the shared pg_control authority'); +ok(-e "$shared_root/global/pgrac_xid_authority", + 'G1: node0 seeded the shared XID authority before native-era load'); + +my $abort_xid = $node0->safe_psql('postgres', q{ +BEGIN; +SELECT txid_current(); +CREATE SCHEMA sxid_abort_shadow; +ROLLBACK; +}); + +$node0->safe_psql('postgres', q{ +CREATE SCHEMA sxid; +CREATE ROLE sxid_login LOGIN PASSWORD 'sxidpass'; +}); + +my $seed_schema_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_namespace WHERE nspname = 'sxid'}); +my $seed_role_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_authid WHERE rolname = 'sxid_login'}); + +is($node0->safe_psql('postgres', "SELECT txid_status($seed_schema_xid)"), + 'committed', 'G1: seed schema catalog xid is committed on the seed'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G1: native-era aborted xid is aborted on the seed'); + +$node0->stop; + +my $auth = read_xid_authority($shared_root); +ok(($auth->{flags} & 0x0001) != 0, 'G2: clean seed shutdown sealed the XID authority'); +cmp_ok($auth->{native_hw}, '>', $seed_schema_xid, + 'G2: native high-water is above the seed schema catalog xid'); +cmp_ok($auth->{native_hw}, '>', $seed_role_xid, + 'G2: native high-water is above the seed role xid'); +ok(-e "$shared_root/global/pgrac_xid_prehistory", + 'G2: clean seed shutdown published XID prehistory'); + +append_strict_two_node_conf($node0, $disks_csv, undef); +$node0->append_conf('postgresql.conf', "cluster.node_id = 0\n"); + +append_common_shared_catalog_conf($node1, $shared_root); +append_strict_two_node_conf($node1, $disks_csv, undef); +$node1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + +append_pgrac_conf($node0, 'sxid', $ic0, $ic1); +append_pgrac_conf($node1, 'sxid', $ic0, $ic1); + +start_background($node1); +$node0->start; +$node1->_update_pid(1); + +wait_sql_eq($node1, 'SELECT 1', '1', 'G3: node1 is up after XID prehistory adopt'); +wait_sql_eq($node0, + "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " + . "FROM pg_cluster_ic_peers WHERE node_id = 1", + 't', 'G3: node0 sees node1 heartbeats'); + +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}, + '1', 'G4: node1 sees the native-era seed schema catalog row'); +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}, + '1', 'G4: node1 sees the seed role catalog row'); +wait_sql_eq($node1, + q{SET ROLE sxid_login; SELECT current_user}, + 'sxid_login', 'G4: node1 can resolve and SET ROLE to the seed role'); + +wait_sql_eq($node1, "SELECT txid_status($seed_schema_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed schema xid'); +wait_sql_eq($node1, "SELECT txid_status($seed_role_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed role xid'); +wait_sql_eq($node1, "SELECT txid_status($abort_xid)", 'aborted', + 'G5: node1 txid_status sees the native-era aborted xid'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G5: node0 still sees the aborted xid after node1 reads'); + +my $state_hw = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_native_hw'}); +my $state_sealed = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_sealed'}); +my $state_adopted = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_prehistory_adopted'}); + +is($state_hw, "$auth->{native_hw}", 'G6: pg_cluster_state exposes authority native high-water'); +is($state_sealed, 't', 'G6: pg_cluster_state exposes sealed XID authority'); +is($state_adopted, 't', 'G6: pg_cluster_state exposes node1 prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/adopted XID prehistory through native high-water/, + 'G6: node1 log records XID prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/merged nextXid with XID authority native high-water/, + 'G6: node1 log records StartupXLOG nextXid max-merge'); + +$node1->stop; +$node0->stop; + +# ------------------------------------------------------------------------- +# Negative lifecycle legs on the formed shared tree. +# ------------------------------------------------------------------------- +$node0->append_conf('postgresql.conf', <start(fail_ok => 1); +ok($reentry_failed, 'N1: cluster.enabled=off cannot re-enter native seed era after formation'); +like(PostgreSQL::Test::Utils::slurp_file($node0->logfile), + qr/native seed era cannot be re-entered/, + 'N1: startup log names native-era re-entry refusal'); + +corrupt_file("$shared_root/global/pgrac_xid_authority"); +corrupt_file("$shared_root/global/pgrac_xid_authority.bak"); +$node1->append_conf('postgresql.conf', "# t/361 force new logfile generation after corrupt authority\n"); +my $corrupt_failed = !$node1->start(fail_ok => 1); +ok($corrupt_failed, 'N2: corrupt shared XID authority fails closed'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/shared XID authority is unavailable|present but corrupt|cluster_xid_authority_unavailable/, + 'N2: startup log names corrupt/unavailable XID authority'); + +# ------------------------------------------------------------------------- +# Unsealed authority negative: seed startup wrote the authority, but the seed +# crashed before clean shutdown could publish prehistory and seal it. +# ------------------------------------------------------------------------- +{ + my $u_shared = make_shared_root(); + my $u_disks = make_voting_disks(); + my $u_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $u_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $u0 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node0'); + my $u1 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node1'); + + $u0->init(allows_streaming => 1); + $u0->start; + $u0->safe_psql('postgres', 'CHECKPOINT'); + $u0->stop; + cold_clone_data_dir($u0, $u1); + + append_common_shared_catalog_conf($u0, $u_shared); + $u0->append_conf('postgresql.conf', <start; + ok(-e "$u_shared/global/pgrac_xid_authority", + 'N3: unsealed fixture created the authority during seed startup'); + $u0->stop('immediate'); + + append_common_shared_catalog_conf($u1, $u_shared); + append_strict_two_node_conf($u1, $u_disks, undef); + $u1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($u1, 'sxid_unsealed', $u_ic0, $u_ic1); + + my $unsealed_failed = !$u1->start(fail_ok => 1); + ok($unsealed_failed, 'N3: joiner refuses an unsealed XID authority'); + like(PostgreSQL::Test::Utils::slurp_file($u1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N3: startup log names the unsealed authority'); +} + +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..b7a6ab9c2c 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -63,6 +63,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_sequence \ test_cluster_shared_catalog \ test_cluster_oid_lease \ + test_cluster_xid_authority \ test_cluster_relmap_authority \ test_cluster_hw \ test_cluster_dl \ @@ -178,7 +179,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -443,6 +444,21 @@ test_cluster_oid_lease: test_cluster_oid_lease.c unit_test.h \ $(CLUSTER_OID_LEASE_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-6.15b D1/D2: test_cluster_xid_authority links cluster_xid_authority.o +# (the shared XID authority + native-era prehistory: torn-safe file I/O, +# seal/flag monotonicity, prehistory publish/adopt byte round trip). fd.c +# openers, pg_fsync, durable_rename and the ereport machinery are stubbed +# locally; CRC32C comes from libpgport. The startup wiring (bootstrap adopt, +# StartupXLOG horizon merge, shutdown-checkpoint publish) is TAP territory. +CLUSTER_XID_AUTHORITY_O = $(top_builddir)/src/backend/cluster/cluster_xid_authority.o \ + $(top_builddir)/src/backend/cluster/cluster_xid_prehistory.o +test_cluster_xid_authority: test_cluster_xid_authority.c unit_test.h \ + $(CLUSTER_XID_AUTHORITY_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_XID_AUTHORITY_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + + # spec-6.14 D5: test_cluster_relmap_authority links cluster_relmap_authority.o # (the two-phase pending/committed authority substrate). fd.c + ereport # stubbed locally; CRC32C from libpgport. The write-path activation (relmapper diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..98998a0f78 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -53,6 +53,7 @@ #include "cluster/cluster_xnode_lever.h" /* spec-6.12 lever counter stub */ #include "cluster/cluster_hw_lease.h" /* spec-6.12d lease counter stub */ #include "cluster/cluster_relmap_authority.h" /* spec-6.14 D5 header-read stub */ +#include "cluster/cluster_xid_authority.h" /* spec-6.15b XID authority dump stubs */ #undef printf #undef fprintf @@ -342,6 +343,21 @@ cluster_relmap_authority_read_header(bool shared_map, Oid dbid, ClusterRelmapAut return false; } +/* spec-6.15b D7 stubs: dump_catalog reads the XID authority observation + * keys; this standalone debug unit does not link the file-I/O authority + * objects. */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return false; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return false; +} + /* spec-4.12 D7 + spec-4.12b D6 stubs: dump_write_fence (cluster_debug.o) reads 8 * counters now, and cluster_startup_phase.o (linked here) references the rejoin * self-fence gate. cluster_write_fence.o is not linked -- provide stubs returning diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..60b4da16e6 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,7 +207,7 @@ UT_TEST(test_gate_raw_compare_traps) } int -main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) +main(void) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ cluster_pi_shadow_shmem_init(); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c new file mode 100644 index 0000000000..66a9c3db32 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -0,0 +1,488 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_xid_authority.c + * Runtime unit tests for the shared XID authority + native-era + * prehistory module (spec-6.15b D1/D2): on-disk layout lock, the pure + * classification and payload-sizing logic, the torn-safe read/write + * round trip with .bak fallback, seal/flag monotonicity, and a real + * prehistory publish -> adopt byte round trip between two fake PGDATA + * pg_xact trees. + * + * Like test_cluster_oid_lease.c this exercises the module for REAL + * against temp directories: fd.c openers map onto open(2)/close(2), + * pg_fsync onto fsync(2), durable_rename onto rename(2). The errstart + * stub aborts on elevel >= ERROR, so every FATAL/PANIC leg (corrupt + * authority at adopt, cap refusal, era re-entry) is TAP territory + * (t/361), not unit territory; reaching an abort here is a real bug. + * + * Covers (spec §4): + * U1 classify truth table (short / magic / CRC / valid), both files + * U2 publish monotonicity (native_hw never lowered) + * U3 flags one-way (SEALED / CLUSTER_ERA never cleared) + * U4 prehistory payload sizing + publish/adopt byte round trip + * U5 header layout locked by offset assertions + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2, §4 U1-U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#include +#include +#include + +#include "cluster/cluster_xid_authority.h" +#include "port/pg_crc32c.h" +#include "storage/fd.h" +#include "utils/elog.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* Global read by cluster_xid_authority.o's file I/O. */ +char *cluster_shared_data_dir = NULL; + +/* ---- Assert + ereport machinery (aborts on ERROR; the read paths never + * ereport, the write paths only PANIC on real I/O failure). ---- */ +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + if (elevel >= ERROR) { + printf("# unexpected ereport(elevel=%d) -- aborting\n", elevel); + abort(); + } + return false; +} +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{} +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} +int +errcode_for_file_access(void) +{ + return 0; +} +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errdetail(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errhint(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +/* ---- functional fd.c stubs: map onto the kernel ---- */ +int +OpenTransientFile(const char *fileName, int fileFlags) +{ + return open(fileName, fileFlags, 0600); +} +int +CloseTransientFile(int fd) +{ + return close(fd); +} +int +pg_fsync(int fd) +{ + return fsync(fd); +} +int +durable_rename(const char *oldfile, const char *newfile, int elevel pg_attribute_unused()) +{ + return (rename(oldfile, newfile) != 0) ? -1 : 0; +} + +/* ---- temp dir scaffolding ---- */ +static char test_root[MAXPGPATH]; + +static void +setup_shared_dir(void) +{ + char globaldir[MAXPGPATH]; + + snprintf(test_root, sizeof(test_root), "/tmp/pgrac_xid_auth_ut_%d", (int)getpid()); + mkdir(test_root, 0700); + snprintf(globaldir, sizeof(globaldir), "%s/global", test_root); + mkdir(globaldir, 0700); + cluster_shared_data_dir = test_root; +} + +static void +unlink_files(void) +{ + const char *rels[] = { CLUSTER_XID_AUTHORITY_REL_PATH, CLUSTER_XID_AUTHORITY_BAK_REL_PATH, + CLUSTER_XID_PREHISTORY_REL_PATH, CLUSTER_XID_PREHISTORY_BAK_REL_PATH }; + char p[MAXPGPATH]; + int i; + + for (i = 0; i < (int)lengthof(rels); i++) { + snprintf(p, sizeof(p), "%s/%s", test_root, rels[i]); + unlink(p); + } +} + +/* Build a fake PGDATA with a pg_xact/0000 of n_pages, filled with byte. */ +static void +make_fake_pgdata(char *pgdata, size_t len, const char *tag, int n_pages, unsigned char byte) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd; + int i; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + snprintf(seg, sizeof(seg), "%s/0000", dir); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + memset(page, byte, sizeof(page)); + for (i = 0; i < n_pages; i++) + (void)!write(fd, page, sizeof(page)); + close(fd); +} + +/* ============================================================ + * U5: on-disk layout locked + * ============================================================ */ + +UT_TEST(test_layout_offsets_locked) +{ + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, flags), 8); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, reserved), 12); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, native_hw_full), 16); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, next_multi), 24); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, crc), 32); + + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, native_hw_full), 8); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, payload_len), 16); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, reserved), 20); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, crc), 24); +} + +/* ============================================================ + * U1: classify truth table + * ============================================================ */ + +UT_TEST(test_classify_short_magic_crc_valid) +{ + ClusterXidAuthorityHeader hdr; + char buf[sizeof(ClusterXidAuthorityHeader)]; + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_authority_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, 3), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = 0xDEADBEEF; + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr.version = CLUSTER_XID_AUTHORITY_VERSION; + hdr.native_hw_full = 816; + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr.crc); + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), CLUSTER_XID_AUTHORITY_VALID); + + buf[16] ^= 0x01; /* flip a native_hw bit -> CRC mismatch */ + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_CRC); +} + +/* ============================================================ + * U4: prehistory payload sizing + * ============================================================ */ + +UT_TEST(test_payload_bytes_boundaries) +{ + const uint32 xids_per_page = BLCKSZ * 4; /* CLOG: 2 bits/xact */ + + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(0), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(1), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(816), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page + 1), (uint32)(2 * BLCKSZ)); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID), + (uint32)(CLUSTER_XID_PREHISTORY_MAX_XID / 4)); + /* over the refusal cap: 0 = "no valid payload" (publish FATALs) */ + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID + 1), 0); +} + +/* ============================================================ + * torn-safe authority round trips + * ============================================================ */ + +UT_TEST(test_seed_if_absent_creates_unsealed) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0); + + /* second seed is a no-op and keeps the existing value */ + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(123456), false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_publish_monotone_and_seal) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + + /* interim (unsealed) publish raises the high-water */ + cluster_xid_authority_publish_native(810, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 810); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* sealing publish raises further and stamps SEALED */ + cluster_xid_authority_publish_native(816, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); + + /* a lower publish never lowers, and never clears SEALED */ + cluster_xid_authority_publish_native(500, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + +UT_TEST(test_mark_cluster_era_one_way) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + cluster_xid_authority_mark_cluster_era(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + /* later publishes keep both flags (never cleared) */ + cluster_xid_authority_publish_native(900, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_primary_corrupt_falls_back_to_bak) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + /* second write rolls the 791 image into .bak */ + cluster_xid_authority_publish_native(816, 1, true); + + /* corrupt the primary in place */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + /* falls back to the rolled 791 image */ + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_present_distinguishes_corrupt_from_absent) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + + /* a lone corrupt primary: present=true, read=false (fail-closed) */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR | O_CREAT | O_TRUNC, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), false); +} + +/* ============================================================ + * U4: prehistory publish -> adopt byte round trip + * ============================================================ */ + +UT_TEST(test_prehistory_publish_adopt_round_trip) +{ + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char seed_page[BLCKSZ]; + char join_page[BLCKSZ]; + int fd; + + unlink_files(); + + /* seed: one clog page of 0xAA; joiner: same-size page of zeros */ + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "join", 1, 0x00); + + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), false); + cluster_xid_prehistory_adopt(pgdata_join, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), true); + + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_seed); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, seed_page, BLCKSZ), BLCKSZ); + close(fd); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); + + /* adopt is idempotent: run again, bytes unchanged */ + cluster_xid_prehistory_adopt(pgdata_join, 816); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); +} + +UT_TEST(test_prehistory_classify_corrupt) +{ + char buf[64]; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + /* publish a real blob, then flip a payload bit -> CRC catches it */ + { + char pgdata_seed[MAXPGPATH]; + char p[MAXPGPATH]; + char *image; + struct stat st; + int fd; + int r; + + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed2", 1, 0x55); + cluster_xid_prehistory_publish(pgdata_seed, 816); + + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_PREHISTORY_REL_PATH); + UT_ASSERT_EQ(stat(p, &st), 0); + image = malloc(st.st_size); + fd = open(p, O_RDONLY, 0); + r = (int)read(fd, image, st.st_size); + close(fd); + UT_ASSERT_EQ(r, (int)st.st_size); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_VALID); + + image[sizeof(ClusterXidPrehistoryHeader) + 100] ^= 0x01; + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_INVALID_CRC); + free(image); + } + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); +} + +int +main(void) +{ + setup_shared_dir(); + + UT_PLAN(10); + UT_RUN(test_layout_offsets_locked); + UT_RUN(test_classify_short_magic_crc_valid); + UT_RUN(test_payload_bytes_boundaries); + UT_RUN(test_seed_if_absent_creates_unsealed); + UT_RUN(test_publish_monotone_and_seal); + UT_RUN(test_mark_cluster_era_one_way); + UT_RUN(test_primary_corrupt_falls_back_to_bak); + UT_RUN(test_present_distinguishes_corrupt_from_absent); + UT_RUN(test_prehistory_publish_adopt_round_trip); + UT_RUN(test_prehistory_classify_corrupt); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From adf62e01a02ad47716f5468e886c38194fbe3f2d Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:33:17 +0800 Subject: [PATCH 30/58] fix(cluster): unseal XID authority when a follow-up native-era run boots A second cluster.enabled=off seed pass on a SEALED authority re-opens the native era: clear SEALED at bootstrap so a crash of that pass leaves joiners fail-closed (53RB5, unsealed) instead of silently adopting the previous pass's stale high-water -- false-invisible for every xid the crashed pass consumed. The pass's own clean shutdown re-publishes and re-seals monotonically. Adds cluster_xid_authority_begin_native_run(), the bootstrap enabled=off arm call, unit coverage (seal kept-hw/cleared-flag/idempotent/re-seal), and t/361 N4 (sealed pass-1 -> unsealing pass-2 boot -> immediate-kill -> joiner refuses adoption). Spec: spec-6.15b-xid-authority-native-era.md --- .../cluster/cluster_catalog_bootstrap.c | 13 ++++ src/backend/cluster/cluster_xid_authority.c | 41 +++++++++++++ src/include/cluster/cluster_xid_authority.h | 13 +++- .../t/361_xid_authority_native_era_2node.pl | 59 +++++++++++++++++++ .../cluster_unit/test_cluster_xid_authority.c | 29 ++++++++- 5 files changed, 153 insertions(+), 2 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 96dd7912c6..453b78038d 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -169,6 +169,19 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, errdetail("The shared XID authority is already marked cluster-era started."), errhint( "Keep cluster.enabled=on for this shared tree, or destroy and re-form it."))); + + /* + * A follow-up native run on a SEALED authority re-opens the era: the + * seal is cleared up front so a crash of this run leaves joiners + * fail-closed (unsealed) instead of adopting the previous pass's + * stale high-water (spec-6.15b §3.1 multi-pass arm, rule 8.A). The + * clean shutdown of this run re-publishes and re-seals. + */ + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) { + cluster_xid_authority_begin_native_run(); + elog(LOG, "cluster shared_catalog: re-opened native seed era " + "(XID authority unsealed for this run)"); + } return; } diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c index 0d73ad4552..5d242c315f 100644 --- a/src/backend/cluster/cluster_xid_authority.c +++ b/src/backend/cluster/cluster_xid_authority.c @@ -338,3 +338,44 @@ cluster_xid_authority_mark_cluster_era(void) hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; write_header(&hdr); } + +/* + * cluster_xid_authority_begin_native_run -- re-open the native era for a + * follow-up cluster.enabled=off seed run (spec-6.15b §3.1 multi-pass arm). + * + * A sealed authority means "the high-water and prehistory are complete as + * of a clean native shutdown". A NEW native run invalidates that + * completeness the moment it allocates its first xid, so the seal is + * cleared up front: if this run crashes, joiners fail closed (unsealed, + * 53RB5) instead of adopting the previous pass's stale high-water -- + * false-invisible for every xid the crashed pass consumed (rule 8.A). + * The clean shutdown of this run re-publishes and re-seals monotonically. + * + * Only legal while CLUSTER_ERA is unset (the caller FATALs re-entry + * first; re-checked here defensively). Formation recipes keep native-era + * boots and first cluster boots strictly ordered; if a racing first + * cluster boot stamps CLUSTER_ERA inside this read-modify-write window, + * the stamp is re-applied by the next cluster boot (mark is re-issued + * whenever unset) and this authority is left unsealed -- which only ever + * blocks adoption, never yields a wrong answer. + */ +void +cluster_xid_authority_begin_native_run(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at native-run open"))); + + if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"))); + + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + return; /* already open (first run, or a prior pass crashed) */ + + hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; + write_header(&hdr); +} diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index ed9471907a..7676d89303 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -61,7 +61,10 @@ #define CLUSTER_XID_AUTHORITY_MAGIC 0x0141D617 #define CLUSTER_XID_AUTHORITY_VERSION 1 -/* A clean native-era shutdown published a complete hw + prehistory. */ +/* A clean native-era shutdown published a complete hw + prehistory. + * Cleared again -- only while CLUSTER_ERA is unset -- when a follow-up + * native-era run boots, so a crash of that run leaves joiners + * fail-closed instead of adopting the previous pass's stale hw. */ #define CLUSTER_XID_AUTHORITY_FLAG_SEALED 0x0001 /* A cluster.enabled=on boot closed the native era forever (one-way). */ #define CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA 0x0002 @@ -169,6 +172,14 @@ extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 n /* One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). */ extern void cluster_xid_authority_mark_cluster_era(void); +/* + * A follow-up native-era (cluster.enabled=off) boot on a sealed authority + * re-opens the era: clears SEALED so a crash of this run never exposes the + * previous pass's stale high-water to joiners. Caller vets CLUSTER_ERA + * first (re-entry is FATAL); no-op when already unsealed. + */ +extern void cluster_xid_authority_begin_native_run(void); + /* ============================================================ * Prehistory publish / adopt (spec-6.15b D2; P2: adopt is complete and * fsynced strictly before StartupCLOG -- both run under the postmaster / diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl index bacd49ac69..f4652c83ca 100644 --- a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -407,4 +407,63 @@ sub append_strict_two_node_conf 'N3: startup log names the unsealed authority'); } +# ------------------------------------------------------------------------- +# Multi-pass seed negative: a SECOND native-era run unseals the authority at +# boot, so crashing that run leaves joiners fail-closed instead of silently +# adopting the first pass's stale high-water (spec-6.15b §3.1 multi-pass arm). +# ------------------------------------------------------------------------- +{ + my $m_shared = make_shared_root(); + my $m_disks = make_voting_disks(); + my $m_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $m_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $m0 = PostgreSQL::Test::Cluster->new('sxid_multipass_node0'); + my $m1 = PostgreSQL::Test::Cluster->new('sxid_multipass_node1'); + + $m0->init(allows_streaming => 1); + $m0->start; + $m0->safe_psql('postgres', 'CHECKPOINT'); + $m0->stop; + cold_clone_data_dir($m0, $m1); + + append_common_shared_catalog_conf($m0, $m_shared); + $m0->append_conf('postgresql.conf', < sealed + $m0->start; + $m0->safe_psql('postgres', 'CREATE SCHEMA sxid_pass1;'); + $m0->stop; + my $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) != 0, + 'N4: first native pass sealed the XID authority'); + + # pass 2: booting a follow-up native run re-opens (unseals) the era + $m0->start; + $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) == 0, + 'N4: follow-up native pass unsealed the XID authority at boot'); + like(PostgreSQL::Test::Utils::slurp_file($m0->logfile), + qr/re-opened native seed era/, + 'N4: seed log names the re-opened native era'); + + # crash pass 2: the stale pass-1 high-water must NOT become adoptable + $m0->stop('immediate'); + append_common_shared_catalog_conf($m1, $m_shared); + append_strict_two_node_conf($m1, $m_disks, undef); + $m1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($m1, 'sxid_multipass', $m_ic0, $m_ic1); + + my $multipass_failed = !$m1->start(fail_ok => 1); + ok($multipass_failed, + 'N4: joiner refuses adoption after a crashed follow-up native pass'); + like(PostgreSQL::Test::Utils::slurp_file($m1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N4: startup log fails closed on the unsealed authority'); +} + done_testing(); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 66a9c3db32..3129bd73e3 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -467,18 +467,45 @@ UT_TEST(test_prehistory_classify_corrupt) CLUSTER_XID_AUTHORITY_INVALID_MAGIC); } +UT_TEST(test_begin_native_run_unseals_before_cluster_era) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* a follow-up native run clears SEALED but keeps the high-water */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* idempotent while open */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* the run's clean shutdown re-publishes and re-seals monotonically */ + cluster_xid_authority_publish_native(900, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + int main(void) { setup_shared_dir(); - UT_PLAN(10); + UT_PLAN(11); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); UT_RUN(test_seed_if_absent_creates_unsealed); UT_RUN(test_publish_monotone_and_seal); UT_RUN(test_mark_cluster_era_one_way); + UT_RUN(test_begin_native_run_unseals_before_cluster_era); UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); From 964e587500a15e0baef7184278954d9c81c5bd29 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:44:43 +0800 Subject: [PATCH 31/58] test(cluster): settle first-contact probes in t/337/t/339 for the join-gate window spec-6.15b D6 flipped both tests to online_join+xid_striping: formation now passes through the join gate, whose home-block rebuild window transiently answers 'block master is recovering ... retry is safe' -- even at connection time, via the pg_authid read. Retry the first-contact probes and first DDLs (bounded, 60s); every semantic assert stays strict. Spec: spec-6.15b-xid-authority-native-era.md --- .../t/337_shared_catalog_ddl_2node.pl | 29 ++++++++++++++--- .../t/339_shared_catalog_faults_2node.pl | 32 +++++++++++++++---- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index 1d45da20c0..0f3073e4fd 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -202,6 +202,24 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time, via the pg_authid read). First-contact probes therefore +# retry; every semantic assert below stays strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + # node1 must actually be up. my $n1_up = 0; for (1 .. 60) @@ -214,7 +232,7 @@ # ---------- # L1: both alive + IC-connected on the shared-sysid cluster. # ---------- -my $a0 = $node0->safe_psql('postgres', 'SELECT 1'); +my ($n0_up, $a0) = psql_retry_ok($node0, 'SELECT 1'); is($a0, '1', 'L1: node0 is up on the shared-catalog 2-node cluster'); is($n1_up, 1, 'L1: node1 is up on the shared-catalog 2-node cluster'); @@ -222,10 +240,10 @@ my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L1: node0 sees node1 heartbeats (IC connected)'); @@ -235,9 +253,10 @@ # node1 ever running the DDL. This is the Stage 2 exit debt # (spec-2.0:135) closure. # ---------- -$node0->safe_psql('postgres', +my ($ddl_ok) = psql_retry_ok($node0, 'CREATE TABLE a1_shared_cat (id int PRIMARY KEY, note text);' . "INSERT INTO a1_shared_cat VALUES (1, 'from-node0');"); +die 'first cross-node DDL never succeeded' unless $ddl_ok; # Poll node1 up to 1s (10 x 100ms). "retry is safe" per the GCS remaster # transient; a settled shared catalog resolves well within the budget. diff --git a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl index c37a1d6225..fd97003b23 100644 --- a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl +++ b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl @@ -243,6 +243,23 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time). First-contact probes retry; semantic asserts stay strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + my $n1_up = 0; for (1 .. 60) { @@ -254,16 +271,17 @@ # ---------- # L0: both alive + IC-connected. # ---------- -is($node0->safe_psql('postgres', 'SELECT 1'), '1', 'L0: node0 is up'); +my ($n0_up_rc, $n0_up_out) = psql_retry_ok($node0, 'SELECT 1'); +is($n0_up_out, '1', 'L0: node0 is up'); is($n1_up, 1, 'L0: node1 is up'); my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L0: node0 sees node1 heartbeats (IC connected)'); @@ -279,15 +297,17 @@ # CREATE t_crash_uncommitted (+ row, NO commit) -> 8.A leg # -> node1's cold merge must divert all of these. # ---------- -$node1->safe_psql('postgres', +my ($survivor_ok) = psql_retry_ok($node1, "CREATE TABLE t_survivor (id int PRIMARY KEY, note text);" . "INSERT INTO t_survivor VALUES (1, 'alive');"); +die 'node1 first DDL never succeeded' unless $survivor_ok; -$node0->safe_psql('postgres', +my ($keep_ok) = psql_retry_ok($node0, "CREATE TABLE t_keep (id int PRIMARY KEY, note text);" . "INSERT INTO t_keep VALUES (1, 'keep');" . "CREATE TABLE t_dropme (id int PRIMARY KEY, note text);" . "INSERT INTO t_dropme VALUES (1, 'doomed');"); +die 'node0 first DDL never succeeded' unless $keep_ok; # Shared-tree relfile of t_dropme, captured BEFORE the drop. my $drop_relpath = $node0->safe_psql('postgres', From a76d08c8ee185958af8a9c97019c523b33dabc65 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:06:21 +0800 Subject: [PATCH 32/58] =?UTF-8?q?fix(cluster):=20marker-async=20review=20r?= =?UTF-8?q?1=20=E2=80=94=20PREPARE=20best-effort=20drain,=20CL-I3=20rechec?= =?UTF-8?q?k,=20P1-1/P1-2=20hard=20assertions,=20t/363=20rename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review r1 dispositions (0 P0 / 3 P1 / 2 P2, all fixed): - P1: join Phase-1 PREPARE TIMEOUT/non-ACK now drains best-effort and still publishes the staged JOIN_PENDING (the pre-async code ignored PREPARE results; aborting reverted the joiner to DEAD and the next tick re-detected it and re-bumped the epoch once per deadline period — the exact bump storm the staged record forbids). The revert helper is removed; t/363 gains a bounded epoch-advance assertion across the 95s hold window. - P1: clean-leave staged-ACK handoff re-runs the CL-I3 dead_gen-aware version_coherent check before apply (the COMMITTING marker wait now spans ticks; the guarded CAS alone misses a dead_gen-only move). - P1: unit hard assertions — fail-stop fence stage bumps the epoch exactly once while the marker is PENDING and publishes only on ACK; node-remove re-entry while PENDING reports no false contest and no re-bump. - P2: node-remove FENCE_ARMING skips REMOVING-marker/stripe-retire pre-work while the staged fence marker is in flight (new cluster_reconfig_node_removed_fence_stage_pending predicate). - P2: PG-style banners (Author/IDENTIFICATION/NOTES) on the new files. - TAP renamed t/358 -> t/363 (358-362 reserved by parallel lanes); nightly shard split accordingly. Unit fixture gains MyBackendType stub (B_INVALID) and controllable async-marker stubs. Local gates (cassert build): cluster_unit 158 binaries, t/363 all legs, smoke+touched-baseline TAP 13 files/327 tests, cluster_regress 13, PG 219/219, clang-format 0 violations, scn-cmp + no-clog-overlay clean. cppcheck reports one finding in test_cluster_pi_shadow.c:210 — byte-identical to the base commit, local cppcheck 2.20 version drift, not introduced by this branch. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- .github/workflows/nightly.yml | 9 +- src/backend/cluster/cluster_clean_leave.c | 52 +++-- src/backend/cluster/cluster_debug.c | 6 +- src/backend/cluster/cluster_guc.c | 4 +- src/backend/cluster/cluster_lmon.c | 7 +- src/backend/cluster/cluster_node_remove.c | 68 ++++--- src/backend/cluster/cluster_qvotec.c | 4 +- src/backend/cluster/cluster_reconfig.c | 174 ++++++++++------ src/backend/cluster/cluster_write_fence.c | 9 +- src/include/cluster/cluster_clean_leave.h | 3 +- src/include/cluster/cluster_lmon.h | 2 +- src/include/cluster/cluster_marker_async.h | 42 ++-- src/include/cluster/cluster_node_remove.h | 3 +- src/include/cluster/cluster_reconfig.h | 11 +- src/include/cluster/cluster_write_fence.h | 3 +- ...63_cluster_2_29a_marker_async_liveness.pl} | 19 +- .../cluster_unit/test_cluster_marker_async.c | 6 +- src/test/cluster_unit/test_cluster_reconfig.c | 188 +++++++++++++++++- 18 files changed, 442 insertions(+), 168 deletions(-) rename src/test/cluster_tap/t/{358_cluster_2_29a_marker_async_liveness.pl => 363_cluster_2_29a_marker_async_liveness.pl} (86%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 307a387f2e..e1b41e3ae8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -177,12 +177,15 @@ jobs: - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), - # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and - # t/358 spec-2.29a marker-async LMON liveness. + # the renumbered t/351-356 family, t/357 multi-xmax alias floor. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-358", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + # t/363 spec-2.29a marker-async LMON liveness (t/358-362 reserved by + # the parallel spec-7.2 / S-xid / S-reconfig lanes; L464 occupancy + # verified against origin branches 2026-07-08). + - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 78a95163d9..56b4b7fc62 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -58,7 +58,7 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" -#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ @@ -813,8 +813,8 @@ cl_poll_lmon_marker_stage(void) now = GetCurrentTimestamp(); kind = cl_marker_phase_kind(cl_lmon_marker_phase); if (!cl_lmon_marker_submitted) { - if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, - kind, cl_lmon_marker_leaving, now)) + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, kind, + cl_lmon_marker_leaving, now)) return CL_LEAVE_ASYNC_PENDING; cl_lmon_marker_submitted = true; return CL_LEAVE_ASYNC_PENDING; @@ -859,11 +859,10 @@ cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) cl_send_committed(leaving, committed_epoch); cl_release_lmon_marker_stage(); } else if (ar == CL_LEAVE_ASYNC_FAILED) { - ereport(LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " - "COMMITTED marker is not yet majority-durable; retrying each tick " - "(leaving node waits)", - leaving, (unsigned long long)committed_epoch))); + ereport(LOG, (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); } } @@ -931,8 +930,8 @@ cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeav } ClusterMarkerPollResult -cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (cl_state == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; @@ -1805,12 +1804,37 @@ cl_coordinator_commit(int32 leaving) return; if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { cl_release_lmon_marker_stage(); + + /* + * spec-2.29a review r1 P1 — CL-I3 re-check at the staged-ACK + * handoff. The COMMITTING marker wait now spans LMON ticks, so a + * real death can bump CSSD dead_generation INSIDE the wait window + * while the fail-stop epoch has not yet advanced (the >=3-node + * window of Hardening v1.0.1 P1-2). The guarded CAS inside + * apply_clean_leave_as_coordinator only catches the epoch move; + * re-run the same dead_gen-aware coherence check the non-staged + * pre-check uses, at the last observable point before the commit + * applies. On failure the leave does not commit and the leaving + * node escalates to fail-stop (identical to the pre-check path). + */ + if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), + cl_state->leave_baseline_dead_gen, + cluster_cssd_get_dead_generation())) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or dead_generation) " + "across the COMMITTING marker wait for node %d; not committing " + "(escalate to fail-stop, CL-I3)", + leaving))); + return; + } + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); + ereport( + LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); return; } cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 6ece9182d4..bcebe9a058 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -544,10 +544,8 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); - emit_row(rsinfo, "lmon", "lmon_last_iter_us", - fmt_int64((int64)cluster_lmon_last_iter_us())); - emit_row(rsinfo, "lmon", "lmon_max_iter_us", - fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", fmt_int64((int64)cluster_lmon_max_iter_us())); emit_row(rsinfo, "lmon", "lmon_slow_iter_count", fmt_int64((int64)cluster_lmon_slow_iter_count())); } diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 869e7c2866..5a53b070a9 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -2580,8 +2580,8 @@ cluster_init_guc(void) gettext_noop("A value greater than zero logs LMON main-loop iterations above this " "duration; the lmon_slow_iter_count counter is maintained even when " "logging is disabled with 0."), - &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, - NULL, NULL); + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, + NULL); DefineCustomIntVariable( "cluster.lck_main_loop_interval", diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 51884d0a80..1af59978cc 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -866,10 +866,9 @@ lmon_record_iteration(TimestampTz iter_started_at) } if (should_log) - ereport(LOG, - (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", - (unsigned long long)(elapsed_us / 1000ULL), - cluster_lmon_slow_iteration_warn_ms))); + ereport(LOG, (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); } diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 830a7a8d66..39e31c4b72 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -227,13 +227,13 @@ cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemo } ClusterMarkerPollResult -cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (nr_state == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; - return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, - &nr_state->marker_result, now, out_result, out_elapsed_us); + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, &nr_state->marker_result, + now, out_result, out_elapsed_us); } bool @@ -312,8 +312,8 @@ nr_release_marker_stage(void) } static bool -nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, - uint64 removed_incarnation, uint64 removal_event_id) +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, + uint64 removal_event_id) { return nr_marker_async.has_staged_event && nr_marker_phase == phase && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch @@ -685,31 +685,43 @@ cluster_node_remove_drive(void) (void)baseline_dead_gen; /* contest is now signalled by out_contest, not derived */ - /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ - (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, - last_incarnation, removal_event_id); - /* - * spec-6.15 D5c (appendix B.3): durably retire the removed node's - * xid stripe slot BEFORE the removal point of no return below. - * Ordering is the identity-reuse defence: once removal commits, a - * later fresh join of the same node_id (new incarnation = a new - * durable owner) must land on a retired slot and refuse (53RB1), - * never resume the old owner's congruence class. Not durable yet - * -> stay FENCE_ARMING and retry next tick (fail-closed; same - * shape as a transient fence/quorum failure). A never-activated - * cluster returns true (nothing to retire). + * spec-2.29a review r1 P2: while the staged fence marker from a + * previous FENCE_ARMING tick is still in flight, skip the pre-work + * below — the epoch was already self-bumped at stage-entry, so + * re-running it would key a SECOND REMOVING marker to the post-bump + * epoch and re-submit the stripe retire during the very contention + * window the async stage relieves. Fall through straight to + * apply(), which polls the stage. */ - if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { - ereport(LOG, (errmsg("cluster node removal: stripe slot retire for node %d not durable " - "yet — retrying before the removal commit", - node_id))); - return; + if (!cluster_reconfig_node_removed_fence_stage_pending()) { + /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ + (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, + last_incarnation, removal_event_id); + + /* + * spec-6.15 D5c (appendix B.3): durably retire the removed node's + * xid stripe slot BEFORE the removal point of no return below. + * Ordering is the identity-reuse defence: once removal commits, a + * later fresh join of the same node_id (new incarnation = a new + * durable owner) must land on a retired slot and refuse (53RB1), + * never resume the old owner's congruence class. Not durable yet + * -> stay FENCE_ARMING and retry next tick (fail-closed; same + * shape as a transient fence/quorum failure). A never-activated + * cluster returns true (nothing to retire). + */ + if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { + ereport(LOG, + (errmsg("cluster node removal: stripe slot retire for node %d not durable " + "yet — retrying before the removal commit", + node_id))); + return; + } + /* D5d: carry the retire to WAL consumers (standby stops gap- + * filling the dead class). The voting-disk retire above is the + * correctness anchor; emission failure is tolerated (logged). */ + cluster_xid_stripe_emit_retire_wal(node_id); } - /* D5d: carry the retire to WAL consumers (standby stops gap- - * filling the dead class). The voting-disk retire above is the - * correctness anchor; emission failure is tolerated (logged). */ - cluster_xid_stripe_emit_retire_wal(node_id); /* * The commit point: guarded epoch bump + fence-arm (majority-durable, at diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 6fd071a17a..916a629399 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -95,8 +95,8 @@ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ #include "cluster/cluster_inject.h" -#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ -#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ +#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ +#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D2: fence marker scan + token refresh */ diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 931ea26b8d..cf03d3e8b7 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -341,6 +341,28 @@ cluster_reconfig_publish_event(const ReconfigEvent *evt) } +static void +cluster_reconfig_log_failstop_epoch_bump(const ReconfigEvent *evt) +{ + char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; + int off = 0; + int n; + + Assert(evt != NULL); + + dead[0] = '\0'; + for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { + if (evt->dead_bitmap[n / 8] & (1 << (n % 8))) + off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); + } + + ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " + "(coordinator node %d, dead node(s) {%s})", + (unsigned long long)evt->old_epoch, (unsigned long long)evt->new_epoch, + (int)evt->coordinator_node_id, dead))); +} + + /* ============================================================ * Counter accessors (Step 2 + Step 3 SRF support). * ============================================================ @@ -1075,10 +1097,9 @@ cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_ return; pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); - ereport(LOG, - (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", - cluster_marker_async_kind_name(kind), target_node, - (unsigned long long)(elapsed_us / 1000ULL)))); + ereport(LOG, (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); } void @@ -1087,10 +1108,9 @@ cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_n { if (ReconfigShmem != NULL) pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); - ereport(LOG, - (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", - cluster_marker_async_kind_name(kind), target_node, - (unsigned long long)(elapsed_us / 1000ULL)))); + ereport(LOG, (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); } uint64 @@ -1234,14 +1254,13 @@ cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) } static bool -cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, - ClusterMarkerAsyncKind kind, int32 target_node, - TimestampTz now) +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, + int32 target_node, TimestampTz now) { if (stage->submitted) return true; - if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, - target_node, now)) + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, target_node, + now)) return false; stage->submitted = true; return true; @@ -1281,6 +1300,7 @@ cluster_reconfig_poll_failstop_fence_stage(void) elapsed_us); if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_log_failstop_epoch_bump(&failstop_fence_stage.event); cluster_reconfig_broadcast_local_procsig(); } else { ereport(LOG, @@ -1324,8 +1344,8 @@ cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 rem return 0; } - cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, - removed_node_id, elapsed_us); + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, removed_node_id, + elapsed_us); if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " "voting-disk majority for epoch %llu; not publishing removal " @@ -1364,24 +1384,12 @@ cluster_reconfig_release_join_prepare_stage(void) join_prepare_stage.submitted = false; } -static void -cluster_reconfig_abort_join_prepare_stage(void) -{ - int i; - - if (ReconfigShmem != NULL) { - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - if (!dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) - continue; - ReconfigShmem->pending_join_bitmap[i / 8] &= (uint8) ~(1u << (i % 8)); - if (cluster_membership_get_state(i) == CLUSTER_MEMBER_JOINING) - cluster_membership_set_state(i, CLUSTER_MEMBER_DEAD); - } - LWLockRelease(&ReconfigShmem->lock); - } - cluster_reconfig_release_join_prepare_stage(); -} +/* (spec-2.29a review r1: the former abort_join_prepare_stage — revert + * JOINING→DEAD + clear pending on PREPARE failure — was removed. Q5=A makes + * PREPARE strictly best-effort: TIMEOUT / non-ACK advances the queue and the + * staged JOIN_PENDING still publishes, so no revert path exists; reverting + * would let compute_join_bitmap re-detect the joiner and re-bump the epoch + * every deadline period — the P1-1 bump storm.) */ static bool cluster_reconfig_submit_join_prepare_current(TimestampTz now) @@ -1422,6 +1430,22 @@ cluster_reconfig_submit_join_prepare_current(TimestampTz now) return true; } +/* + * spec-2.29a review r1 P2 — node-remove driver pre-work gate. + * + * While the staged node-removed fence marker from a previous FENCE_ARMING + * tick is still in flight, the driver must not re-run its pre-work + * (REMOVING marker write / stripe retire): the epoch has already been + * self-bumped by the stage-entry, so a re-run would key a SECOND REMOVING + * marker to the post-bump epoch and add voting-disk writes during the + * exact contention window the async stage exists to relieve. + */ +bool +cluster_reconfig_node_removed_fence_stage_pending(void) +{ + return node_removed_fence_stage.async.has_staged_event; +} + static bool cluster_reconfig_poll_join_prepare_stage(void) { @@ -1445,22 +1469,28 @@ cluster_reconfig_poll_join_prepare_stage(void) &elapsed_us); if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) return true; - if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { - cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, - elapsed_us); - cluster_reconfig_abort_join_prepare_stage(); - return true; - } - cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, - elapsed_us); - if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: PREPARE join marker for node %d did not reach a " - "voting-disk majority; not publishing JOIN_PENDING (will retry)", - target))); - cluster_reconfig_abort_join_prepare_stage(); - return true; + /* + * spec-2.29a Q5=A (review r1 P1): PREPARE is best-effort in OUTCOME — the + * pre-async code ignored the submit result and always published + * JOIN_PENDING (only the Phase-2 COMMITTED marker is a commit point, P1-r5). + * A TIMEOUT / non-ACK PREPARE therefore advances the queue instead of + * aborting: aborting would revert the joiner JOINING→DEAD, and the next + * tick's compute_join_bitmap would re-detect it CSSD-alive and RE-BUMP the + * epoch — exactly the per-tick epoch-bump storm the P1-1 staged record + * forbids. Draining to publish keeps the joiner on a stable JOINING + * (one Phase-1 bump total, regardless of PREPARE outcomes). + */ + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + } else { + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach " + "a voting-disk majority; continuing best-effort (COMMITTED is the " + "commit point)", + target))); } join_prepare_stage.submitted = false; @@ -1584,7 +1614,7 @@ cluster_reconfig_poll_join_commit_stage(void) &admitted_generation) || admitted_incarnation != join_commit_stage.admitted_incarnation || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, - admitted_generation) + admitted_generation) != CLUSTER_JOIN_ACCEPT || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, join_commit_stage.admitted_incarnation, @@ -2578,6 +2608,32 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } + /* + * The async stage is process-local by design and is driven only by LMON + * ticks. Test-only backend callers, such as + * cluster_reconfig_inject_dead_node_test(), would otherwise stage the marker + * in their own process and return with no LMON-visible pending publish + * record. Keep those non-LMON paths on the old bounded wait: they are not + * transport-liveness actors, so this does not reintroduce the BUG-C1 LMON + * park. + */ + if (MyBackendType != B_LMON) { + ClusterFenceMarkerSubmitResult result; + + result = cluster_write_fence_submit_marker(&marker); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " + "majority for epoch %llu; not publishing reconfig event " + "(write-fenced, will retry)", + (unsigned long long)new_epoch))); + return; + } + + cluster_reconfig_publish_event(&evt); + cluster_reconfig_log_failstop_epoch_bump(&evt); + return; + } + failstop_fence_stage.event = evt; failstop_fence_stage.marker = marker; failstop_fence_stage.node_id = coordinator_node_id; @@ -2599,21 +2655,7 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * fixed stack buffer (worst case: 128 node ids x 4 chars) keeps the LMON * tick free of allocator traffic. */ - { - char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; - int off = 0; - int n; - - dead[0] = '\0'; - for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { - if (dead_bitmap[n / 8] & (1 << (n % 8))) - off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); - } - ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " - "(coordinator node %d, dead node(s) {%s})", - (unsigned long long)old_epoch, (unsigned long long)new_epoch, - (int)coordinator_node_id, dead))); - } + cluster_reconfig_log_failstop_epoch_bump(&evt); } @@ -2928,8 +2970,8 @@ cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_no } ClusterMarkerPollResult -cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (ReconfigShmem == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index d4988b042d..8917959359 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -786,14 +786,13 @@ cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenc memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); return cluster_marker_async_submit( a, &cluster_write_fence_shmem->marker_request_seq, - &cluster_write_fence_shmem->marker_completion_seq, - cluster_write_fence_shmem->qvotec_latch, now, - (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); + &cluster_write_fence_shmem->marker_completion_seq, cluster_write_fence_shmem->qvotec_latch, + now, (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); } ClusterMarkerPollResult -cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (cluster_write_fence_shmem == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 2d75de8e2a..1330a40490 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -412,8 +412,7 @@ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, - ClusterMarkerAsyncKind kind, - int32 target_node, + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now); extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 1b6a516f4b..e1099ece40 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -138,7 +138,7 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ - uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ uint64 max_iter_us; /* high-water iteration wall time */ uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h index 792e583b47..56cd836bf5 100644 --- a/src/include/cluster/cluster_marker_async.h +++ b/src/include/cluster/cluster_marker_async.h @@ -3,14 +3,27 @@ * cluster_marker_async.h * Small process-local async FSM for qvotec marker submit mailboxes. * - * The mailbox remains the existing request_seq/completion_seq/result slot. - * This wrapper changes only the waiting shape: submit publishes a request and - * returns; the owning LMON tick polls completion without sleeping. - * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 2026, pgrac contributors * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_marker_async.h + * + * NOTES + * The mailbox remains the existing request_seq/completion_seq/result + * slot. This wrapper changes only the waiting shape: submit publishes + * a request and returns; the owning LMON tick polls completion without + * sleeping (spec-2.29a — the pre-async 2ms pg_usleep spin inside the + * LMON tick starved the CSSD heartbeat relay and caused false-DEAD + * storms during cold formation, BUG-C1). + * + * The staged has_staged_event flag is the P1-1 bump-once contract: a + * pre-bump caller (fail-stop fence / node-remove / join Phase-1) must + * not re-enter its epoch-bump path while a stage is live. + * *------------------------------------------------------------------------- */ #ifndef CLUSTER_MARKER_ASYNC_H @@ -109,8 +122,7 @@ cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) } static inline bool -cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, - pg_atomic_uint64 *completion_seq) +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq) { return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); } @@ -118,8 +130,8 @@ cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, static inline bool cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, - TimestampTz now, uint64 timeout_us, - ClusterMarkerAsyncKind kind, int32 target_node) + TimestampTz now, uint64 timeout_us, ClusterMarkerAsyncKind kind, + int32 target_node) { uint64 seq; @@ -148,8 +160,8 @@ cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq static inline ClusterMarkerPollResult cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, - pg_atomic_uint32 *result_slot, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) + pg_atomic_uint32 *result_slot, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { uint64 elapsed; @@ -168,9 +180,8 @@ cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_se pg_read_barrier(); if (out_result != NULL) *out_result = pg_atomic_read_u32(result_slot); - elapsed = ((uint64)now > (uint64)a->submitted_at) - ? ((uint64)now - (uint64)a->submitted_at) - : 0; + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; if (out_elapsed_us != NULL) *out_elapsed_us = elapsed; a->state = CLUSTER_MARKER_ASYNC_IDLE; @@ -178,9 +189,8 @@ cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_se } if ((uint64)now >= a->deadline_us) { - elapsed = ((uint64)now > (uint64)a->submitted_at) - ? ((uint64)now - (uint64)a->submitted_at) - : 0; + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; if (out_elapsed_us != NULL) *out_elapsed_us = elapsed; a->state = CLUSTER_MARKER_ASYNC_IDLE; diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index 1df3d8c868..ef722e3f62 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -419,8 +419,7 @@ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, - ClusterMarkerAsyncKind kind, - int32 target_node, + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now); extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index 3e6abadc16..4ce28e13fd 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -56,7 +56,7 @@ #include "port/atomics.h" #include "storage/lwlock.h" -#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ #include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ @@ -585,8 +585,7 @@ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, const ClusterJoinCommitMarker *m, - ClusterMarkerAsyncKind kind, - TimestampTz now); + ClusterMarkerAsyncKind kind, TimestampTz now); extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, @@ -774,5 +773,11 @@ extern uint64 cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_n uint64 last_incarnation, bool *out_contest); +/* spec-2.29a review r1 P2: true while the staged node-removed fence marker is + * in flight -- the node-remove driver skips its FENCE_ARMING pre-work + * (REMOVING marker / stripe retire) instead of re-keying it to the live + * (already self-bumped) epoch. */ +extern bool cluster_reconfig_node_removed_fence_stage_pending(void); + #endif /* CLUSTER_RECONFIG_H */ diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index dad13ff00d..ac5a965882 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -521,8 +521,7 @@ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, - ClusterMarkerAsyncKind kind, - int32 target_node, + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now); extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, diff --git a/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl similarity index 86% rename from src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl rename to src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl index e2f19d6454..005c0d4a4b 100644 --- a/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl +++ b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl #------------------------------------------------------------------------- # -# 358_cluster_2_29a_marker_async_liveness.pl +# 363_cluster_2_29a_marker_async_liveness.pl # BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. # # The hold injection freezes qvotec marker completion until released. Before @@ -18,8 +18,10 @@ # # Portions Copyright (c) 2026, pgrac contributors # +# Author: SqlRush +# # IDENTIFICATION -# src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl +# src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl # #------------------------------------------------------------------------- @@ -119,6 +121,8 @@ sub set_injection set_injection($node0, 'cluster-qvotec-marker-service-hold'); my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); +my $epoch_hold_start = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; $node1->start; ok(poll_until($node0, @@ -149,6 +153,17 @@ sub set_injection 'member', 'L4 held/timeout marker does not publish node1 admission'); +# P1-1 bump-once (spec-2.29a review r1): the whole >=95s hold must cost at most +# ONE join Phase-1 epoch bump (staged record; PREPARE timeouts drain best-effort +# and never revert the joiner, so compute_join_bitmap cannot re-detect + re-bump +# per deadline period). Pre-fix abort+revert behavior re-bumped ~once per 3.5s +# deadline (~27 bumps in this window); a <=2 bound separates cleanly without +# being flaky against one incidental episode. +my $epoch_hold_end = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; +cmp_ok($epoch_hold_end - $epoch_hold_start, '<=', 2, + 'L4 epoch advance across the hold window is bounded (no per-deadline bump storm)'); + set_injection($node0, ''); ok(poll_until($node0, q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c index 9823fd69bf..b90fa463a2 100644 --- a/src/test/cluster_unit/test_cluster_marker_async.c +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -7,6 +7,8 @@ * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 2026, pgrac contributors * + * Author: SqlRush + * * IDENTIFICATION * src/test/cluster_unit/test_cluster_marker_async.c * @@ -48,8 +50,8 @@ SetLatch(Latch *latch pg_attribute_unused()) } static void -ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, - pg_atomic_uint64 *completion_seq, pg_atomic_uint32 *result_slot) +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot) { cluster_marker_async_init(a); pg_atomic_init_u64(request_seq, 0); diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 5860c19831..6d6fcbfd0f 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -61,6 +61,14 @@ UT_DEFINE_GLOBALS(); +/* spec-2.29a: cluster_reconfig.c gates the async marker stage on + * MyBackendType == B_LMON. The fixture exercises the pre-2.29a bounded-wait + * semantics (write_fence submit stub returns FAILED synchronously), so run as + * a non-LMON backend; the async FSM itself is covered by + * test_cluster_marker_async.c. */ +#include "miscadmin.h" +BackendType MyBackendType = B_INVALID; + /* ============================================================ * Stubs — link cluster_reconfig.o + cluster_epoch.o standalone. @@ -357,22 +365,44 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +/* spec-2.29a review r1 P1-c: controllable async-marker stubs so the tick-level + * P1-1 invariants (bump-once while PENDING, node-remove zero-false-contest, + * publish deferred to ACK) can be hard-asserted. Defaults preserve the + * pre-existing inert behavior (submit rejected, poll idle). */ +static bool ut_fence_async_submit_ok = false; +static ClusterMarkerPollResult ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; +static uint32 ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +static int ut_fence_async_submit_calls = 0; +static int ut_fence_async_poll_calls = 0; + bool -cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m pg_attribute_unused(), - ClusterMarkerAsyncKind kind pg_attribute_unused(), - int32 target_node pg_attribute_unused(), + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now pg_attribute_unused()) { - return false; + ut_fence_async_submit_calls++; + if (!ut_fence_async_submit_ok) + return false; + /* mirror the real wrapper's FSM effect so is_submitted() sees SUBMITTED */ + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->kind = kind; + a->target_node = target_node; + return true; } ClusterMarkerPollResult -cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), - TimestampTz now pg_attribute_unused(), - uint32 *out_result pg_attribute_unused(), - uint64 *out_elapsed_us pg_attribute_unused()) -{ - return CLUSTER_MARKER_POLL_IDLE; +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now pg_attribute_unused(), + uint32 *out_result, uint64 *out_elapsed_us) +{ + ut_fence_async_poll_calls++; + if (out_result != NULL) + *out_result = ut_fence_async_poll_result; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_ACKED + || ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_TIMEOUT) + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return ut_fence_async_poll_pr; } void cluster_lmon_marker_complete_wakeup(void) @@ -982,6 +1012,142 @@ UT_TEST(test_reconfig_lmon_tick_survivor_does_not_advance_epoch) } +/* ============================================================ + * T-reconfig-2.29a — P1-1 staged-record hard assertions (spec-2.29a + * review r1 P1-c). + * + * (a) fail-stop fence stage: while the marker is PENDING across ticks + * the epoch is bumped EXACTLY ONCE and nothing publishes; the + * staged event publishes only on ACKED+ACK. + * (b) node-remove stage: a tick re-entry while the fence marker is + * PENDING returns 0 with *out_contest == false (no false contest + * against our own already-advanced baseline) and no re-bump. + * ============================================================ */ + +UT_TEST(test_reconfig_failstop_fence_stage_bump_once_while_pending) +{ + uint64 epoch_after_bump; + uint64 apply_before; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD; + ut_dead_generation = 1; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + apply_before = cluster_reconfig_get_apply_counter(); + + /* tick #1: stage-entry — bump once, submit, DO NOT publish. */ + cluster_reconfig_lmon_tick(); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT(epoch_after_bump > 0); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* ticks #2/#3: marker PENDING — the bump path must NOT re-enter + * (P1-1 invariant 0: zero re-bump, zero publish, zero side-effects). */ + cluster_reconfig_lmon_tick(); + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* tick #4: ACKED + ACK — the STAGED event publishes; still no re-bump. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)(apply_before + 1)); + + /* restore fixture defaults for the neighboring tests */ + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + +UT_TEST(test_reconfig_node_remove_stage_no_false_contest) +{ + uint64 baseline; + uint64 epoch_after_bump; + uint64 ret; + bool contest; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + baseline = cluster_epoch_get_current(); + + /* call #1: guarded CAS advances baseline→baseline+1, stages the fence + * marker, returns 0 (not yet committed), contest MUST be false. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT_EQ((unsigned long long)epoch_after_bump, (unsigned long long)(baseline + 1)); + + /* call #2 (tick re-entry with the SAME stale baseline while PENDING): + * without the staged record this would re-run the baseline CAS against + * our own advanced epoch and report a FALSE contest. P1-1: it must + * poll the stage instead — 0, contest==false, no second bump. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + /* call #3: ACKED + ACK — removal publishes at the STAGED epoch. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, (unsigned long long)epoch_after_bump); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + /* ============================================================ * T-reconfig-8 — ProcessInterrupts I6 guard (D4). * @@ -1595,6 +1761,8 @@ main(void) /* T-reconfig-3 — lmon_tick dedup. */ UT_RUN(test_reconfig_lmon_tick_dedups_same_event_id); UT_RUN(test_reconfig_lmon_tick_refires_on_dead_gen_bump); + UT_RUN(test_reconfig_failstop_fence_stage_bump_once_while_pending); + UT_RUN(test_reconfig_node_remove_stage_no_false_contest); /* T-reconfig-4 / 4b — Q2 A'' + I2 + L20 + F11 + empty-dead-set gates. */ UT_RUN(test_reconfig_lmon_tick_skips_when_not_in_quorum); From e71fc34315d614921e62c9321ca6e24a765de007 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:08:35 +0800 Subject: [PATCH 33/58] =?UTF-8?q?fix(cluster):=20review=20fixes=20?= =?UTF-8?q?=E2=80=94=20clean-shutdown-only=20seal,=20WARNING-skip=20on=20u?= =?UTF-8?q?nsupported=20seed,=20.bak=20read=20guard,=20multi-segment=20uni?= =?UTF-8?q?t=20leg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review P2-1: the native-era publish+seal now runs only on a TRUE shutdown checkpoint (END_OF_RECOVERY excluded -- sealing mid-run after crash recovery would expose a stale high-water to joiners), and an unsupported native era (MultiXacts / past the prehistory cap) skips the seal with a WARNING instead of FATALing the checkpointer: the refusal still lands fail-closed on every joiner (53RB5, unsealed) but the seed can shut down cleanly instead of wedging in a FATAL loop. Review P3-4: roll_blob_to_bak PANICs on a read() error instead of treating it as EOF (a truncated .bak was already rejected downstream by CRC, but silently swallowing I/O errors broke the module's discipline). Review P3-3: unit multi-segment prehistory round trip (33 pages across SLRU segments 0000/0001, per-page byte pattern, short-clone extension). Spec: spec-6.15b-xid-authority-native-era.md --- src/backend/access/transam/xlog.c | 48 ++++++++++--- src/backend/cluster/cluster_xid_prehistory.c | 3 + .../cluster_unit/test_cluster_xid_authority.c | 71 ++++++++++++++++++- 3 files changed, 110 insertions(+), 12 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 03d5719b89..1563ea5d3c 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -7586,22 +7586,48 @@ CreateCheckPoint(int flags) * are complete, publish the native-era pg_xact prehistory and seal the * shared XID authority. This does durable file I/O and may fail closed, * so it must stay outside the checkpoint critical section. + * + * Two deliberate exclusions (review P2-1): + * + * - END_OF_RECOVERY checkpoints carry `shutdown` but are NOT a clean + * close of the native run: the seed keeps consuming xids afterwards, + * so sealing here would let a joiner adopt a mid-run stale high-water + * (the same 8.A window the begin_native_run unseal closes for the + * multi-pass case). Only a true shutdown checkpoint seals. + * + * - An unsupported native era (MultiXacts created, or past the + * prehistory cap) SKIPS the seal with a WARNING instead of FATALing: + * the refusal still lands fail-closed on every joiner (53RB5, + * authority unsealed), but the seed itself can shut down cleanly + * instead of wedging every later shutdown/boot in a FATAL loop. */ - if (shutdown && cluster_shared_catalog && !cluster_enabled) + if (shutdown && !(flags & CHECKPOINT_END_OF_RECOVERY) + && cluster_shared_catalog && !cluster_enabled) { uint64 native_hw = U64FromFullTransactionId(checkPoint.nextXid); if (checkPoint.nextMulti > FirstMultiXactId) - ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), - errmsg("native-era seed loads that create MultiXacts are not supported"), - errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", - checkPoint.nextMulti, FirstMultiXactId), - errhint("Recreate the seed without MultiXact-producing operations, or move the load in-protocol."))); - - cluster_xid_prehistory_publish(DataDir, native_hw); - cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); - elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", - (unsigned long long)native_hw); + ereport(WARNING, + (errmsg("native-era seed loads that create MultiXacts are not supported; " + "leaving the shared XID authority unsealed"), + errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", + checkPoint.nextMulti, FirstMultiXactId), + errhint("Joiners will fail closed (53RB5); recreate the seed without " + "MultiXact-producing operations, or move the load in-protocol."))); + else if (cluster_xid_prehistory_payload_bytes(native_hw) == 0) + ereport(WARNING, + (errmsg("native-era xid high-water %llu is outside the prehistory publish " + "range; leaving the shared XID authority unsealed", + (unsigned long long) native_hw), + errhint("Joiners will fail closed (53RB5); split the seed load or move " + "it in-protocol."))); + else + { + cluster_xid_prehistory_publish(DataDir, native_hw); + cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); + elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", + (unsigned long long) native_hw); + } } #endif diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c index 9333c1eaba..0e5ed2018d 100644 --- a/src/backend/cluster/cluster_xid_prehistory.c +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -189,6 +189,9 @@ roll_blob_to_bak(const char *primary, const char *bak, const char *baktmp) ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", baktmp))); } + if (r < 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", primary))); CloseTransientFile(src); if (pg_fsync(dst) != 0) ereport(PANIC, diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 3129bd73e3..6178ebbfcd 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -200,6 +200,40 @@ make_fake_pgdata(char *pgdata, size_t len, const char *tag, int n_pages, unsigne close(fd); } +/* Build a fake PGDATA whose pg_xact spans SLRU segments: n_pages whole CLOG + * pages, 32 per segment file (0000, 0001, ...), page p filled with byte + * (p & 0xFF) so cross-segment ordering is byte-verifiable. */ +static void +make_fake_pgdata_multiseg(char *pgdata, size_t len, const char *tag, int n_pages) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd = -1; + int cur_seg = -1; + int p; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + for (p = 0; p < n_pages; p++) { + int segno = p / 32; + + if (segno != cur_seg) { + if (fd >= 0) + close(fd); + snprintf(seg, sizeof(seg), "%s/%04X", dir, segno); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + cur_seg = segno; + } + memset(page, p & 0xFF, sizeof(page)); + (void)!write(fd, page, sizeof(page)); + } + if (fd >= 0) + close(fd); +} + /* ============================================================ * U5: on-disk layout locked * ============================================================ */ @@ -427,6 +461,40 @@ UT_TEST(test_prehistory_publish_adopt_round_trip) UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); } +UT_TEST(test_prehistory_multi_segment_round_trip) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; /* segment 0000 full (32 pages) + 0001 (1 page) */ + const uint64 hw = (uint64)n_pages * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char want[BLCKSZ]; + char got[BLCKSZ]; + int p; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "seedms", n_pages); + /* short pre-seed clone: only one zeroed page in segment 0000 */ + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "joinms", 1); + + cluster_xid_prehistory_publish(pgdata_seed, hw); + cluster_xid_prehistory_adopt(pgdata_join, hw); + + for (p = 0; p < n_pages; p++) { + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", pgdata_join, p / 32); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, (off_t)(p % 32) * BLCKSZ, SEEK_SET), (off_t)(p % 32) * BLCKSZ); + UT_ASSERT_EQ(read(fd, got, BLCKSZ), BLCKSZ); + close(fd); + memset(want, p & 0xFF, sizeof(want)); + UT_ASSERT_EQ(memcmp(got, want, BLCKSZ), 0); + } +} + UT_TEST(test_prehistory_classify_corrupt) { char buf[64]; @@ -498,7 +566,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(11); + UT_PLAN(12); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -509,6 +577,7 @@ main(void) UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); + UT_RUN(test_prehistory_multi_segment_round_trip); UT_RUN(test_prehistory_classify_corrupt); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From 51cdc674ffe7330429913c68e610f9a066e34085 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:50:02 +0800 Subject: [PATCH 34/58] fix(cluster): dual-copy flag writes, divergent-lineage guard, TAP/doc hardening F1: begin_native_run/mark_cluster_era install the transitioned image as BOTH primary and .bak (write_header_both) -- rolling the pre-transition image into .bak let a later single-copy corruption fall back to a stale SEALED (or pre-CLUSTER_ERA) authority, handing joiners the previous pass's high-water or re-opening era re-entry. F2: divergent-lineage guard -- before the adopt/skip decision the joiner byte-compares its local pg_xact prefix [0, min(own_next, native_hw)) against the sealed prehistory blob at 2-bit precision; any contradiction is FATAL (a same-sysid clone that ran standalone after cloning is not a pre-seed lineage; neither trusting nor overwriting its bits is sound). Bypassed only when local oldestXid has advanced past the native range. t/361: real scram TCP login leg (hba + listener), node0 shared-heap re-reads after joiner scans (poison proof on the heap, not just CLOG), a 53RB5 SQLSTATE pin (%e before %q -- %q stops expansion for the postmaster), the N5 divergent-lineage negative, and the L9 shared_catalog=off dormant leg. Also: on-disk header layouts locked with StaticAssertDecl; consumers LOG when the authority is served from the .bak fallback; user manual trimmed to user-visible effect. Spec: spec-6.15b-xid-authority-native-era.md --- docs/user-guide/configuration.md | 6 +- src/backend/access/transam/xlog.c | 9 +- .../cluster/cluster_catalog_bootstrap.c | 45 ++++++- src/backend/cluster/cluster_xid_authority.c | 63 ++++++++- src/backend/cluster/cluster_xid_prehistory.c | 120 +++++++++++++++++ src/include/cluster/cluster_xid_authority.h | 48 +++++++ .../t/361_xid_authority_native_era_2node.pl | 109 +++++++++++++++- .../cluster_unit/test_cluster_xid_authority.c | 121 +++++++++++++++++- 8 files changed, 512 insertions(+), 9 deletions(-) diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 76d6cc8f73..5165f233d9 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -816,9 +816,9 @@ Provisioning sequence: native-era stop; joiners refuse an unsealed authority with SQLSTATE `53RB5`. 4. Start all nodes with `cluster.enabled = on`, `cluster.online_join = on`, - and `cluster.xid_striping = on`. Joiners adopt the prehistory before - WAL startup seeds `nextXid`, so seed tuples and roles are judged from - first-hand pg_xact truth rather than hint bits. + and `cluster.xid_striping = on`. Joiners adopt the seed's transaction + history at startup, so seed tables, roles, and data are visible on + every node. Do not delete or re-create the authority files to recover a failed join. A missing, unsealed, or corrupt authority/prehistory is intentionally diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 1563ea5d3c..850c468df5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5606,8 +5606,15 @@ StartupXLOG(void) if (cluster_shared_catalog && cluster_enabled) { ClusterXidAuthorityHeader auth; + bool auth_from_bak = false; + bool auth_ok; - if (!cluster_xid_authority_read(&auth) || + auth_ok = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (auth_ok && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + + if (!auth_ok || (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("shared XID authority is unavailable during WAL startup"), diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 453b78038d..f4c1718f6a 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -136,7 +136,14 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, ClusterXidAuthorityHeader auth; bool have_auth; - have_auth = cluster_xid_authority_read(&auth); + { + bool auth_from_bak = false; + + have_auth = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (have_auth && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + } if (!have_auth && cluster_xid_authority_present()) cluster_catalog_xid_authority_corrupt_fatal( "Neither the primary shared XID authority nor its .bak fallback passes validation."); @@ -212,6 +219,42 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, "Re-provision this node or verify cluster.shared_data_dir before startup."))); own_next = U64FromFullTransactionId(ra.checkPointCopy.nextXid); + + /* + * Review F2 (spec Q6 amendment): the sysid gate cannot tell a true + * pre-seed clone from a same-sysid clone that ran STANDALONE after + * cloning -- such a node's pg_xact holds its own outcomes in the + * native range, so skipping would trust divergent local truth and + * adopting would overwrite the node's own outcomes; both are + * silently wrong (8.A). Byte-compare the comparable prefix + * [0, min(own_next, native_hw)) against the sealed blob and fail + * closed on any contradiction. Bypass only when the local + * oldestXid has advanced past the native range (frozen+truncated: + * those bits are never consulted again). + */ + if ((uint64)ra.checkPointCopy.oldestXid <= auth.native_hw_full) { + ClusterXidPrefixVerdict pv; + + pv = cluster_xid_prehistory_prefix_check(DataDir, auth.native_hw_full, + Min(own_next, auth.native_hw_full)); + if (pv == CLUSTER_XID_PREFIX_DIVERGED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("local pg_xact contradicts the sealed native-era prehistory"), + errdetail("divergent native-era lineage: this node's transaction " + "history overlaps the seed's native xid range with " + "different outcomes."), + errhint("This node is not a pre-seed clone of the shared tree; " + "destroy and re-provision it from the seed lineage."))); + if (pv == CLUSTER_XID_PREFIX_UNAVAILABLE) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is unavailable for the lineage check"), + errhint("Restore \"%s\" (or its .bak) from the shared tree's " + "backup.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + } + if (own_next < auth.native_hw_full) { cluster_xid_prehistory_adopt(DataDir, auth.native_hw_full); elog(LOG, diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c index 5d242c315f..08904b7445 100644 --- a/src/backend/cluster/cluster_xid_authority.c +++ b/src/backend/cluster/cluster_xid_authority.c @@ -178,11 +178,26 @@ read_image(const char *path, char *image) */ bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return cluster_xid_authority_read_checked(out, NULL); +} + +/* + * cluster_xid_authority_read_checked -- as above, additionally reporting + * whether the image came from the .bak fallback so consumers can surface a + * LOG-once operator signal (review F8; the read itself stays silent for + * the bootstrap early-read path). + */ +bool +cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak) { char primary_path[MAXPGPATH]; char bak_path[MAXPGPATH]; char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + if (used_bak != NULL) + *used_bak = false; + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) return false; @@ -190,6 +205,8 @@ cluster_xid_authority_read(ClusterXidAuthorityHeader *out) if (read_image(primary_path, image) != CLUSTER_XID_AUTHORITY_VALID) { if (read_image(bak_path, image) != CLUSTER_XID_AUTHORITY_VALID) return false; /* fail-closed: neither trustworthy */ + if (used_bak != NULL) + *used_bak = true; } memcpy(out, image, sizeof(*out)); @@ -257,6 +274,48 @@ write_header(ClusterXidAuthorityHeader *hdr) write_durable(tmp, primary, buffer); } +/* + * write_header_both -- CRC-stamp the header and install the SAME image as + * both primary and .bak (primary first). For FLAG transitions (unseal, + * CLUSTER_ERA stamp) the usual roll-primary-to-.bak is unsafe: it would + * preserve a pre-transition image that the fail-closed read falls back to + * when the primary is later damaged -- a stale SEALED .bak hands a joiner + * the previous pass's high-water (review F1, 8.A), and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Crash + * points: both copies old (transition not yet taken: state still + * consistent), primary new + .bak old (read prefers the valid primary), + * both new. A later single-copy corruption therefore never resurrects the + * pre-transition flags. + */ +static void +write_header_both(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + write_durable(tmp, primary, buffer); + write_durable(baktmp, bak, buffer); +} + /* * cluster_xid_authority_seed_if_absent -- read-then-seed: if the authority is * already present (any trustworthy image) do nothing; otherwise create the @@ -336,7 +395,7 @@ cluster_xid_authority_mark_cluster_era(void) return; hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; - write_header(&hdr); + write_header_both(&hdr); /* review F1 variant: one-way flag in BOTH copies */ } /* @@ -377,5 +436,5 @@ cluster_xid_authority_begin_native_run(void) return; /* already open (first run, or a prior pass crashed) */ hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; - write_header(&hdr); + write_header_both(&hdr); /* review F1: no copy may retain SEALED */ } diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c index 0e5ed2018d..a61550e009 100644 --- a/src/backend/cluster/cluster_xid_prehistory.c +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -451,3 +451,123 @@ cluster_xid_prehistory_was_adopted(void) { return prehistory_adopted_this_boot; } + +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +/* + * read_local_clog_page_optional -- read local pg_xact page `pageno` into + * buf. Returns false when the segment file or the page does not exist + * (short clone: the comparable prefix ends there); PANICs on a real read + * error so an I/O fault is never mistaken for a short prefix. + */ +static bool +read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + int r; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) { + if (errno == ENOENT) + return false; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", seg))); + } + if (lseek(fd, offset, SEEK_SET) != offset) { + CloseTransientFile(fd); + return false; + } + errno = 0; + r = read(fd, buf, BLCKSZ); + CloseTransientFile(fd); + if (r < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", seg))); + return r == BLCKSZ; +} + +/* + * cluster_xid_prehistory_prefix_check -- see header. Streams the sealed + * blob (primary, then .bak) and byte-compares the local pg_xact prefix + * covering xids [0, limit_xid_full) at per-xact (2-bit) precision. + */ +ClusterXidPrefixVerdict +cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, + uint64 limit_xid_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char blob_page[BLCKSZ]; + char local_page[BLCKSZ]; + const char *src_path; + uint32 pages = 0; + uint32 p; + uint64 limit; + int src; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + limit = Min(limit_xid_full, native_hw_full); + if (limit == 0) + return CLUSTER_XID_PREFIX_CONSISTENT; + + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (src >= 0) + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + + for (p = 0; p < pages; p++) { + uint64 page_first_xid = (uint64)p * PREHISTORY_XACTS_PER_PAGE; + uint64 in_scope; + uint32 full_bytes; + uint32 partial_bits; + + if (read(src, blob_page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + if (page_first_xid >= limit) + break; /* comparison scope ended on a page boundary */ + + if (!read_local_clog_page_optional(local_pgdata, p, local_page)) + break; /* short clone: no local bits left to contradict */ + + in_scope = Min((uint64)PREHISTORY_XACTS_PER_PAGE, limit - page_first_xid); + full_bytes = (uint32)(in_scope / 4); + partial_bits = (uint32)(in_scope % 4) * 2; + + if (full_bytes > 0 && memcmp(blob_page, local_page, full_bytes) != 0) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; + } + if (partial_bits > 0) { + unsigned char mask = (unsigned char)((1 << partial_bits) - 1); + + if (((unsigned char)blob_page[full_bytes] & mask) + != ((unsigned char)local_page[full_bytes] & mask)) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; + } + } + } + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_CONSISTENT; +} diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index 7676d89303..c9c51f7ab2 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -81,6 +81,19 @@ typedef struct ClusterXidAuthorityHeader { pg_crc32c crc; /* over all preceding bytes */ } ClusterXidAuthorityHeader; +/* On-disk ABI locked at compile time (review F7; mirrors the recovery + * anchor's precedent). The unit truth table re-checks these at runtime. */ +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, flags) == 8, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, native_hw_full) == 16, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, next_multi) == 24, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, crc) == 32, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidAuthorityHeader) == 40, + "xid authority header size is on-disk ABI"); + /* Relative paths under cluster.shared_data_dir. */ #define CLUSTER_XID_AUTHORITY_REL_PATH "global/pgrac_xid_authority" #define CLUSTER_XID_AUTHORITY_BAK_REL_PATH "global/pgrac_xid_authority.bak" @@ -117,6 +130,15 @@ typedef struct ClusterXidPrehistoryHeader { pg_crc32c crc; /* over header fields above + payload */ } ClusterXidPrehistoryHeader; +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, native_hw_full) == 8, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, payload_len) == 16, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, crc) == 24, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidPrehistoryHeader) == 32, + "xid prehistory header size is on-disk ABI"); + #define CLUSTER_XID_PREHISTORY_REL_PATH "global/pgrac_xid_prehistory" #define CLUSTER_XID_PREHISTORY_BAK_REL_PATH "global/pgrac_xid_prehistory.bak" #define CLUSTER_XID_PREHISTORY_TMP_REL_PATH "global/pgrac_xid_prehistory.tmp" @@ -148,6 +170,9 @@ extern ClusterXidAuthorityValidity cluster_xid_prehistory_classify(const char *b */ extern bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out); +/* As read(), reporting .bak fallback for a consumer-side LOG (review F8). */ +extern bool cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak); + /* * ENOENT-only-absent presence probe (spec-6.14 §3.6 posture): any stat() * failure other than ENOENT reports present, so a transiently unreadable @@ -202,4 +227,27 @@ extern void cluster_xid_prehistory_publish(const char *local_pgdata, uint64 nati extern void cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full); extern bool cluster_xid_prehistory_was_adopted(void); +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +typedef enum ClusterXidPrefixVerdict { + CLUSTER_XID_PREFIX_CONSISTENT = 0, /* local prefix matches the blob */ + CLUSTER_XID_PREFIX_DIVERGED, /* local pg_xact contradicts the blob */ + CLUSTER_XID_PREFIX_UNAVAILABLE, /* no trustworthy blob to compare */ +} ClusterXidPrefixVerdict; + +/* + * Compare the local pg_xact bytes covering xids [0, limit_xid_full) with the + * sealed prehistory blob at 2-bit (per-xact) precision. A local segment or + * page that does not exist ends the comparable prefix (a shorter clone has + * no bits to contradict). Callers FATAL on DIVERGED/UNAVAILABLE: a joiner + * whose own native-era history contradicts the seed's is not a pre-seed + * lineage, and neither skipping (trusting local bits) nor adopting + * (overwriting the joiner's own outcomes) is sound for it. + */ +extern ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, + uint64 native_hw_full, + uint64 limit_xid_full); + #endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl index f4652c83ca..37f99b898f 100644 --- a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -232,6 +232,11 @@ sub append_strict_two_node_conf my $node1 = PostgreSQL::Test::Cluster->new('sxid_node1'); cold_clone_data_dir($node0, $node1); +# F3: real scram TCP login leg needs a host auth line AND a TCP listener on +# the joiner (test nodes default to unix sockets only). +PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pg_hba.conf', + "host all sxid_login 127.0.0.1/32 scram-sha-256\n"); +$node1->append_conf('postgresql.conf', "listen_addresses = '127.0.0.1'\n"); append_common_shared_catalog_conf($node0, $shared_root); $node0->append_conf('postgresql.conf', <connect_ok( + 'host=127.0.0.1 port=' . $node1->port + . ' user=sxid_login password=sxidpass dbname=postgres', + 'G4: seed role logs in over scram TCP on node1'); wait_sql_eq($node1, "SELECT txid_status($seed_schema_xid)", 'committed', 'G5: node1 txid_status sees the native-era committed schema xid'); @@ -319,6 +330,15 @@ sub append_strict_two_node_conf 'G5: node1 txid_status sees the native-era aborted xid'); is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), 'aborted', 'G5: node0 still sees the aborted xid after node1 reads'); +# F4: the poison-stamp proof must re-read the shared HEAP rows on node0 -- +# a wrong HEAP_XMIN_INVALID hint written by node1 lives in the shared +# catalog block, not in node0's CLOG. +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}), + '1', 'G5: node0 still sees the seed schema row after node1 scans (no poison stamp)'); +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}), + '1', 'G5: node0 still sees the seed role row after node1 scans (no poison stamp)'); my $state_hw = $node1->safe_psql('postgres', q{ SELECT value FROM pg_cluster_state @@ -359,12 +379,20 @@ sub append_strict_two_node_conf corrupt_file("$shared_root/global/pgrac_xid_authority"); corrupt_file("$shared_root/global/pgrac_xid_authority.bak"); -$node1->append_conf('postgresql.conf', "# t/361 force new logfile generation after corrupt authority\n"); +# F9: pin the 53RB5 SQLSTATE contract on one leg (startup FATALs surface in +# the server log only, so %e in log_line_prefix carries the SQLSTATE; the +# other negative legs deliberately match message text). +# %e must precede %q: the refusal FATAL comes from the postmaster (a +# non-session process), and %q stops expansion there. +$node1->append_conf('postgresql.conf', + "log_line_prefix = '%m [%p] %e %q%a '\n"); my $corrupt_failed = !$node1->start(fail_ok => 1); ok($corrupt_failed, 'N2: corrupt shared XID authority fails closed'); like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), qr/shared XID authority is unavailable|present but corrupt|cluster_xid_authority_unavailable/, 'N2: startup log names corrupt/unavailable XID authority'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/53RB5/, 'N2: the refusal carries SQLSTATE 53RB5'); # ------------------------------------------------------------------------- # Unsealed authority negative: seed startup wrote the authority, but the seed @@ -466,4 +494,83 @@ sub append_strict_two_node_conf 'N4: startup log fails closed on the unsealed authority'); } +# ------------------------------------------------------------------------- +# Divergent-lineage negative (review F2 / spec Q6 amendment): a same-sysid +# clone that ran STANDALONE after cloning holds its own outcomes in the +# native xid range; neither skipping nor adopting is sound -> FATAL 53RB5. +# ------------------------------------------------------------------------- +{ + my $d_shared = make_shared_root(); + my $d_disks = make_voting_disks(); + my $d_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $d_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $d0 = PostgreSQL::Test::Cluster->new('sxid_diverge_node0'); + my $d1 = PostgreSQL::Test::Cluster->new('sxid_diverge_node1'); + + $d0->init(allows_streaming => 1); + $d0->start; + $d0->safe_psql('postgres', 'CHECKPOINT'); + $d0->stop; + cold_clone_data_dir($d0, $d1); + + # d1 diverges: a plain standalone run committing 30 transactions writes + # d1's own outcomes into the xid range the seed is about to consume. + $d1->start; + $d1->safe_psql('postgres', + join('', map { "BEGIN; CREATE TABLE d_t$_ (i int); COMMIT;\n" } 1 .. 30)); + $d1->stop; + + # d0 seeds: one aborted xid guarantees a byte-level contradiction with + # d1's straight-committed overlap. + append_common_shared_catalog_conf($d0, $d_shared); + $d0->append_conf('postgresql.conf', <start; + $d0->safe_psql('postgres', q{ +BEGIN; +CREATE SCHEMA d_abort_shadow; +ROLLBACK; +}); + $d0->safe_psql('postgres', 'CREATE SCHEMA d_seed;'); + $d0->stop; + + append_common_shared_catalog_conf($d1, $d_shared); + append_strict_two_node_conf($d1, $d_disks, undef); + $d1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($d1, 'sxid_diverge', $d_ic0, $d_ic1); + + my $diverge_failed = !$d1->start(fail_ok => 1); + ok($diverge_failed, 'N5: divergent-lineage joiner is refused'); + like(PostgreSQL::Test::Utils::slurp_file($d1->logfile), + qr/divergent native-era lineage/, + 'N5: startup log names the divergent lineage'); +} + +# ------------------------------------------------------------------------- +# L9 dormant leg: with cluster.shared_catalog=off nothing of the XID +# authority machinery may touch the shared tree. +# ------------------------------------------------------------------------- +{ + my $o_shared = make_shared_root(); + my $o0 = PostgreSQL::Test::Cluster->new('sxid_off_node0'); + + $o0->init(allows_streaming => 1); + $o0->append_conf('postgresql.conf', <start; + $o0->safe_psql('postgres', 'CREATE SCHEMA off_mode_probe;'); + $o0->stop; + ok(!-e "$o_shared/global/pgrac_xid_authority", + 'L9: shared_catalog=off never creates the XID authority'); + ok(!-e "$o_shared/global/pgrac_xid_prehistory", + 'L9: shared_catalog=off never creates the XID prehistory'); +} + done_testing(); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 6178ebbfcd..4a646270e1 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -495,6 +495,122 @@ UT_TEST(test_prehistory_multi_segment_round_trip) } } +UT_TEST(test_unseal_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1: after begin_native_run, NO on-disk copy may still say SEALED -- + * a corrupt primary falling back to a stale SEALED .bak would hand a + * joiner the previous pass's high-water (8.A false-invisible). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_begin_native_run(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + /* read failing entirely (no trustworthy copy) is also acceptable: + * joiners fail closed on unavailable exactly like on unsealed. */ +} + +UT_TEST(test_mark_cluster_era_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1 variant: after mark_cluster_era, no on-disk copy may lack the + * one-way CLUSTER_ERA flag -- a stale .bak without it would let an + * enabled=off boot re-enter the native era on a formed tree. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_mark_cluster_era(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_prefix_check_divergence_truth_table) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + int fd; + + /* F2: identical prefix -> CONSISTENT (both trees one page of 0xAA) */ + unlink_files(); + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "f2seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "f2join", 1, 0xAA); + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* flip one status byte INSIDE the limit -> DIVERGED */ + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 100, SEEK_SET), 100); /* byte 100 = xids 400..403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_DIVERGED); + + /* the same flipped byte OUTSIDE the limit -> CONSISTENT (adopt arm: + * bytes at/after own_next belong to the seed alone) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* partial-byte boundary: xid 400's 2-bit slot enters scope at limit 401 */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401), + CLUSTER_XID_PREFIX_DIVERGED); + + /* missing local pg_xact segment -> comparable prefix is empty -> + * CONSISTENT (a shorter clone has no bits to contradict) */ + unlink(seg); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* no trustworthy blob -> UNAVAILABLE */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_UNAVAILABLE); + + /* multi-segment divergence: 33-page trees, flip a byte in segment 0001 */ + { + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "f2seedms", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "f2joinms", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + CLUSTER_XID_PREFIX_CONSISTENT); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "Z", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + CLUSTER_XID_PREFIX_DIVERGED); + } +} + UT_TEST(test_prehistory_classify_corrupt) { char buf[64]; @@ -566,7 +682,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(12); + UT_PLAN(15); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -574,10 +690,13 @@ main(void) UT_RUN(test_publish_monotone_and_seal); UT_RUN(test_mark_cluster_era_one_way); UT_RUN(test_begin_native_run_unseals_before_cluster_era); + UT_RUN(test_unseal_survives_primary_corruption_via_bak); + UT_RUN(test_mark_cluster_era_survives_primary_corruption_via_bak); UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); UT_RUN(test_prehistory_multi_segment_round_trip); + UT_RUN(test_prefix_check_divergence_truth_table); UT_RUN(test_prehistory_classify_corrupt); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From 0dca50fab30c6af7b273c48872197975bf20e881 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:16:22 +0800 Subject: [PATCH 35/58] =?UTF-8?q?fix(cluster):=20GRD=20recovery=20liveness?= =?UTF-8?q?=20=E2=80=94=20same-episode=20HW=20remaster=20retry,=20WAIT=5FC?= =?UTF-8?q?LUSTER=20watchdog,=20event-scoped=20WAIT=5FEPOCH=20escape,=20de?= =?UTF-8?q?ad-node=20PCM=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fail-stop reconfig could wedge a settled cluster forever on three stacked liveness holes: a BLOCKED HW remaster worker was never relaunched within the same episode (BGW_NEVER_RESTART + launched-latch), WAIT_CLUSTER had no watchdog while its cluster/HW gates were held, and WAIT_EPOCH's strict progress gate wedged permanently when an IC-piggybacked epoch bump landed before the local reconfig event (baseline re-captured as the post-bump value, cur == old forever). - HW remaster: per-dead-node result/attempts/next_attempt shmem state; the FSM relaunches BLOCKED workers with exponential backoff (cap 60s) up to cluster.hw_remaster_retry_max_attempts (SIGHUP-raisable); abnormal worker exit marks BLOCKED via before_shmem_exit; the three silent BLOCKED returns now LOG their cause. wal_threads_dir-unset is detected up front as BLOCKED_STRUCTURAL: never retried, WARNING-once with the config hint, and (level 2) a multi-node shared_catalog=on boot without it now fails fast at startup. - WAIT_CLUSTER: observational watchdog on the rebuild-timeout cadence — WARNING with the missing-survivor list, gate states and per-dead-node retry state, plus a counter; it never unfreezes anything. - WAIT_EPOCH: three proof-carrying layers — abort-to-idle on a new event id, durable observed-epoch adoption, and an event-scoped coordinator witness (REDECLARE_DONE carries the event id; stale-event DONEs are dropped at accounting; the equal-epoch escape additionally requires the coordinator's DONE to have been accounted after the accept snapshot). No timeout-only advance exists; without proof the shards stay frozen. - Dead-node PCM cleanup: the GRD dead-sweep clears the dead node's x_holder/s_holders/pi_holders/master_holder and pending-X residue (LOG summary + pcm/dead_cleanup_entries counter), and a local-master S state with no local residency can now upgrade S->X through the standard invalidate/ACK path instead of failing closed forever — post-kill DDL recovers instead of timing out on cluster locks. - Serve-gate scope alignment: the merged-materialization check in gcs_block phase_for_tag now uses the same thread-recovery scope policy as the unfreeze gate, removing the permanent-RECOVERING mismatch outside the gate's applicable scope (in-scope behavior unchanged). - Observability: hw retry/exhausted + grd cluster_gate_timeout / wait_epoch_escape + pcm dead_cleanup_entries dump keys; S4-reject and HW fail-closed diagnostics. Tests: unit relaunch-decision table (8 rows) + event-scoped DONE rejection/ witness advance + PCM dead-cleanup forms incl. pending-X; t/293 gains the same-episode self-heal and retry-exhaustion legs (unfreeze-extend positive + watchdog WARNING greps); t/337 gains the level-2 fail-fast negative leg; new t/362 runs a 4-node shared_catalog formation, kill -9, and asserts remaster convergence, post-kill DDL, cleanup counters and the 0-wedge invariant end to end. Local gates (cassert): unit 158 binaries, t/293 + t/337 + t/362 + smoke subset, cluster_regress 13/13, PG regress 219/219. Spec: spec-4.6a-grd-recovery-liveness.md --- src/backend/cluster/cluster_debug.c | 33 ++ src/backend/cluster/cluster_gcs_block.c | 41 +- src/backend/cluster/cluster_ges.c | 3 +- src/backend/cluster/cluster_grd.c | 503 ++++++++++++++++-- src/backend/cluster/cluster_guc.c | 17 + src/backend/cluster/cluster_hw_remaster.c | 186 ++++++- src/backend/cluster/cluster_hw_shmem.c | 108 +++- src/backend/cluster/cluster_lms.c | 3 + src/backend/cluster/cluster_pcm_lock.c | 125 +++++ src/backend/cluster/cluster_wal_thread.c | 19 +- src/include/cluster/cluster_grd.h | 24 +- src/include/cluster/cluster_guc.h | 2 + src/include/cluster/cluster_hw.h | 28 +- src/include/cluster/cluster_hw_remaster.h | 102 +++- src/include/cluster/cluster_pcm_lock.h | 6 + src/include/cluster/cluster_thread_recovery.h | 10 + src/test/cluster_tap/t/024_pcm_lock.pl | 7 +- .../cluster_tap/t/293_hw_online_remaster.pl | 245 ++++++++- .../t/337_shared_catalog_ddl_2node.pl | 42 +- .../362_shared_catalog_4node_kill_selfheal.pl | 310 +++++++++++ src/test/cluster_unit/Makefile | 1 + src/test/cluster_unit/test_cluster_debug.c | 95 +++- .../cluster_unit/test_cluster_gcs_block.c | 14 +- src/test/cluster_unit/test_cluster_ges.c | 3 +- src/test/cluster_unit/test_cluster_grd.c | 147 ++++- .../test_cluster_grd_starvation.c | 37 ++ .../test_cluster_hw_remaster_retry.c | 156 ++++++ .../cluster_unit/test_cluster_lock_acquire.c | 41 ++ src/test/cluster_unit/test_cluster_pcm_lock.c | 125 ++++- src/test/perl/PostgreSQL/Test/ClusterQuad.pm | 108 +++- 30 files changed, 2375 insertions(+), 166 deletions(-) create mode 100644 src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl create mode 100644 src/test/cluster_unit/test_cluster_hw_remaster_retry.c diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..e19b3e98e8 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1195,8 +1195,30 @@ static void dump_grd_recovery(ReturnSetInfo *rsinfo) { ClusterGrdRecoveryCounters c; + uint32 state; cluster_grd_recovery_counters_snapshot(&c); + state = cluster_grd_recovery_state_value(); + emit_row(rsinfo, "grd_recovery", "state", cluster_grd_recovery_state_name(state)); + emit_row(rsinfo, "grd_recovery", "state_enum_value", fmt_int32((int32)state)); + emit_row(rsinfo, "grd_recovery", "last_event_id", + fmt_int64((int64)cluster_grd_recovery_last_event_id())); + emit_row(rsinfo, "grd_recovery", "event_old_epoch", + fmt_int64((int64)cluster_grd_recovery_event_old_epoch())); + emit_row(rsinfo, "grd_recovery", "episode_epoch", + fmt_int64((int64)cluster_grd_recovery_episode_epoch_value())); + emit_row(rsinfo, "grd_recovery", "event_coordinator", + fmt_int32((int32)cluster_grd_recovery_event_coordinator())); + emit_row(rsinfo, "grd_recovery", "done_self_epoch", + fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_event_id", + fmt_int64((int64)cluster_grd_recovery_done_event_id_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", + fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", + fmt_int64((int64)cluster_grd_recovery_block_redeclare_epoch())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_done", + fmt_bool(cluster_grd_recovery_block_redeclare_done())); emit_row(rsinfo, "grd_recovery", "remaster_started", fmt_int64((int64)c.remaster_started)); emit_row(rsinfo, "grd_recovery", "remaster_done", fmt_int64((int64)c.remaster_done)); emit_row(rsinfo, "grd_recovery", "remaster_failed", fmt_int64((int64)c.remaster_failed)); @@ -1212,6 +1234,11 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) emit_row(rsinfo, "grd_recovery", "unaffected_holder_survived", fmt_int64((int64)c.unaffected_holder_survived)); emit_row(rsinfo, "grd_recovery", "stale_holder_swept", fmt_int64((int64)c.stale_holder_swept)); + emit_row(rsinfo, "grd_recovery", "cluster_gate_timeout", + fmt_int64((int64)c.cluster_gate_timeout)); + emit_row(rsinfo, "grd_recovery", "wait_epoch_escape", fmt_int64((int64)c.wait_epoch_escape)); + /* spec-4.6a D12 — dead-node PCM residue cleanup (categorized under pcm). */ + emit_row(rsinfo, "pcm", "dead_cleanup_entries", fmt_int64((int64)c.pcm_dead_cleanup_entries)); /* spec-5.16 D5 — join-direction remaster counters (same grd_recovery * category; no new dump category, §8 Q6-A). */ emit_row(rsinfo, "grd_recovery", "join_remaster_started", @@ -2810,6 +2837,12 @@ dump_hw(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_hw_remaster_done_count())); emit_row(rsinfo, "hw", "remaster_blocked_count", fmt_int64((int64)cluster_hw_remaster_blocked_count())); + emit_row(rsinfo, "hw", "remaster_retry_count", + fmt_int64((int64)cluster_hw_remaster_retry_count())); + emit_row(rsinfo, "hw", "remaster_retry_exhausted_count", + fmt_int64((int64)cluster_hw_remaster_retry_exhausted_count())); + emit_row(rsinfo, "hw", "remaster_recoverable", + fmt_int64((int64)(cluster_hw_remaster_recoverable() ? 1 : 0))); /* * spec-6.12d D-obs: space-lease counters. bloat_ratio itself is a diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..d0219deaf9 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -51,11 +51,12 @@ #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */ #include "cluster/cluster_grd.h" /* spec-4.6 D4 — block_path_failclosed counter */ #include "cluster/cluster_grd_outbound.h" -#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ -#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ -#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ -#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ +#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ +#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ +#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ +#include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ +#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_itl.h" /* spec-5.2 D11 — active-ITL writer-transfer guard */ @@ -65,6 +66,7 @@ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" #include "cluster/cluster_shmem.h" +#include "cluster/storage/cluster_shared_fs.h" #include "cluster/cluster_sf_dep.h" #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */ #include "common/hashfn.h" @@ -1221,12 +1223,13 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The survivor may SERVE only - * after the dead origin's merged WAL recovery passes the redo-before- - * unfreeze gate; before that the shared-storage / re-declared version may - * be stale → fail-closed RECOVERING (never a stale page). + * live survivor (recovery-aware routing). The redo-before-serve gate must + * engage on EXACTLY the same scope where online thread recovery can actually + * run. Otherwise a >2-node or GUC-off deployment waits forever on a + * materialization authority that the thread-recovery launcher intentionally + * treats as not applicable. * - * TWO conditions, both required (Q5): + * In applicable 2-node scope, two conditions are both required (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1236,13 +1239,19 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * survivor DID observe (rebuilt pi_watermark_lsn > 0), the dead * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably - * reached us → lost-write → fail-closed. This is the LSN comparison - * (NOT a bool), live in the serve path; required_lsn == 0 (cold) is - * trivially covered and (a) carries the safety. - * - * Once both hold → NORMAL → the re-routed survivor serves (rebuilt-from- - * redeclare for held blocks, lazy minimal view for cold blocks). + * reached us → lost-write → fail-closed. */ + { + bool shared_fs; + int survivors; + + shared_fs = (cluster_shared_storage_backend == CLUSTER_SHARED_FS_BACKEND_CLUSTER_FS); + survivors = cluster_conf_node_count() - 1; + if (!cluster_thread_recovery_materialization_gate_enabled( + cluster_online_thread_recovery, cluster_conf_has_peers(), shared_fs, survivors)) + return GCS_BLOCK_NORMAL; + } + if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->recovery_block_resources_recovering, 1); diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index a81f80ee2b..547f0e5aac 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -552,7 +552,8 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) * record and return (no work queue, no reply, no dedup; the sender * re-announces each tick, the receiver write is a monotonic max). */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { - cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch); + cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, + ges_request_holder_request_id(req)); return; } diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..8977739433 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -46,7 +46,8 @@ #include "cluster/cluster_pcm_lock.h" /* spec-2.36 HC124 pending_x node-dead cleanup */ #include "cluster/cluster_gcs.h" /* spec-4.7 D2 — cluster_gcs_lookup_master */ #include "cluster/cluster_membership.h" /* spec-5.16 D1 — cluster_membership_is_member */ -#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ +#include "cluster/cluster_native_lock_probe.h" +#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ #include "cluster/cluster_signal.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_cssd.h" /* spec-2.16 D8 newly-dead bitmap diff */ @@ -731,9 +732,13 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->recovery_event_old_epoch, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_redeclare_generation, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_barrier_deadline, 0); + pg_atomic_init_u32(&cluster_grd_state->recovery_event_coordinator, 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) + for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_event_id[i], 0); + } pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -747,6 +752,9 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->block_path_failclosed_count, 0); pg_atomic_init_u64(&cluster_grd_state->unaffected_holder_survived_count, 0); pg_atomic_init_u64(&cluster_grd_state->stale_holder_swept_count, 0); + pg_atomic_init_u64(&cluster_grd_state->cluster_gate_timeout_count, 0); + pg_atomic_init_u64(&cluster_grd_state->wait_epoch_escape_count, 0); + pg_atomic_init_u64(&cluster_grd_state->pcm_dead_cleanup_entries, 0); /* spec-5.16 D2/D3b/D5 — online-join remaster fence + counters. */ pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_epoch, 0); @@ -1630,6 +1638,78 @@ cluster_grd_redeclare_episode_epoch(void) * been re-declared to its recovery-aware master — serving it would risk an * 8.A double-grant. Reaching IDLE implies all survivor scans completed. */ +uint32 +cluster_grd_recovery_state_value(void) +{ + if (cluster_grd_state == NULL) + return (uint32)GRD_RECOVERY_IDLE; + return pg_atomic_read_u32(&cluster_grd_state->recovery_state); +} + +const char * +cluster_grd_recovery_state_name(uint32 state) +{ + switch ((ClusterGrdRecoveryState)state) { + case GRD_RECOVERY_IDLE: + return "idle"; + case GRD_RECOVERY_WAIT_EPOCH: + return "wait_epoch"; + case GRD_RECOVERY_WAIT_BARRIER: + return "wait_barrier"; + case GRD_RECOVERY_WAIT_CLUSTER: + return "wait_cluster"; + } + return "unknown"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id) + : 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch) + : 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch) + : 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator) + : 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); +} + +uint64 +cluster_grd_recovery_done_event_id_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[node]); +} + bool cluster_grd_recovery_in_progress(void) { @@ -1686,6 +1766,10 @@ cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) out->unaffected_holder_survived = pg_atomic_read_u64(&cluster_grd_state->unaffected_holder_survived_count); out->stale_holder_swept = pg_atomic_read_u64(&cluster_grd_state->stale_holder_swept_count); + out->cluster_gate_timeout = pg_atomic_read_u64(&cluster_grd_state->cluster_gate_timeout_count); + out->wait_epoch_escape = pg_atomic_read_u64(&cluster_grd_state->wait_epoch_escape_count); + out->pcm_dead_cleanup_entries + = pg_atomic_read_u64(&cluster_grd_state->pcm_dead_cleanup_entries); /* spec-5.16 D5 — join-direction remaster counters. */ out->join_remaster_started = pg_atomic_read_u64(&cluster_grd_state->join_remaster_started_count); @@ -1924,6 +2008,39 @@ grd_recovery_barrier_complete(uint64 gen, uint64 episode_epoch) return true; } +static void +grd_recovery_format_waiting_backend(uint64 gen, uint64 episode_epoch, char *buf, Size buflen) +{ + int beid; + pid_t self_pid = MyProcPid; + + if (buflen == 0) + return; + buf[0] = '\0'; + for (beid = 1; beid <= MaxBackends; beid++) { + PGPROC *proc = BackendIdGetProc((BackendId)beid); + uint32 registered; + uint64 acked; + uint64 acked_epoch; + + if (proc == NULL || proc->pid == 0 || proc->pid == self_pid) + continue; + registered = pg_atomic_read_u32(&proc->cluster_grd_registered_count); + if (registered == 0) + continue; + acked = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked); + acked_epoch = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked_epoch); + if (acked < gen || acked_epoch != episode_epoch) { + snprintf(buf, buflen, + "beid=%d pid=%d registered=%u acked=" UINT64_FORMAT "/" UINT64_FORMAT + " target=" UINT64_FORMAT "/" UINT64_FORMAT, + beid, (int)proc->pid, registered, acked, acked_epoch, gen, episode_epoch); + return; + } + } + snprintf(buf, buflen, "none"); +} + /* * spec-4.6 P0#3 cluster gate — announce "my local rebind barrier is * complete for `epoch`" to every declared peer (fire-and-forget; the @@ -1934,6 +2051,7 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; + uint64 event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -1943,6 +2061,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); + req.holder_request_id_lo = (uint32)(event_id & 0xffffffffu); + req.holder_request_id_hi = (uint32)(event_id >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -1961,19 +2081,33 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) { - uint64 prev; + uint64 current_event_id; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; - /* Monotonic max: late/duplicate announcements never regress. */ - prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); - while (epoch > prev) { - if (pg_atomic_compare_exchange_u64(&cluster_grd_state->recovery_done_epoch[node], &prev, - epoch)) - break; + + /* + * spec-4.6a D4-v2: REDECLARE_DONE is an event-scoped proof. A stale DONE + * from a previous episode can carry the same epoch when cur==old, so epoch + * monotonicity alone is not enough. Drop messages for any event other than + * the one this LMON has accepted; senders re-announce every tick. + */ + current_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + if (event_id == 0 || (current_event_id != 0 && event_id != current_event_id)) + return; + + /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic + * per-event epochs the incoming value always wins, but never regress the + * published value if a delayed lower-epoch frame ever slips through. */ + { + uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); + + if (epoch > prev) + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[node], event_id); } /* @@ -2000,6 +2134,144 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +static void +grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) +{ + va_list ap; + int n; + Size avail; + + if (buflen == 0 || *off >= (int)buflen - 1) + return; + avail = buflen - (Size)*off; + va_start(ap, fmt); + n = vsnprintf(buf + *off, avail, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((Size)n >= avail) + *off = (int)buflen - 1; + else + *off += n; +} + +static void +grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, bool is_join, + char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 done_epoch; + + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + if (is_join && join_fence_is_recipient_for(i, episode_epoch)) + continue; + done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); + if (done_epoch >= episode_epoch + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + continue; + grd_recovery_appendf(buf, buflen, &off, + "%s%d(done=" UINT64_FORMAT "/event=" UINT64_FORMAT ")", any ? "," : "", + i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i])); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static void +grd_recovery_format_hw_dead(const uint64 *dead, char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + ClusterHwRemasterResult result; + + if (!grd_dead_bitmap_test(dead, i) || i == cluster_node_id) + continue; + result = cluster_hw_remaster_result(i); + grd_recovery_appendf(buf, buflen, &off, "%s%d:%s/%u", any ? "," : "", i, + cluster_hw_remaster_result_name(result), + cluster_hw_remaster_attempts(i)); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static bool +grd_recovery_adopt_observed_epoch(const uint64 *dead, uint64 cur_epoch) +{ + uint64 max_epoch = cur_epoch; + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 peer_epoch; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + peer_epoch = cluster_reconfig_get_observed_epoch(i); + if (peer_epoch > max_epoch) + max_epoch = peer_epoch; + } + if (max_epoch <= cur_epoch) + return false; + cluster_epoch_adopt_admitted(max_epoch); + return true; +} + +static void +grd_recovery_wait_cluster_watchdog(const uint64 *dead, uint64 episode_epoch) +{ + TimestampTz now; + TimestampTz deadline; + TimestampTz next_deadline; + char missing[512]; + char hw_dead[512]; + bool thread_gate; + bool hw_gate; + bool is_join; + + deadline = (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline); + now = GetCurrentTimestamp(); + if (deadline == 0 || now <= deadline) + return; + + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + grd_recovery_format_missing_survivors(dead, episode_epoch, is_join, missing, sizeof(missing)); + thread_gate = cluster_thread_recovery_gate_unfreeze(dead, (CLUSTER_MAX_NODES + 63) / 64); + hw_gate = cluster_hw_remaster_gate_unfreeze(); + grd_recovery_format_hw_dead(dead, hw_dead, sizeof(hw_dead)); + + pg_atomic_fetch_add_u64(&cluster_grd_state->cluster_gate_timeout_count, 1); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen"), + errdetail("episode_epoch=" UINT64_FORMAT ", missing_survivors=%s, " + "thread_gate=%s, hw_gate=%s, hw_dead=%s", + episode_epoch, missing, thread_gate ? "held" : "open", hw_gate ? "held" : "open", + hw_dead), + errhint("This watchdog is observational only; it never unfreezes a shard."))); + next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); +} + /* * spec-4.7 D2 — survivor block re-declare scan (Q6-A' worker-centric). @@ -2078,6 +2350,24 @@ grd_block_redeclare_scan_complete(uint64 episode_epoch) return grd_block_redeclare_epoch == episode_epoch && grd_block_redeclare_done; } +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return grd_block_redeclare_cursor; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return grd_block_redeclare_epoch; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return grd_block_redeclare_done; +} + void cluster_grd_recovery_lmon_tick(void) { @@ -2119,6 +2409,18 @@ cluster_grd_recovery_lmon_tick(void) * epoch (the genuine pre-reconfig baseline) — do NOT overwrite it * with evt.old_epoch here. */ pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, ev_id); + pg_atomic_write_u32(&cluster_grd_state->recovery_event_coordinator, + (uint32)evt.coordinator_node_id); + if (evt.coordinator_node_id >= 0 && evt.coordinator_node_id < CLUSTER_MAX_NODES) + pg_atomic_write_u64( + &cluster_grd_state->recovery_done_epoch_at_accept, + pg_atomic_read_u64( + &cluster_grd_state->recovery_done_epoch[evt.coordinator_node_id])); + else + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms)); for (b = 0; b < (CLUSTER_MAX_NODES + 63) / 64; b++) { uint64 word = 0; int j; @@ -2174,35 +2476,116 @@ cluster_grd_recovery_lmon_tick(void) int i; int signaled; + for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) + dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + /* - * Gate on the ACCEPTED epoch having advanced past the episode's - * old epoch: the coordinator bumped it earlier this tick; a - * non-coordinator survivor observes it via IC envelope piggyback - * a tick or two later. Running the rebind before the local - * epoch advances would mint holders the new master rejects. - * - * spec-5.16 (P0, Rule 8.A) — direction-aware. A FAIL survivor captures - * old_epoch BEFORE the coordinator bumps, so it must wait for a STRICT - * advance (cur > old) to prove it adopted the new master view. A JOIN - * observer survivor instead adopts the coordinator's JOIN epoch bump (via the - * joiner_self_tick max-peer-epoch adoption, same LMON tick) BEFORE it readmits - * the rejoiner and publishes its own observer JOIN_COMMITTED event, so by the - * time its FSM accepts that event old_epoch == cur_epoch == the join epoch. - * Requiring a strict advance there wedges the survivor in WAIT_EPOCH forever: - * it never runs the re-declare barrier, never broadcasts REDECLARE_DONE, and - * the coordinator's all-members JOIN barrier (Hardening v1.1) hangs → - * join_remaster_done never advances. For JOIN the survivor already holds the - * epoch it rebinds under (that IS the post-bump master view), so equality is - * sufficient; cur_epoch is monotonic so cur < old never occurs for JOIN. + * Gate on the ACCEPTED epoch having advanced past the episode's old epoch. + * FAIL requires proof of a post-bump epoch; JOIN keeps its existing equality + * allowance because the observer already accepted the coordinator's JOIN epoch. + * For FAIL, deadline expiry is not an escape hatch by itself: we either adopt + * a durable observed epoch or require a coordinator REDECLARE_DONE witness for + * cur==old. Without either proof, stay fail-closed in WAIT_EPOCH. */ - if (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) - == (uint32)GRD_REMASTER_DIR_JOIN - ? cur_epoch < old_epoch - : cur_epoch <= old_epoch) - return; + { + ReconfigEvent latest; + bool is_join; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id != 0 + && latest.event_id + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + grd_recovery_abort_to_idle(); + return; + } - for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) - dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + if (is_join) { + if (cur_epoch < old_epoch) + return; + } else if (cur_epoch <= old_epoch) { + TimestampTz now = GetCurrentTimestamp(); + TimestampTz wait_deadline; + TimestampTz next_deadline; + uint32 coordinator; + uint64 coord_done; + uint64 coord_done_event_id; + uint64 accepted_event_id; + uint64 coord_done_at_accept; + + wait_deadline = (TimestampTz)pg_atomic_read_u64( + &cluster_grd_state->recovery_barrier_deadline); + if (wait_deadline == 0 || now <= wait_deadline) + return; + + if (grd_recovery_adopt_observed_epoch(dead, cur_epoch)) { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH adopted a durable observed epoch; " + "affected shards stay frozen until recovery completes"))); + return; + } + + coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); + accepted_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + coord_done + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) + : 0; + coord_done_event_id + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64( + &cluster_grd_state->recovery_done_event_id[coordinator]) + : 0; + coord_done_at_accept + = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); + /* + * spec-4.6a §2.3 witness (all three conjuncts of the frozen + * text): the coordinator's DONE must (a) be for THIS event, + * (b) name exactly cur as the episode epoch, and (c) have been + * accounted after our P0 accept snapshot. Under strictly + * monotonic per-event epochs (c) is implied by (a)+(b) — any + * pre-accept residue carries a lower epoch — but it stays as a + * belt-and-suspenders guard against accounting bugs. + */ + if (cur_epoch == old_epoch && coord_done == cur_epoch + && coord_done_event_id == accepted_event_id + && coord_done > coord_done_at_accept) { + pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " + "equal-epoch progress"), + errdetail("coordinator=%u, epoch=" UINT64_FORMAT + ", event_id=" UINT64_FORMAT, + coordinator, cur_epoch, accepted_event_id))); + } else { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_event_id=" UINT64_FORMAT + ", accepted_event_id=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_event_id, accepted_event_id, coord_done_at_accept))); + return; + } + } + } /* * P1 freeze affected + P3 scoped sweep + P4 remaster. Direction- @@ -2306,6 +2689,7 @@ cluster_grd_recovery_lmon_tick(void) if (state == (uint32)GRD_RECOVERY_WAIT_BARRIER) { uint64 gen = pg_atomic_read_u64(&cluster_grd_state->recovery_redeclare_generation); uint64 episode_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch); + TimestampTz deadline; /* P0-1 epoch-coherence guard: a SECOND epoch bump landed * mid-episode (e.g. a third node's heartbeat flap re-fired @@ -2344,7 +2728,12 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)); grd_recovery_broadcast_done(episode_epoch); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)deadline); pg_atomic_write_u32(&cluster_grd_state->recovery_state, (uint32)GRD_RECOVERY_WAIT_CLUSTER); ereport(DEBUG1, @@ -2358,6 +2747,12 @@ cluster_grd_recovery_lmon_tick(void) if (GetCurrentTimestamp() > (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline)) { TimestampTz deadline; + char waiting_backend[256]; + bool ges_barrier_complete = grd_recovery_barrier_complete(gen, episode_epoch); + bool block_scan_complete = grd_block_redeclare_scan_complete(episode_epoch); + + grd_recovery_format_waiting_backend(gen, episode_epoch, waiting_backend, + sizeof(waiting_backend)); /* * Barrier deadline expired: fail-closed. Affected shards @@ -2376,6 +2771,12 @@ cluster_grd_recovery_lmon_tick(void) (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), errmsg("cluster GRD holder-rebuild barrier timed out; affected shards " "stay frozen"), + errdetail("gen=" UINT64_FORMAT ", episode_epoch=" UINT64_FORMAT + ", ges_barrier=%s, block_scan=%s, block_cursor=%d" + ", block_epoch=" UINT64_FORMAT ", waiting_backend=%s", + gen, episode_epoch, ges_barrier_complete ? "done" : "waiting", + block_scan_complete ? "done" : "waiting", grd_block_redeclare_cursor, + grd_block_redeclare_epoch, waiting_backend), errhint("A backend has not acked the cooperative rebind within " "cluster.grd_rebuild_timeout_ms; re-broadcasting."))); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), @@ -2409,6 +2810,8 @@ cluster_grd_recovery_lmon_tick(void) for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + grd_recovery_wait_cluster_watchdog(dead, episode_epoch); + /* * spec-4.11 D1 (3b-4b Part 3) — launch one per-episode online thread- * recovery worker for each in-scope dead origin (idempotent per tick; @@ -2461,8 +2864,9 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; - if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) - < episode_epoch) { + if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { all_done = false; break; } @@ -5344,13 +5748,28 @@ cluster_grd_cleanup_on_node_dead(int32 dead_node_id) int swept = 0; int nresids; uint64 pending_x_cleared = 0; + uint64 pcm_holders_cleaned = 0; int i; pending_x_cleared = cluster_pcm_lock_clear_pending_x_for_node(dead_node_id); - if (pending_x_cleared > 0) - ereport(DEBUG1, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " - "cleared " UINT64_FORMAT " PCM pending_x entries", - dead_node_id, pending_x_cleared))); + pcm_holders_cleaned = cluster_pcm_lock_cleanup_on_node_dead(dead_node_id); + + /* + * spec-4.6a D12: dead-node PCM residue is part of the reconfig + * convergence chain -- a leftover holder/pending-X record from the dead + * node blocks post-kill DDL with lock-acquire timeouts long after the + * remaster itself finished. Surface the cleanup as a LOG summary plus an + * assertable counter (pcm/dead_cleanup_entries). + */ + if (pending_x_cleared > 0 || pcm_holders_cleaned > 0) { + if (cluster_grd_state != NULL) + pg_atomic_fetch_add_u64(&cluster_grd_state->pcm_dead_cleanup_entries, + pending_x_cleared + pcm_holders_cleaned); + ereport(LOG, (errmsg("cluster GRD dead-node cleanup for node %d: cleared " UINT64_FORMAT + " PCM pending-X waiter(s) and " UINT64_FORMAT + " PCM holder record(s) left by the dead node", + dead_node_id, pending_x_cleared, pcm_holders_cleaned))); + } if (cluster_grd_entry_htab == NULL) { ereport(DEBUG2, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..b505360e11 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -226,6 +226,8 @@ int cluster_tm_convert_mode = CLUSTER_TM_CONVERT_MODE_CONVERT; /* spec-4.6 D4/D1 — failure-driven remaster tunables. */ int cluster_grd_remaster_wait_ms = 200; /* frozen-shard short wait before 53R9I */ int cluster_grd_rebuild_timeout_ms = 5000; /* holder-rebuild barrier deadline */ +int cluster_hw_remaster_retry_backoff_ms = 1000; +int cluster_hw_remaster_retry_max_attempts = 16; /* spec-5.4 D8 — SQ sequence lock tunables. */ int cluster_sequence_default_cache = 100; /* CREATE-time CACHE injection default */ @@ -2374,6 +2376,21 @@ cluster_init_guc(void) "request with a fresh deadline."), &cluster_grd_rebuild_timeout_ms, 5000, 100, 600000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_backoff_ms", + gettext_noop("Initial backoff before retrying a BLOCKED HW remaster worker (ms)."), + gettext_noop("Range [100, 60000]. Default 1000. Same-episode HW remaster " + "retries use exponential backoff capped at 60 seconds; the adopted " + "shards stay frozen while retrying."), + &cluster_hw_remaster_retry_backoff_ms, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_max_attempts", + gettext_noop("Maximum same-episode retries for a BLOCKED HW remaster worker."), + gettext_noop("Range [0, 1000]. Default 16. Zero disables same-episode retry. " + "Raising the value with SIGHUP lets an exhausted episode resume retrying " + "without waiting for a new reconfig episode."), + &cluster_hw_remaster_retry_max_attempts, 16, 0, 1000, PGC_SIGHUP, 0, NULL, NULL, NULL); /* spec-2.23 D11 NEW: coordinator REPORT collect deadline. */ DefineCustomIntVariable("cluster.lmd_probe_collect_timeout_ms", diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index d1ade2a760..c585268a42 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -61,6 +61,7 @@ #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/fd.h" +#include "storage/ipc.h" #include "utils/wait_event.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ @@ -74,6 +75,65 @@ #include "cluster/cluster_wal_thread.h" /* node id -> thread id */ #include "cluster/storage/cluster_undo_xlog.h" /* xl_hw_reserve, XLOG_HW_RESERVE */ +/* One worker process owns exactly one dead origin / episode. */ +static int hw_worker_dead_node = -1; +static uint64 hw_worker_episode = 0; +static bool hw_worker_armed = false; + +static uint64 +hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) +{ + uint32 backoff_ms; + TimestampTz deadline; + + backoff_ms = cluster_hw_remaster_compute_backoff_ms(cluster_hw_remaster_retry_backoff_ms, + completed_retry_attempts); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), (int)backoff_ms); + return (uint64)deadline; +} + +static void +hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) +{ + if (res == CLUSTER_HW_REMASTER_DONE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_done(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { + uint32 attempts = cluster_hw_remaster_attempts(dead_node); + + cluster_hw_remaster_set_next_attempt_at(dead_node, + hw_remaster_next_attempt_deadline(attempts)); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(dead_node, CLUSTER_HW_REMASTER_NO_DEADLINE); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + cluster_hw_remaster_set_result(dead_node, res); + } +} + +static void +hw_remaster_mark_blocked_on_exit(int code pg_attribute_unused(), Datum arg) +{ + int dead_node = DatumGetInt32(arg); + + if (!hw_worker_armed) + return; + if (dead_node != hw_worker_dead_node) + return; + if (cluster_hw_remaster_launched_episode(dead_node) != hw_worker_episode) + return; + if (cluster_hw_remaster_result(dead_node) != CLUSTER_HW_REMASTER_RUNNING) + return; + + cluster_hw_bump_failclosed(); + hw_remaster_record_terminal(dead_node, CLUSTER_HW_REMASTER_BLOCKED); +} + /* ============================================================ * Minimal per-thread WAL reader over a dead origin's WAL stream. @@ -332,10 +392,24 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; if (dead_node_id < 0 || dead_node_id == cluster_node_id) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; + if (!cluster_hw_remaster_recoverable()) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: cluster.wal_threads_dir is not configured; " + "adopted shards stay fail-closed"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "restart, or restart the dead node so a JOIN rebuild can replace " + "the failure-driven remaster."))); + return CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL; + } dead_tid = cluster_wal_thread_id_for(true, dead_node_id); - if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) + if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d maps to invalid WAL thread id %u; " + "adopted shards stay fail-closed", + dead_node_id, (unsigned)dead_tid))); return CLUSTER_HW_REMASTER_BLOCKED; + } entries = (ClusterHwSnapshotEntry *)palloc(sizeof(ClusterHwSnapshotEntry) * CLUSTER_HW_AUTHORITY_MAX); @@ -384,6 +458,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_wal_state_read_slot(dead_tid, &slot) != CLUSTER_WAL_SLOT_OK || slot.highest_lsn == 0) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d WAL state slot is unusable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } validated_min = (XLogRecPtr)slot.highest_lsn; @@ -391,6 +468,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_thread_recovery_validated_end(dead_tid, lower, validated_min, &upper) != CLUSTER_THREADREC_DONE) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d validated WAL end is unavailable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } @@ -466,29 +546,26 @@ cluster_hw_remaster_worker_main(Datum main_arg) /* The bgworker framework starts the entry point with signals blocked; unblock * before any I/O so a SIGTERM during a stuck shared-storage read / shutdown is - * delivered (the default die handler FATALs, a clean fail-closed: unmarked). */ + * delivered (the default die handler FATALs into the BLOCKED exit callback). */ BackgroundWorkerUnblockSignals(); /* Capture the live reconfig episode this rebuild is for; rebuild_origin uses * it as the staleness fence before it marks any shard rebuilt. */ episode = cluster_grd_redeclare_episode_epoch(); - res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_worker_dead_node = dead_node; + hw_worker_episode = episode; + hw_worker_armed = true; + before_shmem_exit(hw_remaster_mark_blocked_on_exit, Int32GetDatum(dead_node)); - /* Observability (S5/S7): a DONE rebuilt + adopted + marked the shards; a - * BLOCKED kept them frozen (fail-closed). NOT_APPLICABLE (HW inactive) is - * neither. */ - if (res == CLUSTER_HW_REMASTER_DONE) - cluster_hw_bump_remaster_done(); - else if (res == CLUSTER_HW_REMASTER_BLOCKED) - cluster_hw_bump_remaster_blocked(); + res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_remaster_record_terminal(dead_node, res); + hw_worker_armed = false; ereport(LOG, (errmsg("cluster HW remaster worker: dead node %d -> %s", dead_node, - res == CLUSTER_HW_REMASTER_DONE ? "done" - : res == CLUSTER_HW_REMASTER_BLOCKED ? "blocked (shards kept frozen)" - : "not applicable"))); + cluster_hw_remaster_result_name(res)))); /* Returning is a clean exit(0). On BLOCKED / abnormal exit the adopted shards - * stay unmarked, so the serve gate keeps them fail-closed (8.A) and a later - * episode relaunches the rebuild -- no abnormal-exit handler is needed. */ + * stay unmarked, so the serve gate keeps them fail-closed (8.A); the terminal + * result lets the LMON FSM retry within the same episode when appropriate. */ } /* @@ -522,6 +599,7 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod { int node; int max_node; + uint64 now; if (dead == NULL || nwords <= 0) return; @@ -538,31 +616,93 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod max_node = nwords * 64; if (max_node > CLUSTER_MAX_NODES) max_node = CLUSTER_MAX_NODES; + now = (uint64)GetCurrentTimestamp(); for (node = 0; node < max_node; node++) { + ClusterHwRemasterResult result; + ClusterHwRemasterRelaunchDecision d; + uint64 launched; + uint64 next_attempt_at; + uint32 attempts; + if ((dead[node / 64] & (UINT64CONST(1) << (node % 64))) == 0) continue; if (node == cluster_node_id) continue; /* self is never its own dead origin */ - /* Idempotent: launch once per episode per dead origin. */ - if (cluster_hw_remaster_launched_episode(node) == episode_epoch) + + launched = cluster_hw_remaster_launched_episode(node); + if (!cluster_hw_remaster_recoverable() && launched != episode_epoch) { + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, 0); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL); + launched = episode_epoch; + } + + result = cluster_hw_remaster_result(node); + attempts = cluster_hw_remaster_attempts(node); + next_attempt_at = cluster_hw_remaster_next_attempt_at(node); + d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, + next_attempt_at, now, + cluster_hw_remaster_retry_max_attempts); + + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster is structurally blocked for dead node %d; " + "adopted shards stay fail-closed", + node), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root " + "and restart, or restart the dead node so a JOIN rebuild can " + "replace the failure-driven remaster."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + cluster_hw_bump_remaster_retry_exhausted(); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster retries exhausted for dead node %d after %u " + "attempt(s); adopted shards stay fail-closed", + node, attempts), + errhint("Fix the shared HW snapshot/WAL source, then raise " + "cluster.hw_remaster_retry_max_attempts with SIGHUP to resume " + "same-episode retrying."))); continue; - /* - * Claim BEFORE registering so a same-episode re-entry does not double- - * launch; revert on a registration failure so a later tick retries (the - * adopted shards stay fail-closed meanwhile, via the P7 gate). - */ + } + if (d.action != CLUSTER_HW_REMASTER_LAUNCH_INITIAL + && d.action != CLUSTER_HW_REMASTER_LAUNCH_RETRY) + continue; + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, d.next_attempts); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_RUNNING); if (!register_one_worker(node)) { - cluster_hw_remaster_set_launched(node, 0); + cluster_hw_remaster_set_launched( + node, d.action == CLUSTER_HW_REMASTER_LAUNCH_INITIAL ? 0 : launched); + cluster_hw_remaster_set_attempts(node, attempts); + cluster_hw_remaster_set_next_attempt_at(node, next_attempt_at); + cluster_hw_remaster_set_result(node, result); ereport(WARNING, (errmsg("could not register HW remaster worker for dead node %d", node), errhint("Background worker slots are exhausted (max_worker_processes); the " "adopted shards stay fail-closed until the rebuild can run."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_RETRY) { + cluster_hw_bump_remaster_retry(); + ereport(LOG, + (errmsg("cluster HW remaster: retrying dead node %d in episode " UINT64_FORMAT + " (attempt %u/%d)", + node, episode_epoch, d.next_attempts, + cluster_hw_remaster_retry_max_attempts))); } } } + bool cluster_hw_remaster_gate_unfreeze(void) { diff --git a/src/backend/cluster/cluster_hw_shmem.c b/src/backend/cluster/cluster_hw_shmem.c index 14bf3f7586..860241e8e9 100644 --- a/src/backend/cluster/cluster_hw_shmem.c +++ b/src/backend/cluster/cluster_hw_shmem.c @@ -100,6 +100,8 @@ typedef struct ClusterHwShared { pg_atomic_uint64 not_ready_count; /* 53RA6 serve-gate (shard adopted, unrebuilt) */ pg_atomic_uint64 remaster_done_count; /* online-remaster HW rebuild DONE (S5/S7) */ pg_atomic_uint64 remaster_blocked_count; /* online-remaster HW rebuild fail-closed (S5/S7) */ + pg_atomic_uint64 remaster_retry_count; /* same-episode retry launches */ + pg_atomic_uint64 remaster_retry_exhausted_count; /* same-episode retry cap reached */ pg_atomic_uint32 hw_rebuilt_generation[PGRAC_GRD_SHARD_COUNT]; /* §3.1b R4/R9 gate */ /* * remaster_launched_episode[node] -- the reconfig episode for which the GRD @@ -111,6 +113,9 @@ typedef struct ClusterHwShared { * race on this field. */ pg_atomic_uint64 remaster_launched_episode[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_result[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_attempts[CLUSTER_MAX_NODES]; + pg_atomic_uint64 remaster_next_attempt_at[CLUSTER_MAX_NODES]; } ClusterHwShared; /* @@ -196,12 +201,18 @@ cluster_hw_shmem_init(void) pg_atomic_init_u64(&hw_state->not_ready_count, 0); pg_atomic_init_u64(&hw_state->remaster_done_count, 0); pg_atomic_init_u64(&hw_state->remaster_blocked_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_exhausted_count, 0); /* No shard rebuilt yet; 0 matches a never-remastered shard's GRD * master_generation (0), so boot / steady state serves. */ for (s = 0; s < PGRAC_GRD_SHARD_COUNT; s++) pg_atomic_init_u32(&hw_state->hw_rebuilt_generation[s], 0); - for (s = 0; s < CLUSTER_MAX_NODES; s++) + for (s = 0; s < CLUSTER_MAX_NODES; s++) { pg_atomic_init_u64(&hw_state->remaster_launched_episode[s], 0); + pg_atomic_init_u32(&hw_state->remaster_result[s], (uint32)CLUSTER_HW_REMASTER_NONE); + pg_atomic_init_u32(&hw_state->remaster_attempts[s], 0); + pg_atomic_init_u64(&hw_state->remaster_next_attempt_at[s], 0); + } } memset(&hctl, 0, sizeof(hctl)); @@ -408,6 +419,81 @@ cluster_hw_remaster_set_launched(int node_id, uint64 episode) pg_atomic_write_u64(&hw_state->remaster_launched_episode[node_id], episode); } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return CLUSTER_HW_REMASTER_NONE; + return (ClusterHwRemasterResult)pg_atomic_read_u32(&hw_state->remaster_result[node_id]); +} + +void +cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_result[node_id], (uint32)result); +} + +uint32 +cluster_hw_remaster_attempts(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u32(&hw_state->remaster_attempts[node_id]); +} + +void +cluster_hw_remaster_set_attempts(int node_id, uint32 attempts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_attempts[node_id], attempts); +} + +uint64 +cluster_hw_remaster_next_attempt_at(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&hw_state->remaster_next_attempt_at[node_id]); +} + +void +cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u64(&hw_state->remaster_next_attempt_at[node_id], ts); +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result) +{ + switch (result) { + case CLUSTER_HW_REMASTER_NONE: + return "none"; + case CLUSTER_HW_REMASTER_RUNNING: + return "running"; + case CLUSTER_HW_REMASTER_DONE: + return "done"; + case CLUSTER_HW_REMASTER_BLOCKED: + return "blocked"; + case CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL: + return "blocked_structural"; + case CLUSTER_HW_REMASTER_NOT_APPLICABLE: + return "not_applicable"; + } + return "unknown"; +} + +bool +cluster_hw_remaster_recoverable(void) +{ + return !cluster_hw_authority_active() + || (cluster_wal_threads_dir != NULL && cluster_wal_threads_dir[0] != '\0'); +} + /* ============================================================ * Checkpoint write + recovery load (§3.1b R1/R2; PG-core hooked). @@ -747,6 +833,16 @@ cluster_hw_bump_remaster_blocked(void) { HW_BUMP(remaster_blocked_count); } +void +cluster_hw_bump_remaster_retry(void) +{ + HW_BUMP(remaster_retry_count); +} +void +cluster_hw_bump_remaster_retry_exhausted(void) +{ + HW_BUMP(remaster_retry_exhausted_count); +} uint64 cluster_hw_alloc_count(void) @@ -788,3 +884,13 @@ cluster_hw_remaster_blocked_count(void) { return HW_READ(remaster_blocked_count); } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return HW_READ(remaster_retry_count); +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return HW_READ(remaster_retry_exhausted_count); +} diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index c3b9a39c24..8bec6ddd71 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,6 +59,7 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" +#include "cluster/cluster_cssd.h" #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" @@ -1044,6 +1045,8 @@ cluster_lms_native_probe_dispatch(uint32 slot_idx) conf_node = cluster_conf_lookup_node(candidate); if (conf_node == NULL) continue; + if (cluster_cssd_get_peer_state(candidate) == CLUSTER_CSSD_PEER_DEAD) + continue; if (!native_probe_node_bit(candidate, &peer_bit)) { pg_atomic_fetch_add_u64(&cluster_lms_state->native_probe_timeout_count, 1); slot->final_status = CLUSTER_NATIVE_PROBE_FINAL_TIMEOUT; diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 545ac33480..b9de7f6401 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -724,6 +724,96 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) return cleared; } +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node) +{ + HASH_SEQ_STATUS scan; + struct GrdEntry *entry; + uint64 cleaned = 0; + uint32 dead_bit; + + if (cluster_pcm_htab == NULL || dead_node < 0 || dead_node >= 32) + return 0; + + dead_bit = (uint32)1u << (uint32)dead_node; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + hash_seq_init(&scan, cluster_pcm_htab); + while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool changed = false; + bool broadcast_needed = false; + bool master_holder_was_dead; + PcmState before_state; + PcmState after_state; + uint32 s_bitmap; + uint32 pi_bitmap; + + /* Cheap unlocked filter; rechecked under entry_lock below. */ + if (entry->x_holder_node != dead_node + && (pg_atomic_read_u32(&entry->s_holders_bitmap) & dead_bit) == 0 + && (pg_atomic_read_u32(&entry->pi_holders_bitmap) & dead_bit) == 0 + && (!pcm_master_holder_is_valid(entry) + || (int32)entry->master_holder.node_id != dead_node)) + continue; + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + before_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + master_holder_was_dead + = pcm_master_holder_is_valid(entry) && (int32)entry->master_holder.node_id == dead_node; + + if (entry->x_holder_node == dead_node) { + entry->x_holder_node = -1; + changed = true; + } + s_bitmap = pg_atomic_read_u32(&entry->s_holders_bitmap); + if ((s_bitmap & dead_bit) != 0) { + s_bitmap &= ~dead_bit; + pg_atomic_write_u32(&entry->s_holders_bitmap, s_bitmap); + changed = true; + } + pi_bitmap = pg_atomic_read_u32(&entry->pi_holders_bitmap); + if ((pi_bitmap & dead_bit) != 0) { + pg_atomic_write_u32(&entry->pi_holders_bitmap, pi_bitmap & ~dead_bit); + changed = true; + } + + if (entry->x_holder_node >= 0) { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); + if (master_holder_was_dead) + pcm_master_holder_set_node(entry, entry->x_holder_node); + } else if (s_bitmap != 0) { + int32 next_holder = pcm_lowest_set_bit_node(s_bitmap); + + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_S); + if (master_holder_was_dead) { + if (next_holder >= 0) + pcm_master_holder_set_node(entry, next_holder); + else + pcm_master_holder_clear(entry); + } + } else { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_N); + if (pcm_master_holder_is_valid(entry)) + pcm_master_holder_clear(entry); + } + if (master_holder_was_dead) + changed = true; + after_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + if (changed && after_state != before_state) + broadcast_needed = true; + + LWLockRelease(&entry->entry_lock.lock); + + if (broadcast_needed) + ConditionVariableBroadcast(&entry->wait_cv); + if (changed) + cleaned++; + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + + return cleaned; +} + /* ======================================================================== * PGRAC MODIFICATIONS by SqlRush — spec-5.13 D5 (clean-leave PCM release). @@ -2199,6 +2289,41 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + + /* + * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after + * fail-stop. Local-master state=S with other live S holders but no local + * S bit used to fall through to the tag-only X acquire, which fail-closed + * as "no local S residency". This entry point is buffer-aware: the + * caller has already read or initialized the BufferDesc before asking for + * X, so first register a local S residency, then reuse the existing + * local S->X invalidate/upgrade path. If the invalidate cannot be proven, + * drop the temporary S claim and rethrow the same fail-closed error. + */ + if (master_state == PCM_LOCK_MODE_S && cluster_node_id >= 0 && cluster_node_id < 32) { + uint32 self_bit = (uint32)1u << (uint32)cluster_node_id; + + if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { + struct GrdEntry *entry; + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + if (!cluster_gcs_block_local_x_upgrade(tag)) { + cluster_pcm_lock_release(tag); + ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), + errhint("Remote S holders did not all acknowledge in time; " + "retry the statement."))); + } + + entry = pcm_find_entry(tag); + if (entry != NULL) { + pcm_entry_lock_exclusive(entry); + entry->s_holder_refcount_local = 0; + LWLockRelease(&entry->entry_lock.lock); + } + return true; + } + } } cluster_pcm_lock_acquire(tag, mode); diff --git a/src/backend/cluster/cluster_wal_thread.c b/src/backend/cluster/cluster_wal_thread.c index 17edd98586..b5f7cab947 100644 --- a/src/backend/cluster/cluster_wal_thread.c +++ b/src/backend/cluster/cluster_wal_thread.c @@ -45,7 +45,9 @@ #include #include "access/xlog_internal.h" /* XLOGDIR */ +#include "cluster/cluster_conf.h" #include "cluster/cluster_guc.h" +#include "cluster/cluster_hw.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_wal_state.h" /* spec-4.2 ensure() */ @@ -385,8 +387,23 @@ cluster_wal_thread_init(void) cluster_wal_thread_shmem->dir_configured = dir_set ? 1 : 0; } - if (!dir_set) + if (!dir_set) { + if (cluster_shared_catalog && cluster_conf_has_peers()) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cluster.shared_catalog requires cluster.wal_threads_dir in a " + "multi-node cluster"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "initialise each node with pgrac-init --wal-threads-dir."))); + if (!cluster_hw_remaster_recoverable()) + ereport(WARNING, + (errmsg("cluster HW remaster is not recoverable because " + "cluster.wal_threads_dir is unset"), + errhint("A dead node's adopted HW authority cannot be rebuilt from per-thread " + "WAL until cluster.wal_threads_dir is configured and the node is " + "restarted."))); return; /* flat layout: identity stamping only (Q3-A) */ + } /* * Configuration coherence, fail-closed (L58: enumerate the diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..a098d53ed1 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -275,6 +275,8 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 recovery_event_old_epoch; pg_atomic_uint64 recovery_redeclare_generation; pg_atomic_uint64 recovery_barrier_deadline; + pg_atomic_uint32 recovery_event_coordinator; + pg_atomic_uint64 recovery_done_epoch_at_accept; /* * spec-4.6 P0-1 (Fable review) — the epoch this episode is LOCKED to. @@ -294,6 +296,7 @@ typedef struct ClusterGrdShared { * last announced "local rebind barrier complete" (REDECLARE_DONE). * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_event_id[CLUSTER_MAX_NODES]; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -311,6 +314,10 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 block_path_failclosed_count; /* D4 GCS/PCM 53R9K (L12) */ pg_atomic_uint64 unaffected_holder_survived_count; /* L13 sweep-scope guard */ pg_atomic_uint64 stale_holder_swept_count; /* P6 post-barrier sweep (L15) */ + pg_atomic_uint64 cluster_gate_timeout_count; /* WAIT_CLUSTER watchdog only */ + pg_atomic_uint64 wait_epoch_escape_count; /* proof-carrying equal-epoch advance */ + pg_atomic_uint64 + pcm_dead_cleanup_entries; /* spec-4.6a D12: dead-node PCM holder/pending-X records cleaned */ /* spec-5.1b D9 — convert state-machine observability counters. * convert_queue_full reuses the existing converts_full_count above; @@ -609,6 +616,18 @@ typedef enum ClusterGrdRecoveryState { } ClusterGrdRecoveryState; extern void cluster_grd_recovery_lmon_tick(void); +/* spec-4.6a D5/D6 — recovery state observability for pg_cluster_state. */ +extern uint32 cluster_grd_recovery_state_value(void); +extern const char *cluster_grd_recovery_state_name(uint32 state); +extern uint64 cluster_grd_recovery_last_event_id(void); +extern uint64 cluster_grd_recovery_event_old_epoch(void); +extern uint64 cluster_grd_recovery_episode_epoch_value(void); +extern uint32 cluster_grd_recovery_event_coordinator(void); +extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); +extern uint64 cluster_grd_recovery_done_event_id_for(int32 node); +extern int cluster_grd_recovery_block_redeclare_cursor(void); +extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); +extern bool cluster_grd_recovery_block_redeclare_done(void); extern uint64 cluster_grd_redeclare_generation(void); /* spec-4.6 P0-1 — the epoch the current episode is locked to (0 = none). */ extern uint64 cluster_grd_redeclare_episode_epoch(void); @@ -626,7 +645,7 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch); +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); @@ -650,6 +669,9 @@ typedef struct ClusterGrdRecoveryCounters { uint64 block_path_failclosed; uint64 unaffected_holder_survived; uint64 stale_holder_swept; + uint64 cluster_gate_timeout; + uint64 wait_epoch_escape; + uint64 pcm_dead_cleanup_entries; /* spec-5.16 D5 — join-direction remaster counters (distinct from the * failure-driven remaster_* above; §8 Q6-A). */ uint64 join_remaster_started; diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..d091b2be3f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -399,6 +399,8 @@ extern int cluster_ges_reply_wait_max_entries; * 5000ms; SIGHUP). */ extern int cluster_grd_remaster_wait_ms; extern int cluster_grd_rebuild_timeout_ms; +extern int cluster_hw_remaster_retry_backoff_ms; +extern int cluster_hw_remaster_retry_max_attempts; /* spec-5.4 D8: SQ sequence lock tunables. */ extern int cluster_sequence_default_cache; diff --git a/src/include/cluster/cluster_hw.h b/src/include/cluster/cluster_hw.h index 89656a384f..e021c1c445 100644 --- a/src/include/cluster/cluster_hw.h +++ b/src/include/cluster/cluster_hw.h @@ -312,12 +312,30 @@ extern uint32 cluster_hw_shard_rebuilt_generation(uint32 shard_id); extern void cluster_hw_mark_shard_rebuilt(uint32 shard_id, uint32 generation); /* - * Per-dead-origin online-remaster launch idempotency (S5d). The GRD FSM records - * the episode it last launched a rebuild worker for a dead origin under, and - * skips relaunching while the value equals the current episode. + * Per-dead-origin online-remaster launch state (S5d / spec-4.6a). The GRD FSM + * records the episode it last launched a rebuild worker for a dead origin under, + * and the worker records a terminal result. BLOCKED is retryable within the same + * episode; BLOCKED_STRUCTURAL is not. */ +typedef enum ClusterHwRemasterResult { + CLUSTER_HW_REMASTER_NONE = 0, + CLUSTER_HW_REMASTER_RUNNING, + CLUSTER_HW_REMASTER_DONE, + CLUSTER_HW_REMASTER_BLOCKED, + CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, + CLUSTER_HW_REMASTER_NOT_APPLICABLE, +} ClusterHwRemasterResult; + extern uint64 cluster_hw_remaster_launched_episode(int node_id); extern void cluster_hw_remaster_set_launched(int node_id, uint64 episode); +extern ClusterHwRemasterResult cluster_hw_remaster_result(int node_id); +extern void cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result); +extern uint32 cluster_hw_remaster_attempts(int node_id); +extern void cluster_hw_remaster_set_attempts(int node_id, uint32 attempts); +extern uint64 cluster_hw_remaster_next_attempt_at(int node_id); +extern void cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts); +extern const char *cluster_hw_remaster_result_name(ClusterHwRemasterResult result); +extern bool cluster_hw_remaster_recoverable(void); /* * cluster_hw_apply_hwm -- redo / remaster rebuild: create-if-absent and raise @@ -383,6 +401,8 @@ extern uint64 cluster_hw_failclosed_count(void); extern uint64 cluster_hw_not_ready_count(void); extern uint64 cluster_hw_remaster_done_count(void); extern uint64 cluster_hw_remaster_blocked_count(void); +extern uint64 cluster_hw_remaster_retry_count(void); +extern uint64 cluster_hw_remaster_retry_exhausted_count(void); extern void cluster_hw_bump_alloc(void); extern void cluster_hw_bump_authority_create(void); extern void cluster_hw_bump_reserve_wal(void); @@ -391,6 +411,8 @@ extern void cluster_hw_bump_failclosed(void); extern void cluster_hw_bump_not_ready(void); extern void cluster_hw_bump_remaster_done(void); extern void cluster_hw_bump_remaster_blocked(void); +extern void cluster_hw_bump_remaster_retry(void); +extern void cluster_hw_bump_remaster_retry_exhausted(void); #endif /* !FRONTEND */ diff --git a/src/include/cluster/cluster_hw_remaster.h b/src/include/cluster/cluster_hw_remaster.h index e1bf3ba529..e2fcc481f8 100644 --- a/src/include/cluster/cluster_hw_remaster.h +++ b/src/include/cluster/cluster_hw_remaster.h @@ -47,17 +47,91 @@ #ifndef CLUSTER_HW_REMASTER_H #define CLUSTER_HW_REMASTER_H -/* - * Result of an online-remaster HW authority rebuild for one dead origin. Only - * DONE marks the adopted shards rebuilt (opens the serve gate); BLOCKED and - * NOT_APPLICABLE never do, so a shard whose HWM is not provably rebuilt stays - * fail-closed (8.A). - */ -typedef enum ClusterHwRemasterResult { - CLUSTER_HW_REMASTER_DONE = 0, /* rebuilt + adoption snapshot durable + shards marked */ - CLUSTER_HW_REMASTER_BLOCKED, /* fail-closed: keep shards frozen (53RA6) */ - CLUSTER_HW_REMASTER_NOT_APPLICABLE, /* HW authority inactive / no adopted shard / bad input */ -} ClusterHwRemasterResult; +#include "cluster/cluster_hw.h" /* ClusterHwRemasterResult */ + +#define CLUSTER_HW_REMASTER_NO_DEADLINE PG_UINT64_MAX +#define CLUSTER_HW_REMASTER_BACKOFF_CAP_MS 60000 + +typedef enum ClusterHwRemasterLaunchAction { + CLUSTER_HW_REMASTER_LAUNCH_SKIP = 0, + CLUSTER_HW_REMASTER_LAUNCH_INITIAL, + CLUSTER_HW_REMASTER_LAUNCH_RETRY, + CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED, + CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL, +} ClusterHwRemasterLaunchAction; + +typedef struct ClusterHwRemasterRelaunchDecision { + ClusterHwRemasterLaunchAction action; + uint32 next_attempts; + uint64 next_attempt_at; + ClusterHwRemasterResult next_result; +} ClusterHwRemasterRelaunchDecision; + +static inline uint32 +cluster_hw_remaster_compute_backoff_ms(int base_ms, uint32 completed_retry_attempts) +{ + uint64 ms; + uint32 shift; + + if (base_ms < 1) + base_ms = 1; + ms = (uint64)base_ms; + shift = completed_retry_attempts > 0 ? completed_retry_attempts - 1 : 0; + while (shift-- > 0 && ms < CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms <<= 1; + if (ms > CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms = CLUSTER_HW_REMASTER_BACKOFF_CAP_MS; + return (uint32)ms; +} + +static inline ClusterHwRemasterRelaunchDecision +cluster_hw_remaster_relaunch_decide(uint64 launched_episode, uint64 current_episode, + ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int retry_max_attempts) +{ + ClusterHwRemasterRelaunchDecision d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_SKIP; + d.next_attempts = attempts; + d.next_attempt_at = next_attempt_at; + d.next_result = result; + + if (current_episode == 0) + return d; + if (launched_episode != current_episode || result == CLUSTER_HW_REMASTER_NONE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_INITIAL; + d.next_attempts = 0; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; + } + + if (result == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (result != CLUSTER_HW_REMASTER_BLOCKED) + return d; + + if (retry_max_attempts <= 0 || attempts >= (uint32)retry_max_attempts) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE && now < next_attempt_at) + return d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_RETRY; + d.next_attempts = attempts + 1; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; +} #ifndef FRONTEND @@ -81,9 +155,9 @@ extern ClusterHwRemasterResult cluster_hw_remaster_rebuild_origin(int dead_node_ /* * cluster_hw_remaster_worker_main -- dynamic-bgworker entry point (main_arg = the * dead origin node id). Captures the live reconfig episode and drives - * cluster_hw_remaster_rebuild_origin off the LMON tick. An abnormal exit simply - * leaves the shards unmarked -> the serve gate keeps them fail-closed (8.A), so - * no abnormal-exit handler is needed. + * cluster_hw_remaster_rebuild_origin off the LMON tick. A before_shmem_exit + * callback records an abnormal exit as retryable BLOCKED so the same episode can + * self-heal instead of staying pinned on the launch idempotency bit. */ extern void cluster_hw_remaster_worker_main(Datum main_arg); diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index d88d82dc8f..3fb909557f 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -415,6 +415,12 @@ extern void cluster_pcm_lock_clear_pending_x(BufferTag tag); extern int32 cluster_pcm_lock_query_pending_x_requester(BufferTag tag); extern uint64 cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node); +/* PGRAC: spec-4.6a BUG-C2 — failure-path PCM holder cleanup. Removes a + * DEAD node from X/S/PI holder records in this master's PCM directory and + * demotes entries to N when no live holder remains. Idempotent under repeated + * dead-sweep ticks. */ +extern uint64 cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node); + /* PGRAC: spec-5.13 D5 (clean-leave PCM release) — a leaving node clears its * OWN holder records (X / S / PI) from the local PCM directory after the GCS * flush seam has persisted all dirty X blocks (CL-I5). Demotes an entry's diff --git a/src/include/cluster/cluster_thread_recovery.h b/src/include/cluster/cluster_thread_recovery.h index c455b77436..c7be25e6b0 100644 --- a/src/include/cluster/cluster_thread_recovery.h +++ b/src/include/cluster/cluster_thread_recovery.h @@ -376,6 +376,16 @@ cluster_thread_recovery_gate_decide(ClusterThreadRecScope scope, const uint64 *d return false; /* every dead origin materialized -> ready to unfreeze */ } +static inline bool +cluster_thread_recovery_materialization_gate_enabled(bool guc_on, bool has_peers, + bool shared_fs_backend, + int live_survivor_count) +{ + return cluster_thread_recovery_decide_scope(guc_on, has_peers, shared_fs_backend, + live_survivor_count) + == CLUSTER_THREADREC_SCOPE_APPLICABLE; +} + /* * cluster_thread_recovery_replay_epoch_aborts -- the L235 episode-epoch * staleness guard (spec-4.11 3b-4b). The per-thread replay slot stamps the GRD diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..f2565c8fe1 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,15 @@ # ---------- -# L1: pg_cluster_state.pcm category has 22 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5) +# L1: pg_cluster_state.pcm category has 23 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5 +# + spec-4.6a D12 dead_cleanup_entries) # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '22', - 'L1 pg_cluster_state.pcm category has 22 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5)'); + '23', + 'L1 pg_cluster_state.pcm category has 23 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5 + spec-4.6a D12)'); # ---------- diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 9c156e1e53..509c2f50ff 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -30,13 +30,18 @@ # retryable transient, NOT an HW fault -- so this leg retries and SKIPs # (never fails) when only that orthogonal recovery is outstanding. # -# Negative (corrupt the dead master's snapshot -> fail-closed): +# Negative/self-heal extensions (spec-4.6a): # L6 a fresh pair; corrupt node0's HW snapshot on shared storage, then # kill node0. The survivor's rebuild must FAIL CLOSED # (hw.remaster_blocked_count advances) -- it never trusts a corrupt # snapshot, never auto-creates at 0, never reads FileSize as the -# authority. The affected shards stay frozen (P7 gate), so no extend -# can silently re-hand a block. +# authority. +# L7 hide node0's snapshot, kill node0, observe BLOCKED, then restore +# the same file. The same reconfig episode must retry and converge +# to DONE without waiting for a later episode. +# L8 keep the snapshot corrupt with retry_max_attempts=2. The survivor +# must report retry_exhausted once and keep fail-closed; DONE must not +# advance. # # Harness: ClusterPair shared_data + wal_threads_root + 3 voting disks. # Mirrors the kill/reconfig pattern of t/249 / t/274. @@ -181,17 +186,22 @@ sub retry_sql { my ($node, $sql) = @_; my $last = ''; + my $last_retryable = 0; for my $attempt (1 .. 30) { my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); - return (1, $out) if defined $rc && $rc == 0; - $last = $err // ''; + my $combined = join("\n", grep { defined $_ && $_ ne '' } ($out, $err)); + return (1, $out, 0) + if defined $rc && $rc == 0 && $combined !~ /ERROR:/; + + $last = $combined; + $last_retryable = + $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; # retry only the orthogonal post-reconfig settling transients - last - unless $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; + last unless $last_retryable; usleep(500_000); } - return (0, $last); + return (0, $last, $last_retryable); } # L5a (the robust HW proof): the rebuilt authority is SERVEABLE -- the HW serve @@ -210,22 +220,213 @@ sub retry_sql # that is NOT an HW fault. Retry, and SKIP (never fail) when only that orthogonal # recovery is outstanding; a 53RA6 from the HW gate would NOT be retried and would # fail the test. -my ($ext_ok, $ext_res) = +my ($ext_ok, $ext_res, $ext_retryable) = retry_sql($n1, 'INSERT INTO r1 SELECT g,g,g,g FROM generate_series(4001,4500) g'); SKIP: { - skip - "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", - 1 - unless $ext_ok; - my ($cnt_ok, $cnt) = retry_sql($n1, 'SELECT count(*) FROM r1'); - is($cnt, '4500', - 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') - or diag("count result: $cnt"); + if (!$ext_ok) + { + skip + "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", + 1 + if $ext_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("extend result: $ext_res"); + } + else + { + my ($cnt_ok, $cnt, $cnt_retryable) = retry_sql($n1, 'SELECT count(*) FROM r1'); + if (!$cnt_ok) + { + skip + "post-remaster count gated by orthogonal visibility recovery (not HW): $cnt", + 1 + if $cnt_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("count result: $cnt"); + } + else + { + is($cnt, '4500', + 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') + or diag("count result: $cnt"); + } + } } $pair->stop_pair; +# ====================================================================== +# SELF-HEAL: first attempt BLOCKED, same episode retries after repair. +# ====================================================================== + +my $heal = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_selfheal', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +$heal->start_pair; +my $h0 = $heal->node0; +my $h1 = $heal->node1; + +ok(poll_until($h1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L7: self-heal pair in quorum'); +poll_until( + $h1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($h0, $h1, "s$_", 4000) for (1 .. 6); +$h0->safe_psql('postgres', 'CHECKPOINT'); +$h1->safe_psql('postgres', 'CHECKPOINT'); + +my $hsnap = $heal->shared_data_root . "/global/pg_hw_snapshot.0"; +my $hhold = "$hsnap.hold"; +ok(-f $hsnap, 'L7: node0 HW snapshot exists before self-heal injection'); +rename($hsnap, $hhold) or die "rename $hsnap -> $hhold: $!"; + +my $h_done_before = hw_counter($h1, 'remaster_done_count'); +my $h_blocked_before = hw_counter($h1, 'remaster_blocked_count'); +my $h_retry_before = hw_counter($h1, 'remaster_retry_count'); +$heal->kill_node9(0); +ok( poll_until( + $h1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30), + 'L7: self-heal pair declared node0 dead'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_blocked_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_blocked_count'", + 'true', 40), + 'L7: first HW rebuild attempt failed closed while the snapshot was hidden'); + +rename($hhold, $hsnap) or die "rename $hhold -> $hsnap: $!"; +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_retry_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_count'", + 'true', 40), + 'L7: same reconfig episode launched a HW remaster retry'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_done_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_done_count'", + 'true', 60), + 'L7: restored snapshot let the same-episode retry converge to DONE'); +is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, + 'L7: self-heal path did not exhaust the retry budget'); + +# spec-4.6a section 4 (D8): DONE must actually unfreeze (P7) -- the survivor +# extends a table that lived on shards the dead master owned; a frozen shard +# would fail this INSERT with the remastering error. +{ + my ($ext_ok, $ext_res) = (0, ''); + for my $try (1 .. 30) + { + $ext_res = eval { + $h1->safe_psql('postgres', + 'INSERT INTO s1 SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 1; + } ? 'ok' : ($@ // 'error'); + if ($ext_res eq 'ok') { $ext_ok = 1; last; } + sleep 1; + } + ok($ext_ok, 'L7: survivor extend succeeds after same-episode DONE (P7 unfreeze)') + or diag($ext_res); +} + +$heal->stop_pair; + +# ====================================================================== +# NEGATIVE: corrupt snapshot persists -> retries exhaust and stay closed. +# ====================================================================== + +my $exh = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_exhausted', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', + ]); +$exh->start_pair; +my $e0 = $exh->node0; +my $e1 = $exh->node1; + +ok(poll_until($e1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L8: exhausted pair in quorum'); +poll_until( + $e1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($e0, $e1, "x$_", 4000) for (1 .. 6); +$e0->safe_psql('postgres', 'CHECKPOINT'); +$e1->safe_psql('postgres', 'CHECKPOINT'); + +my $esnap = $exh->shared_data_root . "/global/pg_hw_snapshot.0"; +ok(-f $esnap, 'L8: node0 HW snapshot exists before exhaustion injection'); +open(my $efh, '>', $esnap) or die "open $esnap: $!"; +binmode $efh; +print $efh ("\xDE\xAD\xBE\xEF" x 64); +close $efh; + +my $e_done_before = hw_counter($e1, 'remaster_done_count'); +my $e_exh_before = hw_counter($e1, 'remaster_retry_exhausted_count'); +$exh->kill_node9(0); +poll_until( + $e1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30); +ok( poll_until( + $e1, + "SELECT (value::bigint > $e_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L8: persistent corrupt snapshot exhausted the same-episode retry budget'); +is(hw_counter($e1, 'remaster_done_count'), $e_done_before, + 'L8: DONE did not advance after retry exhaustion on corrupt snapshot'); + +# spec-4.6a section 4 (D8/L408): after exhaustion the WAIT_CLUSTER watchdog +# keeps sounding (counter + WARNING line) while shards stay frozen. +ok( poll_until( + $e1, + q{SELECT (value::bigint > 0)::text FROM pg_cluster_state } + . q{WHERE category='grd_recovery' AND key='cluster_gate_timeout'}, + 'true', 30), + 'L8: WAIT_CLUSTER watchdog counter advances while exhausted episode stays frozen'); +{ + my $log = PostgreSQL::Test::Utils::slurp_file($e1->logfile); + like( + $log, + qr/cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen/, + 'L8: watchdog WARNING appears in the survivor log'); +} + +$exh->stop_pair; + # ====================================================================== # NEGATIVE: corrupt the dead master's snapshot -> rebuild fails closed. # ====================================================================== @@ -242,6 +443,9 @@ sub retry_sql 'cluster.ges_retransmit_max_attempts = 0', 'cluster.cssd_heartbeat_interval_ms = 1000', 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', ]); $neg->start_pair; my $m0 = $neg->node0; @@ -277,6 +481,7 @@ sub retry_sql } my $blocked_before = hw_counter($m1, 'remaster_blocked_count'); +my $neg_exh_before = hw_counter($m1, 'remaster_retry_exhausted_count'); $neg->kill_node9(0); poll_until( $m1, @@ -291,6 +496,12 @@ sub retry_sql . "WHERE category='hw' AND key='remaster_blocked_count'", 'true', 40), 'L6: HW rebuild FAILED CLOSED on the corrupt snapshot (remaster_blocked advanced; no auto-create-0)'); +ok( poll_until( + $m1, + "SELECT (value::bigint > $neg_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L6: corrupt snapshot eventually exhausts bounded same-episode retries'); $neg->stop_pair; diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..86970cb280 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -94,6 +94,8 @@ my $shared_root = PostgreSQL::Test::Utils::tempdir(); mkdir "$shared_root/global" or die "mkdir shared global: $!"; +my $wal_root = PostgreSQL::Test::Utils::tempdir(); + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); my @disks; for my $i (0 .. 2) @@ -114,7 +116,7 @@ # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). # ---------- my $node0 = PostgreSQL::Test::Cluster->new('sc_ddl_node0'); -$node0->init(allows_streaming => 1); +$node0->init(allows_streaming => 1, extra => [ '-X', "$wal_root/thread_1" ]); $node0->start; $node0->backup('scb'); $node0->stop; @@ -122,6 +124,24 @@ my $node1 = PostgreSQL::Test::Cluster->new('sc_ddl_node1'); $node1->init_from_backup($node0, 'scb'); +# Relocate node1's backup-copied WAL into its shared thread dir. shared_catalog +# multi-node formation is fail-fast without cluster.wal_threads_dir, and the WAL +# thread validator requires pg_wal to resolve to the configured thread_N dir. +{ + my $pgwal = $node1->data_dir . '/pg_wal'; + my $wal2 = "$wal_root/thread_2"; + mkdir $wal2 or die "mkdir $wal2: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal2/$e") or die "rename $pgwal/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal2, $pgwal) or die "symlink $pgwal -> $wal2: $!"; +} + # Config common to the shared-catalog feature (both nodes, both phases). my $sc_common = <append_conf('postgresql.conf', $cluster_conf); @@ -819,4 +840,23 @@ sub oid_authority_hw qr/cluster\.shared_catalog is off but the shared tree holds/, 'L7: the refusal is the designed off-flip vet FATAL (never a stale serve)'); +# ---------- +# L8 (spec-4.6a D11 level-2 fail-fast): a multi-node shared_catalog=on boot +# without cluster.wal_threads_dir is refused at startup -- that shape cannot +# rebuild a dead node's HW authority from per-thread WAL, so a node failure +# would be permanently unrecoverable (the silent-BLOCKED wedge of +# spec-4.6a section 0). Flip shared_catalog back on and blank the WAL +# threads root; the boot must fail on the D11 vet, not come up degraded. +# ---------- +$node1->append_conf('postgresql.conf', "cluster.shared_catalog = on +"); +$node1->append_conf('postgresql.conf', "cluster.wal_threads_dir = '' +"); +my $nowalret = $node1->start(fail_ok => 1); +is($nowalret, 0, 'L8: multi-node shared_catalog boot without wal_threads_dir fails'); +my $nowallog = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($nowallog, + qr/cluster\.shared_catalog requires cluster\.wal_threads_dir in a multi-node cluster/, + 'L8: the refusal is the spec-4.6a D11 startup FATAL with the config errhint'); + done_testing(); diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl new file mode 100644 index 0000000000..883860c0fb --- /dev/null +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -0,0 +1,310 @@ +#------------------------------------------------------------------------- +# +# 362_shared_catalog_4node_kill_selfheal.pl +# spec-4.6a D9 -- 4-node shared_catalog fail-stop self-heal. +# +# Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) +# so a killed node's HW authority is recoverable from its per-thread WAL. +# After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means +# the test harness is misconfigured, and lack of convergence is a real FAIL. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterQuad; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared_catalog 4-node kill self-heal requires --enable-cluster'; +} + +sub poll_until +{ + my ($node, $query, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) }; + return 1 if defined $last && $last eq $want; + usleep(300_000); + } + diag("poll_until timed out: '$query' last='" . ($last // '(undef)') + . "' want='$want'"); + return 0; +} + +sub retry_sql +{ + my ($node, $sql, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); + return (1, $out // '') if defined $rc && $rc == 0; + $last = $err // ''; + usleep(500_000); + } + return (0, $last); +} + +sub state_counter +{ + my ($node, $cat, $key) = @_; + my $v = eval { + $node->safe_psql('postgres', + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'") + }; + return defined $v && $v ne '' ? $v + 0 : 0; +} + +sub sum_hw +{ + my ($quad, $key, @idx) = @_; + my $sum = 0; + for my $i (@idx) + { + $sum += state_counter($quad->node($i), 'hw', $key); + } + return $sum; +} + +sub logs_contain +{ + my ($quad, $pattern, @idx) = @_; + for my $i (@idx) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + } + return 0; +} + +sub log_until +{ + my ($quad, $idx, $pattern, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($idx)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + usleep(300_000); + } + diag("log_until timed out on node$idx waiting for /$pattern/"); + return 0; +} + +sub startup_env_blocker +{ + my ($quad) = @_; + my $log = ''; + if ($quad) + { + for my $i (0 .. 3) + { + $log .= eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + } + } + return $log =~ /could not create shared memory segment|No space left on device|No space left|SHMMNI|semget/i; +} + +my $quad = eval { + PostgreSQL::Test::ClusterQuad->new_quad( + 'sc4_selfheal', + shared_catalog => 1, + quorum_voting_disks => 3, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.lms_enabled = on', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +}; +if (!$quad) +{ + my $err = $@ || 'unknown constructor failure'; + plan skip_all => "L1 shared_catalog ClusterQuad formation hit local substrate blocker: $err" + if $err =~ /could not create shared memory|No space left|SHMMNI|semget/i; + BAIL_OUT("ClusterQuad(shared_catalog) constructor failed: $err"); +} + +my $started = 1; +for my $i (1 .. 3) +{ + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $quad->node($i)->data_dir, + '-l', $quad->node($i)->logfile, + '-o', "--cluster-name=sc4_selfheal_node$i", 'start'); +} +unless ($quad->node(0)->start(fail_ok => 1)) +{ + $started = 0; +} +for my $i (1 .. 3) +{ + $quad->node($i)->_update_pid(-1); +} +if (!$started) +{ + if (startup_env_blocker($quad)) + { + eval { $quad->stop_quad; }; + plan skip_all => 'L1 shared_catalog ClusterQuad formation hit local shared-memory/resource limit'; + } + my $fatal = logs_contain($quad, qr/cluster\.shared_catalog requires cluster\.wal_threads_dir|structurally blocked|BLOCKED_STRUCTURAL/i, 0 .. 3); + eval { $quad->stop_quad; }; + ok(!$fatal, 'L1: shared_catalog quad must not hit wal_threads_dir/BLOCKED_STRUCTURAL configuration failure'); + BAIL_OUT('ClusterQuad(shared_catalog) failed to start for a non-environment reason'); +} + +# L1: all four members formed with quorum and per-thread WAL configured. +for my $i (0 .. 3) +{ + ok(poll_until($quad->node($i), 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 60), + "L1: node$i reached quorum"); + is(state_counter($quad->node($i), 'hw', 'remaster_recoverable'), 1, + "L1: node$i reports HW remaster recoverable"); +} +for my $to (1 .. 3) +{ + ok($quad->wait_for_peer_state(0, $to, 'connected', 60), + "L1: node0 sees node$to connected"); +} + +my $n0 = $quad->node0; +my $dead = 3; +my @survivors = (0, 1, 2); + +# Seed shared-catalog DDL and force node3 to own real HW state before death. +for my $t (1 .. 2) +{ + my ($ddl_ok, $ddl_res) = retry_sql($n0, + "CREATE TABLE sc4_hw_$t (id int, pad int)", 60); + ok($ddl_ok, "L1: coordinator created shared_catalog table sc4_hw_$t") + or diag($ddl_res); + my ($ok, $res) = retry_sql($quad->node($dead), + "INSERT INTO sc4_hw_$t SELECT g, g FROM generate_series(1,5000) g", 60); + ok($ok, "L1: node$dead extended shared_catalog table sc4_hw_$t before kill") + or diag($res); +} +for my $i (0 .. 3) +{ + retry_sql($quad->node($i), 'CHECKPOINT', 30); +} + +my $done_before = sum_hw($quad, 'remaster_done_count', @survivors); +my $blocked_before = sum_hw($quad, 'remaster_blocked_count', @survivors); +my $retry_before = sum_hw($quad, 'remaster_retry_count', @survivors); + +# L2: kill node3 and require every survivor to see the real DEAD edge. +# Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, +# ordinary catalog reads may legitimately return 53R9I until the barrier opens. +$quad->kill_node9($dead); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/cssd: peer $dead transitioned .* DEAD/, 45), + "L2: survivor node$i declared node$dead dead"); +} + +# L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +for my $i (@survivors) +{ + ok(poll_until($quad->node($i), + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + "L3: survivor node$i remastered all GRD shards off node$dead"); +} +my $done_converged = 0; +my $deadline = time() + 90; +while (time() < $deadline) +{ + if (sum_hw($quad, 'remaster_done_count', @survivors) > $done_before) + { + $done_converged = 1; + last; + } + usleep(500_000); +} +ok($done_converged, 'L3: HW remaster DONE advanced on at least one survivor after kill'); +ok(!logs_contain($quad, qr/blocked_structural|structurally blocked|BLOCKED_STRUCTURAL/i, @survivors), + 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); + +# L6: if a transient BLOCKED occurred, a same-episode retry must also have run. +my $blocked_after = sum_hw($quad, 'remaster_blocked_count', @survivors); +if ($blocked_after > $blocked_before) +{ + ok(sum_hw($quad, 'remaster_retry_count', @survivors) > $retry_before, + 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); +} +else +{ + pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); +} + +# L4: survivor SQL and shared-catalog DDL recover after fail-stop. +for my $i (@survivors) +{ + my ($ok, $res) = retry_sql($quad->node($i), 'SELECT txid_current()', 60); + ok($ok, "L4: survivor node$i accepts txid_current after reconfig") or diag($res); +} +my ($ddl_ok, $ddl_res) = retry_sql($n0, 'CREATE TABLE sc4_after_kill (id int)', 60); +ok($ddl_ok, 'L4: coordinator survivor can run DDL after node kill') or diag($ddl_res); +for my $i (1, 2) +{ + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); +} + +# L4 (spec-4.6a D12, r2-P1-3): the dead-node PCM residue cleanup counter is +# exposed and non-negative on every survivor; when the dead node left any +# holder/pending-X records behind, at least one survivor accounts a cleanup +# (delta assertable; zero stays legal when the dead node held nothing). +{ + my $cleanup_total = 0; + for my $i (@survivors) + { + my $c = state_counter($quad->node($i), 'pcm', 'dead_cleanup_entries'); + ok($c >= 0, "L4: survivor node$i exposes pcm/dead_cleanup_entries"); + $cleanup_total += $c; + } + diag("L4: pcm/dead_cleanup_entries total across survivors = $cleanup_total"); +} + +# L5: watchdog counters are allowed to be zero, but must be readable. +for my $i (@survivors) +{ + my $cluster_gate = state_counter($quad->node($i), 'grd_recovery', 'cluster_gate_timeout'); + my $epoch_escape = state_counter($quad->node($i), 'grd_recovery', 'wait_epoch_escape'); + ok($cluster_gate >= 0 && $epoch_escape >= 0, + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counters"); +} + +$quad->stop_quad; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..521855f18a 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -65,6 +65,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_oid_lease \ test_cluster_relmap_authority \ test_cluster_hw \ + test_cluster_hw_remaster_retry \ test_cluster_dl \ test_cluster_extend_gate \ test_cluster_ir \ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..8b64c01755 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -45,6 +45,7 @@ #include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b catalog counter stubs */ #include "cluster/cluster_debug.h" +#include "cluster/cluster_grd.h" /* ClusterGrdRecoveryCounters */ #include "cluster/cluster_hang.h" /* spec-5.11: ClusterHangDumpData for dump_hang stubs */ #include "cluster/cluster_hang_resolve.h" /* spec-5.12: ClusterHangResolveCounters for dump stubs */ #include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ @@ -564,6 +565,9 @@ uint64 cluster_hw_failclosed_count(void); uint64 cluster_hw_not_ready_count(void); uint64 cluster_hw_remaster_done_count(void); uint64 cluster_hw_remaster_blocked_count(void); +uint64 cluster_hw_remaster_retry_count(void); +uint64 cluster_hw_remaster_retry_exhausted_count(void); +bool cluster_hw_remaster_recoverable(void); uint64 cluster_hw_alloc_count(void) { @@ -604,6 +608,21 @@ cluster_hw_remaster_blocked_count(void) { return 0; } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return 0; +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return 0; +} +bool +cluster_hw_remaster_recoverable(void) +{ + return true; +} /* spec-5.7 D4 dump_dl stubs (cluster_dl.c not linked in this binary). */ uint64 cluster_dl_lease_count(void); @@ -3288,15 +3307,77 @@ cluster_grd_deadlock_chunk_oo_buffer_overflow_count(void) return 0; } -/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter - * snapshot. Zero-fill; layout = 13 × uint64 (must track the - * ClusterGrdRecoveryCounters struct in cluster_grd.h). */ -struct ClusterGrdRecoveryCounters; -void cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out); +uint32 +cluster_grd_recovery_state_value(void) +{ + return (uint32)GRD_RECOVERY_IDLE; +} + +const char * +cluster_grd_recovery_state_name(uint32 state pg_attribute_unused()) +{ + return "idle"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_event_id_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return 0; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return true; +} + +/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter snapshot. */ void -cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out) +cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) { - memset(out, 0, 13 * sizeof(uint64)); + memset(out, 0, sizeof(*out)); } uint32 diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 444accf8f0..67fcc29dc2 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -70,6 +70,7 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_thread_recovery.h" #include "common/hashfn.h" #include "storage/buf_internals.h" #include "storage/lwlock.h" @@ -287,6 +288,16 @@ UT_TEST(test_gcs_block_tag_is_standard_buffer_tag_20b) } +UT_TEST(test_dead_static_master_materialization_gate_scope_matches_thread_recovery) +{ + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(false, true, true, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, false, true, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, false, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, true, 2)); + UT_ASSERT(cluster_thread_recovery_materialization_gate_enabled(true, true, true, 1)); +} + + /* spec-5.2 D2 (U3): pure master-side decision for an X-held N→S read. * node0 = holder/master in DIRECT, node1 = requester. */ UT_TEST(test_xheld_read_ship_decision_truth_table) @@ -514,7 +525,7 @@ UT_TEST(test_clean_xfer_stale_break_predicate) int main(void) { - UT_PLAN(21); + UT_PLAN(22); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -530,6 +541,7 @@ main(void) UT_RUN(test_gcs_block_data_size_equals_blcksz); UT_RUN(test_gcs_block_msg_type_enum_extends_without_gap); UT_RUN(test_gcs_block_tag_is_standard_buffer_tag_20b); + UT_RUN(test_dead_static_master_materialization_gate_scope_matches_thread_recovery); UT_RUN(test_xheld_read_ship_decision_truth_table); UT_RUN(test_forward_payload_read_image_flag_roundtrip); UT_RUN(test_clean_page_xfer_eligible_flag_roundtrip_and_orthogonal); diff --git a/src/test/cluster_unit/test_cluster_ges.c b/src/test/cluster_unit/test_cluster_ges.c index 1df0a6b7fc..dd2a47185a 100644 --- a/src/test/cluster_unit/test_cluster_ges.c +++ b/src/test/cluster_unit/test_cluster_ges.c @@ -296,7 +296,8 @@ cluster_cancel_token_consume(void) * completion. Standalone fixture has no recovery shmem; no-op. */ void cluster_grd_recovery_mark_peer_done(int32 node pg_attribute_unused(), - uint64 epoch pg_attribute_unused()) + uint64 epoch pg_attribute_unused(), + uint64 event_id pg_attribute_unused()) {} ClusterGrdEntryResult diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..c7f805459d 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -55,6 +55,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -85,6 +86,15 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; the GRD fixture has no PCM shmem — no-op (the cleanup + * logic itself is unit-tested in test_cluster_pcm_lock.c). */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -316,6 +326,13 @@ cluster_epoch_get_current(void) return ut_mock_epoch; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch) +{ + if (admitted_epoch > ut_mock_epoch) + ut_mock_epoch = admitted_epoch; +} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -377,10 +394,20 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, return end; } +/* spec-4.6a r2-P1-1: controllable last-event mock so the recovery FSM's P0 + * accept + WAIT_EPOCH event-scoped witness can be unit-driven. Default + * zeroed = pre-existing inert behavior. */ +static ReconfigEvent ut_mock_last_event; void cluster_reconfig_get_last_event(ReconfigEvent *out) { - memset(out, 0, sizeof(*out)); + *out = ut_mock_last_event; +} + +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; } /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the @@ -419,6 +446,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { @@ -432,10 +477,12 @@ SendProcSignal(int pid pg_attribute_unused(), int reason pg_attribute_unused(), return 0; } +/* spec-4.6a r2-P1-1: controllable clock (default 0 = pre-existing value). */ +static int64 ut_mock_now = 0; int64 GetCurrentTimestamp(void) { - return 0; + return ut_mock_now; } void @@ -3903,15 +3950,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10); + cluster_grd_recovery_mark_peer_done(1, 10, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3934,15 +3981,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5); - cluster_grd_recovery_mark_peer_done(1, 5); - cluster_grd_recovery_mark_peer_done(2, 5); + cluster_grd_recovery_mark_peer_done(0, 5, 1); + cluster_grd_recovery_mark_peer_done(1, 5, 1); + cluster_grd_recovery_mark_peer_done(2, 5, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3967,9 +4014,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4020,9 +4067,9 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4035,15 +4082,74 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20); + cluster_grd_recovery_mark_peer_done(0, 20, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20); + cluster_grd_recovery_mark_peer_done(1, 20, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } +/* ============================================================ + * spec-4.6a r2-P1-1 — event-scoped REDECLARE_DONE proof. + * + * A stale DONE (previous event) must be dropped at the accounting + * layer and must never satisfy the WAIT_EPOCH coordinator witness; + * a current-event DONE at exactly cur, accounted after the P0 + * accept snapshot, is the only equal-epoch escape (proof-carrying, + * never timeout-only). + * ============================================================ */ + +UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) +{ + ClusterGrdRecoveryCounters before; + ClusterGrdRecoveryCounters after; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline. ut_mock_epoch already + * holds the post-bump value, reproducing the mis-captured-baseline wedge + * shape (cur == old) from spec-4.6a section 0. */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Stage the fail-stop event: dead node 2, coordinator 0, event 77. */ + ut_mock_last_event.event_id = 77; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); + + /* Stale-event DONE (event 76) is dropped: nothing is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, 76); + UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Current-event DONE is accounted (epoch + event id). */ + cluster_grd_recovery_mark_peer_done(0, 10, 77); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 77); + + /* Witness advance: deadline expired + cur==old + coordinator DONE for + * THIS event at exactly cur, accounted after the accept snapshot. The + * FSM takes the event-scoped equal-epoch escape exactly once. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4055,7 +4161,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4264,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_stale_event_done_rejected_and_witness_advance); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..d40ee60d43 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -61,6 +61,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -91,6 +92,14 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; this fixture has no PCM shmem — no-op. */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -298,6 +307,10 @@ cluster_epoch_get_current(void) return 0; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch pg_attribute_unused()) +{} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -355,6 +368,12 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -391,6 +410,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_hw_remaster_retry.c b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c new file mode 100644 index 0000000000..456c5979f8 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_hw_remaster_retry.c + * Unit tests for spec-4.6a same-episode HW remaster retry decisions. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_hw_remaster_retry.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_hw_remaster.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +static ClusterHwRemasterRelaunchDecision +decide(uint64 launched, uint64 episode, ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int max_attempts) +{ + return cluster_hw_remaster_relaunch_decide(launched, episode, result, attempts, next_attempt_at, + now, max_attempts); +} + +UT_TEST(test_new_episode_initial_launch_resets_state) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(10, 11, CLUSTER_HW_REMASTER_BLOCKED, 7, 1234, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_running_done_and_not_applicable_do_not_relaunch) +{ + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_RUNNING, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_DONE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_NOT_APPLICABLE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_structural_blocked_warns_once_and_never_retries) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, 0, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, CLUSTER_HW_REMASTER_NO_DEADLINE, + 3000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_blocked_waits_until_backoff_deadline) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 4999, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 5000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 1); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_cap_exhausts_once_and_sighup_raise_recovers) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, 0, 5000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 6000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + /* Operator raised cluster.hw_remaster_retry_max_attempts via SIGHUP. */ + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 7000, 3); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 3); +} + +UT_TEST(test_zero_max_attempts_disables_retry) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 0, 5000, 0); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); +} + +/* spec-4.6a section 2.1 truth-table row 8: latched == episode but the result + * slot reads NONE (registration failed and was reverted) -> take the + * first-launch row again; no attempt is charged. */ +UT_TEST(test_none_result_registration_failed_falls_back_to_initial) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_NONE, 3, 999, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_backoff_exponential_cap) +{ + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 0), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 1), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 2), 2000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 7), 60000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(100, 20), 60000); +} + + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_new_episode_initial_launch_resets_state); + UT_RUN(test_running_done_and_not_applicable_do_not_relaunch); + UT_RUN(test_structural_blocked_warns_once_and_never_retries); + UT_RUN(test_blocked_waits_until_backoff_deadline); + UT_RUN(test_cap_exhausts_once_and_sighup_raise_recovers); + UT_RUN(test_zero_max_attempts_disables_retry); + UT_RUN(test_none_result_registration_failed_falls_back_to_initial); + UT_RUN(test_backoff_exponential_cap); + UT_DONE(); + + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_lock_acquire.c b/src/test/cluster_unit/test_cluster_lock_acquire.c index 5636b8fff8..a5ca5a719c 100644 --- a/src/test/cluster_unit/test_cluster_lock_acquire.c +++ b/src/test/cluster_unit/test_cluster_lock_acquire.c @@ -365,6 +365,47 @@ cluster_grd_lookup_master(const ClusterResId *resid pg_attribute_unused()) return -1; } +/* spec-4.6a: the S4-reject diagnostic references the CSSD peer-state view; + * fixture has no CSSD shmem — stub DEAD-free defaults. */ +#include "cluster/cluster_cssd.h" +ClusterCssdPeerState +cluster_cssd_get_peer_state(int32 peer_id pg_attribute_unused()) +{ + return CLUSTER_CSSD_PEER_ALIVE; +} +const char * +cluster_cssd_peer_state_to_string(ClusterCssdPeerState s pg_attribute_unused()) +{ + return "alive"; +} + +/* spec-4.6a: the same diagnostic is the first ereport in this .o — swallow + * elog machinery (mirror test_cluster_grd.c pattern). */ +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} + /* spec-4.6 P0 regression harness — test-controlled S3/S4 outcomes. * stub_reserve_result default NOT_READY keeps the legacy tests on the * pre-reservation-fail path; the S4 default-deny test flips it to OK diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c0ff3b6710..c02894306f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -100,6 +100,9 @@ static Size fake_pcm_keysize = 0; static Size fake_pcm_entrysize = 0; static LWLock *fake_lwlock_held = NULL; static LWLockMode fake_lwlock_mode = LW_EXCLUSIVE; +static LWLock *fake_lwlock_stack[16]; +static LWLockMode fake_lwlock_mode_stack[16]; +static int fake_lwlock_depth = 0; static uint32 fake_init_wait_event_seen = 0; static uint32 fake_lwlock_wait_event_seen = 0; @@ -121,6 +124,7 @@ static struct { static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; +static bool fake_local_x_upgrade_result = false; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -155,6 +159,10 @@ reset_fake_pcm_runtime(int max_entries) fake_pcm_keysize = 0; fake_pcm_entrysize = 0; fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + fake_lwlock_depth = 0; + memset(fake_lwlock_stack, 0, sizeof(fake_lwlock_stack)); + memset(fake_lwlock_mode_stack, 0, sizeof(fake_lwlock_mode_stack)); fake_init_wait_event_seen = 0; fake_lwlock_wait_event_seen = 0; ut_wait_event_info_storage = 0; @@ -167,6 +175,7 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; cluster_pcm_grd_max_entries = max_entries; @@ -191,9 +200,20 @@ bool cluster_read_scache = false; bool cluster_gcs_block_local_x_upgrade(BufferTag tag); bool -cluster_gcs_block_local_x_upgrade(BufferTag tag pg_attribute_unused()) +cluster_gcs_block_local_x_upgrade(BufferTag tag) { - return false; + uint32 holders_bm; + int n; + + if (!fake_local_x_upgrade_result) + return false; + holders_bm = cluster_pcm_lock_query_s_holders_bitmap(tag); + if (cluster_node_id >= 0 && cluster_node_id < 32) + holders_bm &= ~((uint32)1u << (uint32)cluster_node_id); + for (n = 0; n < 32; n++) + if ((holders_bm & ((uint32)1u << n)) != 0) + (void)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, n); + return cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_X_UPGRADE, cluster_node_id); } /* spec-4.7a D4 — stub CSSD peer liveness for the other-live-holder gate. @@ -348,6 +368,10 @@ LWLockInitialize(LWLock *lock pg_attribute_unused(), int tranche_id pg_attribute bool LWLockAcquire(LWLock *lock, LWLockMode mode) { + Assert(fake_lwlock_depth < (int)lengthof(fake_lwlock_stack)); + fake_lwlock_stack[fake_lwlock_depth] = lock; + fake_lwlock_mode_stack[fake_lwlock_depth] = mode; + fake_lwlock_depth++; fake_lwlock_held = lock; fake_lwlock_mode = mode; fake_lwlock_wait_event_seen = ut_wait_event_info_storage; @@ -357,14 +381,28 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) void LWLockRelease(LWLock *lock) { - Assert(fake_lwlock_held == lock); - fake_lwlock_held = NULL; + Assert(fake_lwlock_depth > 0); + Assert(fake_lwlock_stack[fake_lwlock_depth - 1] == lock); + fake_lwlock_depth--; + fake_lwlock_stack[fake_lwlock_depth] = NULL; + if (fake_lwlock_depth > 0) { + fake_lwlock_held = fake_lwlock_stack[fake_lwlock_depth - 1]; + fake_lwlock_mode = fake_lwlock_mode_stack[fake_lwlock_depth - 1]; + } else { + fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + } } bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode) { - return fake_lwlock_held == lock && fake_lwlock_mode == mode; + int i; + + for (i = fake_lwlock_depth - 1; i >= 0; i--) + if (fake_lwlock_stack[i] == lock && fake_lwlock_mode_stack[i] == mode) + return true; + return false; } /* ---------- @@ -1280,6 +1318,79 @@ UT_TEST(test_pcm_d3_not_double_x) } } +UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) +{ + BufferTag tag = make_tag(96); + BufferDesc buf; + bool save = cluster_gcs_block_local_cache; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + cluster_gcs_block_local_cache = true; + fake_local_x_upgrade_result = true; + + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); + UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); + UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); + + cluster_gcs_block_local_cache = save; +} + +UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) +{ + BufferTag stag = make_tag(94); + BufferTag xtag = make_tag(95); + uint32 s_bitmap; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + + /* Dead node 2 was the first S holder, so master_holder points at it. */ + cluster_node_id = 2; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 2); + UT_ASSERT(!cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + + cluster_node_id = 2; + cluster_pcm_lock_acquire(xtag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_X); + + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)2); + + s_bitmap = cluster_pcm_lock_query_s_holders_bitmap(stag); + UT_ASSERT_EQ((int)(s_bitmap & (1u << 2)), 0); + UT_ASSERT((s_bitmap & (1u << 1)) != 0); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(stag), (int)PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 1); + + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_N); + UT_ASSERT(cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + UT_ASSERT((fake_cv_broadcast_count) >= (1)); + + /* Idempotent: a repeated dead-sweep pass has nothing left to clean. */ + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)0); + + /* spec-4.6a D12 (r2-P1-3) pending-X form: a dead requester's parked X + * intent is cleared by the companion HC124 sweep the same dead-sweep hook + * drives, and the sweep is idempotent too. */ + cluster_pcm_lock_set_pending_x(stag, 2, 1234); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), 2); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)1); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), -1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)0); +} + /* spec-5.2a D2 (U1): clean-page X-transfer arm is one-shot. arm(true) sets * the backend-local flag; consume() reads-and-clears it (the acquire path * calls consume() once so the eligibility can never leak into a SUBSEQUENT @@ -1313,7 +1424,7 @@ UT_TEST(test_clean_page_xfer_arm_is_one_shot) int main(void) { - UT_PLAN(39); + UT_PLAN(41); UT_RUN(test_pcm_lock_mode_constant_aliases_match_pcm_state); UT_RUN(test_pcm_lock_transition_count_is_9); UT_RUN(test_pcm_lock_transition_enum_values_are_1_to_9); @@ -1352,6 +1463,8 @@ main(void) UT_RUN(test_pcm_d1_recovering_gate_fail_closed); UT_RUN(test_pcm_d2_rebuild_from_redeclare); UT_RUN(test_pcm_d3_not_double_x); + UT_RUN(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades); + UT_RUN(test_pcm_dead_node_cleanup_drops_holder_records); UT_RUN(test_clean_page_xfer_arm_is_one_shot); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm index 3b3a2bb29f..873852fd76 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm @@ -55,6 +55,29 @@ use PostgreSQL::Test::Utils; our $NODES = 4; + +# Relocate an init_from_backup node's copied pg_wal into its shared WAL thread +# directory, matching the layout initdb -X produces for the seed node. +sub _relocate_backup_pg_wal +{ + my ($node, $wal_threads_root, $thread_id) = @_; + my $pgwal = $node->data_dir . '/pg_wal'; + my $wal_thread = "$wal_threads_root/thread_$thread_id"; + + mkdir $wal_thread or die "mkdir $wal_thread: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal_thread/$e") + or die "rename $pgwal/$e -> $wal_thread/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal_thread, $pgwal) or die "symlink $pgwal -> $wal_thread: $!"; + return; +} + #----------------------------------------------------------------------- # new_quad($class, $cluster_name, %opts) # @@ -66,6 +89,8 @@ our $NODES = 4; # voting-disk files (QVOTEC reaches quorum OK) # wal_threads_root : 1 -> shared per-thread WAL root # shared_data : 1 -> shared data root (cluster_fs backend) +# shared_catalog : 1 -> t/337-style shared-catalog formation; +# implies shared_data + wal_threads_root #----------------------------------------------------------------------- sub new_quad { @@ -91,6 +116,12 @@ sub new_quad "${cluster_name}_node$i", port => $pg_ports[$i]); } + if ($opts{shared_catalog}) + { + $opts{shared_data} = 1; + $opts{wal_threads_root} = 1; + } + # spec-4.6: strict-mode opt-in (mirror ClusterTriple) -- pre-allocate N # shared voting-disk files so QVOTEC reaches quorum_state=OK and the GES # inbound validation (in_quorum, check 4) accepts cross-node traffic. @@ -127,23 +158,75 @@ sub new_quad if ($opts{shared_data}) { $shared_data_root = PostgreSQL::Test::Utils::tempdir(); + if ($opts{shared_catalog}) + { + mkdir "$shared_data_root/global" + or die "mkdir $shared_data_root/global: $!"; + } + } + + if ($opts{shared_catalog}) + { + my $sc_common = <init(allows_streaming => 1, + extra => [ '-X', "$wal_threads_root/thread_1" ]); + $nodes[0]->start; + $nodes[0]->backup('clusterquad_scb'); + $nodes[0]->stop; + + for my $i (1 .. $NODES - 1) + { + $nodes[$i]->init_from_backup($nodes[0], 'clusterquad_scb'); + _relocate_backup_pg_wal($nodes[$i], $wal_threads_root, $i + 1); + } + + # Seed the shared catalog/controlfile/OID authorities in a single-node era, + # then append the real cluster config below. Last GUC value wins. + $nodes[0]->append_conf('postgresql.conf', $sc_common); + $nodes[0]->append_conf('postgresql.conf', <start; + die "shared_catalog seed did not create catalog authority" + unless -e "$shared_data_root/global/pgrac_catalog_authority"; + $nodes[0]->stop; + } + else + { + my $wal_node_index = 0; + for my $node (@nodes) + { + if (defined $wal_threads_root) + { + my $thread_id = $wal_node_index + 1; + $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); + } + else + { + $node->init; + } + $wal_node_index++; + } } - my $wal_node_index = 0; for my $node (@nodes) { if (defined $wal_threads_root) { - my $thread_id = $wal_node_index + 1; - $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); $node->append_conf('postgresql.conf', "cluster.wal_threads_dir = '$wal_threads_root'\n"); } - else - { - $node->init; - } - $wal_node_index++; if (defined $shared_data_root) { @@ -153,6 +236,15 @@ sub new_quad "cluster.shared_data_dir = '$shared_data_root'\n"); $node->append_conf('postgresql.conf', "cluster.smgr_user_relations = on\n"); + if ($opts{shared_catalog}) + { + $node->append_conf('postgresql.conf', + "cluster.controlfile_shared_authority = on\n"); + $node->append_conf('postgresql.conf', + "cluster.shared_catalog = on\n"); + $node->append_conf('postgresql.conf', + "cluster.merged_recovery = on\n"); + } } # Enable cluster + tier1, same baseline as ClusterTriple (spec-2.2). From e7ca433d8c8b2c5d5d3490b9276a5efa569debc8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:17:40 +0800 Subject: [PATCH 36/58] ci(nightly): own shard for the t/362 4-node kill self-heal TAP t/358-361 are reserved by parallel lanes (spec-7.2 / S-xid), t/363 by the S-dead lane; occupancy verified against origin branches. The 4-node formation + kill -9 + convergence run gets its own shard for wall-clock isolation. Spec: spec-4.6a-grd-recovery-liveness.md --- .github/workflows/nightly.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..ae5dcdf16c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -182,6 +182,11 @@ jobs: # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-361 are + # reserved by the parallel spec-7.2 / S-xid lanes, t/363 by S-dead; + # L464 occupancy verified against origin branches 2026-07-08). Own + # shard: 4-node formation + kill + convergence needs the wall clock. + - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 From 8853bc7d7e6214ee0b9f791c4fff5c80179f6e12 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:25:22 +0800 Subject: [PATCH 37/58] test(cluster): suppress pre-existing cppcheck constParameter in pi_shadow unit Same inline suppression + reason as the other cluster_unit main() signatures (test_cluster_grd.c precedent); a newer local cppcheck flags it, the CI baseline version does not. --- src/test/cluster_unit/test_cluster_pi_shadow.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..7d075852dd 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,6 +207,9 @@ UT_TEST(test_gate_raw_compare_traps) } int +/* cppcheck-suppress constParameter + * Reason: main() keeps the standard test harness signature used by the + * other cluster_unit binaries; argv is intentionally unused. */ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ From 0204d22459dd1736fc4cc3fc201d4ca77d50382a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:52:51 +0800 Subject: [PATCH 38/58] =?UTF-8?q?fix(cluster):=20close=20two=20spec-2.29a?= =?UTF-8?q?=20nightly=20regressions=20=E2=80=94=20dump-category=20baseline?= =?UTF-8?q?=20+=20async=20pre-bump=20WAIT=5FEPOCH=20wedge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ship-gate nightly on the marker-async head surfaced two lane-introduced regressions that the fast-gate matrix does not cover: 1. Dump-category baseline miss: reconfig telemetry added a pg_cluster_state category (55 -> 56), but three cross-node TAPs that assert the total (t/204/205/206) were not updated — only the locally-run L405 subset was. Bump them to 56 with the spec-2.29a note. 2. Async pre-bump WAIT_EPOCH wedge (behavior regression): the async fence / join-Phase-1 / node-remove marker staging bumps the membership epoch at stage-entry but only publishes the reconfig event once the voting-disk marker ACKs — now several LMON ticks later instead of the pre-async single-tick spin. In that window the coordinator's OWN GRD recovery IDLE tick re-captured the already-bumped epoch as its WAIT_EPOCH baseline, so the P0 accept that followed the publish read old == cur and froze the affected shards forever — the spec-4.6a section-0 shape, here triggered by the coordinator on itself with no IC piggyback (reproduced deterministically as t/293 fail-stop remaster + t/326 node-leave shard reopen). Fix: while any pre-bump stage is live, the GRD IDLE tick holds its last stable pre-reconfig baseline instead of re-capturing (cluster_reconfig_has_pending_prebump_stage guard). This restores the pre-async bump->publish atomicity as seen by GRD without touching the epoch bump timing or any user-visible reconfig contract. Adds the cluster_grd_recovery_event_old_epoch accessor + a baseline-hold unit test. Local gates (cassert): unit 158 binaries (incl. baseline-hold test), t/293 + t/325 + t/326 family all green (were red), t/204/205/206 + observability TAPs, PG regress 219/219, clang-format/headers/scn-cmp clean. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_grd.c | 28 ++++++++++- src/backend/cluster/cluster_reconfig.c | 24 +++++++++ src/include/cluster/cluster_grd.h | 1 + src/include/cluster/cluster_reconfig.h | 6 +++ .../t/204_cluster_visibility_fork_2node.pl | 4 +- .../t/205_cluster_visibility_scn_2node.pl | 4 +- .../t/206_cluster_itl_writable_2node.pl | 2 +- src/test/cluster_unit/test_cluster_grd.c | 50 ++++++++++++++++++- .../test_cluster_grd_starvation.c | 7 +++ 9 files changed, 118 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..d85321d2b3 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -1638,6 +1638,20 @@ cluster_grd_recovery_in_progress(void) return pg_atomic_read_u32(&cluster_grd_state->recovery_state) != (uint32)GRD_RECOVERY_IDLE; } +/* + * spec-2.29a: the pre-reconfig baseline epoch the WAIT_EPOCH gate compares + * against. Exposed for diagnostics + the IDLE baseline-hold unit test (the + * async pre-bump staging window must not let this re-capture a post-bump + * value; see the IDLE branch of cluster_grd_recovery_lmon_tick). + */ +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch); +} + /* spec-4.6 D4/D5 — recovery counter bump helpers for out-of-module * call sites (S4 stale mapping in cluster_lock_acquire.c; GCS block * fail-closed guard in cluster_gcs_block.c). */ @@ -2108,9 +2122,19 @@ cluster_grd_recovery_lmon_tick(void) * fires, making old_epoch already == the post-bump epoch and * wedging WAIT_EPOCH forever (cur <= old). The last stable * idle epoch is the reliable "before reconfig" value. + * + * spec-2.29a nightly-regression fix: the coordinator's own + * async pre-bump stages (fail-stop / node-remove / join + * Phase-1) bump the epoch several ticks before publishing the + * event. During that window this IDLE tick must NOT re-capture + * the already-bumped epoch as the baseline, or the P0 accept + * that follows the publish reads old == cur and wedges + * WAIT_EPOCH — the coordinator wedging itself, no IC piggyback + * needed. Hold the last stable pre-reconfig baseline instead. */ - pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, - cluster_epoch_get_current()); + if (!cluster_reconfig_has_pending_prebump_stage()) + pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, + cluster_epoch_get_current()); return; } diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index cf03d3e8b7..7edc315c52 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -1253,6 +1253,30 @@ cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) stage->removal_event_id = 0; } +/* + * spec-2.29a nightly-regression fix — pre-bump staging window guard. + * + * The three pre-bump coordinator paths (fail-stop fence, node-remove, + * join Phase-1) advance the membership epoch at stage-entry but only + * publish the reconfig event once the voting-disk marker ACKs, which now + * spans several LMON ticks instead of the pre-async single-tick spin. + * Between the bump and the publish the GRD recovery IDLE tick would + * re-capture recovery_event_old_epoch as the post-bump value, so its P0 + * accept later reads old == cur and wedges WAIT_EPOCH forever (the + * spec-4.6a section 0 shape, here triggered by the coordinator on ITSELF + * — even in a 2-node cluster with no IC piggyback). While any pre-bump + * stage is live the GRD IDLE tick must hold its last stable (genuine + * pre-reconfig) baseline instead of re-capturing. join Phase-2 COMMITTED + * does not pre-bump, so it is intentionally excluded. + */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return failstop_fence_stage.async.has_staged_event + || node_removed_fence_stage.async.has_staged_event + || join_prepare_stage.async.has_staged_event; +} + static bool cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now) diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..08d475a9d6 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -660,6 +660,7 @@ typedef struct ClusterGrdRecoveryCounters { } ClusterGrdRecoveryCounters; extern void cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out); +extern uint64 cluster_grd_recovery_event_old_epoch(void); /* spec-2.29a WAIT_EPOCH baseline */ /* spec-4.6 P0#2 — pre-remaster stale-epoch sweep SCOPED to the affected * (dead-master) shards; affected_shards is a PGRAC_GRD_SHARD_COUNT-bit diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index 4ce28e13fd..df42f84d77 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -409,6 +409,12 @@ extern void cluster_reconfig_shmem_register(void); */ extern void cluster_reconfig_get_last_event(ReconfigEvent *out); +/* spec-2.29a: true while a pre-bump coordinator stage (fail-stop fence / + * node-remove / join Phase-1) has bumped the epoch but not yet published its + * reconfig event -- the GRD recovery IDLE tick must hold its baseline instead + * of re-capturing the post-bump epoch (else WAIT_EPOCH wedges). */ +extern bool cluster_reconfig_has_pending_prebump_stage(void); + /* ============================================================ * Coordinator path APIs (Step 2 D2 wiring). diff --git a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl index f12abe0f14..9bc0e1fccf 100644 --- a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl +++ b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl @@ -325,8 +325,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); my $tt_categories = $pair->node0->safe_psql('postgres', q{ SELECT string_agg(c, ',' ORDER BY c) diff --git a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl index 4e65595eff..56d8cd35f9 100644 --- a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl +++ b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl @@ -296,8 +296,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L14a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L14a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl index 80447a5161..8f4f1c5ce7 100644 --- a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl +++ b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl @@ -287,7 +287,7 @@ # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', 'L13a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', 'L13a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..77e7862d6f 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -383,6 +383,16 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +/* spec-2.29a nightly-regression fix: controllable pre-bump-stage flag so the + * GRD IDLE baseline-hold can be unit-driven (default false = pre-fix + * re-capture behavior; existing tests are unaffected). */ +static bool ut_mock_pending_prebump = false; +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return ut_mock_pending_prebump; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -4044,6 +4054,43 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) } +/* ============================================================ + * spec-2.29a nightly-regression fix — GRD IDLE baseline hold while a + * pre-bump coordinator stage is in flight. + * + * The async fence/join/node-remove marker staging bumps the epoch + * several ticks before publishing. While that window is open the GRD + * IDLE tick must NOT re-capture the post-bump epoch as its WAIT_EPOCH + * baseline, or the P0 accept that follows reads old == cur and wedges. + * ============================================================ */ + +UT_TEST(test_recovery_idle_holds_baseline_during_prebump_stage) +{ + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_pending_prebump = false; + + /* Idle tick with epoch 5 captures baseline 5 (no staging; get_last_event + * returns event_id 0 so the tick stays IDLE and re-captures). */ + ut_mock_epoch = 5; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* A pre-bump stage goes live and the coordinator bumps the epoch to 6 + * before publishing. The IDLE tick must HOLD baseline 5. */ + ut_mock_pending_prebump = true; + ut_mock_epoch = 6; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* Marker ACKs, stage clears; a subsequent idle tick resumes tracking. */ + ut_mock_pending_prebump = false; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 6); + + cluster_enabled = false; +} + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4055,7 +4102,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4205,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_idle_holds_baseline_during_prebump_stage); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..af5f119123 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -355,6 +355,13 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +/* spec-2.29a: no pre-bump staging in the starvation fixture (no-op false). */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return false; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op From 87f793084b2e78b4c27ccc27e0bf269d2fb14173 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:06:25 +0800 Subject: [PATCH 39/58] =?UTF-8?q?fix(cluster):=20clean-leave=20coherence?= =?UTF-8?q?=20excludes=20the=20leaving=20node=20(regression=20=E2=91=A1b,?= =?UTF-8?q?=20others-dead)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec-2.29a marker-async change turned the clean-leave COMMITTING marker into an async, multi-tick wait. In a slow environment (nightly Linux) the leaving node finishes its drain and stops heart-beating inside that wait window, so CSSD marks it SUSPECTED->DEAD and bumps the global dead_generation — and the coherence gate, which compared that SCALAR dead_generation, wrongly read it as a third-party death intruding mid-drain and escalated an otherwise healthy clean leave (t/310 dormant-member legs, t/331 clean_leave x idle). apply_clean_leave_as_coordinator bumps + publishes atomically, so this is NOT the baseline-hold wedge (②a); it is the coherence predicate counting the leaving node's OWN expected DEAD. Fix: compare an others-dead bitmap (the dead set EXCLUDING the leaving node) instead of the scalar dead_generation, at all four coherence sites (CL_COHERENT macro, drive_drain commit-point, staged-ACK re-check, non-staged pre-check). cl_state snapshots the others-dead set at bind; the pure predicate does a bitmap compare (fail-closed on a missing view) and is unit-tested against the reflexive-case matrix. Every third-party incoherence still escalates (epoch move OR a non-leaving node entering the others-dead set); only the leaving node's own expected transition is tolerated. 8.A / CL-I3 protection is unchanged — argued in spec-2.29a §②b with a case matrix. Local gates (cassert): policy unit reflexive-case matrix, cluster_unit 158 binaries, PG regress 219/219, clang-format/headers/scn-cmp clean. NOTE: t/310/331 are timing-sensitive — the false escalation only reproduces in a slow environment, so the fix is verified by nightly, not locally (a local run passes the assertions). A separate pre-existing cassert teardown SIGABRT flake in t/310 (isolated: reproduces without this change) is unrelated. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_clean_leave.c | 122 +++++++++++++----- .../cluster/cluster_clean_leave_policy.c | 32 ++++- src/include/cluster/cluster_clean_leave.h | 22 +++- .../cluster_unit/test_cluster_clean_leave.c | 30 +++-- 4 files changed, 152 insertions(+), 54 deletions(-) diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 56b4b7fc62..6ea7873643 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -382,6 +382,11 @@ cluster_clean_leave_check_pending_in_proc_interrupts(void) * disabled survivor MUST reply LEAVE_DRAIN_NAK rather than go silent, else the * leaving node waits for an ACK that never comes and false-times-out. */ + +/* spec-2.29a ②b: forward decls — the bind sites (below) snapshot the + * others-dead set that the coherence gate (defined near drive_drain) compares. */ +static void cl_others_dead_snapshot(int32 leaving, uint8 *out); + static void cl_announce_handler(const ClusterICEnvelope *env, const void *payload) { @@ -510,6 +515,9 @@ cl_announce_handler(const ClusterICEnvelope *env, const void *payload) * fails the commit closed instead of committing on a stale membership view. */ cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes the leaving node) + * the coherence gate compares against. */ + cl_others_dead_snapshot(leaving, cl_state->leave_baseline_others_dead); pg_atomic_write_u32(&cl_state->survivor_acked, 0); pg_atomic_write_u32(&cl_state->commit_ready_received, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); @@ -1430,6 +1438,8 @@ cl_request_body(void) cl_state->leaving_node_id = cluster_node_id; cl_state->leave_epoch = baseline_epoch; cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes self, the leaver). */ + cl_others_dead_snapshot(cluster_node_id, cl_state->leave_baseline_others_dead); cl_state->barrier_deadline_us = (uint64)GetCurrentTimestamp() + (uint64)cluster_clean_leave_drain_timeout_ms * 1000ULL; /* @@ -1563,6 +1573,49 @@ cluster_clean_leave_request(void) * re-checks version coherence (CL-I3: an external epoch bump = a real death * intruded -> escalate) and the mixed-mode NAK (clean abort). */ + +/* + * spec-2.29a ②b: snapshot the CSSD dead set EXCLUDING the leaving node. The + * coherence gate compares this "others-dead" set rather than the scalar global + * dead_generation, so the leaving node's own expected alive->DEAD transition + * (it stops heart-beating once its drain finishes) never falsely escalates the + * leave. A third-party death still changes this set and escalates (CL-I3). + */ +static void +cl_others_dead_snapshot(int32 leaving, uint8 *out /* CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES */) +{ + int i; + + memset(out, 0, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (i == leaving) + continue; /* the leaving node's own DEAD is expected, not incoherence */ + if (i >= CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES * 8) + break; + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (cluster_cssd_get_peer_state(i) == CLUSTER_CSSD_PEER_DEAD) + out[i / 8] |= (uint8)(1u << (i % 8)); + } +} + +/* + * spec-2.29a ②b: the coherence gate used by every clean-leave step. Coherent + * iff the epoch has not moved AND no THIRD-PARTY death changed the others-dead + * set since the leave was bound. cl_state->leaving_node_id names the node + * whose own DEAD is excluded. + */ +static bool +cl_coherent(uint64 baseline_epoch) +{ + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + return cluster_clean_leave_version_coherent( + baseline_epoch, cluster_epoch_get_current(), cl_state->leave_baseline_others_dead, + now_others_dead, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); +} + void cluster_clean_leave_drive_drain(void) { @@ -1623,16 +1676,14 @@ cluster_clean_leave_drive_drain(void) /* * CL-I3 coherence gate, re-checked before every drain step. Uses the * dead_gen-aware helper, not an epoch-only compare: CSSD increments the - * dead_generation the moment it declares a peer dead, which is STRICTLY - * BEFORE the reconfig coordinator bumps the membership epoch. Checking - * dead_gen too lets us escalate in that earlier window instead of draining - * into a death that has not yet reached the epoch. At commit the guarded - * CAS in apply_clean_leave_as_coordinator is the final authority. + * others-dead set the moment it declares a THIRD-PARTY peer dead, which is + * STRICTLY BEFORE the reconfig coordinator bumps the membership epoch. + * Checking that set too lets us escalate in that earlier window instead of + * draining into a death that has not yet reached the epoch. The leaving + * node's OWN DEAD is excluded (spec-2.29a ②b). At commit the guarded CAS + * in apply_clean_leave_as_coordinator is the final authority. */ -#define CL_COHERENT() \ - cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), \ - cl_state->leave_baseline_dead_gen, \ - cluster_cssd_get_dead_generation()) +#define CL_COHERENT() cl_coherent(baseline_epoch) /* REQUESTED -> QUIESCING: abort local writable backends (53R62). */ if (!CL_COHERENT()) { @@ -1726,14 +1777,22 @@ cl_leaving_barrier_tick(void) * leave is active (cluster_clean_leave_in_progress gates drive_joins + * commit_member), and the leave does not start while a join is pending * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * dead_gen-unchanged bump during a leave can only be the leave's own commit. + * bump with an unchanged others-dead set during a leave can only be the + * leave's own commit (spec-2.29a ②b: the leaving node's own DEAD is excluded + * so its expected heartbeat stop no longer looks like a third-party death). */ if (cluster_epoch_get_current() > baseline_epoch) { - if (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen) { + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + if (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0) { /* - * Commit point observed (our clean-leave epoch was published; no real - * death intruded). The leave can no longer be un-committed, so from - * here the barrier deadline must NOT escalate (Hardening v1.0.1 P1-1). + * Commit point observed (our clean-leave epoch was published; no + * third-party death intruded). The leave can no longer be + * un-committed, so from here the barrier deadline must NOT escalate + * (Hardening v1.0.1 P1-1). */ pg_atomic_write_u32(&cl_state->commit_point_observed, 1); LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); @@ -1817,11 +1876,9 @@ cl_coordinator_commit(int32 leaving) * applies. On failure the leave does not commit and the leaving * node escalates to fail-stop (identical to the pre-check path). */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { + if (!cl_coherent(baseline_epoch)) { ereport(LOG, - (errmsg("cluster clean-leave: version moved (epoch or dead_generation) " + (errmsg("cluster clean-leave: version moved (epoch or third-party death) " "across the COMMITTING marker wait for node %d; not committing " "(escalate to fail-stop, CL-I3)", leaving))); @@ -1862,22 +1919,21 @@ cl_coordinator_commit(int32 leaving) * guarded CAS inside apply_clean_leave_as_coordinator below, which closes the * check-then-bump TOCTOU at >=3 nodes. */ /* - * CL-I3 commit-handoff coherence (epoch AND dead_gen): refuse to commit if a - * real death intruded since this survivor started tracking the leave — either - * the death's fail-stop already bumped the epoch, OR (the >=3-node window, - * Hardening v1.0.1 P1-2) CSSD bumped its dead_generation but the fail-stop - * epoch has NOT yet advanced, which an epoch-only check (and the guarded CAS - * below) would miss and commit on a stale membership view. Uses the same - * unit-tested version_coherent helper as drive_drain so the dead_gen-aware - * coherence holds through the commit handoff; the leaving node then observes - * the eventual foreign event and escalates. + * CL-I3 commit-handoff coherence (epoch AND others-dead): refuse to commit if + * a THIRD-PARTY death intruded since this survivor started tracking the leave + * — either the death's fail-stop already bumped the epoch, OR (the >=3-node + * window, Hardening v1.0.1 P1-2) CSSD added it to the others-dead set but the + * fail-stop epoch has NOT yet advanced, which an epoch-only check (and the + * guarded CAS below) would miss and commit on a stale membership view. Uses + * the same others-dead coherence helper as drive_drain (spec-2.29a ②b: the + * leaving node's own expected DEAD is excluded so it never falsely escalates); + * the leaving node then observes the eventual foreign event and escalates. */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { - ereport(LOG, (errmsg("cluster clean-leave: version moved (epoch or dead_generation) before " - "committing node %d; not committing (escalate to fail-stop, CL-I3)", - leaving))); + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) before " + "committing node %d; not committing (escalate to fail-stop, CL-I3)", + leaving))); return; } diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 0a4eeebd04..095f4b082b 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -136,17 +136,35 @@ cluster_clean_leave_should_abort_writable(bool in_transaction, bool has_top_xid) /* * cluster_clean_leave_version_coherent -- the bound leave is still coherent - * only if neither the cluster epoch nor the CSSD dead_generation moved since - * the leave was bound. Any external bump (a real death intruding mid-drain) - * makes it incoherent → caller must ABORTED_ESCALATE (never complete a clean - * leave on a stale version, which would let a destructive sweep run on a newer - * version and double-grant). + * only if neither the cluster epoch nor the OTHERS-dead set (every dead node + * except the leaving node itself) moved since the leave was bound. Any + * external membership change — a third-party death intruding mid-drain, or a + * third-party fail-stop that already bumped the epoch — makes it incoherent → + * caller must ABORTED_ESCALATE (never complete a clean leave on a stale + * membership view, which would let a destructive sweep run on a newer view + * and double-grant, CL-I3). + * + * spec-2.29a ②b fix: the pre-fix predicate compared the SCALAR global CSSD + * dead_generation, which also counts the leaving node's OWN alive→DEAD + * transition (it stops heart-beating once its drain finishes — the EXPECTED + * terminal state of a clean leave). Under the async COMMITTING marker that + * spans several LMON ticks, a slow environment lets that transition land + * inside the wait window and the scalar check wrongly escalated an otherwise + * healthy leave. Comparing the others-dead bitmap (which excludes the + * leaving node) removes exactly that false positive while keeping every + * third-party incoherence escalation (see the reflexive-case matrix in + * spec-2.29a §②b 8.A argument). */ bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen) + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes) { - return bound_epoch == current_epoch && bound_dead_gen == current_dead_gen; + if (bound_epoch != current_epoch) + return false; + if (bound_others_dead == NULL || current_others_dead == NULL) + return false; /* fail-closed: cannot prove coherence without both views */ + return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; } diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 1330a40490..3d3a3801e4 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -151,11 +151,17 @@ typedef struct ClusterLeaveState { int32 leaving_node_id; /* -1 if none */ uint32 _pad0; uint64 leave_epoch; /* epoch this leave is bound to (CL-I3 immutable) */ - uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound; - * an unchanged value at the epoch bump means OUR - * clean-leave committed (no real death intruded) */ + uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound + * (retained for observability; the coherence gate + * now uses leave_baseline_others_dead, spec-2.29a + * ②b — the scalar counted the leaving node's own + * expected DEAD and falsely escalated) */ uint64 barrier_deadline_us; /* fail-closed deadline (drain_timeout_ms) */ uint8 ack_bitmap[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; /* survivor acks */ + /* spec-2.29a ②b: CSSD dead set (EXCLUDING the leaving node) snapshotted when + * the leave was bound; the coherence gate escalates only if a THIRD-PARTY + * death changed this set mid-drain. */ + uint8 leave_baseline_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; pg_atomic_uint64 ges_drained_count; /* shards/grants drained (observability) */ pg_atomic_uint64 gcs_flushed_count; /* dirty/X pages flushed */ pg_atomic_uint64 shards_remastered; /* shards moved off leaving node */ @@ -337,10 +343,14 @@ extern bool cluster_clean_leave_should_abort_writable(bool in_transaction, bool /* version-coherent leave (U3 / CL-I3 / L235): the leave is still coherent only * if neither the cluster epoch nor the CSSD dead_generation moved since the - * leave was bound; any external bump (a real death intruding) => not coherent - * => the caller must ABORTED_ESCALATE. */ + * leave was bound; any external membership change (a third-party death, or a + * third-party fail-stop that already bumped the epoch) => not coherent => the + * caller must ABORTED_ESCALATE. spec-2.29a ②b: the dead set compared here + * EXCLUDES the leaving node itself (others-dead bitmap), so the leaving node's + * own expected alive→DEAD transition never falsely escalates the leave. */ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen); + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes); /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index b703798b38..24a7ce43a4 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -159,14 +159,28 @@ UT_TEST(test_phase_fsm) UT_TEST(test_version_coherent) { - /* nothing moved -> coherent */ - UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, 3, 3)); - /* epoch bumped by an external death -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 3)); - /* dead_generation moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, 3, 4)); - /* both moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 4)); + /* spec-2.29a ②b: coherence = epoch unchanged AND the others-dead bitmap + * (dead set EXCLUDING the leaving node) unchanged. The reflexive-case + * matrix from the spec §②b 8.A argument, exercised on the pure predicate. */ + uint8 od_none[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + uint8 od_third[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + const int n = CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES; + + od_third[0] = 0x04; /* a THIRD-PARTY node (e.g. node 2) became DEAD */ + + /* (i) nothing moved (leaving node's own DEAD is already excluded upstream, + * so its transition does not appear here) -> coherent */ + UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, od_none, od_none, n)); + /* (iii) epoch bumped by an external fail-stop -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_none, n)); + /* (ii) a third-party death entered the others-dead set, epoch not yet + * bumped (the P1-b window) -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, od_third, n)); + /* (iv) both moved -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_third, n)); + /* fail-closed on a missing view */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, NULL, od_none, n)); + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, NULL, n)); } From 55dd3f34057ddeec164f1fcff5ee47c9e133b909 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:37 +0800 Subject: [PATCH 40/58] fix(cluster): converge REDECLARE_DONE on (episode_epoch, dead_bitmap_hash) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The P6 all-done gate and the WAIT_EPOCH coordinator witness keyed the cross-node DONE on event_id, but event_id folds the sender-local cssd_dead_generation, which drifts with each survivor's private flap observation history and never converges across nodes: any flap asymmetry made the survivors compute different ids for the same episode, drop each other's DONEs, and wedge P6 forever — nondeterministically reintroducing the permanent-freeze shape this branch exists to remove. The quorum-accepted dead SET is what actually converges, so the DONE payload now carries a hash over the dead bitmap alone (same kernel as the event_id hash, riding the same request-id field pair — no envelope change), stamped per episode at P0 accept. Accounting, the P6 gate and the witness compare against the stamp; event_id stays a purely local accept-dedup scope. A late DONE from a previous episode still cannot back a new one: a coordinator re-election changes the dead set (hash mismatch) and a same-node re-death rides a higher epoch through the interposed JOIN bump (epoch conjunct). Pre-accept frames now mismatch the previous stamp and are dropped; senders re-announce every tick, so accounting always lands after the accept snapshot — closing the first-event pre-accept window as a side effect. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R2/R4) --- src/backend/cluster/cluster_debug.c | 4 +- src/backend/cluster/cluster_ges.c | 4 +- src/backend/cluster/cluster_grd.c | 146 ++++++++++++++------- src/include/cluster/cluster_grd.h | 28 +++- src/test/cluster_unit/test_cluster_debug.c | 2 +- src/test/cluster_unit/test_cluster_grd.c | 124 ++++++++++++----- 6 files changed, 216 insertions(+), 92 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index e19b3e98e8..1e50f2b3a9 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1211,8 +1211,8 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) fmt_int32((int32)cluster_grd_recovery_event_coordinator())); emit_row(rsinfo, "grd_recovery", "done_self_epoch", fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); - emit_row(rsinfo, "grd_recovery", "done_self_event_id", - fmt_int64((int64)cluster_grd_recovery_done_event_id_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_bitmap_hash", + fmt_int64((int64)cluster_grd_recovery_done_bitmap_hash_for(cluster_node_id))); emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index 547f0e5aac..f68e407239 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -550,7 +550,9 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) /* spec-4.6 P0#3 cluster gate — fire-and-forget barrier announcement: * record and return (no work queue, no reply, no dedup; the sender - * re-announces each tick, the receiver write is a monotonic max). */ + * re-announces each tick, the receiver write is a monotonic max). + * Amendment v1.2 (R2): the request-id field pair carries the sender's + * dead_bitmap hash (composite convergence key), not an event_id. */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, ges_request_holder_request_id(req)); diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 8977739433..ae87b62dfa 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -737,8 +737,9 @@ cluster_grd_shmem_init(void) /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); - pg_atomic_init_u64(&cluster_grd_state->recovery_done_event_id[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_bitmap_hash[i], 0); } + pg_atomic_init_u64(&cluster_grd_state->recovery_event_bitmap_hash, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -1703,11 +1704,36 @@ cluster_grd_recovery_done_epoch_for(int32 node) } uint64 -cluster_grd_recovery_done_event_id_for(int32 node) +cluster_grd_recovery_done_bitmap_hash_for(int32 node) { if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return 0; - return pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[node]); + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[node]); +} + +uint64 +cluster_grd_recovery_event_bitmap_hash_value(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); +} + +/* + * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). + * + * The cross-node half of the REDECLARE_DONE convergence key: a hash over + * the quorum-accepted dead bitmap ALONE. Same kernel as the event_id hash + * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — + * the generation is per-instance observation history and does NOT converge + * across survivors, which is exactly why event_id cannot key the P6 gate. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere, + * so the composite key degrades to the epoch-only gate there. + */ +uint64 +cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) +{ + return hash_bytes_extended(dead_bitmap, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES, 0); } bool @@ -2051,7 +2077,11 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; - uint64 event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + /* Amendment v1.2 (R2): the DONE payload carries the sender's accepted + * dead-bitmap hash (cross-node convergence key), riding the request-id + * field pair — NOT the sender-local event_id (its dead_generation fold + * diverges across survivors' flap observation histories). */ + uint64 bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -2061,8 +2091,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); - req.holder_request_id_lo = (uint32)(event_id & 0xffffffffu); - req.holder_request_id_hi = (uint32)(event_id >> 32); + req.holder_request_id_lo = (uint32)(bitmap_hash & 0xffffffffu); + req.holder_request_id_hi = (uint32)(bitmap_hash >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -2081,21 +2111,29 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash) { - uint64 current_event_id; + uint64 episode_bitmap_hash; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; /* - * spec-4.6a D4-v2: REDECLARE_DONE is an event-scoped proof. A stale DONE - * from a previous episode can carry the same epoch when cur==old, so epoch - * monotonicity alone is not enough. Drop messages for any event other than - * the one this LMON has accepted; senders re-announce every tick. + * spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE converges on the COMPOSITE + * key (episode_epoch, dead_bitmap_hash). A stale DONE from a previous + * episode can carry the same epoch when cur==old, so epoch monotonicity + * alone is not enough — but the previous episode's dead SET necessarily + * differs (a coordinator re-election adds the old coordinator to it; a + * re-death of the same node rides a higher epoch via the interposed JOIN + * bump), so the bitmap hash is the ABA guard. Flap-history drift changes + * only the per-instance dead_generation, never the quorum dead SET, so + * DONEs for the same episode always match here (the R2 wedge fix). + * Pre-accept frames mismatch the previous episode's stamp and are dropped; + * senders re-announce every tick, so accounting lands after our P0 accept + * (which also closes the R4 pre-accept window by construction). */ - current_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); - if (event_id == 0 || (current_event_id != 0 && event_id != current_event_id)) + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) return; /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic @@ -2107,7 +2145,7 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) if (epoch > prev) pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } - pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[node], event_id); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } /* @@ -2175,13 +2213,13 @@ grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, continue; done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); if (done_epoch >= episode_epoch - && pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) - == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) continue; grd_recovery_appendf(buf, buflen, &off, - "%s%d(done=" UINT64_FORMAT "/event=" UINT64_FORMAT ")", any ? "," : "", - i, done_epoch, - pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i])); + "%s%d(done=" UINT64_FORMAT "/bitmap_hash=" UINT64_FORMAT ")", + any ? "," : "", i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i])); any = true; } if (!any) @@ -2433,6 +2471,12 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } + /* Amendment v1.2 (R2): stamp the composite convergence key's cross-node + * half. Hash over the accepted dead bitmap alone — every survivor that + * accepted the same quorum dead SET computes the same value regardless + * of its private dead_generation observation history. */ + pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, + cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2510,8 +2554,8 @@ cluster_grd_recovery_lmon_tick(void) TimestampTz next_deadline; uint32 coordinator; uint64 coord_done; - uint64 coord_done_event_id; - uint64 accepted_event_id; + uint64 coord_done_bitmap_hash; + uint64 episode_bitmap_hash; uint64 coord_done_at_accept; wait_deadline = (TimestampTz)pg_atomic_read_u64( @@ -2533,29 +2577,33 @@ cluster_grd_recovery_lmon_tick(void) } coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); - accepted_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + episode_bitmap_hash + = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); coord_done = coordinator < CLUSTER_MAX_NODES ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) : 0; - coord_done_event_id + coord_done_bitmap_hash = coordinator < CLUSTER_MAX_NODES ? pg_atomic_read_u64( - &cluster_grd_state->recovery_done_event_id[coordinator]) + &cluster_grd_state->recovery_done_bitmap_hash[coordinator]) : 0; coord_done_at_accept = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); /* - * spec-4.6a §2.3 witness (all three conjuncts of the frozen - * text): the coordinator's DONE must (a) be for THIS event, - * (b) name exactly cur as the episode epoch, and (c) have been + * spec-4.6a §2.3 witness, composite-key form (Amendment v1.2 + * R2): the coordinator's DONE must (a) name this episode's + * quorum dead SET (bitmap hash — the cross-node-consistent + * key; the sender-local event_id folds the per-instance + * dead_generation and diverges under flap asymmetry), (b) + * name exactly cur as the episode epoch, and (c) have been * accounted after our P0 accept snapshot. Under strictly * monotonic per-event epochs (c) is implied by (a)+(b) — any * pre-accept residue carries a lower epoch — but it stays as a * belt-and-suspenders guard against accounting bugs. */ if (cur_epoch == old_epoch && coord_done == cur_epoch - && coord_done_event_id == accepted_event_id + && coord_done_bitmap_hash == episode_bitmap_hash && coord_done > coord_done_at_accept) { pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); ereport(WARNING, @@ -2563,25 +2611,25 @@ cluster_grd_recovery_lmon_tick(void) errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " "equal-epoch progress"), errdetail("coordinator=%u, epoch=" UINT64_FORMAT - ", event_id=" UINT64_FORMAT, - coordinator, cur_epoch, accepted_event_id))); + ", dead_bitmap_hash=" UINT64_FORMAT, + coordinator, cur_epoch, episode_bitmap_hash))); } else { next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); - ereport( - WARNING, - (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), - errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " - "affected shards stay frozen"), - errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT - ", coordinator=%u, coordinator_done=" UINT64_FORMAT - ", coordinator_done_event_id=" UINT64_FORMAT - ", accepted_event_id=" UINT64_FORMAT - ", coordinator_done_at_accept=" UINT64_FORMAT, - cur_epoch, old_epoch, coordinator, coord_done, - coord_done_event_id, accepted_event_id, coord_done_at_accept))); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_bitmap_hash=" UINT64_FORMAT + ", episode_bitmap_hash=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_bitmap_hash, episode_bitmap_hash, + coord_done_at_accept))); return; } } @@ -2728,8 +2776,8 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); - pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[cluster_node_id], - pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)); grd_recovery_broadcast_done(episode_epoch); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), cluster_grd_rebuild_timeout_ms); @@ -2864,9 +2912,13 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; + /* Amendment v1.2 (R2): composite key — the peer's DONE must name + * this episode's epoch AND the same quorum dead SET. Keying on + * event_id here wedged the gate whenever survivors' private + * dead_generation histories diverged (flap asymmetry). */ if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch - || pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) - != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { all_done = false; break; } diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index a098d53ed1..c6fa42ecaf 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -294,9 +294,21 @@ typedef struct ClusterGrdShared { /* spec-4.6 P0#3 cluster gate — per-node epoch for which that node * last announced "local rebind barrier complete" (REDECLARE_DONE). - * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ + * P6 requires done_epoch[s] >= current epoch for EVERY survivor. + * + * spec-4.6a Amendment v1.2 (R2): the cross-node convergence key is the + * COMPOSITE (episode_epoch, dead_bitmap_hash). event_id is NOT globally + * consistent (it folds the per-instance cssd_dead_generation, which + * drifts with each node's own flap observation history), so keying the + * P6 gate on it wedges the cluster whenever two survivors observed a + * different flap history. The quorum-accepted dead SET is what actually + * converges; its hash rides the DONE payload instead. event_id stays a + * purely LOCAL ABA scope (P0 accept dedup). */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; - pg_atomic_uint64 recovery_done_event_id[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_bitmap_hash[CLUSTER_MAX_NODES]; + /* hash of THIS episode's accepted dead_bitmap (LMON writes at P0 accept; + * empty-bitmap hash for JOIN episodes, identical on every node). */ + pg_atomic_uint64 recovery_event_bitmap_hash; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -624,7 +636,11 @@ extern uint64 cluster_grd_recovery_event_old_epoch(void); extern uint64 cluster_grd_recovery_episode_epoch_value(void); extern uint32 cluster_grd_recovery_event_coordinator(void); extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); -extern uint64 cluster_grd_recovery_done_event_id_for(int32 node); +extern uint64 cluster_grd_recovery_done_bitmap_hash_for(int32 node); +extern uint64 cluster_grd_recovery_event_bitmap_hash_value(void); +/* Amendment v1.2 (R2): the cross-node DONE key — hash over the dead bitmap + * ALONE (no dead_generation fold; same kernel as the event_id hash). */ +extern uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap); extern int cluster_grd_recovery_block_redeclare_cursor(void); extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); extern bool cluster_grd_recovery_block_redeclare_done(void); @@ -644,8 +660,10 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind - * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id); + * barrier for `epoch`. Amendment v1.2 (R2): the third argument is the + * sender's dead_bitmap hash (the cross-node half of the composite + * convergence key), NOT the sender-local event_id. */ +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 8b64c01755..8b741f7c42 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -3350,7 +3350,7 @@ cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) } uint64 -cluster_grd_recovery_done_event_id_for(int32 node pg_attribute_unused()) +cluster_grd_recovery_done_bitmap_hash_for(int32 node pg_attribute_unused()) { return 0; } diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index c7f805459d..2e4b813e17 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -3748,6 +3748,30 @@ ut_jr_setup_3node(void) cluster_grd_master_map_init(); } +/* spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE accounting converges on the + * composite key (episode_epoch, dead_bitmap_hash), so a test that feeds DONEs + * must first drive a P0 accept to stamp the episode's bitmap hash. Accept an + * empty-dead-bitmap FAIL event (hash identical to a JOIN episode's, no fence + * side effects; the FSM parks in WAIT_EPOCH awaiting a strict bump, which the + * fence assertions never touch). Returns the stamped hash for the callers' + * mark_peer_done payloads. */ +static uint64 +ut_jr_stamp_episode_bitmap_hash(uint64 event_id) +{ + bool saved_enabled = cluster_enabled; + + cluster_enabled = true; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle tick: baseline capture */ + ut_mock_last_event.event_id = event_id; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + cluster_grd_recovery_lmon_tick(); /* P0 accept stamps the bitmap hash */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_enabled = saved_enabled; + return cluster_grd_recovery_event_bitmap_hash_value(); +} + /* U1 — recompute moves the joiner's home shards back from the survivor. */ UT_TEST(test_jr_u1_recompute_moves_joiner_home) { @@ -3937,9 +3961,12 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(901); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* the rejoiner's home block */ @@ -3950,15 +3977,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3969,10 +3996,13 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(902); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); @@ -3981,15 +4011,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5, 1); - cluster_grd_recovery_mark_peer_done(1, 5, 1); - cluster_grd_recovery_mark_peer_done(2, 5, 1); + cluster_grd_recovery_mark_peer_done(0, 5, done_hash); + cluster_grd_recovery_mark_peer_done(1, 5, done_hash); + cluster_grd_recovery_mark_peer_done(2, 5, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -4002,10 +4032,12 @@ UT_TEST(test_jr_u15_master_side_gate_decision) int n1[1] = { 1 }; BufferTag tag; bool deny; + uint64 done_hash; memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(903); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; @@ -4014,9 +4046,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4059,17 +4091,20 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) int n2[1] = { 2 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); /* declared {0,1,2}, all members */ /* --- Episode 1: node 1 rejoins and completes (all survivors re-declare). --- */ ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(904); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4082,29 +4117,35 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20, 1); + cluster_grd_recovery_mark_peer_done(0, 20, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20, 1); + cluster_grd_recovery_mark_peer_done(1, 20, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } /* ============================================================ - * spec-4.6a r2-P1-1 — event-scoped REDECLARE_DONE proof. + * spec-4.6a Amendment v1.2 (R2) — composite-key REDECLARE_DONE proof. * - * A stale DONE (previous event) must be dropped at the accounting - * layer and must never satisfy the WAIT_EPOCH coordinator witness; - * a current-event DONE at exactly cur, accounted after the P0 - * accept snapshot, is the only equal-epoch escape (proof-carrying, - * never timeout-only). + * The cross-node convergence key is (episode_epoch, dead_bitmap_hash). + * A DONE naming a DIFFERENT dead set (the old-event ABA shape) must be + * dropped at the accounting layer and never satisfy the WAIT_EPOCH + * coordinator witness. A DONE naming the SAME dead set must be + * accounted even though the sender's local event_id differs (the + * dead_generation-drift shape that wedged the event_id key: R2), and + * then satisfies the witness — the only equal-epoch escape + * (proof-carrying, never timeout-only). * ============================================================ */ -UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) +UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) { ClusterGrdRecoveryCounters before; ClusterGrdRecoveryCounters after; + uint8 old_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 episode_hash; + uint64 old_event_hash; ut_jr_setup_3node(); cluster_enabled = true; @@ -4117,27 +4158,38 @@ UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); cluster_grd_recovery_lmon_tick(); - /* Stage the fail-stop event: dead node 2, coordinator 0, event 77. */ + /* Stage the fail-stop event: dead node 2, coordinator 0. The local + * event_id (77) folds this node's own dead_generation; a peer that saw a + * different flap history computes a DIFFERENT id for the same episode. */ ut_mock_last_event.event_id = 77; ut_mock_last_event.coordinator_node_id = 0; ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; ut_mock_last_event.dead_bitmap[0] = 0x04; cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); - - /* Stale-event DONE (event 76) is dropped: nothing is accounted. */ - cluster_grd_recovery_mark_peer_done(0, 10, 76); - UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 0); + episode_hash = cluster_grd_recovery_event_bitmap_hash_value(); + UT_ASSERT_EQ(episode_hash, cluster_grd_dead_bitmap_hash(ut_mock_last_event.dead_bitmap)); + + /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 + * instead of node 2 — e.g. the previous episode before a coordinator + * re-election) is dropped: nothing is accounted. */ + memset(old_dead, 0, sizeof(old_dead)); + old_dead[0] = 0x08; + old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); + cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); - /* Current-event DONE is accounted (epoch + event id). */ - cluster_grd_recovery_mark_peer_done(0, 10, 77); + /* Generation-drift shape (the R2 wedge): the peer's local event_id + * differs, but its DONE names the SAME quorum dead set -> the composite + * key matches and the DONE is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, episode_hash); UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); - UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 77); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), episode_hash); - /* Witness advance: deadline expired + cur==old + coordinator DONE for - * THIS event at exactly cur, accounted after the accept snapshot. The - * FSM takes the event-scoped equal-epoch escape exactly once. */ + /* Witness advance: deadline expired + cur==old + coordinator DONE naming + * THIS dead set at exactly cur, accounted after the accept snapshot. The + * FSM takes the composite-key equal-epoch escape exactly once. */ cluster_grd_recovery_counters_snapshot(&before); ut_mock_now = (int64)60 * 1000000; cluster_grd_recovery_lmon_tick(); @@ -4264,7 +4316,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); - UT_RUN(test_recovery_stale_event_done_rejected_and_witness_advance); + UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From ddefc0003592ae537598ac18601d7633d3fc10ae Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:58 +0800 Subject: [PATCH 41/58] fix(cluster): dead-master block serve proof is unconditional again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert the scope short-circuit that returned NORMAL for a dead static master's blocks wherever online thread recovery cannot run (the default configuration and every >2-node deployment): it skipped both the is_materialized cold-block door and the redo-coverage lost-write door, so a committed write only the dead node saw could be silently read stale from shared storage. No other guard sits on that read path — the GRD freeze ends with the episode, the HW gate covers only extend high-water marks, and the page-SCN checks ride the ship path. Out of scope the posture is now an explicit bounded retryable error (53R9L, hint updated to name the way out: restart the failed node, or enable online thread recovery in a supported scope) for every tag whose static master is dead — including never-written blocks, which the cold-block door cannot distinguish from not-yet-replayed ones — and the door reopens the moment the failed node stops being DEAD. In-scope behavior is byte-identical. The now-orphaned scope helper is removed. t/362 asserts the honest two-outcome contract on fresh connections (success or explicit SQLSTATE — 53R9L or the TT-unknown visibility door — never a hang), rides pre-warmed sessions for convergence observability, and pins the D12 cleanup counter to a positive delta; t/293's post-kill extend leg follows the same posture. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R1/R3) --- src/backend/cluster/cluster_gcs_block.c | 31 ++- src/include/cluster/cluster_thread_recovery.h | 10 - .../cluster_tap/t/293_hw_online_remaster.pl | 30 ++- .../362_shared_catalog_4node_kill_selfheal.pl | 218 +++++++++++++++--- .../cluster_unit/test_cluster_gcs_block.c | 13 +- 5 files changed, 223 insertions(+), 79 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index d0219deaf9..ee200ffafe 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1223,13 +1223,8 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The redo-before-serve gate must - * engage on EXACTLY the same scope where online thread recovery can actually - * run. Otherwise a >2-node or GUC-off deployment waits forever on a - * materialization authority that the thread-recovery launcher intentionally - * treats as not applicable. - * - * In applicable 2-node scope, two conditions are both required (Q5): + * live survivor (recovery-aware routing). Two conditions are both + * required before the on-disk version may be served (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1240,18 +1235,18 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably * reached us → lost-write → fail-closed. + * + * spec-4.6a Amendment v1.2 (R1): this proof is UNCONDITIONAL. Where + * online thread recovery cannot run (GUC off — the default — or any + * >2-node deployment) the materialization authority is never published, + * so a dead master's blocks stay RECOVERING until the failed node + * restarts and runs its own instance recovery: a bounded, retryable + * ERROR on the request path (53R9L), never an unproven serve. A scope + * predicate must never gate a correctness proof — a committed write on + * a cold block that only the dead node saw has NO other guard on this + * read path (GRD freeze ends with the episode; the HW gate covers only + * extend high-water marks; pd_block_scn checks ride the ship path). */ - { - bool shared_fs; - int survivors; - - shared_fs = (cluster_shared_storage_backend == CLUSTER_SHARED_FS_BACKEND_CLUSTER_FS); - survivors = cluster_conf_node_count() - 1; - if (!cluster_thread_recovery_materialization_gate_enabled( - cluster_online_thread_recovery, cluster_conf_has_peers(), shared_fs, survivors)) - return GCS_BLOCK_NORMAL; - } - if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->recovery_block_resources_recovering, 1); diff --git a/src/include/cluster/cluster_thread_recovery.h b/src/include/cluster/cluster_thread_recovery.h index c7be25e6b0..c455b77436 100644 --- a/src/include/cluster/cluster_thread_recovery.h +++ b/src/include/cluster/cluster_thread_recovery.h @@ -376,16 +376,6 @@ cluster_thread_recovery_gate_decide(ClusterThreadRecScope scope, const uint64 *d return false; /* every dead origin materialized -> ready to unfreeze */ } -static inline bool -cluster_thread_recovery_materialization_gate_enabled(bool guc_on, bool has_peers, - bool shared_fs_backend, - int live_survivor_count) -{ - return cluster_thread_recovery_decide_scope(guc_on, has_peers, shared_fs_backend, - live_survivor_count) - == CLUSTER_THREADREC_SCOPE_APPLICABLE; -} - /* * cluster_thread_recovery_replay_epoch_aborts -- the L235 episode-epoch * staleness guard (spec-4.11 3b-4b). The per-thread replay slot stamps the GRD diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 509c2f50ff..2447312952 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -329,23 +329,39 @@ sub retry_sql is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, 'L7: self-heal path did not exhaust the retry budget'); -# spec-4.6a section 4 (D8): DONE must actually unfreeze (P7) -- the survivor -# extends a table that lived on shards the dead master owned; a frozen shard -# would fail this INSERT with the remastering error. +# spec-4.6a section 4 (D8, Amendment v1.2 R1): after the same-episode DONE the +# episode has converged (P7 ran: DONE counter + no exhaustion asserted above), +# but in this out-of-scope deployment (online_thread_recovery off) EVERY tag +# whose static master is the dead node stays fail-closed until that node +# returns — including brand-new blocks, which have no materialization proof +# either (the cold-block door cannot distinguish never-existed from +# not-yet-replayed; main's t/293 L5b documents the same posture). The +# survivor write is therefore the honest two-outcome contract: success (all +# touched tags survivor-mastered) or the explicit bounded 53R9L — never a +# hang and never an unbounded wedge. { my ($ext_ok, $ext_res) = (0, ''); - for my $try (1 .. 30) + for my $try (1 .. 10) { $ext_res = eval { $h1->safe_psql('postgres', - 'INSERT INTO s1 SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 'CREATE TABLE IF NOT EXISTS s1_heal (a int, b int, c int, d int); ' + . 'INSERT INTO s1_heal SELECT g, g, g, g FROM generate_series(1, 4096) g'); 1; } ? 'ok' : ($@ // 'error'); if ($ext_res eq 'ok') { $ext_ok = 1; last; } sleep 1; } - ok($ext_ok, 'L7: survivor extend succeeds after same-episode DONE (P7 unfreeze)') - or diag($ext_res); + if ($ext_ok) + { + ok(1, 'L7: survivor new-table extend succeeded after same-episode DONE (P7 unfreeze)'); + } + else + { + like($ext_res, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + 'L7: survivor extend fail-closed with an explicit SQLSTATE (dead-master tag, out-of-scope posture)'); + } } $heal->stop_pair; diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl index 883860c0fb..a5f6719eea 100644 --- a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -5,6 +5,13 @@ # # Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) # so a killed node's HW authority is recoverable from its per-thread WAL. +# Amendment v1.2 (R1): in this out-of-scope deployment (4 nodes, online +# thread recovery off) the dead master's PRE-EXISTING writes stay +# fail-closed until the failed node restarts — its unproven blocks (bounded +# 53R9L) and equally its transaction verdicts (TT status unknown for its +# xids, the visibility door). Reads and DDL are asserted on the honest +# two-outcome contract (success or explicit SQLSTATE, never a hang); +# new-object DDL/writes prove P7 unfreeze. # After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means # the test harness is misconfigured, and lack of convergence is a real FAIL. # @@ -88,6 +95,52 @@ sub sum_hw return $sum; } +# Amendment v1.2 (R1): after the kill, a NEW backend's parse may cold-read a +# dead-master catalog page and honestly fail closed (53R9L) until the failed +# node returns. The convergence observability therefore rides PRE-WARMED +# long-lived psql sessions (catcache hot; pg_cluster_* reads touch shmem, not +# dead-master blocks), while fresh-connection legs assert the two-outcome +# contract. bg_query returns (1, value) or (0, error-ish) without dying. +sub bg_query +{ + my ($bg, $sql) = @_; + my $out = eval { $bg->query($sql) }; + return (0, $@ // 'query failed') if $@; + return (1, $out // ''); +} + +sub bg_counter +{ + my ($bg, $cat, $key) = @_; + my ($ok, $v) = bg_query($bg, + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'"); + return ($ok && defined $v && $v ne '') ? $v + 0 : 0; +} + +sub bg_sum +{ + my ($bgs, $cat, $key, @idx) = @_; + my $sum = 0; + $sum += bg_counter($bgs->{$_}, $cat, $key) for @idx; + return $sum; +} + +sub bg_poll_until +{ + my ($bg, $sql, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($ok, $out) = bg_query($bg, $sql); + $last = $out; + return 1 if $ok && defined $out && $out eq $want; + usleep(500_000); + } + diag("bg_poll_until timed out: '$sql' last='$last' want='$want'"); + return 0; +} + sub logs_contain { my ($quad, $pattern, @idx) = @_; @@ -217,9 +270,20 @@ sub startup_env_blocker retry_sql($quad->node($i), 'CHECKPOINT', 30); } -my $done_before = sum_hw($quad, 'remaster_done_count', @survivors); -my $blocked_before = sum_hw($quad, 'remaster_blocked_count', @survivors); -my $retry_before = sum_hw($quad, 'remaster_retry_count', @survivors); +# Amendment v1.2 (R1): pre-warmed observability sessions (see helper note). +my %bg; +for my $i (@survivors) +{ + $bg{$i} = $quad->node($i)->background_psql('postgres', on_error_stop => 0); + bg_query($bg{$i}, "SELECT value FROM pg_cluster_state WHERE category='hw' AND key='remaster_done_count'"); + bg_query($bg{$i}, "SELECT count(*) FROM pg_cluster_grd_shards WHERE master_node_id=$dead"); + bg_query($bg{$i}, 'SELECT txid_current()'); +} + +my $done_before = bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors); +my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); +my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); # L2: kill node3 and require every survivor to see the real DEAD edge. # Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, @@ -232,18 +296,26 @@ sub startup_env_blocker } # L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +# The map read is hard-asserted on the warm COORDINATOR session only: the view +# scan on a cold survivor deterministically cold-reads a dead-master catalog +# page out of the small test buffer pool and honestly fails closed (53R9L) +# until node3 returns — the map itself is a deterministic recompute barriered +# by P6, and each cold survivor is instead hard-asserted on its own HW +# remaster worker DONE log line below. +ok(bg_poll_until($bg{0}, + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + 'L3: coordinator remastered all GRD shards off node3 (warm session)'); for my $i (@survivors) { - ok(poll_until($quad->node($i), - "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", - 'true', 60), - "L3: survivor node$i remastered all GRD shards off node$dead"); + ok(log_until($quad, $i, qr/HW remaster worker: dead node $dead -> done/, 90), + "L3: survivor node$i HW remaster worker reported done"); } my $done_converged = 0; my $deadline = time() + 90; while (time() < $deadline) { - if (sum_hw($quad, 'remaster_done_count', @survivors) > $done_before) + if (bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors) > $done_before) { $done_converged = 1; last; @@ -255,10 +327,10 @@ sub startup_env_blocker 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); # L6: if a transient BLOCKED occurred, a same-episode retry must also have run. -my $blocked_after = sum_hw($quad, 'remaster_blocked_count', @survivors); +my $blocked_after = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); if ($blocked_after > $blocked_before) { - ok(sum_hw($quad, 'remaster_retry_count', @survivors) > $retry_before, + ok(bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors) > $retry_before, 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); } else @@ -266,45 +338,127 @@ sub startup_env_blocker pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); } -# L4: survivor SQL and shared-catalog DDL recover after fail-stop. +# L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an +# out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a +# BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a +# hang. sc4_hw_* spans ~20+ pages, so some page's static master is node3 with +# overwhelming probability; the redo-coverage proof cannot be published without +# online thread recovery, so the scan hits 53R9L (resource recovering). If by +# hash luck no scanned page is node3-mastered the scan succeeds — both are the +# honest two-outcome contract; only a hang/timeout or an unexpected error fails. +for my $t (1 .. 2) +{ + my $t0 = time(); + my ($rc, $out, $err) = $quad->node(1)->psql('postgres', + "SELECT count(*) FROM sc4_hw_$t", timeout => 60); + my $elapsed = time() - $t0; + if (defined $rc && $rc == 0) + { + ok(1, "L4a: dead-master table sc4_hw_$t read succeeded (no node$dead-mastered page scanned)"); + diag("L4a: sc4_hw_$t count=$out"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + "L4a: dead-master table sc4_hw_$t read fail-closed with an explicit SQLSTATE"); + } + cmp_ok($elapsed, '<', 60, "L4a: sc4_hw_$t read returned bounded (no hang)"); +} + +# L4: survivor SQL recovers after fail-stop — hard-asserted on the pre-warmed +# sessions; a FRESH connection is the honest two-outcome contract (v1.2 R1): +# success, or an explicit fail-closed SQLSTATE on a dead-master catalog page — +# never a hang. for my $i (@survivors) { - my ($ok, $res) = retry_sql($quad->node($i), 'SELECT txid_current()', 60); - ok($ok, "L4: survivor node$i accepts txid_current after reconfig") or diag($res); + my $got = 0; + my $dl = time() + 60; + while (time() < $dl) + { + my ($ok, $out) = bg_query($bg{$i}, 'SELECT txid_current()'); + if ($ok && defined $out && $out =~ /^\d+$/) { $got = 1; last; } + usleep(500_000); + } + ok($got, "L4: survivor node$i accepts txid_current after reconfig (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($i)->psql('postgres', 'SELECT txid_current()', + timeout => 60); + if (defined $rc && $rc == 0) + { + ok(1, "L4: fresh connection txid_current succeeded on node$i"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); + } + cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); +} +my ($ddl_rc, $ddl_out, $ddl_err) = $n0->psql('postgres', + 'CREATE TABLE sc4_after_kill (id int)', timeout => 60); +my $ddl_ok = defined $ddl_rc && $ddl_rc == 0; +if ($ddl_ok) +{ + ok(1, 'L4: coordinator survivor DDL succeeded after node kill'); +} +else +{ + like($ddl_err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); + diag("L4: DDL fail-closed honestly: $ddl_err"); } -my ($ddl_ok, $ddl_res) = retry_sql($n0, 'CREATE TABLE sc4_after_kill (id int)', 60); -ok($ddl_ok, 'L4: coordinator survivor can run DDL after node kill') or diag($ddl_res); for my $i (1, 2) { - ok(poll_until($quad->node($i), - "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", - 'true', 60), - "L4: survivor node$i sees post-kill shared-catalog DDL"); + if ($ddl_ok) + { + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); + } + else + { + ok(1, "L4: survivor node$i visibility leg not applicable (DDL fail-closed honestly)"); + } } -# L4 (spec-4.6a D12, r2-P1-3): the dead-node PCM residue cleanup counter is -# exposed and non-negative on every survivor; when the dead node left any -# holder/pending-X records behind, at least one survivor accounts a cleanup -# (delta assertable; zero stays legal when the dead node held nothing). +# L4 (spec-4.6a D12, Amendment v1.2 R3): node3 held real PCM/HW state at kill +# time (it extended sc4_hw_1/2 with 5000 rows each), so the dead-node residue +# cleanup MUST account a positive delta across the survivors — a >= 0 +# assertion would stay green with D12 entirely disabled. { my $cleanup_total = 0; for my $i (@survivors) { - my $c = state_counter($quad->node($i), 'pcm', 'dead_cleanup_entries'); - ok($c >= 0, "L4: survivor node$i exposes pcm/dead_cleanup_entries"); - $cleanup_total += $c; + my ($ok, $n) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='pcm' AND key='dead_cleanup_entries'"); + ok($ok && defined $n && $n eq '1', + "L4: survivor node$i exposes the pcm/dead_cleanup_entries key"); + $cleanup_total += bg_counter($bg{$i}, 'pcm', 'dead_cleanup_entries'); } - diag("L4: pcm/dead_cleanup_entries total across survivors = $cleanup_total"); + cmp_ok($cleanup_total - $cleanup_before, '>', 0, + 'L4: dead-node PCM residue cleanup accounted a positive delta after kill'); + diag("L4: pcm/dead_cleanup_entries delta across survivors = " + . ($cleanup_total - $cleanup_before)); } -# L5: watchdog counters are allowed to be zero, but must be readable. +# L5: watchdog counters are allowed to be zero, but the KEYS must exist +# (state_counter reads a missing key as 0, so presence is the real assertion). for my $i (@survivors) { - my $cluster_gate = state_counter($quad->node($i), 'grd_recovery', 'cluster_gate_timeout'); - my $epoch_escape = state_counter($quad->node($i), 'grd_recovery', 'wait_epoch_escape'); - ok($cluster_gate >= 0 && $epoch_escape >= 0, - "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counters"); + my ($ok1, $n1) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='cluster_gate_timeout'"); + my ($ok2, $n2) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='wait_epoch_escape'"); + ok($ok1 && $ok2 && defined $n1 && $n1 eq '1' && defined $n2 && $n2 eq '1', + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counter keys"); } +$bg{$_}->quit for @survivors; + $quad->stop_quad; done_testing(); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 67fcc29dc2..8729fbd744 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -288,16 +288,6 @@ UT_TEST(test_gcs_block_tag_is_standard_buffer_tag_20b) } -UT_TEST(test_dead_static_master_materialization_gate_scope_matches_thread_recovery) -{ - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(false, true, true, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, false, true, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, false, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, true, 2)); - UT_ASSERT(cluster_thread_recovery_materialization_gate_enabled(true, true, true, 1)); -} - - /* spec-5.2 D2 (U3): pure master-side decision for an X-held N→S read. * node0 = holder/master in DIRECT, node1 = requester. */ UT_TEST(test_xheld_read_ship_decision_truth_table) @@ -525,7 +515,7 @@ UT_TEST(test_clean_xfer_stale_break_predicate) int main(void) { - UT_PLAN(22); + UT_PLAN(21); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -541,7 +531,6 @@ main(void) UT_RUN(test_gcs_block_data_size_equals_blcksz); UT_RUN(test_gcs_block_msg_type_enum_extends_without_gap); UT_RUN(test_gcs_block_tag_is_standard_buffer_tag_20b); - UT_RUN(test_dead_static_master_materialization_gate_scope_matches_thread_recovery); UT_RUN(test_xheld_read_ship_decision_truth_table); UT_RUN(test_forward_payload_read_image_flag_roundtrip); UT_RUN(test_clean_page_xfer_eligible_flag_roundtrip_and_orthogonal); From 0d57fa90bd6500743988b9653735330a5660bbcc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:58 +0800 Subject: [PATCH 42/58] fix(cluster): remaster retry ordering, runtime-structural warning, upgrade cleanup - Order the worker's terminal-result store after its backoff-deadline store (pg_memory_barrier) so the relaunch decider never pairs a fresh BLOCKED with a stale deadline and skips one backoff wait. - A structural cause discovered only by the WORKER (SIGHUP race) now leaves the retry deadline unset so the FSM's next tick takes the MARK_STRUCTURAL branch and emits the once-per-episode operator WARNING with the configuration hint. - Wrap the local S->X upgrade in PG_TRY and release the temporary S claim when the ACK wait throws (cancel via CHECK_FOR_INTERRUPTS), not only on the false return; document why the completed upgrade hard-resets the local S refcount (the X grant subsumes all local S declarations). Linker-only exception stubs for the units that link cluster_pcm_lock.o. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R5/R6/R7/R9) --- src/backend/cluster/cluster_hw_remaster.c | 19 ++++++++++++- src/backend/cluster/cluster_pcm_lock.c | 27 +++++++++++++++++-- .../test_cluster_bufmgr_pcm_hook.c | 11 ++++++++ src/test/cluster_unit/test_cluster_pcm_lock.c | 13 +++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index c585268a42..03ee6751a3 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -95,8 +95,16 @@ hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) static void hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) { + /* Amendment v1.2 (R6): the worker stores next_attempt_at BEFORE the + * terminal result, and the LMON relaunch decision reads them in the + * opposite order (result first). Order the two stores so a decider + * that observes the terminal result also observes the matching backoff + * deadline — without the fence a weakly-ordered CPU could let it pair a + * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait + * (bounded self-correcting, but cheap to close outright). */ if (res == CLUSTER_HW_REMASTER_DONE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_done(); } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { @@ -104,14 +112,23 @@ hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) cluster_hw_remaster_set_next_attempt_at(dead_node, hw_remaster_next_attempt_deadline(attempts)); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_blocked(); } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { - cluster_hw_remaster_set_next_attempt_at(dead_node, CLUSTER_HW_REMASTER_NO_DEADLINE); + /* Amendment v1.2 (R7): leave next_attempt_at at 0 (not NO_DEADLINE) + * so the FSM's next tick takes the MARK_STRUCTURAL decision branch + * and emits the episode-once operator WARNING + errhint — covering + * the SIGHUP race where only the WORKER (not the launch-side + * precheck) discovers the structural cause. MARK_STRUCTURAL then + * stamps NO_DEADLINE, making the WARNING once-per-episode. */ + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_blocked(); } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); } } diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index b9de7f6401..88ddaad4cf 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -2221,7 +2221,10 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) errmsg("block-level cache protocol state is being rebuilt " "after reconfiguration"), errhint("The block resource is recovering (survivor re-declare / " - "master rebuild after node failure); retry the transaction."))); + "master rebuild after node failure); retry the transaction. " + "If the failed node stays down, restart it to run its " + "instance recovery, or enable online thread recovery in a " + "supported scope (cluster.online_thread_recovery)."))); CHECK_FOR_INTERRUPTS(); pgstat_report_wait_start(WAIT_EVENT_GCS_BLOCK_RECOVERING); @@ -2305,9 +2308,24 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { struct GrdEntry *entry; + bool upgraded = false; cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); - if (!cluster_gcs_block_local_x_upgrade(tag)) { + /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with + * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out + * of it — release the temporary S claim on that path too, not + * only on the false return. */ + PG_TRY(); + { + upgraded = cluster_gcs_block_local_x_upgrade(tag); + } + PG_CATCH(); + { + cluster_pcm_lock_release(tag); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!upgraded) { cluster_pcm_lock_release(tag); ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), @@ -2318,6 +2336,11 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) entry = pcm_find_entry(tag); if (entry != NULL) { pcm_entry_lock_exclusive(entry); + /* Amendment v1.2 (R9): the completed S->X upgrade replaces + * ALL of this node's S declarations for the tag (the X + * grant subsumes them), so the local S refcount is + * intentionally hard-reset rather than decremented — a + * later release of the X does not owe any S releases. */ entry->s_holder_refcount_local = 0; LWLockRelease(&entry->entry_lock.lock); } diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 540cff5b7d..90151fc3cb 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -181,6 +181,17 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): linker-only exception-stack stubs (the + * linked cluster_pcm_lock.o now wraps the S->X upgrade in PG_TRY). */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c02894306f..08ba568886 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -134,6 +134,19 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): the S->X upgrade is now wrapped in + * PG_TRY/PG_CATCH, which references the exception stack + re-throw. The + * unit's errfinish mock longjmps to its own buffer (never through + * PG_exception_stack), so these exist for the linker only. */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { From 1be30fed70488c2c367c6366142373adadb076c1 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 21:10:46 +0800 Subject: [PATCH 43/58] fix(cluster): r2 P2-1 barrier-tick scalar conjunct + t/274 backend pre-bump shmem bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review-r2 items on top of the ②b others-dead coherence fix: P2-1 — the leaving node's barrier-tick own-commit latch (cl_leaving_barrier_tick) inferred 'my leave committed' from epoch>baseline AND the others-dead bitmap unchanged. The bitmap is not monotone: a third-party node that false-fail- stopped (bumping the epoch) then recovered leaves CSSD hysteresis DEAD->ALIVE, so the bitmap rebounds to its bound value while the scalar dead_generation only advances. If the leaver's first epoch>baseline observation lands after that rebound it would mis-latch a leave the survivor coordinator actually refused, suppress the barrier-deadline escalation, and hang forever in BARRIER_WAIT. Restore the scalar conjunct (epoch>baseline AND others_dead==bound AND dead_gen==baseline) via the pure, unit-tested cluster_clean_leave_own_commit_ latched predicate. This does not reintroduce the ②b false positive: the leaver's OWN alive->DEAD never bumps ITS OWN dead_generation (a node does not observe itself dead), so on the leaver side the scalar stays at baseline through its drain. Survivor-side coherence keeps the bitmap-only check. Adds the rebound negative-leg unit (others_dead unchanged + dead_gen advanced -> escalate). t/274 — a backend-context coordinator (the fail-stop inject test) advances the epoch then SYNCHRONOUSLY waits for the fence marker before publishing (MyBackendType != B_LMON path). During that seconds-long wait the epoch is bumped but unpublished, and a concurrent LMON GRD IDLE tick re-captures the post-bump epoch as its WAIT_EPOCH baseline -> old==cur -> the thread replay slot never reaches DONE -> HG#1b/HG#5 done-poll timeout. The ②a baseline-hold guard only reads LMON-process-local staged flags, invisible to a backend's pre-bump window. Publish it in shmem: ReconfigShmem.prebump_sync_active set before the bump, cleared at every return; the guard reads the shmem bit OR the local stage. 8.A (durable-marker-before-publish) unchanged. Reproduced the wedge locally at 87f793084b (HG#1b slot-DONE poll times out forever), then verified the fix: t/274 goes 18/18 green (exit 2 -> 0). This also fixes the long-standing main flake (the wedge is deterministic once LMON is freed from the sync-marker park; it was merely probabilistic before) — the t/274 known-flake / L66 registration should be revised. Local gates (cassert): cluster_unit 158 binaries (incl. own-commit-latched + version-coherent matrices), t/274 18/18, PG regress 219/219, clang-format/ headers/scn-cmp clean. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_clean_leave.c | 27 ++++++++++++---- .../cluster/cluster_clean_leave_policy.c | 30 +++++++++++++++++ src/backend/cluster/cluster_reconfig.c | 23 +++++++++++++ src/include/cluster/cluster_clean_leave.h | 9 ++++++ src/include/cluster/cluster_reconfig.h | 10 ++++++ .../cluster_unit/test_cluster_clean_leave.c | 32 ++++++++++++++++++- 6 files changed, 124 insertions(+), 7 deletions(-) diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 6ea7873643..31e25bc8a4 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -1777,17 +1777,32 @@ cl_leaving_barrier_tick(void) * leave is active (cluster_clean_leave_in_progress gates drive_joins + * commit_member), and the leave does not start while a join is pending * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * bump with an unchanged others-dead set during a leave can only be the - * leave's own commit (spec-2.29a ②b: the leaving node's own DEAD is excluded - * so its expected heartbeat stop no longer looks like a third-party death). + * bump with an unchanged version during a leave can only be the leave's own + * commit. + * + * spec-2.29a ②b + r2 P2-1: "unchanged version" here means the others-dead + * bitmap AND the scalar dead_generation both unchanged. The bitmap excludes + * the leaving node's own expected DEAD (②b: else its heartbeat stop would + * falsely escalate), but the bitmap is not monotone — a third-party + * false-DEAD→ALIVE rebound restores it while the scalar dead_generation only + * advances (r2 P2-1: else the leaver could mis-latch a refused leave and + * hang). The leaver's own DEAD never bumps its OWN dead_generation, so the + * scalar conjunct is safe on this side (it would NOT be safe on the survivor + * side, which keeps the bitmap-only coherence check). */ if (cluster_epoch_get_current() > baseline_epoch) { uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool others_dead_unchanged; + bool dead_gen_unchanged; cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); - if (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, - CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) - == 0) { + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged + = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(true, others_dead_unchanged, + dead_gen_unchanged)) { /* * Commit point observed (our clean-leave epoch was published; no * third-party death intruded). The leave can no longer be diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 095f4b082b..ece1d0813f 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -167,6 +167,36 @@ cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; } +/* + * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick + * infers "my clean-leave committed" from the membership epoch advancing past + * its bound baseline (the coordinator publishes the CLEAN_LEAVE event into ITS + * own reconfig state, but the epoch piggybacks to the leaver). Latching that + * inference suppresses the barrier-deadline escalation, so a FALSE latch hangs + * the leaver in BARRIER_WAIT forever. + * + * spec-2.29a r2 P2-1: the leaver's own-commit inference needs BOTH a bitmap + * check AND the scalar dead_generation. The others-dead bitmap alone is not + * monotone: a third-party node that false-fail-stopped (bumping the epoch) + * then recovered leaves the CSSD hysteresis DEAD→ALIVE, so the others-dead + * bitmap returns to its bound value while the scalar dead_generation (which + * only advances) does not. If the leaver's first epoch>baseline observation + * lands after that rebound, a bitmap-only check would mis-latch a leave the + * survivor coordinator actually refused. The scalar conjunct closes it — and + * it does NOT reintroduce the ②b false positive, because the leaving node's + * OWN alive→DEAD transition never bumps ITS OWN dead_generation (a node does + * not observe itself dead), so on the leaver side dead_gen stays at baseline + * through its own drain. (The survivor-side coherence sites keep the + * others-dead bitmap: a survivor DOES observe the leaver's DEAD and would be + * falsely escalated by the scalar there.) + */ +bool +cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, + bool dead_gen_unchanged) +{ + return epoch_advanced && others_dead_unchanged && dead_gen_unchanged; +} + /* ============================================================ * leave-intent marker structural validation (D2 / §2.5) diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 7edc315c52..ccaf5991c1 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -177,6 +177,7 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->apply_counter, 0); pg_atomic_init_u64(&ReconfigShmem->dedup_skip_counter, 0); pg_atomic_init_u64(&ReconfigShmem->procsig_broadcast_count, 0); + pg_atomic_init_u32(&ReconfigShmem->prebump_sync_active, 0); /* spec-2.29a r2 t/274 */ /* spec-5.14 D6 — touched_peers observability counters. */ pg_atomic_init_u64(&ReconfigShmem->touched_abort_count, 0); pg_atomic_init_u64(&ReconfigShmem->touched_stamp_count, 0); @@ -1272,6 +1273,11 @@ cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) bool cluster_reconfig_has_pending_prebump_stage(void) { + /* spec-2.29a r2 t/274: the shmem bit covers a BACKEND-context coordinator's + * bump→publish window (LMON cannot see that process's local stage); the + * three LMON-local staged flags cover the LMON-driven paths. */ + if (ReconfigShmem != NULL && pg_atomic_read_u32(&ReconfigShmem->prebump_sync_active) != 0) + return true; return failstop_fence_stage.async.has_staged_event || node_removed_fence_stage.async.has_staged_event || join_prepare_stage.async.has_staged_event; @@ -2533,6 +2539,16 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( CLUSTER_INJECTION_POINT("cluster-reconfig-epoch-bump-pre"); + /* + * spec-2.29a r2 t/274: mark the pre-bump→publish window BEFORE bumping the + * epoch, so a concurrent LMON GRD IDLE tick holds its WAIT_EPOCH baseline + * while this (possibly backend-context, synchronous) path bumps but has not + * yet published. Cleared at every return below. Harmless on the LMON / + * fence-off paths (they either hand off to the local staged flag or publish + * within this same call with no intervening tick). + */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 1); + /* D18: atomic CAS-loop increment. Returns pre/post snapshots. */ cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); @@ -2650,11 +2666,13 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( "majority for epoch %llu; not publishing reconfig event " "(write-fenced, will retry)", (unsigned long long)new_epoch))); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); return; } cluster_reconfig_publish_event(&evt); cluster_reconfig_log_failstop_epoch_bump(&evt); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); return; } @@ -2665,6 +2683,8 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, CLUSTER_MARKER_KIND_FENCE_FAILSTOP, coordinator_node_id, GetCurrentTimestamp()); + /* LMON path: the local staged flag now covers the pending window. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); return; /* fail-closed until the staged marker is majority-durable */ } @@ -2680,6 +2700,9 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * tick free of allocator traffic. */ cluster_reconfig_log_failstop_epoch_bump(&evt); + + /* spec-2.29a r2 t/274: fence-off path published within this call. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); } diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 3d3a3801e4..14975b9e7e 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -352,6 +352,15 @@ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 curr const uint8 *bound_others_dead, const uint8 *current_others_dead, int nbytes); +/* spec-2.29a r2 P2-1: the leaving node's barrier-tick own-commit latch. Needs + * BOTH the others-dead bitmap unchanged AND the scalar dead_generation + * unchanged — the bitmap alone is not monotone under a third-party + * false-DEAD→ALIVE rebound and could mis-latch a refused leave (hang). The + * leaver's own DEAD never bumps its own dead_generation, so the scalar + * conjunct does not reintroduce the ②b false positive. */ +extern bool cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, + bool dead_gen_unchanged); + /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the * leaving node is the expected declared peer, and the dead_bitmap names only the diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index df42f84d77..1aa106f690 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -226,6 +226,16 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 apply_counter; /* total events observed */ pg_atomic_uint64 dedup_skip_counter; /* duplicate event_id skipped */ pg_atomic_uint64 procsig_broadcast_count; /* PROCSIG broadcast tally */ + /* spec-2.29a r2 t/274: a backend-context coordinator (e.g. the fail-stop + * inject test) advances the epoch then SYNCHRONOUSLY waits for the fence + * marker before publishing. During that (seconds-long) wait the epoch is + * bumped but the reconfig event is not yet published, so a concurrently + * running LMON GRD IDLE tick would re-capture the post-bump epoch as its + * WAIT_EPOCH baseline and wedge (old==cur). The LMON-process-local staged + * flags cannot see a backend's pre-bump window, so publish it in shmem: + * set before the bump, cleared after publish / on failure. The GRD IDLE + * baseline-hold guard reads this OR the local stage. */ + pg_atomic_uint32 prebump_sync_active; /* spec-5.14 D6 — touched_peers fail-stop observability (Q8 A': these * live in this existing region; the region count is unchanged). */ diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index 24a7ce43a4..53dfe32045 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -184,6 +184,35 @@ UT_TEST(test_version_coherent) } +/* ============================================================ + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r2 P2-1) + * ============================================================ */ + +UT_TEST(test_own_commit_latched) +{ + /* own commit latches only when the epoch advanced AND both the others-dead + * bitmap and the scalar dead_generation are unchanged. */ + + /* clean commit: epoch advanced, nothing else moved -> latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); + + /* epoch has not advanced yet -> not our commit (keep waiting) */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + + /* a third-party death is currently in the others-dead set -> escalate */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, true)); + + /* r2 P2-1 rebound case: the others-dead bitmap has REBOUND to its bound + * value (a third-party false-DEAD then recovered) but the scalar + * dead_generation ADVANCED — the non-monotone bitmap must NOT be trusted; + * the scalar conjunct forces escalate instead of a false latch/hang. */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, true, false)); + + /* both diverged -> escalate */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, false)); +} + + /* ============================================================ * U5 — writable-only quiesce gate * ============================================================ */ @@ -396,10 +425,11 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(8); + UT_PLAN(9); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); + UT_RUN(test_own_commit_latched); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); From ee2c09fd34ead1a615034923d80b54a2d021df8a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 07:59:22 +0800 Subject: [PATCH 44/58] test(cluster): pin evidence-over-inference own-commit latch (known-red) The leaving node's barrier-tick own-commit latch must be backed by direct evidence (the durable COMMITTED marker confirmation for THIS leave attempt) and must be immune to third-party transient false-DEAD flap noise: the scalar dead_generation is monotone, so a flap during the leave window advances it forever and the r2 P2-1 three-conjunct inference then refuses a healthy committed leave until the barrier deadline escalates it (nightly t/331 C1/C4 false-escalation, run 28948167577). The reworked U3b matrix pins: (a) evidence, no noise -> latch (d) evidence + flap noise -> still latch (RED on current code) (c) no evidence, any noise state -> never latch (deadline escalation stays armed; the r2 P2-1 refused-leave mis-latch wedge stays shut) Known-red TDD commit: leg (d) fails on the current three-conjunct predicate; the follow-up commit flips the latch to marker evidence. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- .../cluster_unit/test_cluster_clean_leave.c | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index 53dfe32045..5da4135ddc 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -185,31 +185,44 @@ UT_TEST(test_version_coherent) /* ============================================================ - * U3b — leaver barrier-tick own-commit latch (spec-2.29a r2 P2-1) + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r3, evidence + * over inference) * ============================================================ */ UT_TEST(test_own_commit_latched) { - /* own commit latches only when the epoch advanced AND both the others-dead - * bitmap and the scalar dead_generation are unchanged. */ - - /* clean commit: epoch advanced, nothing else moved -> latch */ + /* The latch verdict is EVIDENCE-only: the first argument is "the durable + * COMMITTED marker for THIS leave attempt was confirmed" (the coordinator's + * nonce-bound LEAVE_COMMITTED attestation). The two coherence observations + * (others-dead bitmap / scalar dead_generation) stay in the signature as + * contract inputs the verdict must IGNORE — a third-party transient + * false-DEAD flap on the leaver's local CSSD view advances the (monotone) + * dead_generation and can transiently disturb the bitmap, and neither may + * refuse a latch that marker evidence backs (the t/331 C1/C4 false- + * escalation), nor may any combination latch without evidence (the r2 P2-1 + * refused-leave mis-latch hang). */ + + /* (a) evidence present, no flap noise -> latch */ UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); - /* epoch has not advanced yet -> not our commit (keep waiting) */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); - - /* a third-party death is currently in the others-dead set -> escalate */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, true)); + /* (d) PINNING LEG (t/331 C1/C4 regression): a third-party flap advanced the + * scalar dead_generation (it never rebounds) while the bitmap rebounded to + * its bound value — WITH marker evidence the leave still latches; the flap + * must not false-escalate a committed leave. */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, false)); - /* r2 P2-1 rebound case: the others-dead bitmap has REBOUND to its bound - * value (a third-party false-DEAD then recovered) but the scalar - * dead_generation ADVANCED — the non-monotone bitmap must NOT be trusted; - * the scalar conjunct forces escalate instead of a false latch/hang. */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, true, false)); + /* (d) flap currently visible in the others-dead bitmap too -> still latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, true)); + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, false)); - /* both diverged -> escalate */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, false)); + /* (c) no evidence -> never latch, whatever the coherence observations say + * (a refused leave never produces a COMMITTED marker, so the barrier + * deadline escalation stays armed and bounds the wait — the r2 P2-1 + * mis-latch wedge stays closed). */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, false)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, false)); } From 1bd762925d6088c39d3793ed4c570a14789c6f7e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 08:05:30 +0800 Subject: [PATCH 45/58] fix(cluster): repair torn dual-copy xid-authority header on restart A crash between write_header_both's two durable renames leaves the transitioned flag (unseal / CLUSTER_ERA stamp) in the primary copy only, with a stale pre-transition .bak behind it. Nothing on the restart path rewrote the pair -- the bootstrap gates skipped the transition call when the primary already showed the new flag, and the transition functions early-returned on the same observation -- so the stale .bak survived the whole native run. Any later transient primary read failure then fell back to it: a stale SEALED .bak hands joiners the previous pass's high-water (false-invisible + xid reissue), and a stale pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Make the two flag transitions idempotent re-asserts: the bootstrap calls them unconditionally on every boot of their respective arm, and they rewrite BOTH copies until both validate with the post-transition flags (a missing or invalid copy counts as unsettled and is restored too). Once both copies are settled the call is a no-write no-op, so the steady-state boot cost is two header reads. All fail-closed semantics (53RB5 refusals, CLUSTER_ERA re-entry FATAL, corrupt-authority PANIC) are unchanged. Red-first unit coverage: torn-window tests for both transitions (complete the transition, re-install the old image as .bak, re-run the boot path, assert the .bak is repaired and that a primary read failure falls back to the repaired image, not the stale one). Spec: spec-6.15b-xid-authority-native-era.md --- .../cluster/cluster_catalog_bootstrap.c | 27 +++- src/backend/cluster/cluster_xid_authority.c | 68 +++++++++- src/include/cluster/cluster_xid_authority.h | 10 +- .../cluster_unit/test_cluster_xid_authority.c | 127 +++++++++++++++++- 4 files changed, 216 insertions(+), 16 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index f4c1718f6a..69591fd2e6 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -183,12 +183,19 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, * fail-closed (unsealed) instead of adopting the previous pass's * stale high-water (spec-6.15b §3.1 multi-pass arm, rule 8.A). The * clean shutdown of this run re-publishes and re-seals. + * + * Called unconditionally (not gated on the primary's SEALED flag): + * the transition re-asserts BOTH on-disk copies, so a crash between + * a previous unseal's two renames -- a stale SEALED .bak behind an + * already-unsealed primary, which a later primary read failure + * would resurrect -- is repaired here instead of surviving the run + * (review r3-X1). Once both copies are open this is a no-write + * no-op. */ - if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) { - cluster_xid_authority_begin_native_run(); + cluster_xid_authority_begin_native_run(); + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) elog(LOG, "cluster shared_catalog: re-opened native seed era " "(XID authority unsealed for this run)"); - } return; } @@ -268,13 +275,21 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, (unsigned long long)own_next, (unsigned long long)auth.native_hw_full); } - if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) { - cluster_xid_authority_mark_cluster_era(); + /* + * One-way CLUSTER_ERA stamp, re-run unconditionally on every cluster + * boot: the transition re-asserts BOTH on-disk copies, so a crash + * between a previous stamp's two renames -- a stale pre-CLUSTER_ERA + * .bak behind an already-stamped primary, which a later primary read + * failure would hand to an enabled=off boot as a re-entry bypass -- is + * repaired here instead of surviving the run (review r3-X1). Once + * both copies are stamped this is a no-write no-op. + */ + cluster_xid_authority_mark_cluster_era(); + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) elog(LOG, "cluster shared_catalog: marked XID authority cluster era started at native " "high-water %llu", (unsigned long long)auth.native_hw_full); - } } void diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c index 08904b7445..6a16671c57 100644 --- a/src/backend/cluster/cluster_xid_authority.c +++ b/src/backend/cluster/cluster_xid_authority.c @@ -171,6 +171,46 @@ read_image(const char *path, char *image) return cluster_xid_authority_classify(image, CLUSTER_XID_AUTHORITY_FILE_SIZE); } +/* + * copy_flags_settled -- does the on-disk copy at relpath validate AND carry + * all `must_set` flag bits with all `must_clear` bits clear? + */ +static bool +copy_flags_settled(const char *relpath, uint32 must_set, uint32 must_clear) +{ + ClusterXidAuthorityHeader hdr; + char path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (!build_path(path, sizeof(path), relpath)) + return false; + if (read_image(path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; + memcpy(&hdr, image, sizeof(hdr)); + return (hdr.flags & must_set) == must_set && (hdr.flags & must_clear) == 0; +} + +/* + * both_copies_flags_settled -- is a one-way flag transition complete in BOTH + * on-disk copies? Flag transitions install the same image as primary and + * .bak (primary first), so a crash between the two durable renames leaves a + * pre-transition .bak behind an already-transitioned primary. Nothing else + * on the restart path rewrites the pair, so without a re-assert the stale + * copy survives the whole run, and any later transient primary read failure + * resurrects the pre-transition flags through the .bak fallback -- a stale + * SEALED .bak hands joiners the previous pass's high-water and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard (review + * r3-X1, 8.A). The transition entry points therefore skip their write only + * when this returns true; a missing or invalid copy is "not settled" too, + * which restores the damaged copy for free. + */ +static bool +both_copies_flags_settled(uint32 must_set, uint32 must_clear) +{ + return copy_flags_settled(CLUSTER_XID_AUTHORITY_REL_PATH, must_set, must_clear) + && copy_flags_settled(CLUSTER_XID_AUTHORITY_BAK_REL_PATH, must_set, must_clear); +} + /* * cluster_xid_authority_read -- fail-closed read of the shared authority. * Tries primary then .bak; returns false when neither is trustworthy. Never @@ -284,8 +324,10 @@ write_header(ClusterXidAuthorityHeader *hdr) * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Crash * points: both copies old (transition not yet taken: state still * consistent), primary new + .bak old (read prefers the valid primary), - * both new. A later single-copy corruption therefore never resurrects the - * pre-transition flags. + * both new. The primary-new/.bak-old window does not self-repair on its + * own: the transition entry points are re-run by every boot and re-assert + * both copies until both_copies_flags_settled holds (review r3-X1), so a + * later single-copy corruption never resurrects the pre-transition flags. */ static void write_header_both(ClusterXidAuthorityHeader *hdr) @@ -378,7 +420,11 @@ cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, b /* * cluster_xid_authority_mark_cluster_era -- one-way CLUSTER_ERA stamp (first - * cluster.enabled=on boot; spec §3.1). No-op when already stamped. A + * cluster.enabled=on boot; spec §3.1). Idempotent re-assert: re-run by + * every cluster boot, it rewrites BOTH copies until both carry the flag, so + * a crash between write_header_both's two renames (stamped primary, stale + * .bak) is repaired on the next boot instead of surviving the run (review + * r3-X1); once both copies are settled it is a no-write no-op. A * missing/corrupt authority PANICs: the catalog bootstrap seeds it before * any caller can reach this point, so absence here is real damage. */ @@ -391,8 +437,9 @@ cluster_xid_authority_mark_cluster_era(void) ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("shared XID authority is missing or corrupt at cluster-era stamp"))); - if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) - return; + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) != 0 + && both_copies_flags_settled(CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0)) + return; /* transition complete in both copies */ hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; write_header_both(&hdr); /* review F1 variant: one-way flag in BOTH copies */ @@ -417,6 +464,12 @@ cluster_xid_authority_mark_cluster_era(void) * the stamp is re-applied by the next cluster boot (mark is re-issued * whenever unset) and this authority is left unsealed -- which only ever * blocks adoption, never yields a wrong answer. + * + * Idempotent re-assert (review r3-X1): re-run by every native-era boot, + * it rewrites BOTH copies until neither retains SEALED, so a crash + * between write_header_both's two renames (unsealed primary, stale + * SEALED .bak) is repaired on the next boot instead of surviving the + * run; once both copies are settled it is a no-write no-op. */ void cluster_xid_authority_begin_native_run(void) @@ -432,8 +485,9 @@ cluster_xid_authority_begin_native_run(void) (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("native seed era cannot be re-entered on a formed shared catalog tree"))); - if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) - return; /* already open (first run, or a prior pass crashed) */ + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0 + && both_copies_flags_settled(0, CLUSTER_XID_AUTHORITY_FLAG_SEALED)) + return; /* open, and no copy retains SEALED */ hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; write_header_both(&hdr); /* review F1: no copy may retain SEALED */ diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index c9c51f7ab2..3287f77c22 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -194,14 +194,20 @@ extern bool cluster_xid_authority_seed_if_absent(uint64 initial_native_hw); extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, bool seal); -/* One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). */ +/* + * One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). Idempotent + * re-assert: rewrites BOTH copies until both carry the flag (repairing a + * torn previous stamp); no-write no-op once both are stamped. + */ extern void cluster_xid_authority_mark_cluster_era(void); /* * A follow-up native-era (cluster.enabled=off) boot on a sealed authority * re-opens the era: clears SEALED so a crash of this run never exposes the * previous pass's stale high-water to joiners. Caller vets CLUSTER_ERA - * first (re-entry is FATAL); no-op when already unsealed. + * first (re-entry is FATAL). Idempotent re-assert: rewrites BOTH copies + * until neither retains SEALED (repairing a torn previous unseal); + * no-write no-op once both are open. */ extern void cluster_xid_authority_begin_native_run(void); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 4a646270e1..cb745fd2a0 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -544,6 +544,129 @@ UT_TEST(test_mark_cluster_era_survives_primary_corruption_via_bak) CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); } +/* + * write_raw_image / read_raw_image -- install / fetch one fixed-size + * authority image at `path` (raw open/write, no CRC games: the images + * moved around here are byte copies of real, valid on-disk images). + */ +static void +write_raw_image(const char *path, const char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(write(fd, image, len), (long long)len); + close(fd); +} + +static void +read_raw_image(const char *path, char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(read(fd, image, len), (long long)len); + close(fd); +} + +UT_TEST(test_unseal_torn_crash_repairs_stale_sealed_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1: a crash BETWEEN write_header_both's two renames leaves + * primary=UNSEALED(new) / .bak=SEALED(old high-water). Every native-era + * boot re-runs the transition, which must re-assert BOTH copies; before + * the fix it early-returned on the already-unsealed primary and the + * stale SEALED .bak survived the whole run -- any later primary read + * failure resurrected the previous pass's high-water (8.A). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* capture the pre-transition (SEALED) primary image */ + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + /* complete the transition, then re-install the old SEALED image as + * .bak, simulating the crash window between the two durable renames */ + cluster_xid_authority_begin_native_run(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next native-era boot re-runs the transition */ + cluster_xid_authority_begin_native_run(); + + /* the .bak copy itself must have been repaired to an unsealed image */ + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* a later primary read failure must fall back to the REPAIRED image, + * not resurrect the previous pass's sealed high-water */ + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); +} + +UT_TEST(test_mark_cluster_era_torn_crash_repairs_stale_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1 variant: the same torn-crash window across the + * CLUSTER_ERA stamp leaves a .bak without the one-way flag; a .bak + * fallback would then let an enabled=off boot re-enter the native era + * on a formed tree. The stamp is re-run by every cluster boot and + * must repair the lagging copy. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + cluster_xid_authority_mark_cluster_era(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next cluster boot re-runs the stamp */ + cluster_xid_authority_mark_cluster_era(); + + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + UT_TEST(test_prefix_check_divergence_truth_table) { const uint64 xids_per_page = (uint64)BLCKSZ * 4; @@ -682,7 +805,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(15); + UT_PLAN(17); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -692,6 +815,8 @@ main(void) UT_RUN(test_begin_native_run_unseals_before_cluster_era); UT_RUN(test_unseal_survives_primary_corruption_via_bak); UT_RUN(test_mark_cluster_era_survives_primary_corruption_via_bak); + UT_RUN(test_unseal_torn_crash_repairs_stale_sealed_bak); + UT_RUN(test_mark_cluster_era_torn_crash_repairs_stale_bak); UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); From d8b1d6d414b249d755c3b8a93ae62277d80bea0d Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:04:50 +0800 Subject: [PATCH 46/58] fix(cluster): close prefix-check front-truncation blind spot (fail-closed) The divergent-lineage prefix check broke out of its comparison loop and returned CONSISTENT on the FIRST missing local pg_xact page. That is only sound for a limit that ends where the local tree ends; pg_xact front truncation (SimpleLruTruncate removes whole low segments once every xid they cover is frozen past oldestXid) means a joiner can have segment 0000 legitimately absent while later segments are present AND divergent -- the first-page ENOENT then declared the clone CONSISTENT without ever comparing the surviving pages, and the adopt arm overwrote the joiner's own live-xid outcome pages with the seed's bits (false-visible / false-invisible). Pass the caller's recovery-anchor oldestXid into the check and start the comparison at oldestXid's own 2-bit slot: bits below it are frozen truth that CLOG never consults again and whose on-disk survival is an accident of truncation timing, so they are exempt whether the pages survive or not (neither truncation nor tuple freezing rewrites surviving CLOG bytes, so this can never hide live evidence). Within the comparable range [oldestXid, min(own_next, native_hw)) a missing local page is an anomaly -- pg_xact always covers [oldestXid, nextXid) on a well-formed node -- and now returns UNAVAILABLE (fail-closed 53RB5 at the caller), never CONSISTENT. Sub-byte boundaries at oldestXid are masked at 2-bit precision. Red-first unit coverage: front-truncated divergent clone must DIVERGE (returned CONSISTENT pre-fix), divergence strictly below oldestXid stays CONSISTENT (frozen exemption, mid-byte lead mask), oldestXid at the divergent slot DIVERGES, and a hole inside the comparable range is UNAVAILABLE (the old truth-table leg expecting CONSISTENT for a missing segment encoded exactly the blind spot and now expects UNAVAILABLE). Spec: spec-6.15b-xid-authority-native-era.md --- .../cluster/cluster_catalog_bootstrap.c | 20 ++- src/backend/cluster/cluster_xid_prehistory.c | 123 ++++++++++++++---- src/include/cluster/cluster_xid_authority.h | 23 ++-- .../cluster_unit/test_cluster_xid_authority.c | 86 ++++++++++-- 4 files changed, 203 insertions(+), 49 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 69591fd2e6..413af44ebb 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -234,16 +234,21 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, * native range, so skipping would trust divergent local truth and * adopting would overwrite the node's own outcomes; both are * silently wrong (8.A). Byte-compare the comparable prefix - * [0, min(own_next, native_hw)) against the sealed blob and fail - * closed on any contradiction. Bypass only when the local + * [oldestXid, min(own_next, native_hw)) against the sealed blob and + * fail closed on any contradiction. Bypass only when the local * oldestXid has advanced past the native range (frozen+truncated: - * those bits are never consulted again). + * those bits are never consulted again). Below that bypass the + * check starts at oldestXid itself (review r3-X2): segments frozen + * and truncated away are no alibi for the surviving range, and a + * missing local page inside the comparable range fails closed + * (UNAVAILABLE) instead of passing as a shorter clone. */ if ((uint64)ra.checkPointCopy.oldestXid <= auth.native_hw_full) { ClusterXidPrefixVerdict pv; pv = cluster_xid_prehistory_prefix_check(DataDir, auth.native_hw_full, - Min(own_next, auth.native_hw_full)); + Min(own_next, auth.native_hw_full), + (uint64)ra.checkPointCopy.oldestXid); if (pv == CLUSTER_XID_PREFIX_DIVERGED) ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), @@ -257,8 +262,11 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("shared XID prehistory is unavailable for the lineage check"), - errhint("Restore \"%s\" (or its .bak) from the shared tree's " - "backup.", + errdetail("Either no sealed prehistory blob passes validation, or the " + "local pg_xact has a hole inside the comparable native " + "range."), + errhint("Restore \"%s\" (or its .bak) from the shared tree's backup, " + "or re-provision this node if its local pg_xact is damaged.", CLUSTER_XID_PREHISTORY_REL_PATH))); } diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c index a61550e009..490804ccac 100644 --- a/src/backend/cluster/cluster_xid_prehistory.c +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -458,9 +458,11 @@ cluster_xid_prehistory_was_adopted(void) /* * read_local_clog_page_optional -- read local pg_xact page `pageno` into - * buf. Returns false when the segment file or the page does not exist - * (short clone: the comparable prefix ends there); PANICs on a real read - * error so an I/O fault is never mistaken for a short prefix. + * buf. Returns false when the segment file or the page does not exist; + * PANICs on a real read error so an I/O fault is never mistaken for a + * missing page. The caller decides what a missing page means: below + * page(oldestXid) it is a legitimately truncated frozen segment, at or + * above it is an anomaly (review r3-X2). */ static bool read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf) @@ -492,14 +494,85 @@ read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf return r == BLCKSZ; } +/* + * clog_slot_mask -- byte mask covering the 2-bit CLOG status slots + * [lo, hi) within one byte (0 <= lo < hi <= 4). Slot n of a byte holds + * xid bits at (n * CLOG_BITS_PER_XACT), low-order first (clog.c + * TransactionIdToBIndex). + */ +static inline unsigned char +clog_slot_mask(uint32 lo, uint32 hi) +{ + unsigned int m; + + m = ((hi >= 4) ? 0xFFu : ((1u << (hi * 2)) - 1u)) & ~((1u << (lo * 2)) - 1u); + return (unsigned char)m; +} + +/* + * clog_page_range_equal -- compare blob vs local CLOG page content for the + * page-local 2-bit slots [from_xact, to_xact) only. Sub-byte boundaries + * are masked so bits outside the range never influence the verdict. + */ +static bool +clog_page_range_equal(const char *blob_page, const char *local_page, uint32 from_xact, + uint32 to_xact) +{ + uint32 first_byte = from_xact / 4; + uint32 last_byte = (to_xact - 1) / 4; + + Assert(from_xact < to_xact && to_xact <= PREHISTORY_XACTS_PER_PAGE); + + if (first_byte == last_byte) + return ((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, to_xact - first_byte * 4)) + == 0; + + if (from_xact % 4 != 0) { + if (((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, 4)) + != 0) + return false; + first_byte++; + } + if (to_xact % 4 != 0) { + if (((unsigned char)(blob_page[last_byte] ^ local_page[last_byte]) + & clog_slot_mask(0, to_xact % 4)) + != 0) + return false; + last_byte--; + } + return first_byte > last_byte + || memcmp(blob_page + first_byte, local_page + first_byte, last_byte - first_byte + 1) + == 0; +} + /* * cluster_xid_prehistory_prefix_check -- see header. Streams the sealed * blob (primary, then .bak) and byte-compares the local pg_xact prefix - * covering xids [0, limit_xid_full) at per-xact (2-bit) precision. + * covering xids [oldest_xid_full, min(limit_xid_full, native_hw_full)) at + * per-xact (2-bit) precision. + * + * Frozen-prefix exemption (review r3-X2): SimpleLruTruncate removes whole + * pg_xact SEGMENTS once every page they cover precedes page(oldestXid) + * (slru.c SlruMayDeleteSegment), so a joiner that froze past >=1 segment + * legitimately has no file where the blob still has pages -- the old + * break-on-first-missing-local-page treated exactly that shape as a + * tail-short clone and returned CONSISTENT without ever comparing the + * surviving pages. Neither truncation nor tuple freezing rewrites the + * CLOG bytes that survive, but bits below oldestXid are dead content + * anyway (every tuple with such an xmin/xmax is frozen, so CLOG is never + * consulted for them again) and whether they still exist is an accident + * of truncation timing; lineage evidence therefore starts at oldestXid's + * own 2-bit slot, and everything below it is skipped whether the page + * survives or not. Within the comparable range [oldestXid, limit) a + * MISSING local page is an anomaly -- pg_xact always covers + * [page(oldestXid), page(nextXid)] on a well-formed node -- and returns + * UNAVAILABLE (fail-closed), never CONSISTENT. */ ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, - uint64 limit_xid_full) + uint64 limit_xid_full, uint64 oldest_xid_full) { ClusterXidPrehistoryHeader hdr; char primary[MAXPGPATH]; @@ -508,8 +581,10 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f char local_page[BLCKSZ]; const char *src_path; uint32 pages = 0; + uint32 start_page; uint32 p; uint64 limit; + uint64 oldest; int src; if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) @@ -527,6 +602,11 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f if (limit == 0) return CLUSTER_XID_PREFIX_CONSISTENT; + oldest = Min(oldest_xid_full, limit); + if (oldest >= limit) + return CLUSTER_XID_PREFIX_CONSISTENT; /* everything comparable is frozen */ + start_page = (uint32)(oldest / PREHISTORY_XACTS_PER_PAGE); + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) { if (src >= 0) @@ -536,9 +616,8 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f for (p = 0; p < pages; p++) { uint64 page_first_xid = (uint64)p * PREHISTORY_XACTS_PER_PAGE; - uint64 in_scope; - uint32 full_bytes; - uint32 partial_bits; + uint64 cmp_from; + uint64 cmp_to; if (read(src, blob_page, BLCKSZ) != BLCKSZ) { CloseTransientFile(src); @@ -546,26 +625,24 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f } if (page_first_xid >= limit) break; /* comparison scope ended on a page boundary */ + if (p < start_page) + continue; /* wholly frozen page: skipped, present or not */ - if (!read_local_clog_page_optional(local_pgdata, p, local_page)) - break; /* short clone: no local bits left to contradict */ + cmp_from = Max(page_first_xid, oldest); + cmp_to = Min(page_first_xid + PREHISTORY_XACTS_PER_PAGE, limit); + /* oldest < limit and p >= start_page guarantee a non-empty range */ + Assert(cmp_from < cmp_to); - in_scope = Min((uint64)PREHISTORY_XACTS_PER_PAGE, limit - page_first_xid); - full_bytes = (uint32)(in_scope / 4); - partial_bits = (uint32)(in_scope % 4) * 2; - - if (full_bytes > 0 && memcmp(blob_page, local_page, full_bytes) != 0) { + if (!read_local_clog_page_optional(local_pgdata, p, local_page)) { + /* hole inside the comparable range: fail closed (r3-X2) */ CloseTransientFile(src); - return CLUSTER_XID_PREFIX_DIVERGED; + return CLUSTER_XID_PREFIX_UNAVAILABLE; } - if (partial_bits > 0) { - unsigned char mask = (unsigned char)((1 << partial_bits) - 1); - if (((unsigned char)blob_page[full_bytes] & mask) - != ((unsigned char)local_page[full_bytes] & mask)) { - CloseTransientFile(src); - return CLUSTER_XID_PREFIX_DIVERGED; - } + if (!clog_page_range_equal(blob_page, local_page, (uint32)(cmp_from - page_first_xid), + (uint32)(cmp_to - page_first_xid))) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; } } CloseTransientFile(src); diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index 3287f77c22..0e73859d46 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -244,16 +244,23 @@ typedef enum ClusterXidPrefixVerdict { } ClusterXidPrefixVerdict; /* - * Compare the local pg_xact bytes covering xids [0, limit_xid_full) with the - * sealed prehistory blob at 2-bit (per-xact) precision. A local segment or - * page that does not exist ends the comparable prefix (a shorter clone has - * no bits to contradict). Callers FATAL on DIVERGED/UNAVAILABLE: a joiner - * whose own native-era history contradicts the seed's is not a pre-seed - * lineage, and neither skipping (trusting local bits) nor adopting - * (overwriting the joiner's own outcomes) is sound for it. + * Compare the local pg_xact bytes covering xids [oldest_xid_full, + * limit_xid_full) with the sealed prehistory blob at 2-bit (per-xact) + * precision. Bits below oldest_xid_full are frozen truth that CLOG never + * consults again and that SimpleLruTruncate may already have removed + * (whole segments); they are exempt whether the pages survive or not. A + * local page missing INSIDE the comparable range is fail-closed + * UNAVAILABLE, never CONSISTENT: pg_xact covers [oldestXid, nextXid) on a + * well-formed node, so a hole is an anomaly, and a front-truncated + * divergent clone must not pass as a "shorter clone" (review r3-X2). + * Callers FATAL on DIVERGED/UNAVAILABLE: a joiner whose own native-era + * history contradicts the seed's is not a pre-seed lineage, and neither + * skipping (trusting local bits) nor adopting (overwriting the joiner's + * own outcomes) is sound for it. */ extern ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, - uint64 limit_xid_full); + uint64 limit_xid_full, + uint64 oldest_xid_full); #endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index cb745fd2a0..19b98855fa 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -680,7 +680,7 @@ UT_TEST(test_prefix_check_divergence_truth_table) make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "f2seed", 1, 0xAA); make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "f2join", 1, 0xAA); cluster_xid_prehistory_publish(pgdata_seed, 816); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), CLUSTER_XID_PREFIX_CONSISTENT); /* flip one status byte INSIDE the limit -> DIVERGED */ @@ -690,27 +690,29 @@ UT_TEST(test_prefix_check_divergence_truth_table) UT_ASSERT_EQ(lseek(fd, 100, SEEK_SET), 100); /* byte 100 = xids 400..403 */ (void)!write(fd, "X", 1); close(fd); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), CLUSTER_XID_PREFIX_DIVERGED); /* the same flipped byte OUTSIDE the limit -> CONSISTENT (adopt arm: * bytes at/after own_next belong to the seed alone) */ - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400, 0), CLUSTER_XID_PREFIX_CONSISTENT); /* partial-byte boundary: xid 400's 2-bit slot enters scope at limit 401 */ - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401, 0), CLUSTER_XID_PREFIX_DIVERGED); - /* missing local pg_xact segment -> comparable prefix is empty -> - * CONSISTENT (a shorter clone has no bits to contradict) */ + /* missing local page inside the comparable range -> fail-closed + * UNAVAILABLE, never CONSISTENT: with oldestXid=0 nothing below the + * hole is frozen, so "shorter clone" is not a possible innocent + * explanation (review r3-X2) */ unlink(seg); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), - CLUSTER_XID_PREFIX_CONSISTENT); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_UNAVAILABLE); /* no trustworthy blob -> UNAVAILABLE */ unlink_files(); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), CLUSTER_XID_PREFIX_UNAVAILABLE); /* multi-segment divergence: 33-page trees, flip a byte in segment 0001 */ @@ -722,18 +724,77 @@ UT_TEST(test_prefix_check_divergence_truth_table) make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "f2seedms", n_pages); make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "f2joinms", n_pages); cluster_xid_prehistory_publish(pgdata_seed, hw); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), CLUSTER_XID_PREFIX_CONSISTENT); snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); fd = open(seg, O_RDWR, 0600); UT_ASSERT(fd >= 0); (void)!write(fd, "Z", 1); close(fd); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), CLUSTER_XID_PREFIX_DIVERGED); } } +UT_TEST(test_prefix_check_front_truncation) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + const uint64 page32_first = 32 * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + unsigned char b; + int fd; + + /* review r3-X2 (a): front truncation is no alibi. SimpleLruTruncate + * removed the joiner's segment 0000 (oldestXid frozen past it), but the + * surviving page 32 diverges inside the comparable range -> DIVERGED. + * Pre-fix, break-on-first-missing-local-page returned CONSISTENT and + * the adopt arm then overwrote the joiner's own live-xid outcomes. */ + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "ftseed", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "ftjoin", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + UT_ASSERT_EQ(unlink(seg), 0); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); /* byte 600 = slots 2400..2403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first), + CLUSTER_XID_PREFIX_DIVERGED); + + /* (b) frozen exemption: restore byte 600 (page fill 0x20), then flip + * ONLY slot 2400's two low bits (0x20 -> 0x21). With oldestXid at slot + * 2401 the divergence sits strictly below the boundary, mid-byte: the + * lead mask must exclude it -> CONSISTENT (missing segment 0000 below + * page(oldestXid) is equally exempt). */ + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); + b = 0x21; + (void)!write(fd, &b, 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2401), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* same bytes with oldestXid AT the divergent slot -> DIVERGED (the + * boundary slot itself is comparable) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_DIVERGED); + + /* review r3-X2 (c): a hole AT/ABOVE page(oldestXid) inside the + * comparable range is an anomaly -> UNAVAILABLE (fail-closed), never + * CONSISTENT */ + UT_ASSERT_EQ(unlink(seg), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_UNAVAILABLE); +} + UT_TEST(test_prehistory_classify_corrupt) { char buf[64]; @@ -805,7 +866,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(17); + UT_PLAN(18); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -822,6 +883,7 @@ main(void) UT_RUN(test_prehistory_publish_adopt_round_trip); UT_RUN(test_prehistory_multi_segment_round_trip); UT_RUN(test_prefix_check_divergence_truth_table); + UT_RUN(test_prefix_check_front_truncation); UT_RUN(test_prehistory_classify_corrupt); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From ea2cdc99ee8f5ff5b855f8a222365a51e7e03f38 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:21:29 +0800 Subject: [PATCH 47/58] fix(cluster): clean-leave own-commit latch consumes marker evidence, not inference Nightly t/331 C1 (clean_leave x idle) / C4 (leave_remove x idle) regressed at the r2 P2-1 head (run 28948167577; parent green): the leaver's barrier-tick own-commit latch inferred its commit from epoch>baseline + others-dead bitmap unchanged + scalar dead_generation unchanged. The scalar is monotone, so a third-party transient SUSPECTED->DEAD->ALIVE flap on the leaver's local CSSD view during the leave window advances it forever; the latch then refuses a leave the coordinator ACTUALLY committed and the immediate-escalate arm (or the barrier deadline) aborts it: phase ends ABORTED_ESCALATE -> IDLE, never 'committed', breaking the C1/C4 drains+commits assertions. Reverting to bitmap-only would re-open the r2 P2-1 wedge (rebound mis-latch of a REFUSED leave suppresses escalation forever), so neither inference is usable. Replace inference with direct evidence: latch <=> the durable COMMITTED marker for THIS leave attempt is confirmed. The evidence already reaches the leaver with zero extra IO: the coordinator sends the nonce-bound LEAVE_COMMITTED only after its qvotec ACKs the COMMITTED marker majority-durable (there is no runtime voting-disk read path on the leaver, and none is needed - the LMON tick budget this spec protects is untouched). cl_committed_handler now routes through a pure identity gate (self-addressed + currently leaving + per-attempt nonce + committed epoch past the bound baseline; fail-closed on any mismatch, so a stale confirmation - and through it a stale COMMITTED marker from a previous leave of the same node - can never false-latch) and records the attested committed epoch E before publishing the evidence flag; the barrier tick consumes E instead of re-reading its possibly-stale local epoch view. The latch and the P1-V0.7 exit gate collapse into one step (the evidence IS the durable-truth-source confirmation). Escalation semantics are unchanged: no evidence by the barrier deadline -> the existing cl_escalate path, which is exactly what bounds a refused leave (it never gets a COMMITTED marker). Third-party flaps no longer matter: the coherence observations are now contract inputs the predicate must ignore (pinned by the U3b unit matrix, red-first in the parent commit) and feed only a flap-noise LOG at the latch. Survivor-side coherence sites (drive_drain / staged-ACK / pre-check) are untouched. Unit: U3b matrix flipped green; new U3c pins the evidence identity gate (match / stale-nonce / misrouted / not-leaving / epoch-not-advanced). Local gates: t/331 x4 all green (6/6), t/310 24/24, t/363 18/18, t/274 18/18 (prebump shmem-bit stays green), cluster_unit 158 binaries, cluster_regress 13/13, PG core 219/219, clang-format-18 + comment-headers clean. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_clean_leave.c | 176 +++++++++--------- .../cluster/cluster_clean_leave_policy.c | 89 ++++++--- src/include/cluster/cluster_clean_leave.h | 32 +++- .../cluster_unit/test_cluster_clean_leave.c | 37 +++- 4 files changed, 218 insertions(+), 116 deletions(-) diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 31e25bc8a4..24be2345d8 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -122,6 +122,8 @@ cluster_clean_leave_shmem_init(void) pg_atomic_init_u32(&cl_state->commit_point_observed, 0); pg_atomic_init_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_init_u32(&cl_state->committed_marker_durable, 0); + /* spec-2.29a r3 (evidence latch). */ + cl_state->committed_confirmed_epoch = 0; /* Hardening v1.0.2 (P1 preflight / P2 nonce). */ pg_atomic_init_u64(&cl_state->leave_attempt_nonce, 0); pg_atomic_init_u32(&cl_state->preflight_pending, 0); @@ -555,11 +557,15 @@ cl_commit_ready_handler(const ClusterICEnvelope *env, const void *payload) } /* - * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate): - * the survivor coordinator has made the COMMITTED marker majority-durable and - * signals that the durable truth-source now exists, so this leaving node may - * proceed to COMMITTED and exit. Only acted on if it is about OUR leave; idem- - * potent (the coordinator re-sends each tick until we are gone). + * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate; + * spec-2.29a r3 evidence latch): the survivor coordinator has made the COMMITTED + * marker majority-durable and attests that the durable truth-source now exists. + * This confirmation is the leaver's own-commit MARKER EVIDENCE: the barrier tick + * latches on it (and only on it) and proceeds to COMMITTED/exit. Identity is + * checked by the pure evidence gate (self-addressed + currently leaving + + * per-attempt nonce + committed epoch past the bound baseline) — any mismatch is + * dropped fail-closed. Idempotent (the coordinator re-sends each tick until we + * are gone). */ static void cl_committed_handler(const ClusterICEnvelope *env, const void *payload) @@ -571,17 +577,19 @@ cl_committed_handler(const ClusterICEnvelope *env, const void *payload) return; if (!cluster_clean_leave_announce_payload_valid(p)) return; - if (p->leaving_node_id != cluster_node_id) - return; /* only the leaving node consumes its own COMMITTED confirmation */ - /* Hardening v1.0.2 (P2): bind to THIS attempt — a stale LEAVE_COMMITTED from a - * prior same-epoch attempt must not prematurely confirm the current one. Only - * meaningful once we are the leaver actively awaiting confirmation. */ - if (p->leave_nonce != pg_atomic_read_u64(&cl_state->leave_attempt_nonce)) + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + if (!cluster_clean_leave_committed_evidence_matches( + p->leaving_node_id, p->leave_nonce, p->leave_epoch, cluster_node_id, + cl_state->leaving_node_id, pg_atomic_read_u64(&cl_state->leave_attempt_nonce), + cl_state->leave_epoch)) { + LWLockRelease(&cl_state->lock); return; - if (cl_state->leaving_node_id != cluster_node_id) - return; /* we are not currently leaving */ - + } + /* the committed epoch E is recorded BEFORE the evidence flag is published, + * inside the same critical section the barrier tick reads it back under. */ + cl_state->committed_confirmed_epoch = p->leave_epoch; pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 1); + LWLockRelease(&cl_state->lock); } /* @@ -1467,6 +1475,7 @@ cl_request_body(void) pg_atomic_write_u32(&cl_state->commit_point_observed, 0); pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); + cl_state->committed_confirmed_epoch = 0; /* spec-2.29a r3: per-attempt evidence */ LWLockRelease(&cl_state->lock); cl_set_phase(CLUSTER_LEAVE_REQUESTED); @@ -1755,83 +1764,72 @@ cl_leaving_barrier_tick(void) { uint64 baseline_epoch = cl_state->leave_epoch; int32 coordinator; + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool committed_evidence; + bool others_dead_unchanged; + bool dead_gen_unchanged; /* - * 1. observe the commit. The survivor coordinator runs the actual two-phase - * commit and publishes the CLEAN_LEAVE event into ITS OWN reconfig state — - * the leaving node's last_applied never carries it. What DOES propagate to - * the leaving node is the membership epoch (every IC envelope piggybacks it). - * So the leaving node detects its own commit by the epoch advancing past the - * bound baseline. Discriminate from a real death intruding (CL-I3) by the - * CSSD dead_generation: a cooperative leave does NOT mark anyone CSSD-dead, so - * an unchanged dead_generation at the bump means OUR clean-leave committed; a - * changed one means a real death (escalate). + * 1. observe the commit — EVIDENCE over inference (spec-2.29a r3). The + * survivor coordinator runs the actual two-phase commit, publishes the + * CLEAN_LEAVE event into ITS OWN reconfig state, drives the COMMITTED marker + * to voting-disk majority-durability, and only then sends the nonce-bound + * LEAVE_COMMITTED (re-sent each tick while we are alive). That confirmation + * — validated by the pure identity gate in cl_committed_handler — is the ONLY + * basis on which this node latches its own commit: latch <=> a valid + * COMMITTED marker for THIS leave attempt exists. * - * Hardening v1.0.4 (P2): this "epoch bump + dead_gen unchanged == OUR commit" - * inference is sound ONLY because there is no OTHER dead_gen-unchanged reconfig - * in flight. A spec-5.15 online JOIN also bumps the epoch with dead_gen - * unchanged and would be mis-observed here as the leave's commit (then the node - * stops escalating but never gets the real LEAVE_COMMITTED -> BARRIER_WAIT - * hang). That collision is removed STRUCTURALLY by the one-membership-reconfig- - * at-a-time serialization: the join driver does not bump the epoch while a clean - * leave is active (cluster_clean_leave_in_progress gates drive_joins + - * commit_member), and the leave does not start while a join is pending - * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * bump with an unchanged version during a leave can only be the leave's own - * commit. + * Two inference generations preceded this and each failed one way (see the + * predicate comment in cluster_clean_leave_policy.c): "epoch advanced + + * others-dead bitmap unchanged" could mis-latch a REFUSED leave after a + * third-party false-DEAD rebound (r2 P2-1 wedge); adding the monotone scalar + * dead_generation conjunct then false-escalated a healthy committed leave + * whenever a transient third-party flap advanced the leaver's local + * dead_generation during the leave window (nightly t/331 C1/C4). Marker + * evidence is immune to both: flaps cannot erase a durable marker, and a + * refused leave never produces one — no latch, so the barrier deadline below + * still bounds the wait (fail-closed escalation, unchanged semantics). * - * spec-2.29a ②b + r2 P2-1: "unchanged version" here means the others-dead - * bitmap AND the scalar dead_generation both unchanged. The bitmap excludes - * the leaving node's own expected DEAD (②b: else its heartbeat stop would - * falsely escalate), but the bitmap is not monotone — a third-party - * false-DEAD→ALIVE rebound restores it while the scalar dead_generation only - * advances (r2 P2-1: else the leaver could mis-latch a refused leave and - * hang). The leaver's own DEAD never bumps its OWN dead_generation, so the - * scalar conjunct is safe on this side (it would NOT be safe on the survivor - * side, which keeps the bitmap-only coherence check). + * The coherence observations are still taken, but ONLY for the flap-noise + * LOG at the latch (the predicate contract pins that they never affect the + * verdict). A real third-party death intruding mid-leave is refused on the + * survivor side (cl_coherent pre-check + guarded CAS, CL-I3), so no evidence + * arrives here and the deadline escalates this leaver — same outcome as the + * old immediate escalate arm, now bounded by the barrier deadline instead. + * + * Latching collapses the old two-step gate: the evidence IS the P1-V0.7 + * durable-truth-source confirmation, so the leave can no longer be + * un-committed (deadline must not escalate past here, Hardening v1.0.1 + * P1-1) AND the node may exit (COMMITTED) in the same tick. */ - if (cluster_epoch_get_current() > baseline_epoch) { - uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; - bool others_dead_unchanged; - bool dead_gen_unchanged; - - cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); - others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, - CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) - == 0); - dead_gen_unchanged - = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); - if (cluster_clean_leave_own_commit_latched(true, others_dead_unchanged, - dead_gen_unchanged)) { - /* - * Commit point observed (our clean-leave epoch was published; no - * third-party death intruded). The leave can no longer be - * un-committed, so from here the barrier deadline must NOT escalate - * (Hardening v1.0.1 P1-1). - */ - pg_atomic_write_u32(&cl_state->commit_point_observed, 1); - LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); - cl_state->leave_epoch = cluster_epoch_get_current(); /* the committed epoch E */ - LWLockRelease(&cl_state->lock); + committed_evidence = (pg_atomic_read_u32(&cl_state->committed_durable_confirmed) != 0); + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(committed_evidence, others_dead_unchanged, + dead_gen_unchanged)) { + uint64 committed_epoch; + + pg_atomic_write_u32(&cl_state->commit_point_observed, 1); + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + committed_epoch = cl_state->committed_confirmed_epoch; /* the committed epoch E */ + cl_state->leave_epoch = committed_epoch; + LWLockRelease(&cl_state->lock); - /* - * P1-V0.7 exit gate: reach COMMITTED ("may exit") ONLY after the - * coordinator confirms the COMMITTED marker is majority-durable - * (LEAVE_COMMITTED). Until then stay in BARRIER_WAIT and re-tick — the - * durable truth-source must exist before this node departs, else a - * survivor restart could not rebuild the clean-departed fact. - */ - if (pg_atomic_read_u32(&cl_state->committed_durable_confirmed)) { - cl_set_phase(CLUSTER_LEAVE_COMMITTED); - CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); - ereport(LOG, - (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " - "majority-durable; this node has drained and may exit", - (unsigned long long)cl_state->leave_epoch))); - } - } else { - cl_escalate(); /* a real death changed the version mid-leave (CL-I3) */ - } + if (!others_dead_unchanged || !dead_gen_unchanged) + ereport(LOG, (errmsg("cluster clean-leave: third-party liveness flap observed at the " + "commit latch (others-dead %s, dead_generation %s); durable " + "COMMITTED marker evidence overrides it", + others_dead_unchanged ? "unchanged" : "changed", + dead_gen_unchanged ? "unchanged" : "advanced"))); + + cl_set_phase(CLUSTER_LEAVE_COMMITTED); + CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); + ereport(LOG, (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " + "majority-durable; this node has drained and may exit", + (unsigned long long)committed_epoch))); return; } @@ -1842,8 +1840,12 @@ cl_leaving_barrier_tick(void) } /* 3. fail-closed deadline — ONLY before the commit point (P1-1: a committed - * leave is never un-committed; post-commit we wait for the durable marker, - * bounded by disk health, and never escalate). */ + * leave is never un-committed). With the r3 evidence latch this arm is what + * bounds EVERY no-evidence outcome: a refused leave, a foreign death that + * moved the version (the coordinator then refuses, CL-I3), or a lost/never- + * sent confirmation all end here instead of hanging in BARRIER_WAIT. The + * commit_point_observed guard is belt-and-braces: the latch above transitions + * out of BARRIER_WAIT in the same tick it sets the flag. */ if (!pg_atomic_read_u32(&cl_state->commit_point_observed) && (uint64)GetCurrentTimestamp() > cl_state->barrier_deadline_us) { cl_escalate(); diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index ece1d0813f..91a2c4d768 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -169,32 +169,79 @@ cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, /* * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick - * infers "my clean-leave committed" from the membership epoch advancing past - * its bound baseline (the coordinator publishes the CLEAN_LEAVE event into ITS - * own reconfig state, but the epoch piggybacks to the leaver). Latching that - * inference suppresses the barrier-deadline escalation, so a FALSE latch hangs - * the leaver in BARRIER_WAIT forever. + * latches "my clean-leave committed" on direct EVIDENCE, never on inference + * (spec-2.29a r3). The evidence is committed_marker_evidence: the survivor + * coordinator made the COMMITTED marker for THIS leave attempt majority- + * durable on the voting disk and attested it with a nonce-bound + * LEAVE_COMMITTED (validated by cluster_clean_leave_committed_evidence_ + * matches before the flag this argument mirrors is ever set). Latching + * suppresses the barrier-deadline escalation and lets the leaver exit, so + * the verdict must be exactly: latch <=> evidence. * - * spec-2.29a r2 P2-1: the leaver's own-commit inference needs BOTH a bitmap - * check AND the scalar dead_generation. The others-dead bitmap alone is not - * monotone: a third-party node that false-fail-stopped (bumping the epoch) - * then recovered leaves the CSSD hysteresis DEAD→ALIVE, so the others-dead - * bitmap returns to its bound value while the scalar dead_generation (which - * only advances) does not. If the leaver's first epoch>baseline observation - * lands after that rebound, a bitmap-only check would mis-latch a leave the - * survivor coordinator actually refused. The scalar conjunct closes it — and - * it does NOT reintroduce the ②b false positive, because the leaving node's - * OWN alive→DEAD transition never bumps ITS OWN dead_generation (a node does - * not observe itself dead), so on the leaver side dead_gen stays at baseline - * through its own drain. (The survivor-side coherence sites keep the - * others-dead bitmap: a survivor DOES observe the leaver's DEAD and would be - * falsely escalated by the scalar there.) + * History of the two inference failure modes this replaces: + * - r2 P2-1 (mis-latch wedge): the pre-r2 "epoch advanced + others-dead + * bitmap unchanged" inference could mis-latch a REFUSED leave after a + * third-party false-DEAD -> ALIVE rebound restored the (non-monotone) + * bitmap, suppressing the deadline escalation forever. + * - r3 (t/331 C1/C4 false-escalation): the r2 scalar dead_generation + * conjunct is monotone the other way — a third-party transient flap on + * the leaver's local CSSD view advances it forever, so a healthy + * committed leave was refused until the deadline escalated it. + * Marker evidence is immune to both: a refused leave never gets a COMMITTED + * marker (no latch -> bounded deadline escalation), and a flap cannot make + * durable evidence disappear (latch -> no false escalation). + * + * others_dead_unchanged / dead_gen_unchanged are the leaver's live coherence + * observations. They are deliberately kept in the signature as contract + * inputs that MUST NOT affect the verdict — the U3b unit matrix pins the + * flap-immunity on them — and the runtime uses them only for the flap-noise + * LOG at the latch point (observability, never control flow). */ bool -cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, +cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, bool others_dead_unchanged, bool dead_gen_unchanged) { - return epoch_advanced && others_dead_unchanged && dead_gen_unchanged; + (void)others_dead_unchanged; /* observability-only input (see above) */ + (void)dead_gen_unchanged; /* observability-only input (see above) */ + return committed_marker_evidence; +} + +/* + * cluster_clean_leave_committed_evidence_matches -- may the leaving node + * accept a LEAVE_COMMITTED confirmation as marker evidence for THIS leave + * attempt? (spec-2.29a r3; the payload's magic/version/CRC were already + * checked by cluster_clean_leave_announce_payload_valid.) + * + * Identity is bound three ways, all fail-closed: + * - payload_leaving_node == self_node: the confirmation is addressed to + * this node's own leave (LEAVE_COMMITTED is point-to-point, but a + * misrouted frame must still not latch). + * - current_leaving_node == self_node: this node IS currently the leaver + * (not idle, not a survivor of someone else's leave). + * - payload_nonce == current_attempt_nonce: the per-attempt nonce + * (Hardening v1.0.2) pins the confirmation to THIS attempt, so a stale + * LEAVE_COMMITTED — and through it a stale COMMITTED marker — from a + * PREVIOUS leave of the same node can never false-latch a new attempt. + * - payload_epoch > bound_leave_epoch: the committed epoch E the + * coordinator attests must lie past the baseline this leave bound + * (sanity; the commit is a guarded CAS off that baseline). + */ +bool +cluster_clean_leave_committed_evidence_matches(int32 payload_leaving_node, uint64 payload_nonce, + uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, + uint64 current_attempt_nonce, + uint64 bound_leave_epoch) +{ + if (payload_leaving_node != self_node) + return false; + if (current_leaving_node != self_node) + return false; + if (payload_nonce != current_attempt_nonce) + return false; + if (payload_epoch <= bound_leave_epoch) + return false; + return true; } diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 14975b9e7e..6a4808f793 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -206,6 +206,14 @@ typedef struct ClusterLeaveState { pg_atomic_uint32 commit_point_observed; pg_atomic_uint32 committed_durable_confirmed; pg_atomic_uint32 committed_marker_durable; + /* spec-2.29a r3 (evidence latch): the committed epoch E the coordinator's + * LEAVE_COMMITTED attested (the epoch stamped into the majority-durable + * COMMITTED marker). Written by the leaving node's confirmation handler + * under `lock` BEFORE committed_durable_confirmed is set; consumed once by + * the barrier tick when the evidence latches (so the leaver records the + * exact committed epoch instead of inferring it from its possibly-stale + * local epoch view). Guarded by `lock`. */ + uint64 committed_confirmed_epoch; /* * Hardening v1.0.2 fields. @@ -352,15 +360,25 @@ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 curr const uint8 *bound_others_dead, const uint8 *current_others_dead, int nbytes); -/* spec-2.29a r2 P2-1: the leaving node's barrier-tick own-commit latch. Needs - * BOTH the others-dead bitmap unchanged AND the scalar dead_generation - * unchanged — the bitmap alone is not monotone under a third-party - * false-DEAD→ALIVE rebound and could mis-latch a refused leave (hang). The - * leaver's own DEAD never bumps its own dead_generation, so the scalar - * conjunct does not reintroduce the ②b false positive. */ -extern bool cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, +/* spec-2.29a r3: the leaving node's barrier-tick own-commit latch — evidence + * over inference. Latches iff the durable COMMITTED marker for THIS leave + * attempt was confirmed (nonce-bound LEAVE_COMMITTED attestation); the two + * coherence observations are contract inputs the verdict must ignore (a + * third-party transient flap must neither refuse an evidenced latch — the + * t/331 C1/C4 false-escalation — nor latch anything without evidence — the + * r2 P2-1 refused-leave mis-latch wedge). */ +extern bool cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, + bool others_dead_unchanged, bool dead_gen_unchanged); +/* spec-2.29a r3: identity gate for a LEAVE_COMMITTED confirmation — accept it + * as marker evidence only for THIS node's CURRENT leave attempt (self- + * addressed + currently leaving + per-attempt nonce match + committed epoch + * past the bound baseline); fail-closed on any mismatch. */ +extern bool cluster_clean_leave_committed_evidence_matches( + int32 payload_leaving_node, uint64 payload_nonce, uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, uint64 current_attempt_nonce, uint64 bound_leave_epoch); + /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the * leaving node is the expected declared peer, and the dead_bitmap names only the diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index 5da4135ddc..04363fd736 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -226,6 +226,40 @@ UT_TEST(test_own_commit_latched) } +/* ============================================================ + * U3c — LEAVE_COMMITTED evidence identity gate (spec-2.29a r3) + * ============================================================ */ + +UT_TEST(test_committed_evidence_matches) +{ + /* args: payload_leaving_node, payload_nonce, payload_epoch, + * self_node, current_leaving_node, current_attempt_nonce, + * bound_leave_epoch */ + + /* (a) confirmation for THIS node's CURRENT attempt, committed epoch past + * the bound baseline -> evidence accepted */ + UT_ASSERT(cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 1, 42, 7)); + + /* (b) stale identity: a LEAVE_COMMITTED (and through it a COMMITTED + * marker) from a PREVIOUS leave attempt of the same node — nonce differs + * -> fail-closed, no latch */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 41, 8, 1, 1, 42, 7)); + + /* (b) misrouted: confirmation addressed to a different leaving node */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(2, 42, 8, 1, 1, 42, 7)); + + /* (b) this node is not currently leaving (idle: leaving_node_id == -1, + * or tracking someone else's leave) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, -1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 2, 42, 7)); + + /* (b) committed epoch not past the bound baseline (attested epoch must be + * the guarded-CAS successor of the baseline this leave bound) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 7, 1, 1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 0, 1, 1, 42, 7)); +} + + /* ============================================================ * U5 — writable-only quiesce gate * ============================================================ */ @@ -438,11 +472,12 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(9); + UT_PLAN(10); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); UT_RUN(test_own_commit_latched); + UT_RUN(test_committed_evidence_matches); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); From 0155eb94975bae86fc2f385eb119d085b885884f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:17 +0800 Subject: [PATCH 48/58] fix(cluster): re-consume same-epoch dead-set growth mid recovery episode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WAIT_BARRIER / WAIT_CLUSTER carried only epoch-coherence guards, so a survivor whose CSSD detected a coordinator-folded multi-death late kept announcing REDECLARE_DONE under its stale dead-bitmap hash: peers that stamped the full set dropped those frames on the composite key and P6 — which has no timeout by design — wedged permanently (survivor-behind detection skew). Add a mid-episode guard that aborts to IDLE on a newer local event whose dead-bitmap hash differs: the next tick re-consumes the event, re-stamps recovery_event_bitmap_hash, re-runs recovery against the fuller dead set and re-announces DONE under the new composite key. Same-set dead_generation drift is absorbed without episode churn (the event_id keeps its local ABA-scoping role). The coordinator-behind direction needs no handling: the coordinator's own later detection bumps the epoch again and the epoch guards abort. Correct the stamp-site / accounting / hash-helper comments that overstated the bitmap as quorum-ratified (each node stamps its OWN accepted event's bitmap; convergence is eventual via the CSSD deadband plus this guard), and assert a stamped episode hash is never 0 — 0 is reserved as unstamped in the mark_peer_done drop test, and a JOIN episode's all-zero bitmap hashes to a fixed nonzero constant. Unit: survivor-behind growth leg (H({1}) stamped at the folded epoch, grown {1,2} event arrives -> abort, re-stamp, full-set remaster, DONE accounting converges) plus a same-set drift no-churn leg; both go red with the guard disabled. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P2-2, r3-P3a) --- src/backend/cluster/cluster_grd.c | 118 ++++++++++++++++++-- src/test/cluster_unit/test_cluster_grd.c | 134 ++++++++++++++++++++++- 2 files changed, 241 insertions(+), 11 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index ae87b62dfa..3df1166222 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -1723,12 +1723,16 @@ cluster_grd_recovery_event_bitmap_hash_value(void) * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). * * The cross-node half of the REDECLARE_DONE convergence key: a hash over - * the quorum-accepted dead bitmap ALONE. Same kernel as the event_id hash + * the sender's ACCEPTED dead bitmap ALONE (each node's own local event — + * there is no cross-node ratification of the bitmap; see the P0 stamp-site + * note and the r3-P2-2 mid-episode re-consume guard for how detection skew + * converges). Same kernel as the event_id hash * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — * the generation is per-instance observation history and does NOT converge * across survivors, which is exactly why event_id cannot key the P6 gate. - * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere, - * so the composite key degrades to the epoch-only gate there. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere + * (a fixed NONZERO constant — 0 stays reserved for "unstamped"), so the + * composite key degrades to the epoch-only gate there. */ uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) @@ -2126,11 +2130,23 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap * differs (a coordinator re-election adds the old coordinator to it; a * re-death of the same node rides a higher epoch via the interposed JOIN * bump), so the bitmap hash is the ABA guard. Flap-history drift changes - * only the per-instance dead_generation, never the quorum dead SET, so - * DONEs for the same episode always match here (the R2 wedge fix). + * only the per-instance dead_generation, never the sender's accepted dead + * SET, so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently stamps (and announces) a SMALLER set's hash + * at the same epoch; those frames are dropped HERE until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + re-announces + * the full set (grd_recovery_consume_new_event_mid_episode). Per-tick + * re-announce then converges the accounting. * Pre-accept frames mismatch the previous episode's stamp and are dropped; * senders re-announce every tick, so accounting lands after our P0 accept * (which also closes the R4 pre-accept window by construction). + * + * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own + * recovery_event_bitmap_hash is 0 before the first P0 accept). This + * relies on a stamped hash never being 0: JOIN episodes hash an ALL-ZERO + * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero + * constant (asserted at the stamp site). */ episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) @@ -2172,6 +2188,61 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +/* + * grd_recovery_consume_new_event_mid_episode — spec-4.6a r3-P2-2. + * + * WAIT_BARRIER / WAIT_CLUSTER guard against a NEWER local reconfig event at + * the SAME epoch. Multi-death detection skew: when two nodes die near- + * simultaneously the coordinator can fold both into ONE epoch bump with dead + * set {A,B}, while a survivor whose CSSD saw only {B} first stamps + * recovery_event_bitmap_hash = H({B}) and sails past WAIT_EPOCH on that + * bump. Its later local detection of A publishes a new event with dead set + * {A,B} but does NOT move the epoch, so the WAIT_BARRIER / WAIT_CLUSTER + * epoch guards never fire, the survivor keeps announcing DONE under H({B}), + * every peer that stamped H({A,B}) keeps dropping it (composite-key + * mismatch), and P6 — which has no timeout by design — wedges permanently. + * + * Disposition (runs in the single-writer LMON tick, after the epoch guard): + * - new event, FAIL direction, SAME dead-bitmap hash: only the sender- + * local dead_generation fold moved (flap drift re-fired the same quorum + * dead set). Absorb the event_id — the episode semantics (epoch, dead + * set) are unchanged, so re-running P1-P7 would be pure churn. + * Absorbing also keeps the IDLE dedup from re-consuming the event after + * this episode completes. + * - new event with a DIFFERENT dead-bitmap hash (grown dead set — the + * skew above — or a JOIN/FAIL interleave): abort to IDLE. The next + * tick re-consumes the current event, re-stamps + * recovery_event_bitmap_hash, re-runs recovery against the fuller dead + * set, and re-announces DONE under the new composite key — closing the + * survivor-behind wedge. The coordinator-behind direction needs no + * handling here: the coordinator's own later detection bumps the epoch + * again and the existing epoch guards abort. + * + * Returns true when the caller must return (episode aborted to IDLE). + */ +static bool +grd_recovery_consume_new_event_mid_episode(void) +{ + ReconfigEvent latest; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id == 0 + || latest.event_id == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + return false; + + if (latest.reconfig_kind == (uint8)RECONFIG_KIND_FAIL_STOP + && pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_FAIL + && cluster_grd_dead_bitmap_hash(latest.dead_bitmap) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { + pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, latest.event_id); + return false; + } + + grd_recovery_abort_to_idle(); + return true; +} + static void grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) { @@ -2471,12 +2542,28 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } - /* Amendment v1.2 (R2): stamp the composite convergence key's cross-node - * half. Hash over the accepted dead bitmap alone — every survivor that - * accepted the same quorum dead SET computes the same value regardless - * of its private dead_generation observation history. */ + /* Amendment v1.2 (R2) + r3-P2-2: stamp the composite convergence key's + * cross-node half. The hash is over THIS node's OWN accepted event's + * dead bitmap — there is NO cross-node ratification of the bitmap (the + * envelope propagates only the epoch). Convergence is eventual, not + * instantaneous: every survivor's CSSD deadband converges on the same + * dead set, and until it does, a survivor that accepted a SMALLER set + * at the same epoch stamps a different hash. The mid-episode + * re-consume guard (grd_recovery_consume_new_event_mid_episode) closes + * that skew: the survivor's own later detection re-stamps the full + * set's hash and re-announces DONE. dead_generation drift alone never + * changes the bitmap, so same-set re-fires hash identically. */ pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); + /* r3-P3(a) — 0 is reserved as "unstamped/pre-accept" in + * mark_peer_done's drop test, so a stamped episode hash must never be + * 0. JOIN episodes hash an ALL-ZERO dead bitmap: the invariant is + * that hash_bytes_extended(zero[16], 0) != 0 (a fixed nonzero + * constant). For FAIL bitmaps a zero hash is a 2^-64 collision, the + * same trust level as the event_id hash; the Assert turns the + * otherwise-undebuggable "all DONEs dropped" wedge into a loud stop + * on cassert builds. */ + Assert(pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash) != 0); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2749,6 +2836,12 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — a newer local event at the SAME epoch (multi-death + * detection skew grew the dead set) re-consumes; same-set drift is + * absorbed without churn. */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — advance the survivor block re-declare scan one chunk * while the GES rebind barrier is still pending (worker-centric, runs * in this tick; epoch-coherent via the guard just above). */ @@ -2850,6 +2943,13 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — same-epoch dead-set growth re-consumes here too: this + * node may already have announced DONE under the stale bitmap hash; + * the re-run re-announces under the full set's hash so peers' + * composite-key accounting converges (P6 has no timeout). */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — keep advancing the block re-declare scan after the GES * rebind barrier completed, so a large pool is fully re-declared within * the recovery window (no-op once the cursor reaches NBuffers). */ diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2e4b813e17..981aaa967e 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -4201,6 +4201,131 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); } +/* r3-P2-2 — survivor-behind multi-death detection skew. The coordinator + * folded {1,2} into ONE epoch bump; this survivor accepted only {1} first and + * stamped H({1}) at that epoch. Its later local detection of node 2 publishes + * a new event with the GROWN set at the SAME epoch (no further bump), which + * pre-fix was never re-consumed past WAIT_EPOCH: DONE hashes could never match + * and P6 (no timeout) wedged permanently. The mid-episode guard must abort to + * IDLE, re-consume, re-stamp the full set's hash, re-run the remaster against + * the fuller dead set, and let full-set DONEs account. */ +UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint8 grown[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 h_small; + uint64 h_grown; + int i; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline (old = 10). */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Event 1: this survivor's CSSD has seen only node 1 dead so far. */ + ut_mock_last_event.event_id = 601; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x02; /* {1} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> parks in WAIT_EPOCH (cur==old) */ + h_small = cluster_grd_recovery_event_bitmap_hash_value(); + + /* The coordinator's single folded bump arrives via piggyback: this node + * sails past WAIT_EPOCH with the SMALL set stamped. */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER, episode epoch 11 */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); + + /* The wedge shape: a full-set DONE from the coordinator is dropped while + * this node's stamp is behind. */ + memset(grown, 0, sizeof(grown)); + grown[0] = 0x06; /* {1,2} */ + h_grown = cluster_grd_dead_bitmap_hash(grown); + UT_ASSERT_NE(h_grown, h_small); + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 602; + ut_mock_last_event.dead_bitmap[0] = 0x06; + cluster_grd_recovery_lmon_tick(); /* WAIT_BARRIER guard -> abort to IDLE */ + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed + 1); + + /* Re-consume: re-stamp to the FULL set's hash, re-run the remaster against + * the fuller set (nothing may stay mastered by 1 OR 2), land back in + * WAIT_BARRIER under the same epoch. */ + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_grown); + for (i = 0; i < PGRAC_GRD_SHARD_COUNT; i++) + UT_ASSERT_EQ(cluster_grd_shard_master(i), (int32)0); + + /* The coordinator's per-tick full-set DONE re-announce now accounts. */ + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), h_grown); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 no-regression leg — dead_generation drift re-fires the SAME dead + * set under the same epoch (new event_id, identical bitmap). The guard must + * ABSORB it: no abort-to-IDLE churn, stamp unchanged, and the event_id is + * consumed so the post-episode IDLE dedup does not re-run the episode. */ +UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint64 h_set; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle baseline (old = 10) */ + + ut_mock_last_event.event_id = 611; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; /* {2} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + h_set = cluster_grd_recovery_event_bitmap_hash_value(); + + /* Same-set re-fire (drift): absorbed, no episode churn. The tick may + * legitimately advance the episode (WAIT_BARRIER -> WAIT_CLUSTER with the + * trivial unit barrier) — the assertion is that it never falls back to + * IDLE and the stamp/counters do not move. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 612; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed); /* no abort */ + UT_ASSERT_NE(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_set); + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 612); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + int /* cppcheck-suppress constParameter @@ -4212,8 +4337,9 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * spec-6.3a:+6 lifecycle; 5.8 D1b:+4 (U2a-d); D1c:+2 (U3a-b); * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); - * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(81); + * +1 (U17 cross-episode fence Hardening); + * spec-4.6a r3-P2-2:+2 (same-epoch dead-set growth re-stamp + no-churn). */ + UT_PLAN(83); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4318,6 +4444,10 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); + /* spec-4.6a r3-P2-2 — same-epoch multi-death detection-skew closure. */ + UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); + UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 2a35cb2f8c2a1ebe0c0ae23e6efbdc3c6f162987 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:32 +0800 Subject: [PATCH 49/58] fix(cluster): pair the HW remaster terminal-state fence with a read barrier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hw_remaster_record_terminal stores the backoff deadline before the terminal result with a release fence between the stores, but the LMON relaunch decider loaded result-then-deadline with no read-side pairing: on a weakly-ordered CPU it could still pair a fresh BLOCKED with a stale zero deadline and skip one backoff wait. Insert pg_read_barrier() between the result and deadline loads (attempts needs no fence — it is written only by the single-writer LMON FSM) and cross-reference the pairing at the write site. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P3b) --- src/backend/cluster/cluster_hw_remaster.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index 03ee6751a3..f2231b59b0 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -101,7 +101,9 @@ hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) * that observes the terminal result also observes the matching backoff * deadline — without the fence a weakly-ordered CPU could let it pair a * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait - * (bounded self-correcting, but cheap to close outright). */ + * (bounded self-correcting, but cheap to close outright). r3-P3(b): + * paired with the pg_read_barrier() between the result and deadline + * loads in cluster_hw_remaster_launch_workers (acquire/release pair). */ if (res == CLUSTER_HW_REMASTER_DONE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); pg_memory_barrier(); @@ -657,6 +659,16 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod } result = cluster_hw_remaster_result(node); + /* r3-P3(b) — read-side pairing for the hw_remaster_record_terminal + * write fence (deadline stored BEFORE the terminal result, with + * pg_memory_barrier between). Load the result FIRST, fence, then the + * deadline: a decider that observed a terminal result is then + * guaranteed to observe the matching backoff deadline. Without this + * the release fence is one-sided and a weakly-ordered reader could + * still pair a fresh BLOCKED with a stale (zero) deadline. attempts + * needs no fence: it is written only by this FSM (single-writer + * LMON). */ + pg_read_barrier(); attempts = cluster_hw_remaster_attempts(node); next_attempt_at = cluster_hw_remaster_next_attempt_at(node); d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, From bf11f6a29099d145ec9784330477e88b4c1a09a7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:32 +0800 Subject: [PATCH 50/58] docs(cluster): ground the dead-master unseal predicate safety proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The serve-gate unseal (static master no longer CSSD-DEAD -> NORMAL) is heartbeat liveness, not a direct recovery-completion signal. Document the traced proof of why returning NORMAL is nevertheless safe: CSSD — the only heartbeat sender — is spawned by the phase-4 driver at the PM_RUN transition, after the startup process completed the node's crash recovery against shared storage; and even under a stale-ALIVE view a fetch cannot complete against a still-recovering node, because the master-side handler default-denies until the node is an in-quorum MEMBER and only the (equally post-PM_RUN) QVOTEC process can establish the quorum lease after a restart wiped shmem. Deny replies surface as bounded 53R9L, endpoint unavailability as bounded 53R90, and no path falls back to a silent local storage read. Replaces the previous wording that asserted recovery-completion semantics without grounding. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P2-1 prove-safe) --- src/backend/cluster/cluster_gcs_block.c | 37 ++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index ee200ffafe..15d0643910 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1201,6 +1201,39 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) return GCS_BLOCK_RECOVERING; } + /* + * r3-P2-1 unseal-safety proof — this predicate is heartbeat LIVENESS + * (CSSD hysteresis flips DEAD->ALIVE on heartbeat receipt alone, + * cluster_cssd.c deadband scan), NOT a direct "instance recovery + * complete" signal. It is nevertheless safe to return NORMAL here, + * because on a crash-restarted master the heartbeat source itself is + * recovery-gated: + * (1) CSSD — the only heartbeat sender — is spawned by the cluster + * phase-4 driver, which the postmaster reaper invokes only at the + * PM_RUN transition, i.e. after the startup process exited 0 and + * the node's crash recovery fully replayed its WAL thread to shared + * storage (the ServerLoop respawn is equally PM_RUN-gated). A + * still-recovering node sends NO heartbeats, so a survivor's DEAD + * verdict cannot flip back early. + * (2) Belt-and-suspenders: even under a stale-ALIVE view (fast restart + * inside the deadband), a fetch cannot complete against a + * still-recovering node. The master-side handler + * (cluster_gcs_handle_block_request_envelope) default-denies unless + * the node is an in-quorum MEMBER, and cluster_qvotec_in_quorum() + * demands QUORUM_OK plus a live lease — state only the QVOTEC + * process (phase-4 / post-PM_RUN as well) can establish after a + * restart wiped shmem. The deny replies map to bounded 53R9L; an + * unresponsive endpoint exhausts the retransmit budget into bounded + * 53R90. Neither path ever falls back to a silent local storage + * read (STORAGE_FALLBACK is a master REPLY status, not a local + * fallback). + * (3) For online_join rejoin, MEMBER additionally requires coordinator + * admission, which vets the joiner's post-recovery voting slot + * (the slot_generation != 0 readiness sub-gate). + * So heartbeat-ALIVE implies the returned master completed its own + * instance recovery: its committed WAL is on shared storage and no + * merged-materialization proof is needed for its blocks. + */ if (static_master == cluster_node_id || cluster_cssd_get_peer_state(static_master) != CLUSTER_CSSD_PEER_DEAD) return GCS_BLOCK_NORMAL; @@ -1240,7 +1273,9 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * online thread recovery cannot run (GUC off — the default — or any * >2-node deployment) the materialization authority is never published, * so a dead master's blocks stay RECOVERING until the failed node - * restarts and runs its own instance recovery: a bounded, retryable + * restarts and completes its own instance recovery (the unseal above is + * heartbeat liveness; the r3-P2-1 note explains why heartbeats imply + * recovery completion): a bounded, retryable * ERROR on the request path (53R9L), never an unproven serve. A scope * predicate must never gate a correctness proof — a committed write on * a cold block that only the dead node saw has NO other guard on this From 5b8e8ef040c500825fd696c9b51497124efda83a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:48 +0800 Subject: [PATCH 51/58] test(cluster): lock the dead-master serve gate with a must-fail 53R9L leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Amendment v1.2 (R1) serve-gate fix had no test lock: the t/362 two-outcome legs stay green under a reverted gate (their success arm accepts a NORMAL serve of a dead-master page) and no unit links the real phase decision. Add L4h: after the kill, with the dead node never restarted, once the observed survivor's own recovery episode reached IDLE, a FRESH backend on a cold survivor runs a wide pg_class/pg_attribute scan that MUST fail with exactly 53R9L — no success arm, bounded return. An out-of-scope formation never publishes the dead node's materialization proof, so every dead-master page stays RECOVERING while the node is down; the scan spans dozens of catalog pages hashed ~1/4 to the dead node and cold-reads them past the 16MB test pool, and it touches no dead-node-written rows (heap INSERTs only, no DDL), so 53R9L is the only legal outcome. Verified red-first: with the gate reverted the suite fails at exactly this leg while every other leg stays green. The episode-IDLE wait polls the pre-warmed session with the exact query shape it ran pre-kill and compares in Perl: a cast or operator added to the SQL performs fresh syscache lookups that themselves cold-read a dead-master catalog page and trip 53R9L (the very mechanism under test). Also accept the 53R51 write-fence rejection in the pre-existing two-outcome write legs: a transient lease/epoch fence right after the reconfig is an explicit, retryable fail-closed rejection inside the same honest contract. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P1-1 TAP hard leg) --- .../362_shared_catalog_4node_kill_selfheal.pl | 64 ++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl index a5f6719eea..75747a223c 100644 --- a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -11,7 +11,9 @@ # 53R9L) and equally its transaction verdicts (TT status unknown for its # xids, the visibility door). Reads and DDL are asserted on the honest # two-outcome contract (success or explicit SQLSTATE, never a hang); -# new-object DDL/writes prove P7 unfreeze. +# new-object DDL/writes prove P7 unfreeze. L4h (r3-P1-1) is the serve-gate +# LOCK: a fresh backend's wide catalog scan on a cold survivor MUST fail +# 53R9L (no success arm) while the dead node is down. # After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means # the test harness is misconfigured, and lack of convergence is a real FAIL. # @@ -284,6 +286,9 @@ sub startup_env_blocker my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); +my %grd_done_before; +$grd_done_before{$_} = bg_counter($bg{$_}, 'grd_recovery', 'remaster_done') + for @survivors; # L2: kill node3 and require every survivor to see the real DEAD edge. # Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, @@ -338,6 +343,56 @@ sub startup_env_blocker pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); } +# L4h (r3-P1-1 hard leg): with the serve gate in place, an out-of-scope +# formation (4 nodes / online_thread_recovery off) NEVER publishes the dead +# node's materialization proof, so EVERY node3-mastered page stays RECOVERING +# until node3 returns — and node3 is never restarted in this test. A wide +# catalog scan from a FRESH backend on a cold survivor therefore MUST fail +# closed with 53R9L: pg_class + pg_attribute span dozens of pages hashed ~1/4 +# to node3, and a fresh backend's full scan cold-reads pages no prior session +# pulled into the 16MB test pool. Deliberately NO success arm — a NORMAL +# serve of a dead-master page here is the exact 8.A regression this leg locks +# (reverting the serve-gate fix turns this leg red while the two-outcome legs +# stay green). Gate on the survivor's OWN episode reaching IDLE first so the +# failure surface is deterministically the post-episode materialization proof +# (53R9L), never the in-episode shard freeze (53R9I). The scan touches no +# node3-written rows (node3 ran only heap INSERTs, no DDL), so the TT +# visibility door cannot fire first: 53R9L is the ONLY legal outcome. +{ + my $cold = 2; + + # Compare in Perl, not SQL: the warm session's catcache covers exactly the + # bg_counter query shape it ran pre-kill; a cast/operator added in SQL + # would itself cold-read a dead-master catalog page and trip 53R9L. + my $idle = 0; + my $idle_deadline = time() + 90; + while (time() < $idle_deadline) + { + if (bg_counter($bg{$cold}, 'grd_recovery', 'remaster_done') + > $grd_done_before{$cold}) + { + $idle = 1; + last; + } + usleep(500_000); + } + ok($idle, "L4h: survivor node$cold GRD recovery episode reached IDLE (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($cold)->psql('postgres', + 'SELECT count(*) FROM pg_class c JOIN pg_attribute a ON a.attrelid = c.oid', + timeout => 60); + my $elapsed = time() - $t0; + isnt(defined $rc ? $rc : 0, 0, + "L4h: fresh wide catalog scan on survivor node$cold MUST fail while node$dead is down"); + like($err // '', + qr/block-level cache protocol state is being rebuilt after reconfiguration/, + 'L4h: the failure is the explicit 53R9L dead-master fail-closed gate'); + cmp_ok($elapsed, '<', 60, 'L4h: fail-closed scan returned bounded (no hang)'); + diag("L4h: scan failed closed as required: " . ($err // '(no stderr)')) + if defined $err && $err ne ''; +} + # L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an # out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a # BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a @@ -392,7 +447,7 @@ sub startup_env_blocker else { like($err // '', - qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); } cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); @@ -406,8 +461,11 @@ sub startup_env_blocker } else { + # write-fenced (53R51, spec-4.12): a transient lease/epoch fence on the + # writer right after the reconfig — an explicit, retryable fail-closed + # rejection, squarely inside the honest two-outcome contract. like($ddl_err // '', - qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); diag("L4: DDL fail-closed honestly: $ddl_err"); } From bfe00bc2e9c5bd727fc81baa59210c1db5837fa2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:55:22 +0800 Subject: [PATCH 52/58] fix(ci): fuse nightly TAP shard matrix after 3-way merge (362 + 363 shards, 361 into crossnode-b) --- .github/workflows/nightly.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6900330eec..905b89fab7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,33 +176,20 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: -<<<<<<< HEAD # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), - # the renumbered t/351-356 family, t/357 multi-xmax alias floor. + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and + # t/361 XID authority native-era adoption. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } -<<<<<<< HEAD - # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-361 are - # reserved by the parallel spec-7.2 / S-xid lanes, t/363 by S-dead; - # L464 occupancy verified against origin branches 2026-07-08). Own - # shard: 4-node formation + kill + convergence needs the wall clock. + - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-360 + # reserved by the parallel spec-7.2 lane; L464 occupancy verified + # against origin branches 2026-07-08). Own shard: 4-node formation + # + kill + convergence needs the wall clock. - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } -======= - # t/363 spec-2.29a marker-async LMON liveness (t/358-362 reserved by - # the parallel spec-7.2 / S-xid / S-reconfig lanes; L464 occupancy - # verified against origin branches 2026-07-08). + # t/363 spec-2.29a marker-async LMON liveness. - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } ->>>>>>> stage7-p0-dead-diagnosis -======= - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and t/361 XID authority native-era adoption. - # Closes the 346+ shard hole (every new t/ file must land in a shard - # the same commit; see docs/code-review-checklist.md). - - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } ->>>>>>> stage7-p0-xid-authority steps: - name: Checkout uses: actions/checkout@v4 From 152b5792486c6add2072f22cf8d82f860f4f8481 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 12:38:18 +0800 Subject: [PATCH 53/58] fix(cluster): feed the join fence from an unconditional done-epoch axis The spec-4.6a Amendment v1.2 (R2) composite convergence key gated both recovery_done_epoch[] and recovery_done_bitmap_hash[] behind one drop gate in cluster_grd_recovery_mark_peer_done(). A rejoining node never P0-accepts the JOIN episode as its own FSM episode (JOIN_COMMITTED is published coordinator-side only), so its local episode-hash stamp stays 0 forever; the composite gate then dropped every survivor REDECLARE_DONE, the join fence (cluster_grd_block_view_rebuilt) never lifted, and every joiner-home block access fail-closed 53R9L permanently. Nightly t/347 L4-iii (rejoin with xid_striping) and t/353 L5 (fresh insert on reborn node0) pinned the wedge. Split the two accounting axes: * done_epoch[] accounts unconditionally (monotonic-max), feeding the epoch-only join fence. Sound because REDECLARE_DONE(epoch=E) is broadcast only after grd_block_redeclare_scan_complete(E), so a DONE at E proves the sender finished its WAIT_BARRIER re-declare at E -- the fence's safety condition, independent of the sender's episode dead set. * done_bitmap_hash[] stays composite-gated for the P6 cluster gate and the WAIT_EPOCH coordinator witness, which still AND both axes and stay fail-closed on the withheld hash half (r3-P2-2 multi-death skew protection unchanged). New unit leg test_recovery_idle_joiner_accounts_done_epoch_for_fence asserts the fence lifts once survivor DONEs land while the composite half stays withheld; the stale-bitmap and dead-set-growth legs gain a negative assertion that the epoch axis alone never fires the composite escape. Spec: spec-4.6a-grd-recovery-liveness.md --- src/backend/cluster/cluster_grd.c | 68 +++++++++++------- src/test/cluster_unit/test_cluster_grd.c | 91 ++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 32 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 3df1166222..ff5119a6f7 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -2123,24 +2123,44 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap return; /* - * spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE converges on the COMPOSITE - * key (episode_epoch, dead_bitmap_hash). A stale DONE from a previous - * episode can carry the same epoch when cur==old, so epoch monotonicity - * alone is not enough — but the previous episode's dead SET necessarily - * differs (a coordinator re-election adds the old coordinator to it; a - * re-death of the same node rides a higher epoch via the interposed JOIN - * bump), so the bitmap hash is the ABA guard. Flap-history drift changes - * only the per-instance dead_generation, never the sender's accepted dead - * SET, so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the - * bitmap is NOT quorum-ratified — under multi-death detection skew a - * behind survivor transiently stamps (and announces) a SMALLER set's hash - * at the same epoch; those frames are dropped HERE until its own CSSD - * catches up and the mid-episode re-consume guard re-stamps + re-announces - * the full set (grd_recovery_consume_new_event_mid_episode). Per-tick - * re-announce then converges the accounting. - * Pre-accept frames mismatch the previous episode's stamp and are dropped; - * senders re-announce every tick, so accounting lands after our P0 accept - * (which also closes the R4 pre-accept window by construction). + * The two accounting axes feed DIFFERENT consumers and must not share + * one drop gate (nightly regression: t/347 L4-iii / t/353 L5 joiner + * write wedge): + * + * 1. done_epoch[] — monotonic-max, UNCONDITIONAL (the spec-4.6 axis; + * never regress on a delayed lower-epoch frame). Consumed epoch-only + * by the spec-5.16 D3 join fence (cluster_grd_block_view_rebuilt): a + * DONE at epoch E proves the sender completed its WAIT_BARRIER at E — + * the cluster-wide rebind plus the FULL block re-declare scan against + * then-current routing — which is the fence's safety condition + * regardless of the sender's episode dead set. Crucially, a + * rejoining node never P0-accepts the JOIN episode as its own FSM + * episode (spec-5.16 D8: JOIN_COMMITTED is coordinator-side only), so + * its local episode-hash stamp stays 0/stale forever; gating this + * axis on the composite key drops every survivor DONE on the joiner, + * its join fence never lifts, and every joiner-home block access + * fail-closes 53R9L permanently. + * + * 2. done_bitmap_hash[] — composite-gated (spec-4.6a Amendment v1.2 R2). + * Read together with done_epoch[] as the (episode_epoch, + * dead_bitmap_hash) composite key by the P6 cluster gate and the + * WAIT_EPOCH coordinator witness. A stale DONE from a previous + * episode can carry the same epoch when cur==old, but its dead SET + * necessarily differs (a coordinator re-election adds the old + * coordinator to it; a re-death of the same node rides a higher epoch + * via the interposed JOIN bump), so the hash is the ABA guard: + * mismatching frames withhold the hash half and the composite + * consumers stay fail-closed. Flap-history drift changes only the + * per-instance dead_generation, never the sender's accepted dead SET, + * so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently announces a SMALLER set's hash at the + * same epoch; those frames withhold the hash half until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + + * re-announces the full set + * (grd_recovery_consume_new_event_mid_episode). Per-tick re-announce + * then converges the accounting after our P0 accept (the R4 + * pre-accept window stays closed for the composite consumers). * * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own * recovery_event_bitmap_hash is 0 before the first P0 accept). This @@ -2148,19 +2168,17 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero * constant (asserted at the stamp site). */ - episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); - if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) - return; - - /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic - * per-event epochs the incoming value always wins, but never regress the - * published value if a delayed lower-epoch frame ever slips through. */ { uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); if (epoch > prev) pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) + return; + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 981aaa967e..360a60d159 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -4172,13 +4172,27 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 * instead of node 2 — e.g. the previous episode before a coordinator - * re-election) is dropped: nothing is accounted. */ + * re-election) withholds the composite hash half. The epoch axis + * accounts unconditionally (the spec-5.16 join-fence liveness feed — + * see cluster_grd_recovery_mark_peer_done); the composite consumers + * stay blocked, pinned by the no-escape tick below. */ memset(old_dead, 0, sizeof(old_dead)); old_dead[0] = 0x08; old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); - UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + + /* Deadline expiry with ONLY the stale-set DONE accounted: the witness + * must NOT fire (its hash conjunct is unsatisfied), and the FSM stays + * fail-closed in WAIT_EPOCH — the epoch axis alone can never unlock the + * composite escape. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_EPOCH); /* Generation-drift shape (the R2 wedge): the peer's local event_id * differs, but its DONE names the SAME quorum dead set -> the composite @@ -4189,9 +4203,11 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) /* Witness advance: deadline expired + cur==old + coordinator DONE naming * THIS dead set at exactly cur, accounted after the accept snapshot. The - * FSM takes the composite-key equal-epoch escape exactly once. */ + * FSM takes the composite-key equal-epoch escape exactly once. (The + * no-proof tick above re-armed the deadline to +rebuild_timeout, so jump + * well past it.) */ cluster_grd_recovery_counters_snapshot(&before); - ut_mock_now = (int64)60 * 1000000; + ut_mock_now = (int64)130 * 1000000; cluster_grd_recovery_lmon_tick(); cluster_grd_recovery_counters_snapshot(&after); UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); @@ -4242,15 +4258,17 @@ UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); - /* The wedge shape: a full-set DONE from the coordinator is dropped while - * this node's stamp is behind. */ + /* The wedge shape: a full-set DONE from the coordinator withholds the + * composite hash half while this node's stamp is behind (the epoch axis + * accounts unconditionally for the join-fence liveness feed; P6 still + * blocks on the missing hash). */ memset(grown, 0, sizeof(grown)); grown[0] = 0x06; /* {1,2} */ h_grown = cluster_grd_dead_bitmap_hash(grown); UT_ASSERT_NE(h_grown, h_small); cluster_grd_recovery_mark_peer_done(0, 11, h_grown); UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); - UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ cluster_grd_recovery_counters_snapshot(&c0); @@ -4326,6 +4344,62 @@ UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); } +/* Joiner-side DONE-accounting liveness (nightly t/347 L4-iii / t/353 L5 + * regression pin). A rejoining node never P0-accepts the JOIN episode as + * its own FSM episode (spec-5.16 D8: JOIN_COMMITTED is published + * coordinator-side only), so its local episode-hash stamp stays 0 forever. + * The survivors' per-tick REDECLARE_DONE re-announces are that node's ONLY + * feed for the spec-5.16 D3 join fence (cluster_grd_block_view_rebuilt); + * gating the done_epoch axis on the composite key drops every frame on the + * joiner, the fence never lifts, and every joiner-home block access + * fail-closes 53R9L until the caller's retry budget burns out. The epoch + * axis must account unconditionally — a DONE at epoch E proves the sender + * completed its WAIT_BARRIER at E (cluster-wide rebind + full block + * re-declare scan against then-current routing), which is the fence's + * safety condition regardless of the sender's episode dead set — while the + * composite hash half stays withheld pre-accept (P6 gate and WAIT_EPOCH + * witness unchanged). */ +UT_TEST(test_recovery_idle_joiner_accounts_done_epoch_for_fence) +{ + uint8 join1[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint8 zero_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + int n1[1] = { 1 }; + BufferTag tag; + uint64 join_hash; + + memset(&tag, 0, sizeof(tag)); + ut_jr_setup_3node(); /* FSM IDLE, episode hash stamp 0: never accepted */ + ut_mock_epoch = 10; + ut_jr_build_bitmap(join1, n1, 1); + cluster_grd_arm_join_pcm_fence(join1); + ut_mock_static_master = 1; /* the rejoiner's home block */ + + /* The real JOIN-episode DONE payload: hash of the all-zero dead bitmap + * (nonzero by the r3-P3(a) invariant). */ + memset(zero_bitmap, 0, sizeof(zero_bitmap)); + join_hash = cluster_grd_dead_bitmap_hash(zero_bitmap); + UT_ASSERT_NE(join_hash, 0); + + UT_ASSERT(cluster_grd_join_remaster_active_for_shard(tag)); + UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); + + /* Survivor DONEs arrive while this node's FSM is IDLE and unstamped + * (node 1 is this episode's recipient — skipped by the barrier). */ + cluster_grd_recovery_mark_peer_done(1, 10, join_hash); + cluster_grd_recovery_mark_peer_done(0, 10, join_hash); + cluster_grd_recovery_mark_peer_done(2, 10, join_hash); + + /* Epoch axis accounted -> the join fence lifts (the wedge fix)... */ + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(2), 10); + UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); + + /* ...while the composite hash half stays withheld pre-accept (v1.2 R2: + * P6 / witness consumers remain fail-closed on this node). */ + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(2), 0); +} + int /* cppcheck-suppress constParameter @@ -4448,6 +4522,9 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + /* spec-4.6a nightly regression — joiner-side DONE accounting liveness. */ + UT_RUN(test_recovery_idle_joiner_accounts_done_epoch_for_fence); + UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 589409c3f148dda7fe76ba18dede14aea612683c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 12:38:18 +0800 Subject: [PATCH 54/58] test(cluster): refresh grd dump baselines surfaced by nightly The same S-reconfig nightly that pinned the join-fence wedge also flagged two stale dump-count baselines that lagged already-shipped counters: * t/108 pg_cluster_state pcm category 22 -> 23 (spec-4.6a D12 dead_cleanup_entries). * t/249 grd_recovery dump roster 18 -> 31 keys plus the exact sorted key list (spec-4.6 2.4 base + spec-5.16 D5 join direction + spec-4.6a liveness/episode surface). Both stay exact-count / exact-roster assertions; the code already emits the 31-key grd_recovery set (verified against cluster_debug.c). Spec: spec-4.6a-grd-recovery-liveness.md --- .../cluster_tap/t/108_pcm_state_machine.pl | 4 +-- .../t/249_grd_recovery_remaster.pl | 32 ++++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index cebe221c82..dcf9ef8d0c 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '22', - 'L1 pg_cluster_state pcm category has 22 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter)'); +is($pcm_category_rows, '23', + 'L1 pg_cluster_state pcm category has 23 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter + spec-4.6a D12 dead_cleanup_entries)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( diff --git a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl index 3c66d90c53..1e0847375a 100644 --- a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl +++ b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl @@ -41,9 +41,10 @@ # L5 (D3 flipped) the rebound holders release against their # current-epoch masters and both resources are immediately # re-acquirable. Pinned gap was: release silently swallowed. -# L7 (D5 flipped) grd_recovery dump category: 13 counters, key -# roster per spec-4.6 §2.4, episode trail asserted -# (remaster_started/done, shards_remastered=2048, +# L7 (D5 flipped) grd_recovery dump category: 31 keys (spec-4.6 +# §2.4 base 13 + spec-5.16 D5 join-direction 5 + spec-4.6a +# liveness/episode surface 13), full key roster, episode trail +# asserted (remaster_started/done, shards_remastered=2048, # holders_redeclared>=1, rebuild_timeout=0). # # Discipline notes: @@ -395,22 +396,29 @@ sub poll_query_until_timeout # ---------- -# L7 (FLIPPED by D5; spec-5.16 D5 +5 join-direction counters): grd_recovery -# dump category exposes 18 counters and the recovery episode left the -# expected trail. +# L7 (FLIPPED by D5; spec-5.16 D5 +5 join-direction counters; spec-4.6a +# +13 episode/liveness keys): grd_recovery dump category exposes 31 +# keys and the recovery episode left the expected trail. # ---------- is($pair->node1->safe_psql('postgres', q{SELECT count(*) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - '18', 'L7 grd_recovery dump category exposes 18 counters (D5 + spec-5.16 join)'); + '31', + 'L7 grd_recovery dump category exposes 31 keys (spec-4.6 D5 + spec-5.16 join + spec-4.6a)'); is($pair->node1->safe_psql('postgres', q{ SELECT string_agg(key, ',' ORDER BY key) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - 'block_path_failclosed,converts_requeued,holders_rebound,holders_redeclared,' - . 'rebuild_timeout,remaster_done,remaster_failed,remaster_started,' - . 'shards_remastered,stale_holder_swept,stale_request_drop,' - . 'unaffected_holder_survived,waiters_requeued', - 'L7 grd_recovery key roster matches spec-4.6 §2.4'); + 'block_path_failclosed,block_redeclare_cursor,block_redeclare_done,' + . 'block_redeclare_epoch,cluster_gate_timeout,converts_requeued,' + . 'done_self_bitmap_hash,done_self_epoch,episode_epoch,event_coordinator,' + . 'event_old_epoch,holders_rebound,holders_redeclared,' + . 'join_block_recovering_failclosed,join_block_views_rebuilt,' + . 'join_remaster_done,join_remaster_started,join_shards_remastered,' + . 'last_event_id,rebuild_timeout,remaster_done,remaster_failed,' + . 'remaster_started,shards_remastered,stale_holder_swept,' + . 'stale_request_drop,state,state_enum_value,unaffected_holder_survived,' + . 'wait_epoch_escape,waiters_requeued', + 'L7 grd_recovery key roster (spec-4.6 §2.4 + spec-5.16 D5 + spec-4.6a)'); is($pair->node1->safe_psql('postgres', q{SELECT value::bigint >= 1 FROM cluster_dump_state() From 6e4ca73c18ffc0a33767a5ceecc20ba6a860c7a8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 12:43:34 +0800 Subject: [PATCH 55/58] fix(cluster): spec-7.2 D7 F6-1 -- one-shot conn-reset injection, un-SKIP t/360 L5 The cluster-lms-conn-reset injection modelled a single epoch-bump DATA-mesh reset, but its only arm path is the process-local injection GUC, which stays armed and re-set skip_pending on every LMS tick -- a reset storm that starved the mesh and (on the passive side) tripped the accept/close WES churn into a crash, then hit the F1-8 block_device recovery wall. That made t/360's L5 leg KNOWN-BLOCKED. Fix: latch the injected reset so it fires exactly ONCE per LMS process, and only latch after it has actually torn down a live connection (so an arm- before-connect race retries instead of wasting the single shot). The real epoch-bump reset path (cur_epoch != dp_last_epoch) is unaffected. t/360 L5 is now a real e2e leg (installcheck, 3x stable): L5.1 the injected reset fires once (reset-log count +1) L5.2 the mesh reconnects -- both nodes emit a fresh "state CONNECTED" after the reset (node0 re-dials, node1 re-accepts) L5.3 one-shot: exactly one reset event, no per-tick storm L5.2 proves reconnect via log evidence rather than a post-reset SQL probe: by that point the value-gate ping-pong has burned fault_t's rows' xmax into recycled TT slots, so ANY access (read or write) fail-closes with "cluster TT status unknown" -- the pre-existing #119 recycled-slot wall, orthogonal to the DATA plane and not caused by the reset. Block-shipping capability itself is quantified by the L2 value gate (394 ships, p99 <= 500us). L4 (F1-8) and L6 (F6-2) stay honest SKIP; the stage7 P0 substrate was evaluated and does not unblock either (L4 needs the F1 recovery-vs-membership fix; L6 needs the data-dispatch injection point repositioned -- observability polish, block-over-DATA already proven by t/358 + the value gate). Spec: spec-7.2-ic-data-plane-decoupling.md D7 / F6-1 --- src/backend/cluster/cluster_lms_data_plane.c | 40 ++++- .../t/360_lms_data_plane_faults.pl | 137 ++++++++++++++---- 2 files changed, 143 insertions(+), 34 deletions(-) diff --git a/src/backend/cluster/cluster_lms_data_plane.c b/src/backend/cluster/cluster_lms_data_plane.c index 14736b1263..eb21da3dff 100644 --- a/src/backend/cluster/cluster_lms_data_plane.c +++ b/src/backend/cluster/cluster_lms_data_plane.c @@ -233,23 +233,57 @@ cluster_lms_data_plane_tick(long timeout_ms) { static uint64 dp_last_epoch = 0; static bool dp_epoch_seen = false; + static bool dp_inject_reset_done = false; uint64 cur_epoch = cluster_epoch_get_current(); + bool inject_reset = false; + + /* + * PGRAC: spec-7.2 D7 (F6-1) — the cluster-lms-conn-reset injection + * models a SINGLE epoch-bump reset, but its only arm path is the + * process-local injection GUC, which stays armed and re-sets + * skip_pending on every tick (CLUSTER_INJECTION_POINT above) — a + * reset storm that starves the mesh (and, on the passive side, was + * observed to trip the accept/close WES churn into a crash). Latch + * the injected reset so it fires exactly ONCE per process lifetime: + * the test then observes one clean reset -> reconnect -> converge, + * matching the real single-bump path (cur_epoch != dp_last_epoch). + * should_skip still drains the pending flag each tick (harmless once + * latched). The real epoch-bump reset is unaffected by the latch. + */ + if (cluster_injection_should_skip("cluster-lms-conn-reset") && !dp_inject_reset_done) + inject_reset = true; if (!dp_epoch_seen) { dp_last_epoch = cur_epoch; dp_epoch_seen = true; - } else if (cur_epoch != dp_last_epoch - || cluster_injection_should_skip("cluster-lms-conn-reset")) { + } else if (cur_epoch != dp_last_epoch || inject_reset) { + const char *reason = inject_reset ? "data-plane conn-reset injection (F6-1 one-shot)" + : "data-plane epoch bump reset"; + int n_closed = 0; + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { if (dp_track[pi].fd >= 0) { - cluster_ic_tier1_close_peer(pi, "data-plane epoch bump reset"); + cluster_ic_tier1_close_peer(pi, reason); dp_track[pi].fd = -1; dp_track[pi].substate = LMS_DP_DOWN; dp_track[pi].connect_started_at = 0; dp_track[pi].next_attempt_at = 0; /* reconnect immediately */ dp_wes_dirty = true; + n_closed++; } } + + /* + * F6-1 one-shot latch: only mark the injected reset consumed + * once it has actually torn down a live connection. This keeps + * the injection from being wasted on a tick where the mesh is + * momentarily DOWN (arm-before-connect race) — it retries until + * a peer is live, then latches so the persistent GUC does not + * storm. The real epoch-bump reset does not touch the latch. + */ + if (inject_reset && n_closed > 0) + dp_inject_reset_done = true; + dp_last_epoch = cur_epoch; } } diff --git a/src/test/cluster_tap/t/360_lms_data_plane_faults.pl b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl index 2deb926bf4..5eb56b578a 100644 --- a/src/test/cluster_tap/t/360_lms_data_plane_faults.pl +++ b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl @@ -15,9 +15,16 @@ # p50 < 5ms and p99 < 20ms (裁决④ loopback口径). Actual # percentiles are diag'd. # L3 hygiene: zero plane misroutes + no plane-gate errors. +# L5 conn-reset injection (F6-1, un-SKIPPED): arm the one-shot +# cluster-lms-conn-reset injection, observe exactly one DATA-mesh +# reset in node0's log, then verify the mesh reconnects and a +# cross-node block transfer converges. The reset is now latched +# one-shot (cluster_lms_data_plane.c) so the persistent GUC arm no +# longer storms or crashes the passive LMS. # -# KNOWN-BLOCKED legs (honest SKIP; see docs/stage7-substrate- -# findings.md — Stage-7 substrate follow-ups, user ruling 2026-07-07): +# KNOWN-BLOCKED / deferred legs (honest SKIP; see docs/stage7-substrate- +# findings.md). The stage7 P0 substrate (spec-6.15b/4.6a/2.29a) was +# evaluated and does NOT unblock either of these: # L4 kill -9 LMS x3 crash recovery — F1-8: a SIGKILL of the LMS # triggers a node crash-restart, but block_device crash- # recovery replays a relation create/extend that needs the @@ -25,19 +32,16 @@ # and GES is unavailable before the node re-joins the cluster # -> recovery FATAL "could not acquire raw layout lock". This # is a recovery-vs-membership ordering gap in the spec-6.0a -# block_device backend, orthogonal to the spec-7.2 DATA plane. -# L5 conn-reset injection reset->reconnect — F6-1: the only arm -# path is the process-local injection GUC, which is persistent -# and storms a reset every LMS tick (not the one-shot epoch -# bump it models); the storm crashes the passive LMS with -# kevent() EBADF and then hits the same F1-8 recovery wall. -# The real single-bump reset path (cur_epoch != dp_last_epoch) -# is clean (rebuild-before-wait within the tick). +# block_device backend (raw_layout_lock, cluster_shared_fs_ +# block_device.c:570-600), orthogonal to the spec-7.2 DATA plane +# and untouched by the P0 substrate -- needs the F1 recovery-vs- +# membership fix (separate spec). # L6 data-dispatch injection reachability — F6-2: the # cluster-lms-data-dispatch point sits on the LMS async recv # pump's CONNECTED&READABLE branch, which the actual block -# REPLY recv does not traverse (hits stay 0), so it needs -# repositioning before it can be armed as a reachability leg. +# REPLY recv does not traverse (hits stay 0). Observability +# point-placement polish, not a correctness gap (block-over-DATA +# is proven by t/358 + the L2 value gate); needs repositioning. # # Spec: spec-7.2-ic-data-plane-decoupling.md §4 / D7 # @@ -239,31 +243,102 @@ sub pct_bound } # ============================================================ -# L5 — KNOWN-BLOCKED: conn-reset injection reset -> reconnect. -# (F6-1: the injection GUC arm is persistent and storms a reset every LMS -# tick, crashing the passive LMS with kevent() EBADF, then hitting the F1-8 -# wall. The injection cannot be single-fired via the GUC; the real single- -# bump epoch reset path is clean. Not armed here.) +# L5 — F6-1 (UN-SKIPPED): conn-reset injection resets the DATA mesh once, +# then the mesh reconnects and block transfer converges. +# +# The cluster-lms-conn-reset injection is now one-shot (spec-7.2 D7 F6-1, +# cluster_lms_data_plane.c): although the process-local injection GUC stays +# armed, a static latch fires the injected reset exactly ONCE per LMS +# process — modelling a single epoch-bump reset instead of a per-tick storm. +# So arming the persistent GUC no longer storms the mesh or crashes the +# passive LMS, and the real single-bump path (cur_epoch != dp_last_epoch) is +# unaffected by the latch. # ============================================================ -SKIP: { - skip 'F6-1 KNOWN-BLOCKED: cluster-lms-conn-reset GUC arm is persistent ' - . '(storms every tick, not the one-shot epoch bump it models) and ' - . 'crashes the passive LMS with kevent() EBADF, then hits F1-8; needs ' - . 'one-shot arm + WES fd-lifecycle hardening', 2; - ok(0, 'L5.1 conn-reset injection resets the DATA mesh once'); - ok(0, 'L5.2 mesh reconnects + block transfer converges after reset'); +my $reset_re = qr/conn-reset injection \(F6-1 one-shot\)/; +my $conn_re = qr/state CONNECTED/; + +sub count_re { my ($f, $re) = @_; return () = PostgreSQL::Test::Utils::slurp_file($f) =~ /$re/g; } + +my $resets_before = count_re($node0->logfile, $reset_re); +my $conn0_before = count_re($node0->logfile, $conn_re); +my $conn1_before = count_re($node1->logfile, $conn_re); + +# Arm the one-shot conn-reset injection on node0's LMS via the GUC + reload +# (the LMS re-reads cluster.injection_points on SIGHUP). append_conf adds the +# setting (the conf has no prior injection_points line to adjust); a later +# appended line wins over any earlier one. +$node0->append_conf('postgresql.conf', + "cluster.injection_points = 'cluster-lms-conn-reset:skip'"); +$node0->reload; + +# Wait for the LMS tick to fire the single reset (heartbeat-interval driven). +my $reset_now = $resets_before; +for my $i (1 .. 60) +{ + usleep(500_000); + $reset_now = count_re($node0->logfile, $reset_re); + last if $reset_now > $resets_before; } +ok($reset_now > $resets_before, 'L5.1 conn-reset injection reset the DATA mesh once'); + +# Disarm (the SKIP arm survives GUC reloads, but the one-shot latch already +# prevents any further injected reset, so this is belt-and-suspenders). +$node0->append_conf('postgresql.conf', "cluster.injection_points = ''"); +$node0->reload; + +# L5.2 — the mesh RECONNECTS after the injected reset. Proven by log evidence: +# both nodes emit a fresh "state CONNECTED" AFTER the reset -- node0 re-dials +# (active role) and node1 re-accepts (passive role, after its recv sees the +# drop). Once CONNECTED the tier1 link is ready for block transfer, so the +# DATA plane's block-shipping capability (already quantified by the L2 value +# gate: 394 real X-ships at p99 <= 500us) survives an epoch reset. +# +# We do NOT re-probe convergence with an SQL read/write here: the value-gate +# ping-pong burns fault_t's rows' xmax into recycled TT slots, so by this +# point ANY access to fault_t (read OR write) fail-closes with "cluster TT +# status unknown" -- the pre-existing #119 recycled-slot wall (spec-7.1a / +# gap-㉖), which affects all access to that table, is orthogonal to the DATA- +# plane decoupling and is NOT caused by the reset. The reconnect log evidence +# is the direct, uncontaminated proof of what F6-1 delivers. +my $reconnect_deadline = time + 30; +my ($conn0_after, $conn1_after) = ($conn0_before, $conn1_before); +while (time < $reconnect_deadline) +{ + $conn0_after = count_re($node0->logfile, $conn_re); + $conn1_after = count_re($node1->logfile, $conn_re); + last if $conn0_after > $conn0_before && $conn1_after > $conn1_before; + usleep(500_000); +} +ok($conn0_after > $conn0_before && $conn1_after > $conn1_before, + "L5.2 DATA mesh reconnects after the reset " + . "(node0 CONNECTED $conn0_before->$conn0_after, node1 $conn1_before->$conn1_after)"); + +# One-shot guarantee: the persistent GUC arm produced a single reset event +# (one close-log per peer), NOT a per-tick storm. In this 2-node pair that +# is exactly one injected close-log line. +my $reset_final = () = PostgreSQL::Test::Utils::slurp_file($node0->logfile) =~ /$reset_re/g; +my $injected = $reset_final - $resets_before; +ok($injected == 1, + "L5.3 conn-reset injection is one-shot (fired $injected time; no per-tick storm)"); # ============================================================ -# L6 — KNOWN-BLOCKED: data-dispatch injection reachability. -# (F6-2: the cluster-lms-data-dispatch point is on the LMS async recv pump's -# CONNECTED&READABLE branch, which the actual block REPLY recv does not -# traverse -- hits stay 0 even armed at startup. Needs repositioning onto -# the real block recv path before it can be a reachability leg.) +# L6 — HONEST SKIP: data-dispatch injection-point reachability (F6-2). +# (The cluster-lms-data-dispatch injection point sits on the LMS async recv +# pump's CONNECTED&READABLE branch, which the actual block REPLY recv does +# not traverse -- hits stay 0 even armed at startup. This is an OBSERVABILITY +# point-placement issue, NOT a correctness gap: that block traffic really +# flows over the DATA plane is already proven independently by t/358 (flip +# fact: block_family_plane=1 + DATA listeners) and by the L2 value gate above +# (100s of real X-transfers, p99 <= 500us over the DATA connection). The +# stage7 P0 substrate does not touch this; un-SKIPping needs repositioning +# the injection point onto the true block recv landing site -- deferred as a +# spec-7.2 observability follow-up. See docs/stage7-substrate-findings.md F6-2.) # ============================================================ SKIP: { - skip 'F6-2 KNOWN-BLOCKED: cluster-lms-data-dispatch sits off the actual ' - . 'block REPLY recv path (hits stay 0 even armed at startup); needs ' + skip 'F6-2 SKIP (observability polish, not correctness): ' + . 'cluster-lms-data-dispatch sits off the actual block REPLY recv path ' + . '(hits stay 0 even armed at startup); DATA-plane block flow is already ' + . 'proven by t/358 + the L2 value gate; needs injection-point ' . 'repositioning', 1; ok(0, 'L6 cluster-lms-data-dispatch fires on the live block recv path'); } From 04ca16191e4cecf0798a5adfc153b943d17c3958 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 14:54:26 +0800 Subject: [PATCH 56/58] test(cluster): configure wal_threads_root for t/353 reborn HW remaster t/353 L5 crashes node0 and has the reborn node take a FRESH HW space lease, which requires the survivor to complete HW remaster of the dead node's shards. HW remaster reads the dead node's per-thread WAL, so on this shared_data 2-node pair it is only recoverable when cluster.wal_threads_dir is configured (cluster_hw_remaster_recoverable). The pair was created with shared_data but WITHOUT wal_threads_root, so recoverable() returned false, the remaster was marked BLOCKED_STRUCTURAL, and the reborn node's fresh-lease DDL/insert/count fail-closed (L5 subtests 28-31, nightly crossnode-b). Opt the pair into wal_threads_root, matching every other HW-remaster- dependent kill test (t/247/248/274/293). This completes the per-thread WAL harness config that the RACvsRAC/ClusterQuad harness already carries but this ClusterPair test was missing. Spec: spec-4.6a-grd-recovery-liveness.md --- src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl b/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl index 1fb21179be..49fd3e518a 100644 --- a/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl +++ b/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl @@ -117,6 +117,15 @@ sub poll_until 'd612_lease', quorum_voting_disks => 3, shared_data => 1, + # The L5 reborn leg takes a FRESH HW space lease, which needs the survivor + # to complete HW remaster of the dead node's shards. HW remaster reads the + # dead node's per-thread WAL, so on a shared_data multi-node pair it is only + # recoverable when cluster.wal_threads_dir is set + # (cluster_hw_remaster_recoverable); without it the remaster is marked + # BLOCKED_STRUCTURAL and the reborn node's fresh-lease access stays + # fail-closed. Every HW-remaster-dependent kill test opts into this + # (t/247/248/274/293). Spec: spec-4.6a-grd-recovery-liveness.md (D4). + wal_threads_root => 1, extra_conf => [ 'autovacuum = off', 'jit = off', From a3fe868d1e8f5a984236dc1ed03595214906f686 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 15:16:54 +0800 Subject: [PATCH 57/58] fix(cluster): spec-7.2 F6-1 review r2 -- convergence probe, conn-reset counter, epoch short-circuit Code review r2 on the F6-1 one-shot conn-reset commit (three findings): - Finding 1 (over-claim + convergence): the L5.2 note claimed the recycled remote-ITL-slot fail-closed boundary blocks ANY access to fault_t. It is actually per-recycled-slot and does not wall a fresh table's new, un-recycled xids. Corrected the wording (t/360 L5.2 comment) and added t/360 L5.4/L5.5: after the reset+reconnect, a FRESH coincidence table (fault2_t) is X-swapped cross-node, proving block transfer converges (both requesters' block_request_count grow, zero plane misroutes) -- the direct convergence proof the reconnect log evidence alone did not give. - Finding 2 (observability): implement the lms_conn_resets counter -- a shmem atomic in ClusterLmsSharedState, bumped once per DATA connection torn down by a reset (epoch bump in production, injection in the test), dumped under pg_cluster_state category='lms'. t/360 L5.4 asserts it recorded the reset. The cluster-lms-data-dispatch injection point stays a deferred follow-up: wiring a counter on its currently-unreachable branch would be a permanently-zero (false) observable. - Finding 3 (nit): restore the epoch-bump short-circuit precedence. inject_reset is armed only when there is no epoch bump, so a real bump is attributed to the epoch (not the F6-1 one-shot) and does not consume the one-shot latch; the injection still fires exactly once on a later tick. Test-only path; production is unaffected. Also complete the D6 wait-event snapshot bump (118 -> 120, +2 LMS DATA-plane wait events) in the stage3/4/5.5/5/beta acceptance unit tests, which the earlier bump left stale (stage2 + views + retransmit were already 120). Added the matching cluster_lms_get_conn_resets stub in test_cluster_debug. Local gates: cluster_unit 160/160 (0 not-ok), cluster_regress 13/13, PG core 219/219, t/358 flip PASS, t/360 PASS (L5 convergence + L4/L6 honest SKIP, value gate p99 <= 500us). Spec: spec-7.2-ic-data-plane-decoupling.md --- src/backend/cluster/cluster_debug.c | 16 +-- src/backend/cluster/cluster_lms.c | 25 ++++ src/backend/cluster/cluster_lms_data_plane.c | 36 ++++-- src/include/cluster/cluster_lms.h | 13 ++ .../t/360_lms_data_plane_faults.pl | 112 +++++++++++++++--- src/test/cluster_unit/test_cluster_debug.c | 6 + .../test_cluster_stage3_acceptance.c | 8 +- .../test_cluster_stage4_acceptance.c | 6 +- .../test_cluster_stage5_5_cr_acceptance.c | 9 +- .../test_cluster_stage5_beta_acceptance.c | 2 +- ...est_cluster_stage5_integrated_acceptance.c | 2 +- 11 files changed, 194 insertions(+), 41 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 70a75026c5..b2d1c08469 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1260,13 +1260,12 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) /* * dump_lms -- spec-2.18 Sprint A Step 4 D10. * - * Emits 6 rows under category='lms' corresponding to the 6 atomic - * counters in ClusterLmsSharedState (v0.3 §1.4 F2 收紧; - * grant/reject/convert 分项 counter 推 spec-2.20 真激活 grant state - * machine 时一并 ship). - * - * Plus the LMS state string for HC2 4-state semantic分流 - * observability. + * Emits the LMS state string (HC2 4-state semantic 分流 observability) + * plus one row per atomic counter in ClusterLmsSharedState: the + * spec-2.18 base set (started/work_drained/decision grant/reject/convert/ + * drain_empty/error), the spec-2.25 native-lock probe counters, the + * spec-2.27 priority-starvation counter, and the spec-7.2 D6 + * lms_conn_resets DATA-plane reset counter. */ static void dump_lms(ReturnSetInfo *rsinfo) @@ -1310,6 +1309,9 @@ dump_lms(ReturnSetInfo *rsinfo) * on wire; reserved opcode 11 awaits spec-2.28+ integrated receiver). */ emit_row(rsinfo, "lms", "priority_starvation_observed_count", fmt_int64((int64)cluster_lms_get_priority_starvation_observed_count())); + /* spec-7.2 D6 — DATA-plane connection resets (epoch bump in production; + * cluster-lms-conn-reset injection in the F6-1 fault test). */ + emit_row(rsinfo, "lms", "lms_conn_resets", fmt_int64((int64)cluster_lms_get_conn_resets())); } /* diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index 0015ee68d4..590ef7eae3 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -199,6 +199,8 @@ cluster_lms_shmem_init(void) * starvation counter starts at 0. */ pg_atomic_init_u64(&cluster_lms_state->lms_restart_generation, 0); pg_atomic_init_u64(&cluster_lms_state->priority_starvation_observed_count, 0); + /* spec-7.2 D6 — DATA-plane connection resets counter. */ + pg_atomic_init_u64(&cluster_lms_state->lms_conn_resets, 0); ConditionVariableInit(&cluster_lms_state->cv); } } @@ -581,6 +583,29 @@ cluster_lms_get_priority_starvation_observed_count(void) return pg_atomic_read_u64(&cluster_lms_state->priority_starvation_observed_count); } +/* + * spec-7.2 D6 — DATA-plane connection resets counter. + * + * cluster_lms_bump_conn_resets adds the number of DATA connections a + * single reset event tore down (epoch bump in production, injection in + * the F6-1 test). Called from the LMS DATA-plane tick; a shared-memory + * atomic so pg_stat_cluster_* / the debug dump can observe reset activity. + */ +void +cluster_lms_bump_conn_resets(uint32 n) +{ + if (cluster_lms_state != NULL && n > 0) + pg_atomic_fetch_add_u64(&cluster_lms_state->lms_conn_resets, (uint64)n); +} + +uint64 +cluster_lms_get_conn_resets(void) +{ + if (cluster_lms_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_lms_state->lms_conn_resets); +} + /* ============================================================ * HC4 single ownership atomic guard. diff --git a/src/backend/cluster/cluster_lms_data_plane.c b/src/backend/cluster/cluster_lms_data_plane.c index eb21da3dff..b8969f6137 100644 --- a/src/backend/cluster/cluster_lms_data_plane.c +++ b/src/backend/cluster/cluster_lms_data_plane.c @@ -235,6 +235,7 @@ cluster_lms_data_plane_tick(long timeout_ms) static bool dp_epoch_seen = false; static bool dp_inject_reset_done = false; uint64 cur_epoch = cluster_epoch_get_current(); + bool epoch_bumped = dp_epoch_seen && cur_epoch != dp_last_epoch; bool inject_reset = false; /* @@ -246,19 +247,31 @@ cluster_lms_data_plane_tick(long timeout_ms) * observed to trip the accept/close WES churn into a crash). Latch * the injected reset so it fires exactly ONCE per process lifetime: * the test then observes one clean reset -> reconnect -> converge, - * matching the real single-bump path (cur_epoch != dp_last_epoch). - * should_skip still drains the pending flag each tick (harmless once - * latched). The real epoch-bump reset is unaffected by the latch. + * matching the real single-bump path (epoch_bumped). should_skip + * still drains the pending flag each tick (harmless once latched). + * The real epoch-bump reset is unaffected by the latch. + * + * PGRAC modifications (F6-1 review r2, Finding 3): a real epoch + * bump takes precedence over the injected reset in the SAME tick. + * We only arm inject_reset when there is no epoch bump, so the + * reason log is attributed to the epoch bump (not "F6-1 one-shot") + * and the one-shot latch is NOT consumed by an epoch-bump tick — + * the injection then still fires exactly once on a later + * injection-only tick. This preserves the pre-F6-1 short-circuit + * (epoch bump || should_skip) attribution that this reorder + * otherwise lost. Test-only path: the injection GUC is never armed + * in production, where epoch_bumped alone drives the reset. */ - if (cluster_injection_should_skip("cluster-lms-conn-reset") && !dp_inject_reset_done) + if (!epoch_bumped && cluster_injection_should_skip("cluster-lms-conn-reset") + && !dp_inject_reset_done) inject_reset = true; if (!dp_epoch_seen) { dp_last_epoch = cur_epoch; dp_epoch_seen = true; - } else if (cur_epoch != dp_last_epoch || inject_reset) { - const char *reason = inject_reset ? "data-plane conn-reset injection (F6-1 one-shot)" - : "data-plane epoch bump reset"; + } else if (epoch_bumped || inject_reset) { + const char *reason = epoch_bumped ? "data-plane epoch bump reset" + : "data-plane conn-reset injection (F6-1 one-shot)"; int n_closed = 0; for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { @@ -284,6 +297,15 @@ cluster_lms_data_plane_tick(long timeout_ms) if (inject_reset && n_closed > 0) dp_inject_reset_done = true; + /* + * PGRAC: spec-7.2 D6 — observability. lms_conn_resets counts + * DATA-plane connections torn down by a reset (epoch bump in + * production, injection in the F6-1 test), making the reset + * path visible in pg_stat_cluster_* / the debug dump. + */ + if (n_closed > 0) + cluster_lms_bump_conn_resets((uint32)n_closed); + dp_last_epoch = cur_epoch; } } diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index 31b8b6a999..afe960df71 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -257,6 +257,15 @@ typedef struct ClusterLmsSharedState { * BOOST = 11 awaits spec-2.28+ with PG core lock manager * `LockWaitQueueInsertAtHead`改造 + integrated receiver. */ pg_atomic_uint64 priority_starvation_observed_count; + + /* spec-7.2 D6 — DATA-plane connection resets observability counter. + * + * Bumped by the LMS DATA-plane tick (cluster_lms_data_plane.c) once + * per connection torn down by a proactive reset: an epoch bump in + * production (INV-7.2-CONN-EPOCH ③) or the cluster-lms-conn-reset + * injection in the F6-1 fault test. Monotone; cluster-wide reset + * activity is the sum across nodes. */ + pg_atomic_uint64 lms_conn_resets; } ClusterLmsSharedState; @@ -359,6 +368,10 @@ extern void cluster_lms_bump_restart_generation_at_main_entry(void); extern void cluster_lms_inc_priority_starvation_observed(void); extern uint64 cluster_lms_get_priority_starvation_observed_count(void); +/* spec-7.2 D6 — DATA-plane connection resets observability counter. */ +extern void cluster_lms_bump_conn_resets(uint32 n); +extern uint64 cluster_lms_get_conn_resets(void); + /* * State enum -> canonical lowercase string ("disabled", "not_started", * "starting", "ready", "draining", "stopped"). Out-of-range returns diff --git a/src/test/cluster_tap/t/360_lms_data_plane_faults.pl b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl index 5eb56b578a..3c78f09b49 100644 --- a/src/test/cluster_tap/t/360_lms_data_plane_faults.pl +++ b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl @@ -17,10 +17,15 @@ # L3 hygiene: zero plane misroutes + no plane-gate errors. # L5 conn-reset injection (F6-1, un-SKIPPED): arm the one-shot # cluster-lms-conn-reset injection, observe exactly one DATA-mesh -# reset in node0's log, then verify the mesh reconnects and a -# cross-node block transfer converges. The reset is now latched -# one-shot (cluster_lms_data_plane.c) so the persistent GUC arm no -# longer storms or crashes the passive LMS. +# reset in node0's log (L5.1), verify the mesh reconnects (L5.2) +# and that the reset is one-shot (L5.3), then prove block transfer +# CONVERGES after the reset (L5.4/L5.5, spec §4 line 193 "重连收敛"): +# the lms_conn_resets D6 counter recorded the reset, and a FRESH +# post-reset coincidence table (new, un-recycled xids that never hit +# the #119 wall) is X-swapped cross-node with wire block REQUESTs on +# both sides. The reset is now latched one-shot (cluster_lms_data_ +# plane.c) so the persistent GUC arm no longer storms or crashes the +# passive LMS. # # KNOWN-BLOCKED / deferred legs (honest SKIP; see docs/stage7-substrate- # findings.md). The stage7 P0 substrate (spec-6.15b/4.6a/2.29a) was @@ -97,6 +102,14 @@ sub gcs_int return defined($v) && $v ne '' ? int($v) : 0; } +sub lms_int +{ + my ($node, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='lms' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + sub poll_write_ok { my ($node, $sql, $timeout_s, $label) = @_; @@ -289,17 +302,19 @@ sub pct_bound # L5.2 — the mesh RECONNECTS after the injected reset. Proven by log evidence: # both nodes emit a fresh "state CONNECTED" AFTER the reset -- node0 re-dials # (active role) and node1 re-accepts (passive role, after its recv sees the -# drop). Once CONNECTED the tier1 link is ready for block transfer, so the -# DATA plane's block-shipping capability (already quantified by the L2 value -# gate: 394 real X-ships at p99 <= 500us) survives an epoch reset. +# drop). Once CONNECTED the tier1 link is ready for block transfer; L5.4/L5.5 +# below prove that convergence directly with a real cross-node transfer. # -# We do NOT re-probe convergence with an SQL read/write here: the value-gate -# ping-pong burns fault_t's rows' xmax into recycled TT slots, so by this -# point ANY access to fault_t (read OR write) fail-closes with "cluster TT -# status unknown" -- the pre-existing #119 recycled-slot wall (spec-7.1a / -# gap-㉖), which affects all access to that table, is orthogonal to the DATA- -# plane decoupling and is NOT caused by the reset. The reconnect log evidence -# is the direct, uncontaminated proof of what F6-1 delivers. +# We do NOT re-probe convergence on fault_t itself: the L2 value-gate ping- +# pong recycled fault_t's rows' xmax into reused TT slots, so any access to +# THOSE recycled rows fail-closes with "cluster TT status unknown" -- the pre- +# existing #119 recycled-slot wall (spec-7.1a / gap-㉖). That wall is PER- +# recycled-slot: it walls only the rows whose xids the ping-pong recycled, and +# does NOT wall a fresh table's new, un-recycled xids. It is orthogonal to the +# DATA-plane decoupling and is NOT caused by the reset. So the convergence +# probe (L5.4/L5.5) uses a FRESH coincidence table whose new tuples never hit +# #119 -- the direct "block transfer recovers" proof the reconnect log evidence +# alone did not give. my $reconnect_deadline = time + 30; my ($conn0_after, $conn1_after) = ($conn0_before, $conn1_before); while (time < $reconnect_deadline) @@ -321,6 +336,75 @@ sub pct_bound ok($injected == 1, "L5.3 conn-reset injection is one-shot (fired $injected time; no per-tick storm)"); +# ============================================================ +# L5.4 / L5.5 — F6-1 CONVERGENCE (spec §4 line 193 "重连收敛"): after the reset +# + reconnect, a real cross-node block transfer converges. +# +# The reconnect log evidence (L5.2) shows the link is back; this leg proves the +# DATA plane actually SHIPS BLOCKS again. We use a FRESH coincidence table +# (fault2_t) created AFTER the reset: its tuples carry new, un-recycled xids, so +# they never hit the #119 recycled-slot wall that walls fault_t (that wall is +# per-recycled-slot, not table-wide -- see the L5.2 note). The nodes then swap +# X on it exactly as t/358 L2/L3 do; convergence = both writes succeed AND both +# requesters' block_request_count grow (real wire REQUESTs after the reset) AND +# zero plane misroutes. The lms_conn_resets D6 counter also confirms the reset +# was observable. +# ============================================================ +cmp_ok(lms_int($node0, 'lms_conn_resets'), '>', 0, + 'L5.4 node0 lms_conn_resets D6 counter recorded the DATA-plane reset'); + +# Fresh coincidence table on new xids (t/347 OID-align pattern; independent of +# the reset -- just re-aligns the OID counters the run has since drifted). +my ($q0, $q1) = ('', ''); +for my $attempt (1 .. 8) +{ + $node0->safe_psql('postgres', + 'CREATE TABLE fault2_t (aid int, bal int) WITH (fillfactor = 50)'); + $node1->safe_psql('postgres', + 'CREATE TABLE fault2_t (aid int, bal int) WITH (fillfactor = 50)'); + $q0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('fault2_t')}); + $q1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('fault2_t')}); + last if $q0 eq $q1; + my ($m0) = $q0 =~ /(\d+)$/; + my ($m1) = $q1 =~ /(\d+)$/; + my ($lag, $burn) = $m0 < $m1 ? ($node0, $m1 - $m0) : ($node1, $m0 - $m1); + $lag->safe_psql('postgres', + "SELECT lo_unlink(lo_create(0)) FROM generate_series(1, $burn)"); + $node0->safe_psql('postgres', 'DROP TABLE fault2_t'); + $node1->safe_psql('postgres', 'DROP TABLE fault2_t'); +} +is($q0, $q1, 'L5.4 post-reset shared-table coincidence (fresh un-recycled xids)'); +$node0->safe_psql('postgres', + 'INSERT INTO fault2_t SELECT g, 0 FROM generate_series(1, 200) g'); +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node1->safe_psql('postgres', 'CHECKPOINT'); + +# Cross-node X swap: node0 holds -> node1 ships in, node1 holds -> node0 ships +# back. The full swap guarantees each node issues wire block REQUESTs for its +# remote-mastered half (t/358 L2/L3 shape; a single-row leg alone is master- +# placement dependent per HC72 master==self). +my $req0_before = gcs_int($node0, 'block_request_count'); +my $req1_before = gcs_int($node1, 'block_request_count'); + +ok(timed_update_retry($node0, 'UPDATE fault2_t SET bal = bal + 1', 20), + 'L5.5 setup: node0 X-holds the fresh post-reset table'); +usleep(500_000); +ok(timed_update_retry($node1, 'UPDATE fault2_t SET bal = bal + 1 WHERE aid = 7', 20), + 'L5.5 node1 ships a node0-held block over the reconnected DATA plane'); +usleep(500_000); +ok(timed_update_retry($node1, 'UPDATE fault2_t SET bal = bal + 1', 20), + 'L5.5 setup: node1 takes the table X (reverse ship)'); +usleep(500_000); +ok(timed_update_retry($node0, 'UPDATE fault2_t SET bal = bal + 1 WHERE aid = 11', 20), + 'L5.5 node0 ships a node1-held block back (bidirectional convergence)'); + +cmp_ok(gcs_int($node1, 'block_request_count'), '>', $req1_before, + 'L5.5 node1 issued wire block requests after the reset (transfer converged)'); +cmp_ok(gcs_int($node0, 'block_request_count'), '>', $req0_before, + 'L5.5 node0 issued wire block requests after the reset (transfer converged)'); +is(gcs_int($node0, 'plane_misroute_reject'), 0, 'L5.5 node0 zero plane misroutes post-reset'); +is(gcs_int($node1, 'plane_misroute_reject'), 0, 'L5.5 node1 zero plane misroutes post-reset'); + # ============================================================ # L6 — HONEST SKIP: data-dispatch injection-point reachability (F6-2). # (The cluster-lms-data-dispatch injection point sits on the LMS async recv diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 3981b4a75e..c91d184974 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -3703,6 +3703,12 @@ cluster_lms_get_priority_starvation_observed_count(void) { return 0; } +/* spec-7.2 D6 stub — DATA-plane connection resets observability counter. */ +uint64 +cluster_lms_get_conn_resets(void) +{ + return 0; +} const char * cluster_lms_state_to_string(int s pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_stage3_acceptance.c b/src/test/cluster_unit/test_cluster_stage3_acceptance.c index 448d2b6c59..7cc2ed5299 100644 --- a/src/test/cluster_unit/test_cluster_stage3_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage3_acceptance.c @@ -375,14 +375,14 @@ UT_TEST(test_stage3_sqlstate_mvcc_surface_encodable) /* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 118 ===== */ -UT_TEST(test_stage3_wait_events_count_snapshot_118) +UT_TEST(test_stage3_wait_events_count_snapshot_120) { /* spec-4.2 D5 value (95 + 2 wal-state registry I/O). Update-required contract: a future spec * adding a wait event MUST bump this snapshot (it is current state, not * "==93 forever"). spec-4.6 D4: 97 → 98; spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering); spec-4.11 D5: 99 → 100 - * (+ ClusterThreadRecovery). spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * (+ ClusterThreadRecovery). spec-6.13 D8 snapshot: 118. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -475,7 +475,7 @@ main(void) UT_RUN(test_undo_4_8ab_redo_determinism_converges); UT_RUN(test_stage3_capability_dump_category_names); UT_RUN(test_stage3_sqlstate_mvcc_surface_encodable); - UT_RUN(test_stage3_wait_events_count_snapshot_118); + UT_RUN(test_stage3_wait_events_count_snapshot_120); UT_RUN(test_stage3_tt_enum_values_locked); UT_RUN(test_stage3_retention_active_retains_invariant); UT_RUN(test_stage3_bind_opcode_reserved); diff --git a/src/test/cluster_unit/test_cluster_stage4_acceptance.c b/src/test/cluster_unit/test_cluster_stage4_acceptance.c index 3e0617f786..130708cc8d 100644 --- a/src/test/cluster_unit/test_cluster_stage4_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage4_acceptance.c @@ -203,13 +203,13 @@ UT_TEST(test_stage4_sqlstate_recovery_fence_surface_encodable) /* ===== L5 — wait-events count snapshot ===== */ -UT_TEST(test_stage4_wait_events_count_snapshot_118) +UT_TEST(test_stage4_wait_events_count_snapshot_120) { /* Current Stage 4 surface value; spec-6.2 adds the latest 4 Smart Fusion * authority waits and spec-6.13 adds 2 RDMA tier3 waits. update-required * contract: a future spec adding cluster wait events MUST bump this snapshot * (and the dump/test baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -312,7 +312,7 @@ main(void) UT_RUN(test_stage4_undo_opcodes_preserved_and_info_mask_clear); UT_RUN(test_stage4_recovery_dump_category_names); UT_RUN(test_stage4_sqlstate_recovery_fence_surface_encodable); - UT_RUN(test_stage4_wait_events_count_snapshot_118); + UT_RUN(test_stage4_wait_events_count_snapshot_120); UT_RUN(test_stage4_write_fence_enums_locked); UT_RUN(test_stage4_thread_recovery_scope_enum_complete); UT_RUN(test_stage4_undo_writeback_boundary_enum_complete); diff --git a/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c index 181c36b0f3..11f18a489e 100644 --- a/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c @@ -182,13 +182,14 @@ UT_TEST(test_stage5_5_cross_instance_coordinator_enums_locked) /* ===== L6 — wait-events count snapshot ===== */ -UT_TEST(test_stage5_5_wait_events_count_snapshot_118) +UT_TEST(test_stage5_5_wait_events_count_snapshot_120) { /* The whole CR read-path band (5.51-5.57) adds NO new wait events — it reuses * the spec-3.9 ClusterCRConstruct event. spec-6.0a adds 7 block_device * wait events after that band; spec-6.1 adds 2 RDMA wait events; spec-6.2 - * adds 4 Smart Fusion authority waits; spec-6.13 adds 2 RDMA tier3 waits. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * adds 4 Smart Fusion authority waits; spec-6.13 adds 2 RDMA tier3 waits; + * spec-7.2 D6 adds 2 LMS DATA-plane waits. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -200,6 +201,6 @@ main(void) UT_RUN(test_stage5_5_cr_dump_category_names); UT_RUN(test_stage5_5_admission_policy_enum_locked); UT_RUN(test_stage5_5_cross_instance_coordinator_enums_locked); - UT_RUN(test_stage5_5_wait_events_count_snapshot_118); + UT_RUN(test_stage5_5_wait_events_count_snapshot_120); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c index 63750488ed..b5e83402dc 100644 --- a/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c @@ -191,7 +191,7 @@ UT_TEST(test_beta_wait_events_count) /* Current Stage 5 beta surface value. update-required contract: a future * spec adding cluster wait events MUST bump this snapshot (and the dump/test * baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } diff --git a/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c index c672b41494..f998e5ea74 100644 --- a/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c @@ -192,7 +192,7 @@ UT_TEST(test_stage5_wait_events_count_and_multinode_set) * authority waits and spec-6.13 adds 2 RDMA tier3 waits. update-required * contract: a future spec adding cluster wait events MUST bump this snapshot * (and the dump/test baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ /* The multi-node write-path wait events MG-B aggregates for the M2 share * must all be present and pairwise distinct (a reorder/removal would change From b62331ebc6206f144230ece6883a04a7a064f486 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 16:38:46 +0800 Subject: [PATCH 58/58] fix(cluster): add rule-11 Author/IDENTIFICATION banner to t/358 The spec-7.2 D3+D4 flip e2e test file was missing the mandatory 'Author: SqlRush ' header (CLAUDE.md rule 11); it never surfaced on the 7.2 lane because that branch was not pushed to a main-based fast-gate. Add IDENTIFICATION + Author + copyright to match the sibling t/360 banner. Spec: spec-7.2 --- src/test/cluster_tap/t/358_data_plane_flip_e2e.pl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl b/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl index 6b4bbb8b69..cbe507003f 100644 --- a/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl +++ b/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl @@ -24,6 +24,13 @@ # # Spec: spec-7.2-ic-data-plane-decoupling.md (flip commit) # +# IDENTIFICATION +# src/test/cluster_tap/t/358_data_plane_flip_e2e.pl +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# #------------------------------------------------------------------------- use strict;