diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..905b89fab7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,12 +176,20 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and + # t/361 XID authority native-era adoption. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-360 + # reserved by the parallel spec-7.2 lane; L464 occupancy verified + # against origin branches 2026-07-08). Own shard: 4-node formation + # + kill + convergence needs the wall clock. + - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } + # t/363 spec-2.29a marker-async LMON liveness. + - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docs/reference/system-views.md b/docs/reference/system-views.md index 649aa954c8..9ceab9b723 100644 --- a/docs/reference/system-views.md +++ b/docs/reference/system-views.md @@ -210,6 +210,9 @@ must not be read as evidence that early transfer is active. | `smart_fusion` | `origin_suspect_count` | Pending dependencies associated with an origin suspected dead or unavailable. | | `smart_fusion` | `dep_lost_failclosed_count` | Missing or malformed dependency evidence rejected fail-closed. | | `smart_fusion` | `retry_failclosed_count` | Retryable Smart Fusion fail-closed outcomes, primarily commit-brake timeout. | +| `catalog` | `xid_authority_native_hw` | Sealed native-era xid high-water for shared-catalog formation; `0` when the authority is inactive or unreadable. | +| `catalog` | `xid_authority_sealed` | `t` when the seed completed a clean native-era shutdown and published adoptable XID prehistory. | +| `catalog` | `xid_prehistory_adopted` | Process-local marker showing whether this postmaster boot adopted native-era pg_xact prehistory. | Example: diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 007c2cb90e..5165f233d9 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -776,6 +776,9 @@ Requirements and behaviour: * `cluster.node_id` must be between 0 and 15; startup is refused otherwise. Striped clusters are therefore limited to **16 nodes**. * `cluster.online_join` must be `on`; startup is refused otherwise. +* `cluster.shared_catalog = on` with more than one declared node requires + `cluster.xid_striping = on`; startup fails closed with SQLSTATE `53RB5` + if a multi-node shared-catalog cluster is formed without striping. * All nodes must run with the same `cluster.xid_striping` setting. The cluster records its striping state durably on the voting disk; a node whose configuration disagrees with that recorded state (or @@ -791,6 +794,50 @@ cluster.xid_striping = on cluster.online_join = on ``` +### Shared-catalog native-era XID authority + +A shared-catalog cluster that is formed from a single seed node has a +short native era: the seed runs with `cluster.enabled = off` while it +creates initial catalog rows, roles, schemas, and seed data. Those +transactions consume the seed node's local xids before other nodes have +joined. During the seed node's clean shutdown, pgrac publishes two +formation files under `cluster.shared_data_dir/global`: +`pgrac_xid_authority` (the sealed native xid high-water) and +`pgrac_xid_prehistory` (the seed node's pg_xact truth for those xids). + +Provisioning sequence: + +1. Create or clone joiner data directories before the seed load if they + must share the seed node's system identifier. +2. Start the seed with `cluster.shared_catalog = on`, + `cluster.controlfile_shared_authority = on`, `cluster.enabled = off`, + and a fixed `cluster.node_id`; perform the seed load. +3. Stop the seed cleanly. Do not use immediate shutdown for the final + native-era stop; joiners refuse an unsealed authority with SQLSTATE + `53RB5`. +4. Start all nodes with `cluster.enabled = on`, `cluster.online_join = on`, + and `cluster.xid_striping = on`. Joiners adopt the seed's transaction + history at startup, so seed tables, roles, and data are visible on + every node. + +Do not delete or re-create the authority files to recover a failed join. +A missing, unsealed, or corrupt authority/prehistory is intentionally +fail-closed (`53RB5`); restore the shared tree files from backup or +re-provision the cluster. After the first `cluster.enabled = on` boot, +the native era is permanently closed and a later `cluster.enabled = off` +boot against the same shared tree is refused. + +Observe formation state with: + +```sql +SELECT key, value + FROM pg_cluster_state + WHERE category = 'catalog' + AND key IN ('xid_authority_native_hw', + 'xid_authority_sealed', + 'xid_prehistory_adopted'); +``` + ### `cluster.xid_herding_slack` | | | diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670985611f..850c468df5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -201,6 +201,7 @@ #include "cluster/cluster_guc.h" /* PGRAC: spec-5.6 cluster_controlfile_shared_authority */ #include "cluster/cluster_hw_snapshot.h" /* PGRAC: spec-5.7 D3 HW authority checkpoint snapshot */ #include "cluster/cluster_xid_stripe_xlog.h" /* PGRAC: spec-6.15 D5d checkpoint re-emit */ +#include "cluster/cluster_xid_authority.h" /* PGRAC: spec-6.15b native-era XID authority */ #include "cluster/cluster_recovery_anchor.h" /* PGRAC: spec-5.6a per-node recovery anchor */ #include "cluster/cluster_lms.h" /* PGRAC: spec-5.6 GES-ready boundary for CF X */ #endif @@ -5593,6 +5594,42 @@ StartupXLOG(void) */ if (cluster_recovery_anchor_active() && !InRecovery) checkPoint = cluster_recovery_anchor_get()->checkPointCopy; + + /* + * PGRAC (spec-6.15b D5): shared_catalog nodes must reconcile their + * transaction horizon with the sealed native-era XID authority before + * ShmemVariableCache is seeded and before StartupCLOG computes its latest + * page. This deliberately consumes a dedicated never-lowered authority, + * not the shared pg_control CheckPoint copy (spec-5.6a keeps that + * per-writer). + */ + if (cluster_shared_catalog && cluster_enabled) + { + ClusterXidAuthorityHeader auth; + bool auth_from_bak = false; + bool auth_ok; + + auth_ok = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (auth_ok && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + + if (!auth_ok || + (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable during WAL startup"), + errdetail("StartupXLOG cannot seed nextXid without a sealed native-era high-water."), + errhint("Complete a clean native-era seed shutdown before starting shared_catalog nodes."))); + + if (U64FromFullTransactionId(checkPoint.nextXid) < auth.native_hw_full) + { + uint64 own_next = U64FromFullTransactionId(checkPoint.nextXid); + + checkPoint.nextXid = FullTransactionIdFromU64(auth.native_hw_full); + elog(LOG, "cluster shared_catalog: merged nextXid with XID authority native high-water %llu (own was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } + } #endif /* initialize shared memory variables from the checkpoint record */ @@ -7549,6 +7586,59 @@ CreateCheckPoint(int flags) */ END_CRIT_SECTION(); +#ifdef USE_PGRAC_CLUSTER + /* + * PGRAC (spec-6.15b D3): after the shutdown checkpoint record, + * control-file update, CLOG flush, and per-node recovery anchor publish + * are complete, publish the native-era pg_xact prehistory and seal the + * shared XID authority. This does durable file I/O and may fail closed, + * so it must stay outside the checkpoint critical section. + * + * Two deliberate exclusions (review P2-1): + * + * - END_OF_RECOVERY checkpoints carry `shutdown` but are NOT a clean + * close of the native run: the seed keeps consuming xids afterwards, + * so sealing here would let a joiner adopt a mid-run stale high-water + * (the same 8.A window the begin_native_run unseal closes for the + * multi-pass case). Only a true shutdown checkpoint seals. + * + * - An unsupported native era (MultiXacts created, or past the + * prehistory cap) SKIPS the seal with a WARNING instead of FATALing: + * the refusal still lands fail-closed on every joiner (53RB5, + * authority unsealed), but the seed itself can shut down cleanly + * instead of wedging every later shutdown/boot in a FATAL loop. + */ + if (shutdown && !(flags & CHECKPOINT_END_OF_RECOVERY) + && cluster_shared_catalog && !cluster_enabled) + { + uint64 native_hw = U64FromFullTransactionId(checkPoint.nextXid); + + if (checkPoint.nextMulti > FirstMultiXactId) + ereport(WARNING, + (errmsg("native-era seed loads that create MultiXacts are not supported; " + "leaving the shared XID authority unsealed"), + errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", + checkPoint.nextMulti, FirstMultiXactId), + errhint("Joiners will fail closed (53RB5); recreate the seed without " + "MultiXact-producing operations, or move the load in-protocol."))); + else if (cluster_xid_prehistory_payload_bytes(native_hw) == 0) + ereport(WARNING, + (errmsg("native-era xid high-water %llu is outside the prehistory publish " + "range; leaving the shared XID authority unsealed", + (unsigned long long) native_hw), + errhint("Joiners will fail closed (53RB5); split the seed load or move " + "it in-protocol."))); + else + { + cluster_xid_prehistory_publish(DataDir, native_hw); + cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); + elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", + (unsigned long long) native_hw); + } + } + +#endif + /* * Let smgr do post-checkpoint cleanup (eg, deleting old files). */ diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 7c3daf86a5..c0b7a009c7 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3273,8 +3273,16 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl /* * ShmemVariableCache->nextXid must be beyond record's xid. + * + * PGRAC: a merged-recovery FOREIGN record belongs to a peer's xid + * authority, not this node's allocator. Advancing the local nextXid here + * would make StartupCLOG/TrimCLOG expect local pg_xact pages for peer + * striped xids, while the peer outcome truth lives in pg_xact_remote. */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); +#ifdef USE_PGRAC_CLUSTER + if (!(cluster_recmerge_window_active && cluster_recmerge_apply_foreign)) +#endif + AdvanceNextFullTransactionIdPastXid(record->xl_xid); /* * Before replaying this record, check if this record causes the current diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 5ea41237f7..409f537050 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -143,6 +143,8 @@ OBJS = \ cluster_lmd_wait_state.o \ cluster_lmon.o \ cluster_lms.o \ + cluster_lms_data_plane.o \ + cluster_lms_outbound.o \ cluster_lns.o \ cluster_membership.o \ cluster_mrp.o \ @@ -220,6 +222,8 @@ OBJS = \ cluster_views.o \ cluster_wal_state.o \ cluster_wal_thread.o \ + cluster_xid_authority.o \ + cluster_xid_prehistory.o \ cluster_xid_stripe.o \ cluster_xid_stripe_boot.o \ cluster_xid_stripe_xlog.o \ diff --git a/src/backend/cluster/cluster_backup.c b/src/backend/cluster/cluster_backup.c index 512632c5dc..8a28d52ca8 100644 --- a/src/backend/cluster/cluster_backup.c +++ b/src/backend/cluster/cluster_backup.c @@ -1099,6 +1099,7 @@ cluster_backup_coord_request(ClusterBackupWireOp op, const char *backup_id, } cluster_backup_state->coordinator_send_pending = true; LWLockRelease(&cluster_backup_state->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ cluster_lmon_wakeup(); started_at = GetCurrentTimestamp(); @@ -1278,6 +1279,7 @@ cluster_backup_lmon_queue_peer_reply(const ClusterBackupWireAck *reply, int32 de cluster_backup_state->peer_reply_dest = dest; cluster_backup_state->peer_reply_pending = true; LWLockRelease(&cluster_backup_state->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ } static bool @@ -1301,6 +1303,7 @@ cluster_backup_lmon_start_restore_point_prepare(const ClusterBackupWireRequest * cluster_backup_lmon_prepare_started_at = cluster_backup_restore_point_fence_start(); cluster_backup_lmon_restore_point_prepare_pending = true; cluster_backup_lmon_set_prepare_pending(true); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ return false; } @@ -1712,6 +1715,7 @@ cluster_backup_request_handler(const ClusterICEnvelope *env, const void *payload cluster_backup_state->peer_command_pending = true; } LWLockRelease(&cluster_backup_state->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } @@ -1896,6 +1900,8 @@ cluster_backup_lmon_process_peer_command(void) void cluster_backup_lmon_tick(void) { + bool in_flight; + if (cluster_backup_state == NULL || !cluster_enabled) return; @@ -1906,6 +1912,26 @@ cluster_backup_lmon_tick(void) cluster_backup_lmon_advance_restore_point_prepare(); cluster_backup_lmon_send_peer_reply(); cluster_backup_maybe_auto_restore_point(); + + /* + * PGRAC: spec-7.2 D1 -- the backup protocol is multi-phase and its + * advance steps are time-driven (fence quiesce), not producer-driven. + * While any operation is in flight, self-re-mark the duty so the + * lazy gate keeps the legacy every-iteration cadence until the + * protocol settles; steady state (no backup) reaps the lazy skip. + */ + if (cluster_backup_lmon_restore_point_prepare_pending) + in_flight = true; + else { + LWLockAcquire(&cluster_backup_state->lock.lock, LW_SHARED); + in_flight = cluster_backup_state->peer_reply_pending + || cluster_backup_state->coordinator_send_pending + || cluster_backup_state->peer_command_pending + || cluster_backup_state->peer_restore_point_prepare_pending; + LWLockRelease(&cluster_backup_state->lock.lock); + } + if (in_flight) + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_BACKUP); } static void diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 9fee3e481c..413af44ebb 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -35,6 +35,10 @@ */ #include "postgres.h" +#include +#include + +#include "access/transam.h" #include "access/xlog.h" #include "catalog/pg_control.h" #include "cluster/cluster_catalog_bootstrap.h" @@ -44,13 +48,263 @@ #include "cluster/cluster_inject.h" #include "cluster/cluster_mode.h" #include "cluster/cluster_oid_lease.h" +#include "cluster/cluster_recovery_anchor.h" #include "cluster/cluster_startup_phase.h" +#include "cluster/cluster_xid_authority.h" #include "miscadmin.h" +#include "storage/fd.h" + +static bool +cluster_catalog_backup_label_present(void) +{ + char label_path[MAXPGPATH]; + struct stat st; + + snprintf(label_path, sizeof(label_path), "%s/%s", DataDir, BACKUP_LABEL_FILE); + return stat(label_path, &st) == 0; +} + +static void +cluster_catalog_xid_authority_corrupt_fatal(const char *detail) +{ + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable at catalog bootstrap"), + errdetail("%s", detail), + errhint("Restore \"%s/%s\" (or its .bak) from a backup of the shared tree; " + "do not delete or re-seed it from a stale xid high-water.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH))); +} + + +/* + * Early D6 topology sniff: cluster_conf_load() runs later when shmem exists, + * but shared_catalog bootstrap must reject multi-node xid_striping=off before + * it seeds/adopts any authority. This deliberately counts only [node.N] + * section headers; the real parser remains the startup SSOT and will still + * validate syntax, required fields, and node_id consistency later. + */ +static int +cluster_catalog_declared_node_count_early(void) +{ + const char *path = (cluster_config_file != NULL && cluster_config_file[0] != '\0') + ? cluster_config_file + : "pgrac.conf"; + FILE *f; + char line[1024]; + int nodes = 0; + + f = AllocateFile(path, "r"); + if (f == NULL) + return 1; + + while (fgets(line, sizeof(line), f) != NULL) { + const char *p = line; + + while (*p != '\0' && isspace((unsigned char)*p)) + p++; + if (strncmp(p, "[node.", 6) == 0) + nodes++; + } + FreeFile(f); + + return nodes > 0 ? nodes : 1; +} + +static void +cluster_catalog_vet_xid_striping_for_shared_catalog(void) +{ + int nodes; + + if (!cluster_enabled || cluster_xid_striping) + return; + + nodes = cluster_catalog_declared_node_count_early(); + if (nodes > 1) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", nodes), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); +} + +static void +cluster_catalog_prepare_xid_authority(const ControlFileData *cf, + ClusterCatalogMigrateResult migrate_result) +{ + ClusterXidAuthorityHeader auth; + bool have_auth; + + { + bool auth_from_bak = false; + + have_auth = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (have_auth && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + } + if (!have_auth && cluster_xid_authority_present()) + cluster_catalog_xid_authority_corrupt_fatal( + "Neither the primary shared XID authority nor its .bak fallback passes validation."); + + if (!have_auth) { + if (migrate_result != CLUSTER_CATALOG_MIGRATE_SEEDED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is absent for an existing shared catalog tree"), + errdetail("The shared catalog marker exists, but \"%s/%s\" does not.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH), + errhint("Re-seed the shared tree with a build carrying spec-6.15b."))); + + if (cluster_xid_authority_seed_if_absent( + U64FromFullTransactionId(cf->checkPointCopy.nextXid))) + elog(LOG, "cluster shared_catalog: seeded XID authority native high-water at %llu", + (unsigned long long)U64FromFullTransactionId(cf->checkPointCopy.nextXid)); + + if (!cluster_xid_authority_read(&auth)) + cluster_catalog_xid_authority_corrupt_fatal( + "The shared XID authority was just seeded but cannot be read back."); + } + + if (!cluster_enabled) { + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"), + errdetail("The shared XID authority is already marked cluster-era started."), + errhint( + "Keep cluster.enabled=on for this shared tree, or destroy and re-form it."))); + + /* + * A follow-up native run on a SEALED authority re-opens the era: the + * seal is cleared up front so a crash of this run leaves joiners + * fail-closed (unsealed) instead of adopting the previous pass's + * stale high-water (spec-6.15b §3.1 multi-pass arm, rule 8.A). The + * clean shutdown of this run re-publishes and re-seals. + * + * Called unconditionally (not gated on the primary's SEALED flag): + * the transition re-asserts BOTH on-disk copies, so a crash between + * a previous unseal's two renames -- a stale SEALED .bak behind an + * already-unsealed primary, which a later primary read failure + * would resurrect -- is repaired here instead of surviving the run + * (review r3-X1). Once both copies are open this is a no-write + * no-op. + */ + cluster_xid_authority_begin_native_run(); + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) + elog(LOG, "cluster shared_catalog: re-opened native seed era " + "(XID authority unsealed for this run)"); + return; + } + + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is not sealed"), + errdetail("The seed node has not completed a clean native-era shutdown."), + errhint( + "Start the seed with cluster.enabled=off, stop it cleanly, then start joiners."))); + + if (cluster_catalog_backup_label_present()) + elog(LOG, "cluster shared_catalog: skipped XID prehistory adopt on backup_label boot"); + else { + ClusterRecoveryAnchor ra; + uint64 own_next; + + if (!cluster_recovery_anchor_read(cf->system_identifier, &ra, NULL)) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("per-node recovery anchor is unavailable for XID prehistory adopt"), + errdetail("The joiner cannot determine its own pre-adopt nextXid without \"%s\".", + cluster_recovery_anchor_path() != NULL ? cluster_recovery_anchor_path() + : "(unset)"), + errhint( + "Re-provision this node or verify cluster.shared_data_dir before startup."))); + + own_next = U64FromFullTransactionId(ra.checkPointCopy.nextXid); + + /* + * Review F2 (spec Q6 amendment): the sysid gate cannot tell a true + * pre-seed clone from a same-sysid clone that ran STANDALONE after + * cloning -- such a node's pg_xact holds its own outcomes in the + * native range, so skipping would trust divergent local truth and + * adopting would overwrite the node's own outcomes; both are + * silently wrong (8.A). Byte-compare the comparable prefix + * [oldestXid, min(own_next, native_hw)) against the sealed blob and + * fail closed on any contradiction. Bypass only when the local + * oldestXid has advanced past the native range (frozen+truncated: + * those bits are never consulted again). Below that bypass the + * check starts at oldestXid itself (review r3-X2): segments frozen + * and truncated away are no alibi for the surviving range, and a + * missing local page inside the comparable range fails closed + * (UNAVAILABLE) instead of passing as a shorter clone. + */ + if ((uint64)ra.checkPointCopy.oldestXid <= auth.native_hw_full) { + ClusterXidPrefixVerdict pv; + + pv = cluster_xid_prehistory_prefix_check(DataDir, auth.native_hw_full, + Min(own_next, auth.native_hw_full), + (uint64)ra.checkPointCopy.oldestXid); + if (pv == CLUSTER_XID_PREFIX_DIVERGED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("local pg_xact contradicts the sealed native-era prehistory"), + errdetail("divergent native-era lineage: this node's transaction " + "history overlaps the seed's native xid range with " + "different outcomes."), + errhint("This node is not a pre-seed clone of the shared tree; " + "destroy and re-provision it from the seed lineage."))); + if (pv == CLUSTER_XID_PREFIX_UNAVAILABLE) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is unavailable for the lineage check"), + errdetail("Either no sealed prehistory blob passes validation, or the " + "local pg_xact has a hole inside the comparable native " + "range."), + errhint("Restore \"%s\" (or its .bak) from the shared tree's backup, " + "or re-provision this node if its local pg_xact is damaged.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + } + + if (own_next < auth.native_hw_full) { + cluster_xid_prehistory_adopt(DataDir, auth.native_hw_full); + elog(LOG, + "cluster shared_catalog: adopted XID prehistory through native high-water %llu " + "(own nextXid was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } else + elog(LOG, + "cluster shared_catalog: skipped XID prehistory adopt; own nextXid %llu >= native " + "high-water %llu", + (unsigned long long)own_next, (unsigned long long)auth.native_hw_full); + } + + /* + * One-way CLUSTER_ERA stamp, re-run unconditionally on every cluster + * boot: the transition re-asserts BOTH on-disk copies, so a crash + * between a previous stamp's two renames -- a stale pre-CLUSTER_ERA + * .bak behind an already-stamped primary, which a later primary read + * failure would hand to an enabled=off boot as a re-entry bypass -- is + * repaired here instead of surviving the run (review r3-X1). Once + * both copies are stamped this is a no-write no-op. + */ + cluster_xid_authority_mark_cluster_era(); + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) + elog(LOG, + "cluster shared_catalog: marked XID authority cluster era started at native " + "high-water %llu", + (unsigned long long)auth.native_hw_full); +} void cluster_catalog_startup_prepare(void) { ControlFileData cf; + ClusterCatalogMigrateResult migrate_result; /* Postmaster-once: only the postmaster seeds; forked backends inherit. */ if (IsUnderPostmaster) @@ -66,6 +320,8 @@ cluster_catalog_startup_prepare(void) return; /* off: stock per-node catalog */ } + cluster_catalog_vet_xid_striping_for_shared_catalog(); + /* * shared_catalog=on requires the shared pg_control authority (D1 vet), so * it is the single source of the cluster-wide next-OID high-water. Reading @@ -113,7 +369,8 @@ cluster_catalog_startup_prepare(void) * the (D3-flipped) shared smgr route. system_identifier is the shared * pg_control's cluster-wide value both seed and join agree on. */ - cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + migrate_result = cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + cluster_catalog_prepare_xid_authority(&cf, migrate_result); } /* diff --git a/src/backend/cluster/cluster_catalog_migrate.c b/src/backend/cluster/cluster_catalog_migrate.c index 961179b008..bb17838229 100644 --- a/src/backend/cluster/cluster_catalog_migrate.c +++ b/src/backend/cluster/cluster_catalog_migrate.c @@ -588,7 +588,7 @@ vet_seed_no_tablespaces(const char *local_pgdata) FreeDir(dir); } -void +ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) { ClusterCatalogAuthorityMarker existing; @@ -624,7 +624,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) elog(LOG, "cluster shared_catalog: adopted shared catalog authority (sysid " UINT64_FORMAT ")", system_identifier); - return; + return CLUSTER_CATALOG_MIGRATE_ADOPTED; } /* @@ -645,6 +645,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) "cluster shared_catalog: seeded shared catalog authority under \"%s\" " "(sysid " UINT64_FORMAT ")", cluster_shared_data_dir, system_identifier); + return CLUSTER_CATALOG_MIGRATE_SEEDED; } /* diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index d6152a4e28..24be2345d8 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -58,7 +58,8 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" -#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ #include "cluster/cluster_reconfig.h" /* apply_clean_leave + record/is_clean_departed + join_in_progress */ @@ -121,6 +122,8 @@ cluster_clean_leave_shmem_init(void) pg_atomic_init_u32(&cl_state->commit_point_observed, 0); pg_atomic_init_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_init_u32(&cl_state->committed_marker_durable, 0); + /* spec-2.29a r3 (evidence latch). */ + cl_state->committed_confirmed_epoch = 0; /* Hardening v1.0.2 (P1 preflight / P2 nonce). */ pg_atomic_init_u64(&cl_state->leave_attempt_nonce, 0); pg_atomic_init_u32(&cl_state->preflight_pending, 0); @@ -381,6 +384,11 @@ cluster_clean_leave_check_pending_in_proc_interrupts(void) * disabled survivor MUST reply LEAVE_DRAIN_NAK rather than go silent, else the * leaving node waits for an ACK that never comes and false-times-out. */ + +/* spec-2.29a ②b: forward decls — the bind sites (below) snapshot the + * others-dead set that the coherence gate (defined near drive_drain) compares. */ +static void cl_others_dead_snapshot(int32 leaving, uint8 *out); + static void cl_announce_handler(const ClusterICEnvelope *env, const void *payload) { @@ -509,6 +517,9 @@ cl_announce_handler(const ClusterICEnvelope *env, const void *payload) * fails the commit closed instead of committing on a stale membership view. */ cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes the leaving node) + * the coherence gate compares against. */ + cl_others_dead_snapshot(leaving, cl_state->leave_baseline_others_dead); pg_atomic_write_u32(&cl_state->survivor_acked, 0); pg_atomic_write_u32(&cl_state->commit_ready_received, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); @@ -546,11 +557,15 @@ cl_commit_ready_handler(const ClusterICEnvelope *env, const void *payload) } /* - * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate): - * the survivor coordinator has made the COMMITTED marker majority-durable and - * signals that the durable truth-source now exists, so this leaving node may - * proceed to COMMITTED and exit. Only acted on if it is about OUR leave; idem- - * potent (the coordinator re-sends each tick until we are gone). + * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate; + * spec-2.29a r3 evidence latch): the survivor coordinator has made the COMMITTED + * marker majority-durable and attests that the durable truth-source now exists. + * This confirmation is the leaver's own-commit MARKER EVIDENCE: the barrier tick + * latches on it (and only on it) and proceeds to COMMITTED/exit. Identity is + * checked by the pure evidence gate (self-addressed + currently leaving + + * per-attempt nonce + committed epoch past the bound baseline) — any mismatch is + * dropped fail-closed. Idempotent (the coordinator re-sends each tick until we + * are gone). */ static void cl_committed_handler(const ClusterICEnvelope *env, const void *payload) @@ -562,17 +577,19 @@ cl_committed_handler(const ClusterICEnvelope *env, const void *payload) return; if (!cluster_clean_leave_announce_payload_valid(p)) return; - if (p->leaving_node_id != cluster_node_id) - return; /* only the leaving node consumes its own COMMITTED confirmation */ - /* Hardening v1.0.2 (P2): bind to THIS attempt — a stale LEAVE_COMMITTED from a - * prior same-epoch attempt must not prematurely confirm the current one. Only - * meaningful once we are the leaver actively awaiting confirmation. */ - if (p->leave_nonce != pg_atomic_read_u64(&cl_state->leave_attempt_nonce)) + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + if (!cluster_clean_leave_committed_evidence_matches( + p->leaving_node_id, p->leave_nonce, p->leave_epoch, cluster_node_id, + cl_state->leaving_node_id, pg_atomic_read_u64(&cl_state->leave_attempt_nonce), + cl_state->leave_epoch)) { + LWLockRelease(&cl_state->lock); return; - if (cl_state->leaving_node_id != cluster_node_id) - return; /* we are not currently leaving */ - + } + /* the committed epoch E is recorded BEFORE the evidence flag is published, + * inside the same critical section the barrier tick reads it back under. */ + cl_state->committed_confirmed_epoch = p->leave_epoch; pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 1); + LWLockRelease(&cl_state->lock); } /* @@ -745,6 +762,125 @@ cluster_clean_leave_ic_send_ack(int32 dest_node_id, int32 leaving_node_id, uint6 /* qvotec-side per-process handshake cursors (mirror the fence-marker ones). */ static uint64 cl_qvotec_last_processed_marker_seq = 0; static uint64 cl_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync cl_lmon_marker_async; +static ClusterLeaveIntentMarker cl_lmon_marker; +static int cl_lmon_marker_phase = 0; +static int32 cl_lmon_marker_leaving = -1; +static uint64 cl_lmon_marker_epoch = 0; +static bool cl_lmon_marker_submitted = false; + +static void cl_build_marker(ClusterLeaveIntentMarker *m, uint8 marker_phase, int32 leaving, + uint64 epoch); + +typedef enum ClusterLeaveAsyncMarkerResult { + CL_LEAVE_ASYNC_PENDING = 0, + CL_LEAVE_ASYNC_ACKED, + CL_LEAVE_ASYNC_FAILED +} ClusterLeaveAsyncMarkerResult; + +static ClusterMarkerAsyncKind +cl_marker_phase_kind(int phase) +{ + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED; + return CLUSTER_MARKER_KIND_UNKNOWN; +} + +static void +cl_release_lmon_marker_stage(void) +{ + cluster_marker_async_release_stage(&cl_lmon_marker_async); + memset(&cl_lmon_marker, 0, sizeof(cl_lmon_marker)); + cl_lmon_marker_phase = 0; + cl_lmon_marker_leaving = -1; + cl_lmon_marker_epoch = 0; + cl_lmon_marker_submitted = false; +} + +static bool +cl_start_lmon_marker_stage(const ClusterLeaveIntentMarker *m, int phase, int32 leaving, + uint64 epoch) +{ + if (cl_lmon_marker_async.has_staged_event) + return false; + cl_lmon_marker = *m; + cl_lmon_marker_phase = phase; + cl_lmon_marker_leaving = leaving; + cl_lmon_marker_epoch = epoch; + cl_lmon_marker_async.has_staged_event = true; + cl_lmon_marker_submitted = false; + return true; +} + +static ClusterLeaveAsyncMarkerResult +cl_poll_lmon_marker_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_LEAVE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + if (!cl_lmon_marker_async.has_staged_event) + return CL_LEAVE_ASYNC_FAILED; + + now = GetCurrentTimestamp(); + kind = cl_marker_phase_kind(cl_lmon_marker_phase); + if (!cl_lmon_marker_submitted) { + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, kind, + cl_lmon_marker_leaving, now)) + return CL_LEAVE_ASYNC_PENDING; + cl_lmon_marker_submitted = true; + return CL_LEAVE_ASYNC_PENDING; + } + + pr = cluster_clean_leave_poll_marker_async(&cl_lmon_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return CL_LEAVE_ASYNC_PENDING; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, cl_lmon_marker_leaving, elapsed_us); + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + + cluster_reconfig_note_marker_slow_ack(kind, cl_lmon_marker_leaving, elapsed_us); + if (result != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + return CL_LEAVE_ASYNC_ACKED; +} + +static void +cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) +{ + ClusterLeaveIntentMarker cm; + ClusterLeaveAsyncMarkerResult ar; + + if (cl_lmon_marker_async.has_staged_event) { + if (cl_lmon_marker_phase != CLUSTER_LEAVE_MARKER_PHASE_COMMITTED + || cl_lmon_marker_leaving != leaving || cl_lmon_marker_epoch != committed_epoch) + return; + } else { + cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); + (void)cl_start_lmon_marker_stage(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + committed_epoch); + } + + ar = cl_poll_lmon_marker_stage(); + if (ar == CL_LEAVE_ASYNC_ACKED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, committed_epoch); + cl_release_lmon_marker_stage(); + } else if (ar == CL_LEAVE_ASYNC_FAILED) { + ereport(LOG, (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); + } +} ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) @@ -787,6 +923,38 @@ cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) } } +bool +cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (cl_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cl_state->marker_request_seq, + &cl_state->marker_completion_seq)) + return false; + + cl_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &cl_state->marker_request_seq, + &cl_state->marker_completion_seq, cl_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (cl_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cl_state->marker_completion_seq, &cl_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512) { @@ -818,6 +986,7 @@ cluster_clean_leave_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&cl_state->marker_completion_seq, cl_qvotec_inflight_marker_seq); cl_qvotec_last_processed_marker_seq = cl_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -1277,6 +1446,8 @@ cl_request_body(void) cl_state->leaving_node_id = cluster_node_id; cl_state->leave_epoch = baseline_epoch; cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes self, the leaver). */ + cl_others_dead_snapshot(cluster_node_id, cl_state->leave_baseline_others_dead); cl_state->barrier_deadline_us = (uint64)GetCurrentTimestamp() + (uint64)cluster_clean_leave_drain_timeout_ms * 1000ULL; /* @@ -1304,6 +1475,7 @@ cl_request_body(void) pg_atomic_write_u32(&cl_state->commit_point_observed, 0); pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); + cl_state->committed_confirmed_epoch = 0; /* spec-2.29a r3: per-attempt evidence */ LWLockRelease(&cl_state->lock); cl_set_phase(CLUSTER_LEAVE_REQUESTED); @@ -1410,6 +1582,49 @@ cluster_clean_leave_request(void) * re-checks version coherence (CL-I3: an external epoch bump = a real death * intruded -> escalate) and the mixed-mode NAK (clean abort). */ + +/* + * spec-2.29a ②b: snapshot the CSSD dead set EXCLUDING the leaving node. The + * coherence gate compares this "others-dead" set rather than the scalar global + * dead_generation, so the leaving node's own expected alive->DEAD transition + * (it stops heart-beating once its drain finishes) never falsely escalates the + * leave. A third-party death still changes this set and escalates (CL-I3). + */ +static void +cl_others_dead_snapshot(int32 leaving, uint8 *out /* CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES */) +{ + int i; + + memset(out, 0, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (i == leaving) + continue; /* the leaving node's own DEAD is expected, not incoherence */ + if (i >= CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES * 8) + break; + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (cluster_cssd_get_peer_state(i) == CLUSTER_CSSD_PEER_DEAD) + out[i / 8] |= (uint8)(1u << (i % 8)); + } +} + +/* + * spec-2.29a ②b: the coherence gate used by every clean-leave step. Coherent + * iff the epoch has not moved AND no THIRD-PARTY death changed the others-dead + * set since the leave was bound. cl_state->leaving_node_id names the node + * whose own DEAD is excluded. + */ +static bool +cl_coherent(uint64 baseline_epoch) +{ + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + return cluster_clean_leave_version_coherent( + baseline_epoch, cluster_epoch_get_current(), cl_state->leave_baseline_others_dead, + now_others_dead, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); +} + void cluster_clean_leave_drive_drain(void) { @@ -1470,16 +1685,14 @@ cluster_clean_leave_drive_drain(void) /* * CL-I3 coherence gate, re-checked before every drain step. Uses the * dead_gen-aware helper, not an epoch-only compare: CSSD increments the - * dead_generation the moment it declares a peer dead, which is STRICTLY - * BEFORE the reconfig coordinator bumps the membership epoch. Checking - * dead_gen too lets us escalate in that earlier window instead of draining - * into a death that has not yet reached the epoch. At commit the guarded - * CAS in apply_clean_leave_as_coordinator is the final authority. + * others-dead set the moment it declares a THIRD-PARTY peer dead, which is + * STRICTLY BEFORE the reconfig coordinator bumps the membership epoch. + * Checking that set too lets us escalate in that earlier window instead of + * draining into a death that has not yet reached the epoch. The leaving + * node's OWN DEAD is excluded (spec-2.29a ②b). At commit the guarded CAS + * in apply_clean_leave_as_coordinator is the final authority. */ -#define CL_COHERENT() \ - cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), \ - cl_state->leave_baseline_dead_gen, \ - cluster_cssd_get_dead_generation()) +#define CL_COHERENT() cl_coherent(baseline_epoch) /* REQUESTED -> QUIESCING: abort local writable backends (53R62). */ if (!CL_COHERENT()) { @@ -1551,60 +1764,72 @@ cl_leaving_barrier_tick(void) { uint64 baseline_epoch = cl_state->leave_epoch; int32 coordinator; + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool committed_evidence; + bool others_dead_unchanged; + bool dead_gen_unchanged; /* - * 1. observe the commit. The survivor coordinator runs the actual two-phase - * commit and publishes the CLEAN_LEAVE event into ITS OWN reconfig state — - * the leaving node's last_applied never carries it. What DOES propagate to - * the leaving node is the membership epoch (every IC envelope piggybacks it). - * So the leaving node detects its own commit by the epoch advancing past the - * bound baseline. Discriminate from a real death intruding (CL-I3) by the - * CSSD dead_generation: a cooperative leave does NOT mark anyone CSSD-dead, so - * an unchanged dead_generation at the bump means OUR clean-leave committed; a - * changed one means a real death (escalate). + * 1. observe the commit — EVIDENCE over inference (spec-2.29a r3). The + * survivor coordinator runs the actual two-phase commit, publishes the + * CLEAN_LEAVE event into ITS OWN reconfig state, drives the COMMITTED marker + * to voting-disk majority-durability, and only then sends the nonce-bound + * LEAVE_COMMITTED (re-sent each tick while we are alive). That confirmation + * — validated by the pure identity gate in cl_committed_handler — is the ONLY + * basis on which this node latches its own commit: latch <=> a valid + * COMMITTED marker for THIS leave attempt exists. + * + * Two inference generations preceded this and each failed one way (see the + * predicate comment in cluster_clean_leave_policy.c): "epoch advanced + + * others-dead bitmap unchanged" could mis-latch a REFUSED leave after a + * third-party false-DEAD rebound (r2 P2-1 wedge); adding the monotone scalar + * dead_generation conjunct then false-escalated a healthy committed leave + * whenever a transient third-party flap advanced the leaver's local + * dead_generation during the leave window (nightly t/331 C1/C4). Marker + * evidence is immune to both: flaps cannot erase a durable marker, and a + * refused leave never produces one — no latch, so the barrier deadline below + * still bounds the wait (fail-closed escalation, unchanged semantics). * - * Hardening v1.0.4 (P2): this "epoch bump + dead_gen unchanged == OUR commit" - * inference is sound ONLY because there is no OTHER dead_gen-unchanged reconfig - * in flight. A spec-5.15 online JOIN also bumps the epoch with dead_gen - * unchanged and would be mis-observed here as the leave's commit (then the node - * stops escalating but never gets the real LEAVE_COMMITTED -> BARRIER_WAIT - * hang). That collision is removed STRUCTURALLY by the one-membership-reconfig- - * at-a-time serialization: the join driver does not bump the epoch while a clean - * leave is active (cluster_clean_leave_in_progress gates drive_joins + - * commit_member), and the leave does not start while a join is pending - * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * dead_gen-unchanged bump during a leave can only be the leave's own commit. + * The coherence observations are still taken, but ONLY for the flap-noise + * LOG at the latch (the predicate contract pins that they never affect the + * verdict). A real third-party death intruding mid-leave is refused on the + * survivor side (cl_coherent pre-check + guarded CAS, CL-I3), so no evidence + * arrives here and the deadline escalates this leaver — same outcome as the + * old immediate escalate arm, now bounded by the barrier deadline instead. + * + * Latching collapses the old two-step gate: the evidence IS the P1-V0.7 + * durable-truth-source confirmation, so the leave can no longer be + * un-committed (deadline must not escalate past here, Hardening v1.0.1 + * P1-1) AND the node may exit (COMMITTED) in the same tick. */ - if (cluster_epoch_get_current() > baseline_epoch) { - if (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen) { - /* - * Commit point observed (our clean-leave epoch was published; no real - * death intruded). The leave can no longer be un-committed, so from - * here the barrier deadline must NOT escalate (Hardening v1.0.1 P1-1). - */ - pg_atomic_write_u32(&cl_state->commit_point_observed, 1); - LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); - cl_state->leave_epoch = cluster_epoch_get_current(); /* the committed epoch E */ - LWLockRelease(&cl_state->lock); + committed_evidence = (pg_atomic_read_u32(&cl_state->committed_durable_confirmed) != 0); + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(committed_evidence, others_dead_unchanged, + dead_gen_unchanged)) { + uint64 committed_epoch; + + pg_atomic_write_u32(&cl_state->commit_point_observed, 1); + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + committed_epoch = cl_state->committed_confirmed_epoch; /* the committed epoch E */ + cl_state->leave_epoch = committed_epoch; + LWLockRelease(&cl_state->lock); - /* - * P1-V0.7 exit gate: reach COMMITTED ("may exit") ONLY after the - * coordinator confirms the COMMITTED marker is majority-durable - * (LEAVE_COMMITTED). Until then stay in BARRIER_WAIT and re-tick — the - * durable truth-source must exist before this node departs, else a - * survivor restart could not rebuild the clean-departed fact. - */ - if (pg_atomic_read_u32(&cl_state->committed_durable_confirmed)) { - cl_set_phase(CLUSTER_LEAVE_COMMITTED); - CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); - ereport(LOG, - (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " - "majority-durable; this node has drained and may exit", - (unsigned long long)cl_state->leave_epoch))); - } - } else { - cl_escalate(); /* a real death changed the version mid-leave (CL-I3) */ - } + if (!others_dead_unchanged || !dead_gen_unchanged) + ereport(LOG, (errmsg("cluster clean-leave: third-party liveness flap observed at the " + "commit latch (others-dead %s, dead_generation %s); durable " + "COMMITTED marker evidence overrides it", + others_dead_unchanged ? "unchanged" : "changed", + dead_gen_unchanged ? "unchanged" : "advanced"))); + + cl_set_phase(CLUSTER_LEAVE_COMMITTED); + CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); + ereport(LOG, (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " + "majority-durable; this node has drained and may exit", + (unsigned long long)committed_epoch))); return; } @@ -1615,8 +1840,12 @@ cl_leaving_barrier_tick(void) } /* 3. fail-closed deadline — ONLY before the commit point (P1-1: a committed - * leave is never un-committed; post-commit we wait for the durable marker, - * bounded by disk health, and never escalate). */ + * leave is never un-committed). With the r3 evidence latch this arm is what + * bounds EVERY no-evidence outcome: a refused leave, a foreign death that + * moved the version (the coordinator then refuses, CL-I3), or a lost/never- + * sent confirmation all end here instead of hanging in BARRIER_WAIT. The + * commit_point_observed guard is belt-and-braces: the latch above transitions + * out of BARRIER_WAIT in the same tick it sets the flag. */ if (!pg_atomic_read_u32(&cl_state->commit_point_observed) && (uint64)GetCurrentTimestamp() > cl_state->barrier_deadline_us) { cl_escalate(); @@ -1640,79 +1869,97 @@ cl_coordinator_commit(int32 leaving) uint64 new_epoch; ClusterLeaveIntentMarker m; + if (cl_lmon_marker_async.has_staged_event) { + ClusterLeaveAsyncMarkerResult ar = cl_poll_lmon_marker_stage(); + int phase = cl_lmon_marker_phase; + uint64 staged_epoch = cl_lmon_marker_epoch; + + if (ar == CL_LEAVE_ASYNC_PENDING) + return; + if (ar == CL_LEAVE_ASYNC_FAILED) + return; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { + cl_release_lmon_marker_stage(); + + /* + * spec-2.29a review r1 P1 — CL-I3 re-check at the staged-ACK + * handoff. The COMMITTING marker wait now spans LMON ticks, so a + * real death can bump CSSD dead_generation INSIDE the wait window + * while the fail-stop epoch has not yet advanced (the >=3-node + * window of Hardening v1.0.1 P1-2). The guarded CAS inside + * apply_clean_leave_as_coordinator only catches the epoch move; + * re-run the same dead_gen-aware coherence check the non-staged + * pre-check uses, at the last observable point before the commit + * applies. On failure the leave does not commit and the leaving + * node escalates to fail-stop (identical to the pre-check path). + */ + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) " + "across the COMMITTING marker wait for node %d; not committing " + "(escalate to fail-stop, CL-I3)", + leaving))); + return; + } + + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); + if (new_epoch == 0) { + ereport( + LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); + return; + } + cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + new_epoch); + (void)cl_poll_lmon_marker_stage(); + ereport(LOG, + (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", + leaving, (unsigned long long)new_epoch))); + return; + } + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, staged_epoch); + cl_release_lmon_marker_stage(); + return; + } + cl_release_lmon_marker_stage(); + return; + } + /* CL-I3 pre-check: refuse to commit a clean leave on a version a real death * already bumped — the leaving node will then observe a non-CLEAN_LEAVE event * and escalate. This is a cheap early-out; the authoritative guard is the * guarded CAS inside apply_clean_leave_as_coordinator below, which closes the * check-then-bump TOCTOU at >=3 nodes. */ /* - * CL-I3 commit-handoff coherence (epoch AND dead_gen): refuse to commit if a - * real death intruded since this survivor started tracking the leave — either - * the death's fail-stop already bumped the epoch, OR (the >=3-node window, - * Hardening v1.0.1 P1-2) CSSD bumped its dead_generation but the fail-stop - * epoch has NOT yet advanced, which an epoch-only check (and the guarded CAS - * below) would miss and commit on a stale membership view. Uses the same - * unit-tested version_coherent helper as drive_drain so the dead_gen-aware - * coherence holds through the commit handoff; the leaving node then observes - * the eventual foreign event and escalates. + * CL-I3 commit-handoff coherence (epoch AND others-dead): refuse to commit if + * a THIRD-PARTY death intruded since this survivor started tracking the leave + * — either the death's fail-stop already bumped the epoch, OR (the >=3-node + * window, Hardening v1.0.1 P1-2) CSSD added it to the others-dead set but the + * fail-stop epoch has NOT yet advanced, which an epoch-only check (and the + * guarded CAS below) would miss and commit on a stale membership view. Uses + * the same others-dead coherence helper as drive_drain (spec-2.29a ②b: the + * leaving node's own expected DEAD is excluded so it never falsely escalates); + * the leaving node then observes the eventual foreign event and escalates. */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { - ereport(LOG, (errmsg("cluster clean-leave: version moved (epoch or dead_generation) before " - "committing node %d; not committing (escalate to fail-stop, CL-I3)", - leaving))); + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) before " + "committing node %d; not committing (escalate to fail-stop, CL-I3)", + leaving))); return; } /* (1) COMMITTING(E) marker (coordinator's own slot, before the bump; NOT a * trust basis). Not durable -> do not commit. */ cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, baseline_epoch + 1); - if (cluster_clean_leave_submit_marker(&m) != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster clean-leave: COMMITTING marker not durable for node %d; " - "not committing (the leaving node will retry/escalate)", - leaving))); - return; - } - - /* (2) the real commit point: guarded epoch bump (only if still baseline) + - * publish CLEAN_LEAVE + record clean-departed. Returns 0 if a concurrent - * reconfig moved the epoch off the baseline after the COMMITTING marker — - * fail-closed, do NOT write the COMMITTED marker (CL-I3). */ - new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); - if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); - return; /* CL-I3: stale baseline (or cluster disabled / bad id) */ - } - - /* - * (3) COMMITTED(E) marker (after the bump; the ONLY rebuild trust basis). - * Post-commit it MUST become majority-durable — the durable truth-source must - * exist BEFORE the leaving node departs (P1-V0.7 exit gate; §2.5). Try once - * here. If it reaches majority, mark durable + tell the leaving node it may - * exit (LEAVE_COMMITTED). If NOT, do not give up: committed_marker_durable - * stays 0, cl_survivor_tick retries the marker every tick, and the leaving - * node stays in BARRIER_WAIT (no LEAVE_COMMITTED) until it is durable — it - * never departs without a durable truth-source. The leave is already - * committed (epoch bumped); we never revert. - */ - cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); - if (cluster_clean_leave_submit_marker(&m) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - cl_send_committed(leaving, new_epoch); - } else { - ereport( - LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the COMMITTED " - "marker is not yet majority-durable; retrying each tick (leaving node waits)", - leaving, (unsigned long long)new_epoch))); - } - - ereport(LOG, (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", - leaving, (unsigned long long)new_epoch))); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, + baseline_epoch + 1); + (void)cl_poll_lmon_marker_stage(); } /* survivor (incl. coordinator) side of another node's leave. */ @@ -1773,13 +2020,9 @@ cl_survivor_tick(int32 leaving) if (coordinator == cluster_node_id && cluster_reconfig_is_clean_departed(leaving)) { uint64 committed_epoch = cluster_reconfig_get_clean_departed_epoch(leaving); - if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) { - ClusterLeaveIntentMarker cm; + if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) + cl_drive_committed_marker_stage(leaving, committed_epoch); - cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); - if (cluster_clean_leave_submit_marker(&cm) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - } /* Re-send LEAVE_COMMITTED every tick once durable until the leaver is gone * (best-effort IC): the leaving node will not depart until it receives one, * and step 3 holds the slot until it is CSSD-dead, so delivery is assured. */ diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 0a4eeebd04..91a2c4d768 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -136,17 +136,112 @@ cluster_clean_leave_should_abort_writable(bool in_transaction, bool has_top_xid) /* * cluster_clean_leave_version_coherent -- the bound leave is still coherent - * only if neither the cluster epoch nor the CSSD dead_generation moved since - * the leave was bound. Any external bump (a real death intruding mid-drain) - * makes it incoherent → caller must ABORTED_ESCALATE (never complete a clean - * leave on a stale version, which would let a destructive sweep run on a newer - * version and double-grant). + * only if neither the cluster epoch nor the OTHERS-dead set (every dead node + * except the leaving node itself) moved since the leave was bound. Any + * external membership change — a third-party death intruding mid-drain, or a + * third-party fail-stop that already bumped the epoch — makes it incoherent → + * caller must ABORTED_ESCALATE (never complete a clean leave on a stale + * membership view, which would let a destructive sweep run on a newer view + * and double-grant, CL-I3). + * + * spec-2.29a ②b fix: the pre-fix predicate compared the SCALAR global CSSD + * dead_generation, which also counts the leaving node's OWN alive→DEAD + * transition (it stops heart-beating once its drain finishes — the EXPECTED + * terminal state of a clean leave). Under the async COMMITTING marker that + * spans several LMON ticks, a slow environment lets that transition land + * inside the wait window and the scalar check wrongly escalated an otherwise + * healthy leave. Comparing the others-dead bitmap (which excludes the + * leaving node) removes exactly that false positive while keeping every + * third-party incoherence escalation (see the reflexive-case matrix in + * spec-2.29a §②b 8.A argument). */ bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen) + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes) +{ + if (bound_epoch != current_epoch) + return false; + if (bound_others_dead == NULL || current_others_dead == NULL) + return false; /* fail-closed: cannot prove coherence without both views */ + return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; +} + +/* + * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick + * latches "my clean-leave committed" on direct EVIDENCE, never on inference + * (spec-2.29a r3). The evidence is committed_marker_evidence: the survivor + * coordinator made the COMMITTED marker for THIS leave attempt majority- + * durable on the voting disk and attested it with a nonce-bound + * LEAVE_COMMITTED (validated by cluster_clean_leave_committed_evidence_ + * matches before the flag this argument mirrors is ever set). Latching + * suppresses the barrier-deadline escalation and lets the leaver exit, so + * the verdict must be exactly: latch <=> evidence. + * + * History of the two inference failure modes this replaces: + * - r2 P2-1 (mis-latch wedge): the pre-r2 "epoch advanced + others-dead + * bitmap unchanged" inference could mis-latch a REFUSED leave after a + * third-party false-DEAD -> ALIVE rebound restored the (non-monotone) + * bitmap, suppressing the deadline escalation forever. + * - r3 (t/331 C1/C4 false-escalation): the r2 scalar dead_generation + * conjunct is monotone the other way — a third-party transient flap on + * the leaver's local CSSD view advances it forever, so a healthy + * committed leave was refused until the deadline escalated it. + * Marker evidence is immune to both: a refused leave never gets a COMMITTED + * marker (no latch -> bounded deadline escalation), and a flap cannot make + * durable evidence disappear (latch -> no false escalation). + * + * others_dead_unchanged / dead_gen_unchanged are the leaver's live coherence + * observations. They are deliberately kept in the signature as contract + * inputs that MUST NOT affect the verdict — the U3b unit matrix pins the + * flap-immunity on them — and the runtime uses them only for the flap-noise + * LOG at the latch point (observability, never control flow). + */ +bool +cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, bool others_dead_unchanged, + bool dead_gen_unchanged) +{ + (void)others_dead_unchanged; /* observability-only input (see above) */ + (void)dead_gen_unchanged; /* observability-only input (see above) */ + return committed_marker_evidence; +} + +/* + * cluster_clean_leave_committed_evidence_matches -- may the leaving node + * accept a LEAVE_COMMITTED confirmation as marker evidence for THIS leave + * attempt? (spec-2.29a r3; the payload's magic/version/CRC were already + * checked by cluster_clean_leave_announce_payload_valid.) + * + * Identity is bound three ways, all fail-closed: + * - payload_leaving_node == self_node: the confirmation is addressed to + * this node's own leave (LEAVE_COMMITTED is point-to-point, but a + * misrouted frame must still not latch). + * - current_leaving_node == self_node: this node IS currently the leaver + * (not idle, not a survivor of someone else's leave). + * - payload_nonce == current_attempt_nonce: the per-attempt nonce + * (Hardening v1.0.2) pins the confirmation to THIS attempt, so a stale + * LEAVE_COMMITTED — and through it a stale COMMITTED marker — from a + * PREVIOUS leave of the same node can never false-latch a new attempt. + * - payload_epoch > bound_leave_epoch: the committed epoch E the + * coordinator attests must lie past the baseline this leave bound + * (sanity; the commit is a guarded CAS off that baseline). + */ +bool +cluster_clean_leave_committed_evidence_matches(int32 payload_leaving_node, uint64 payload_nonce, + uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, + uint64 current_attempt_nonce, + uint64 bound_leave_epoch) { - return bound_epoch == current_epoch && bound_dead_gen == current_dead_gen; + if (payload_leaving_node != self_node) + return false; + if (current_leaving_node != self_node) + return false; + if (payload_nonce != current_attempt_nonce) + return false; + if (payload_epoch <= bound_leave_epoch) + return false; + return true; } diff --git a/src/backend/cluster/cluster_conf.c b/src/backend/cluster/cluster_conf.c index 28a9f4e27d..e7028eb59f 100644 --- a/src/backend/cluster/cluster_conf.c +++ b/src/backend/cluster/cluster_conf.c @@ -58,6 +58,7 @@ #ifdef USE_PGRAC_CLUSTER #include "cluster/cluster_elog.h" /* CLUSTER_LOG */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_config_file */ +#include "cluster/cluster_ic.h" /* CLUSTER_IC_TIER_* (spec-7.2 data_addr gate) */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (stage 0.27) */ #endif @@ -78,7 +79,11 @@ PG_FUNCTION_INFO_V1(cluster_get_nodes); StaticAssertDecl(sizeof(ClusterNodeRole) == sizeof(int32), "ClusterNodeRole must be int32-sized for shmem ABI stability"); -StaticAssertDecl(sizeof(ClusterConf) <= 65536, "ClusterConf must fit in 64 KiB"); +/* PGRAC: spec-7.2 D2 — budget raised 64 -> 128 KiB when data_addr joined + * the per-node entry (128 slots x 128B pushed the old bound; the region + * is shmem-only, so the budget is an anchor against accidental bloat, + * not a wire/disk contract). cluster-conf-design.md §3.2 amended. */ +StaticAssertDecl(sizeof(ClusterConf) <= 131072, "ClusterConf must fit in 128 KiB"); /* ============================================================ @@ -478,6 +483,17 @@ apply_node_field(ClusterNodeInfo *n, const char *key, const char *value, const c strlcpy(n->public_addr, value, sizeof(n->public_addr)); return true; } + /* PGRAC: spec-7.2 D2 — optional DATA-plane listener address. Empty + * (or absent) = data plane off for this node; never derived from + * interconnect_addr (r1-F2: no port guessing). */ + if (strcmp(key, "data_addr") == 0) { + if (*value != '\0' && !validate_addr_format(value)) { + *out_err = "data_addr must be empty or in host:port form"; + return false; + } + strlcpy(n->data_addr, value, sizeof(n->data_addr)); + return true; + } if (strcmp(key, "role") == 0) { ClusterNodeRole role; @@ -596,6 +612,29 @@ post_validate(const char *path) "cluster_conf: node %d in \"%s\" is missing required field interconnect_addr", n->node_id, path))); } + + /* + * PGRAC: spec-7.2 flip — with the GCS block family on the DATA + * plane, a multi-node cluster on a real interconnect tier + * cannot serve block transfers without a per-node data_addr. + * Fail-closed at startup (r1-F2: no port guessing, no silent + * CONTROL fallback) instead of hanging the first cross-node + * block request. Single-node fallback and stub/mock tiers are + * exempt (no remote block traffic exists there). + */ + if (ClusterConfShmem->node_count > 1 && cluster_enabled + && (cluster_interconnect_tier == CLUSTER_IC_TIER_1 + || cluster_interconnect_tier == CLUSTER_IC_TIER_2 + || cluster_interconnect_tier == CLUSTER_IC_TIER_3) + && n->data_addr[0] == '\0') { + ereport(FATAL, (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("cluster_conf: node %d in \"%s\" is missing required field " + "data_addr (multi-node DATA plane, spec-7.2)", + n->node_id, path), + errhint("Declare data_addr = host:port for every [node.N] section; " + "the GCS block family ships over the LMS-owned DATA plane."))); + } + if (n->rdma_port == 0) n->rdma_port = 1; if (n->node_id == cluster_node_id) diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 7aafcf3e5a..eb431793dc 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -53,6 +53,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope */ #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" /* PGRAC: spec-7.2 D1 READY-publish wakeup */ #include "cluster/cluster_shmem.h" #include "cluster/cluster_tt_durable.h" /* resolve_by_xid (D-i4 complete scan) */ #include "cluster/cluster_tt_slot.h" /* max_recycle_horizon (D-i4 bound) */ @@ -540,6 +541,12 @@ cluster_lms_cr_drain(void) pg_write_barrier(); pg_atomic_write_u32(&slot->state, CLUSTER_LMS_CR_READY); + /* PGRAC: spec-7.2 D1 -- wake the shipper (LMON) right away; + * without this the READY result sat until LMON's next natural + * wakeup (typ. 100-250ms, worst case one heartbeat). + * Publish-before-signal: READY store above precedes the kick. */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SHIP_READY); + cluster_lmon_wakeup(); continue; } @@ -580,6 +587,10 @@ cluster_lms_cr_drain(void) pg_write_barrier(); pg_atomic_write_u32(&slot->state, CLUSTER_LMS_CR_READY); + /* PGRAC: spec-7.2 D1 -- wake the shipper (see the undo/verdict + * READY publish above for the rationale). */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SHIP_READY); + cluster_lmon_wakeup(); } } diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..b2d1c08469 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -130,6 +130,7 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_catalog_stats.h" /* catalog counters (spec-6.14 D10b) */ #include "cluster/cluster_oid_lease.h" /* catalog category (spec-6.14 D10) */ #include "cluster/cluster_relmap_authority.h" /* relmap authority state (spec-6.14 D5) */ +#include "cluster/cluster_xid_authority.h" /* XID authority state (spec-6.15b D7) */ #include "cluster/cluster_remote_xact.h" /* remote outcome counters (spec-4.5a D11) */ #include "cluster/cluster_ic.h" /* ClusterICOps_Active, ClusterICTier */ #include "cluster/cluster_ic_tier1.h" /* listener metadata accessors (Hardening v1.0.1 F3) */ @@ -544,6 +545,10 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_slow_iter_count", + fmt_int64((int64)cluster_lmon_slow_iter_count())); } /* @@ -1195,8 +1200,30 @@ static void dump_grd_recovery(ReturnSetInfo *rsinfo) { ClusterGrdRecoveryCounters c; + uint32 state; cluster_grd_recovery_counters_snapshot(&c); + state = cluster_grd_recovery_state_value(); + emit_row(rsinfo, "grd_recovery", "state", cluster_grd_recovery_state_name(state)); + emit_row(rsinfo, "grd_recovery", "state_enum_value", fmt_int32((int32)state)); + emit_row(rsinfo, "grd_recovery", "last_event_id", + fmt_int64((int64)cluster_grd_recovery_last_event_id())); + emit_row(rsinfo, "grd_recovery", "event_old_epoch", + fmt_int64((int64)cluster_grd_recovery_event_old_epoch())); + emit_row(rsinfo, "grd_recovery", "episode_epoch", + fmt_int64((int64)cluster_grd_recovery_episode_epoch_value())); + emit_row(rsinfo, "grd_recovery", "event_coordinator", + fmt_int32((int32)cluster_grd_recovery_event_coordinator())); + emit_row(rsinfo, "grd_recovery", "done_self_epoch", + fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_bitmap_hash", + fmt_int64((int64)cluster_grd_recovery_done_bitmap_hash_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", + fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", + fmt_int64((int64)cluster_grd_recovery_block_redeclare_epoch())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_done", + fmt_bool(cluster_grd_recovery_block_redeclare_done())); emit_row(rsinfo, "grd_recovery", "remaster_started", fmt_int64((int64)c.remaster_started)); emit_row(rsinfo, "grd_recovery", "remaster_done", fmt_int64((int64)c.remaster_done)); emit_row(rsinfo, "grd_recovery", "remaster_failed", fmt_int64((int64)c.remaster_failed)); @@ -1212,6 +1239,11 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) emit_row(rsinfo, "grd_recovery", "unaffected_holder_survived", fmt_int64((int64)c.unaffected_holder_survived)); emit_row(rsinfo, "grd_recovery", "stale_holder_swept", fmt_int64((int64)c.stale_holder_swept)); + emit_row(rsinfo, "grd_recovery", "cluster_gate_timeout", + fmt_int64((int64)c.cluster_gate_timeout)); + emit_row(rsinfo, "grd_recovery", "wait_epoch_escape", fmt_int64((int64)c.wait_epoch_escape)); + /* spec-4.6a D12 — dead-node PCM residue cleanup (categorized under pcm). */ + emit_row(rsinfo, "pcm", "dead_cleanup_entries", fmt_int64((int64)c.pcm_dead_cleanup_entries)); /* spec-5.16 D5 — join-direction remaster counters (same grd_recovery * category; no new dump category, §8 Q6-A). */ emit_row(rsinfo, "grd_recovery", "join_remaster_started", @@ -1228,13 +1260,12 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) /* * dump_lms -- spec-2.18 Sprint A Step 4 D10. * - * Emits 6 rows under category='lms' corresponding to the 6 atomic - * counters in ClusterLmsSharedState (v0.3 §1.4 F2 收紧; - * grant/reject/convert 分项 counter 推 spec-2.20 真激活 grant state - * machine 时一并 ship). - * - * Plus the LMS state string for HC2 4-state semantic分流 - * observability. + * Emits the LMS state string (HC2 4-state semantic 分流 observability) + * plus one row per atomic counter in ClusterLmsSharedState: the + * spec-2.18 base set (started/work_drained/decision grant/reject/convert/ + * drain_empty/error), the spec-2.25 native-lock probe counters, the + * spec-2.27 priority-starvation counter, and the spec-7.2 D6 + * lms_conn_resets DATA-plane reset counter. */ static void dump_lms(ReturnSetInfo *rsinfo) @@ -1278,6 +1309,9 @@ dump_lms(ReturnSetInfo *rsinfo) * on wire; reserved opcode 11 awaits spec-2.28+ integrated receiver). */ emit_row(rsinfo, "lms", "priority_starvation_observed_count", fmt_int64((int64)cluster_lms_get_priority_starvation_observed_count())); + /* spec-7.2 D6 — DATA-plane connection resets (epoch bump in production; + * cluster-lms-conn-reset injection in the F6-1 fault test). */ + emit_row(rsinfo, "lms", "lms_conn_resets", fmt_int64((int64)cluster_lms_get_conn_resets())); } /* @@ -1475,6 +1509,10 @@ dump_reconfig_join(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_reconfig_get_join_timeout_count())); emit_row(rsinfo, "reconfig_join", "clean_departed_cleared_count", fmt_int64((int64)cluster_reconfig_get_clean_departed_cleared_count())); + emit_row(rsinfo, "reconfig", "marker_slow_ack_count", + fmt_int64((int64)cluster_reconfig_get_marker_slow_ack_count())); + emit_row(rsinfo, "reconfig", "marker_timeout_count", + fmt_int64((int64)cluster_reconfig_get_marker_timeout_count())); } /* @@ -1831,6 +1869,32 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_block_wal_flush_before_ship_count())); emit_row(rsinfo, "gcs", "block_ship_bytes_total", fmt_int64((int64)cluster_gcs_get_block_ship_bytes_total())); + /* PGRAC: spec-7.2 flip — plane facts: which plane owns the block + * family (0 = CONTROL pre-flip, 1 = DATA) and the total frames the + * dispatch plane gate dropped (both tier1 instances). */ + emit_row(rsinfo, "gcs", "block_family_plane", + fmt_int64(cluster_gcs_block_family_on_data_plane() ? 1 : 0)); + emit_row( + rsinfo, "gcs", "plane_misroute_reject", + fmt_int64((int64)(cluster_ic_tier1_get_plane_misroute_reject(CLUSTER_IC_PLANE_CONTROL) + + cluster_ic_tier1_get_plane_misroute_reject(CLUSTER_IC_PLANE_DATA)))); + /* PGRAC: spec-7.2 D6 — requester ship-latency histogram (16 rows, + * keys ship_hist_us_le_ + ship_hist_us_inf). */ + { + int hb; + + for (hb = 0; hb < CLUSTER_GCS_SHIP_HIST_BUCKETS; hb++) { + char histkey[48]; + uint64 bound = cluster_gcs_block_ship_hist_bound_us(hb); + + if (bound == UINT64_MAX) + snprintf(histkey, sizeof(histkey), "ship_hist_us_inf"); + else + snprintf(histkey, sizeof(histkey), "ship_hist_us_le_" UINT64_FORMAT, bound); + emit_row(rsinfo, "gcs", histkey, + fmt_int64((int64)cluster_gcs_block_ship_hist_count(hb))); + } + } /* spec-6.13 D8: RDMA tier3/direct-land copy path observability. */ emit_row(rsinfo, "gcs", "scratch_copy_count", fmt_int64((int64)cluster_gcs_get_scratch_copy_count())); @@ -2810,6 +2874,12 @@ dump_hw(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_hw_remaster_done_count())); emit_row(rsinfo, "hw", "remaster_blocked_count", fmt_int64((int64)cluster_hw_remaster_blocked_count())); + emit_row(rsinfo, "hw", "remaster_retry_count", + fmt_int64((int64)cluster_hw_remaster_retry_count())); + emit_row(rsinfo, "hw", "remaster_retry_exhausted_count", + fmt_int64((int64)cluster_hw_remaster_retry_exhausted_count())); + emit_row(rsinfo, "hw", "remaster_recoverable", + fmt_int64((int64)(cluster_hw_remaster_recoverable() ? 1 : 0))); /* * spec-6.12d D-obs: space-lease counters. bloat_ratio itself is a @@ -3035,6 +3105,19 @@ dump_catalog(ReturnSetInfo *rsinfo) emit_row(rsinfo, "catalog", "relmap_shared_pending_owner_node", fmt_int64(rmok ? (int64)rmhdr.owner_node : -1)); + /* spec-6.15b D7: native-era XID authority and this-boot adopt marker. */ + { + ClusterXidAuthorityHeader xahdr; + bool xaok = cluster_shared_catalog && cluster_xid_authority_read(&xahdr); + + emit_row(rsinfo, "catalog", "xid_authority_native_hw", + fmt_int64(xaok ? (int64)xahdr.native_hw_full : 0)); + emit_row(rsinfo, "catalog", "xid_authority_sealed", + fmt_bool(xaok && (xahdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) != 0)); + emit_row(rsinfo, "catalog", "xid_prehistory_adopted", + fmt_bool(cluster_xid_prehistory_was_adopted())); + } + /* * D10b shared-catalog data-plane counters: cluster-path visibility * resolutions of catalog-page tuples, their fail-closed (53R97-family) diff --git a/src/backend/cluster/cluster_gcs.c b/src/backend/cluster/cluster_gcs.c index df04a899c2..052eec4a01 100644 --- a/src/backend/cluster/cluster_gcs.c +++ b/src/backend/cluster/cluster_gcs.c @@ -499,8 +499,14 @@ gcs_send_envelope_or_loopback(uint8 msg_type, int32 dest_node, const void *paylo /* * GCS requests can be produced from bufmgr/content-lock backend paths. - * Tier1 sockets are owned by LMON, so non-LMON request producers enqueue - * to the existing GRD outbound ring and wait on the normal reply slot. + * CONTROL-plane sockets are owned by LMON, so non-LMON request + * producers enqueue to the existing GRD outbound ring and wait on the + * normal reply slot. PGRAC: spec-7.2 D3 — this branch is also the + * staging rule for LMS: a migrated DATA-plane handler that must emit + * GCS_REQUEST (a CONTROL message; the ㉕ X->S downgrade notify and + * the BAST-nudge branch) lands here by the same != B_LMON test and + * goes through the ring for LMON to send — a direct send would trip + * the router's plane gate by design. */ if (msg_type == PGRAC_IC_MSG_GCS_REQUEST && MyBackendType != B_LMON) return cluster_grd_outbound_enqueue_backend_msg(msg_type, (uint32)dest_node, payload, @@ -725,8 +731,15 @@ cluster_gcs_send_transition_nowait(BufferTag tag, PcmLockTransition transition_i pg_atomic_fetch_add_u64(&ClusterGcs->encode_payload_bytes, sizeof(payload)); pg_atomic_fetch_add_u64(&ClusterGcs->send_request_count, 1); - return cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_REQUEST, master_node, &payload, - sizeof(payload)) + /* + * PGRAC: spec-7.2 D3 — route through the loopback/ring-aware helper + * instead of a raw send: post-flip this fire-and-forget notify is + * emitted from the LMS-context FORWARD/BAST paths, where GCS_REQUEST + * (CONTROL plane) must stage into the outbound ring for LMON (the + * D0-①b staging case). LMON callers keep the direct send. + */ + return gcs_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_REQUEST, master_node, &payload, + sizeof(payload)) == CLUSTER_IC_SEND_DONE; } diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..0fc41b7d18 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -51,11 +51,12 @@ #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */ #include "cluster/cluster_grd.h" /* spec-4.6 D4 — block_path_failclosed counter */ #include "cluster/cluster_grd_outbound.h" -#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ -#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ -#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ -#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ +#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ +#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ +#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ +#include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ +#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_itl.h" /* spec-5.2 D11 — active-ITL writer-transfer guard */ @@ -65,6 +66,7 @@ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" #include "cluster/cluster_shmem.h" +#include "cluster/storage/cluster_shared_fs.h" #include "cluster/cluster_sf_dep.h" #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */ #include "common/hashfn.h" @@ -281,10 +283,54 @@ typedef struct ClusterGcsBlockShared { * comparable version unit (per-thread WAL makes * cross-node LSN comparison meaningless) */ } pi_note_ring[CLUSTER_GCS_PI_NOTE_RING_SIZE]; + + /* PGRAC: spec-7.2 D6 — requester-side block-ship latency histogram. + * Bucketed at the single normal-exit funnel of + * cluster_gcs_send_block_request_and_wait (GRANTED / STORAGE_FALLBACK / + * READ_IMAGE completions only; ereport exits lose the sample, mirroring + * the xp scopes). This is the ruler for the spec-7.2 value gate + * (ship p99 < 20ms, p50 < 5ms) and the 7.7/7.8 wait-closure legs. */ + pg_atomic_uint64 ship_latency_hist[CLUSTER_GCS_SHIP_HIST_BUCKETS]; } ClusterGcsBlockShared; static ClusterGcsBlockShared *ClusterGcsBlock = NULL; + +/* PGRAC: spec-7.2 D6 — ship-latency histogram bucket upper bounds (us). + * 15 bounds -> 16 buckets; the last bucket is the +inf overflow. */ +static const uint64 gcs_ship_hist_bounds_us[CLUSTER_GCS_SHIP_HIST_BUCKETS - 1] + = { 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, + 200000, 500000, 1000000, 2000000, 5000000, 10000000, 30000000 }; + +/* + * PGRAC: spec-7.2 D3/D4 — registry probe: is the GCS block family on + * the DATA plane? REPLY stands in for all five (they flip atomically, + * H-5). Both LMON tick sites (ship_ready / pi_discard) and the LMS + * data-plane loop consult this so the flip commit only edits the six + * registration structs and everything pivots at once. + */ +bool +cluster_gcs_block_family_on_data_plane(void) +{ + const ClusterICMsgTypeInfo *info = cluster_ic_get_msg_type_info(PGRAC_IC_MSG_GCS_BLOCK_REPLY); + + return info != NULL && (ClusterICPlane)info->plane == CLUSTER_IC_PLANE_DATA; +} + +/* Record one completed ship into the histogram (requester context). */ +static void +gcs_block_ship_hist_record(TimestampTz started_at) +{ + uint64 elapsed_us; + int b = 0; + + if (ClusterGcsBlock == NULL) + return; + elapsed_us = (uint64)(GetCurrentTimestamp() - started_at); + while (b < CLUSTER_GCS_SHIP_HIST_BUCKETS - 1 && elapsed_us > gcs_ship_hist_bounds_us[b]) + b++; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->ship_latency_hist[b], 1); +} static ClusterGcsBlockBackendBlock *gcs_block_backend_blocks = NULL; @@ -381,6 +427,9 @@ cluster_gcs_block_shmem_init(void) if (!found) { memset(ClusterGcsBlock, 0, sizeof(*ClusterGcsBlock)); + /* PGRAC: spec-7.2 D6 — ship-latency histogram buckets. */ + for (i = 0; i < CLUSTER_GCS_SHIP_HIST_BUCKETS; i++) + pg_atomic_init_u64(&ClusterGcsBlock->ship_latency_hist[i], 0); pg_atomic_init_u64(&ClusterGcsBlock->block_request_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->block_reply_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->block_timeout_count, 0); @@ -1199,6 +1248,39 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) return GCS_BLOCK_RECOVERING; } + /* + * r3-P2-1 unseal-safety proof — this predicate is heartbeat LIVENESS + * (CSSD hysteresis flips DEAD->ALIVE on heartbeat receipt alone, + * cluster_cssd.c deadband scan), NOT a direct "instance recovery + * complete" signal. It is nevertheless safe to return NORMAL here, + * because on a crash-restarted master the heartbeat source itself is + * recovery-gated: + * (1) CSSD — the only heartbeat sender — is spawned by the cluster + * phase-4 driver, which the postmaster reaper invokes only at the + * PM_RUN transition, i.e. after the startup process exited 0 and + * the node's crash recovery fully replayed its WAL thread to shared + * storage (the ServerLoop respawn is equally PM_RUN-gated). A + * still-recovering node sends NO heartbeats, so a survivor's DEAD + * verdict cannot flip back early. + * (2) Belt-and-suspenders: even under a stale-ALIVE view (fast restart + * inside the deadband), a fetch cannot complete against a + * still-recovering node. The master-side handler + * (cluster_gcs_handle_block_request_envelope) default-denies unless + * the node is an in-quorum MEMBER, and cluster_qvotec_in_quorum() + * demands QUORUM_OK plus a live lease — state only the QVOTEC + * process (phase-4 / post-PM_RUN as well) can establish after a + * restart wiped shmem. The deny replies map to bounded 53R9L; an + * unresponsive endpoint exhausts the retransmit budget into bounded + * 53R90. Neither path ever falls back to a silent local storage + * read (STORAGE_FALLBACK is a master REPLY status, not a local + * fallback). + * (3) For online_join rejoin, MEMBER additionally requires coordinator + * admission, which vets the joiner's post-recovery voting slot + * (the slot_generation != 0 readiness sub-gate). + * So heartbeat-ALIVE implies the returned master completed its own + * instance recovery: its committed WAL is on shared storage and no + * merged-materialization proof is needed for its blocks. + */ if (static_master == cluster_node_id || cluster_cssd_get_peer_state(static_master) != CLUSTER_CSSD_PEER_DEAD) return GCS_BLOCK_NORMAL; @@ -1221,12 +1303,8 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The survivor may SERVE only - * after the dead origin's merged WAL recovery passes the redo-before- - * unfreeze gate; before that the shared-storage / re-declared version may - * be stale → fail-closed RECOVERING (never a stale page). - * - * TWO conditions, both required (Q5): + * live survivor (recovery-aware routing). Two conditions are both + * required before the on-disk version may be served (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1236,12 +1314,20 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * survivor DID observe (rebuilt pi_watermark_lsn > 0), the dead * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably - * reached us → lost-write → fail-closed. This is the LSN comparison - * (NOT a bool), live in the serve path; required_lsn == 0 (cold) is - * trivially covered and (a) carries the safety. + * reached us → lost-write → fail-closed. * - * Once both hold → NORMAL → the re-routed survivor serves (rebuilt-from- - * redeclare for held blocks, lazy minimal view for cold blocks). + * spec-4.6a Amendment v1.2 (R1): this proof is UNCONDITIONAL. Where + * online thread recovery cannot run (GUC off — the default — or any + * >2-node deployment) the materialization authority is never published, + * so a dead master's blocks stay RECOVERING until the failed node + * restarts and completes its own instance recovery (the unseal above is + * heartbeat liveness; the r3-P2-1 note explains why heartbeats imply + * recovery completion): a bounded, retryable + * ERROR on the request path (53R9L), never an unproven serve. A scope + * predicate must never gate a correctness proof — a committed write on + * a cold block that only the dead node saw has NO other guard on this + * read path (GRD freeze ends with the episode; the HW gate covers only + * extend high-water marks; pd_block_scn checks ride the ship path). */ if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) @@ -1330,6 +1416,8 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans ClusterXpScope xp_recv; bool xp_is_read; bool xp_is_index; + /* PGRAC: spec-7.2 D6 — ship-latency histogram start stamp. */ + TimestampTz ship_started_at; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), @@ -1381,6 +1469,11 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans else xp_idx.active = false; + /* PGRAC: spec-7.2 D6 — ship-latency histogram start stamp (always on, + * unlike the GUC-gated xp scopes above; the histogram is the value- + * gate ruler so it must not depend on profiling being enabled). */ + ship_started_at = GetCurrentTimestamp(); + cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_reserve_slot(tag, (uint8)transition_id, current_master, &request_id); @@ -1970,6 +2063,13 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans cluster_xp_end(&xp_idx); cluster_xp_end(&xp_req); + /* PGRAC: spec-7.2 D6 — record the completed ship into the latency + * histogram (GRANTED / STORAGE_FALLBACK / READ_IMAGE all delivered a + * usable page; the terminal-denied tail below ereports and loses the + * sample, mirroring the xp scopes). */ + if (granted || granted_storage_fallback || read_image) + gcs_block_ship_hist_record(ship_started_at); + /* spec-5.2 D2: GRANTED / STORAGE_FALLBACK record durable ownership (the * caller mirrors PCM state); READ_IMAGE is a one-shot non-durable read so * the caller must leave buf->pcm_state == N. */ @@ -3336,6 +3436,24 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo return; } + /* + * PGRAC: spec-7.2 D5 — master-side fail-stop episode fence, symmetric + * with the requester-side acquire gate (cluster_pcm_lock.c). While a + * dead static master's block resources are mid-episode (survivor + * re-declare not yet complete, or merged replay not yet materialized), + * serving a request here could grant a block whose surviving holder + * has not re-declared yet (phantom-holder overtake window). The + * requester-side gate cannot close this alone: a remote requester with + * a stale view routes here directly. Fail-closed before any dedup / + * state change; DENIED_RESOURCE_RECOVERING -> sender maps to 53R9L + * (retry-safe). phase_for_tag counts the hit itself. + */ + if (cluster_gcs_block_phase_for_tag(req->tag) == GCS_BLOCK_RECOVERING) { + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_RESOURCE_RECOVERING, + InvalidXLogRecPtr, NULL); + return; + } + /* HC75 range guard — out of range never enters dedup HTAB. */ if (req->transition_id < PCM_TRANS_N_TO_S || req->transition_id > PCM_TRANS_S_TO_X_CLEANOUT) { gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, @@ -5405,31 +5523,48 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo /* ============================================================ * Dispatch table registration. + * + * PGRAC: spec-7.2 flip — the five block-family msg_types (REQUEST / + * REPLY / FORWARD / INVALIDATE / INVALIDATE_ACK) are registered on + * the DATA plane: the LMS-owned tier1 instance carries their frames, + * the LMS loop dispatches them, and the producer mask admits the LMS + * family for the drain-and-send leg. All five flip in this one edit + * (H-5: no half-migrated window; the registry probe above pivots the + * LMON tick sites and the LMS loop automatically). REDECLARE alone + * stays on the CONTROL plane (r4): recovery re-declare must survive a + * DATA-mesh teardown mid-episode, and the REDECLARE -> REDECLARE_DONE + * pair may not be split across planes. * ============================================================ */ static const ClusterICMsgTypeInfo gcs_block_request_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REQUEST, .name = "gcs_block_request", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_request_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; static const ClusterICMsgTypeInfo gcs_block_reply_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REPLY, .name = "gcs_block_reply", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_reply_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; /* PGRAC: spec-2.35 D8 — holder-side forward handler msg_type registration. */ static const ClusterICMsgTypeInfo gcs_block_forward_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_FORWARD, .name = "gcs_block_forward", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_forward_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; @@ -6146,20 +6281,25 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c } +/* PGRAC: spec-7.2 flip — DATA plane (see the dispatch-table comment). */ static const ClusterICMsgTypeInfo gcs_block_invalidate_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE, .name = "gcs_block_invalidate", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_invalidate_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; static const ClusterICMsgTypeInfo gcs_block_invalidate_ack_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK, .name = "gcs_block_invalidate_ack", - .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, .broadcast_ok = false, .handler = cluster_gcs_handle_block_invalidate_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, }; @@ -6246,12 +6386,18 @@ cluster_gcs_handle_block_redeclare_envelope(const ClusterICEnvelope *env, const } +/* PGRAC: spec-7.2 flip (r4) — REDECLARE stays on the CONTROL plane: + * survivor re-declare and its DONE barrier belong to the LMON-owned + * recovery episode and must not depend on the DATA mesh being up. + * None of the five migrated handlers emits REDECLARE (D0-①b audit), + * so no cross-plane staging leg is required here. */ static const ClusterICMsgTypeInfo gcs_block_redeclare_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REDECLARE, .name = "gcs_block_redeclare", .allowed_producer_mask = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON, .broadcast_ok = false, .handler = cluster_gcs_handle_block_redeclare_envelope, + .plane = CLUSTER_IC_PLANE_CONTROL, }; @@ -6321,6 +6467,25 @@ cluster_gcs_get_block_ship_bytes_total(void) return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->block_ship_bytes_total) : 0; } +/* PGRAC: spec-7.2 D6 — ship-latency histogram accessors (dump + tests). */ +uint64 +cluster_gcs_block_ship_hist_bound_us(int bucket) +{ + if (bucket < 0 || bucket >= CLUSTER_GCS_SHIP_HIST_BUCKETS) + return 0; + if (bucket == CLUSTER_GCS_SHIP_HIST_BUCKETS - 1) + return UINT64_MAX; /* +inf overflow bucket */ + return gcs_ship_hist_bounds_us[bucket]; +} + +uint64 +cluster_gcs_block_ship_hist_count(int bucket) +{ + if (ClusterGcsBlock == NULL || bucket < 0 || bucket >= CLUSTER_GCS_SHIP_HIST_BUCKETS) + return 0; + return pg_atomic_read_u64(&ClusterGcsBlock->ship_latency_hist[bucket]); +} + /* PGRAC: spec-6.13 D8 — RDMA tier3/direct-land copy observability. */ uint64 cluster_gcs_get_scratch_copy_count(void) diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index a81f80ee2b..f68e407239 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -550,9 +550,12 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) /* spec-4.6 P0#3 cluster gate — fire-and-forget barrier announcement: * record and return (no work queue, no reply, no dedup; the sender - * re-announces each tick, the receiver write is a monotonic max). */ + * re-announces each tick, the receiver write is a monotonic max). + * Amendment v1.2 (R2): the request-id field pair carries the sender's + * dead_bitmap hash (composite convergence key), not an event_id. */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { - cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch); + cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, + ges_request_holder_request_id(req)); return; } diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..0adbe640bf 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -46,7 +46,8 @@ #include "cluster/cluster_pcm_lock.h" /* spec-2.36 HC124 pending_x node-dead cleanup */ #include "cluster/cluster_gcs.h" /* spec-4.7 D2 — cluster_gcs_lookup_master */ #include "cluster/cluster_membership.h" /* spec-5.16 D1 — cluster_membership_is_member */ -#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ +#include "cluster/cluster_native_lock_probe.h" +#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ #include "cluster/cluster_signal.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_cssd.h" /* spec-2.16 D8 newly-dead bitmap diff */ @@ -731,9 +732,14 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->recovery_event_old_epoch, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_redeclare_generation, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_barrier_deadline, 0); + pg_atomic_init_u32(&cluster_grd_state->recovery_event_coordinator, 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) + for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_bitmap_hash[i], 0); + } + pg_atomic_init_u64(&cluster_grd_state->recovery_event_bitmap_hash, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -747,6 +753,9 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->block_path_failclosed_count, 0); pg_atomic_init_u64(&cluster_grd_state->unaffected_holder_survived_count, 0); pg_atomic_init_u64(&cluster_grd_state->stale_holder_swept_count, 0); + pg_atomic_init_u64(&cluster_grd_state->cluster_gate_timeout_count, 0); + pg_atomic_init_u64(&cluster_grd_state->wait_epoch_escape_count, 0); + pg_atomic_init_u64(&cluster_grd_state->pcm_dead_cleanup_entries, 0); /* spec-5.16 D2/D3b/D5 — online-join remaster fence + counters. */ pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_epoch, 0); @@ -1630,6 +1639,99 @@ cluster_grd_redeclare_episode_epoch(void) * been re-declared to its recovery-aware master — serving it would risk an * 8.A double-grant. Reaching IDLE implies all survivor scans completed. */ +uint32 +cluster_grd_recovery_state_value(void) +{ + if (cluster_grd_state == NULL) + return (uint32)GRD_RECOVERY_IDLE; + return pg_atomic_read_u32(&cluster_grd_state->recovery_state); +} + +const char * +cluster_grd_recovery_state_name(uint32 state) +{ + switch ((ClusterGrdRecoveryState)state) { + case GRD_RECOVERY_IDLE: + return "idle"; + case GRD_RECOVERY_WAIT_EPOCH: + return "wait_epoch"; + case GRD_RECOVERY_WAIT_BARRIER: + return "wait_barrier"; + case GRD_RECOVERY_WAIT_CLUSTER: + return "wait_cluster"; + } + return "unknown"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id) + : 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch) + : 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator) + : 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); +} + +uint64 +cluster_grd_recovery_done_bitmap_hash_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[node]); +} + +uint64 +cluster_grd_recovery_event_bitmap_hash_value(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); +} + +/* + * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). + * + * The cross-node half of the REDECLARE_DONE convergence key: a hash over + * the sender's ACCEPTED dead bitmap ALONE (each node's own local event — + * there is no cross-node ratification of the bitmap; see the P0 stamp-site + * note and the r3-P2-2 mid-episode re-consume guard for how detection skew + * converges). Same kernel as the event_id hash + * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — + * the generation is per-instance observation history and does NOT converge + * across survivors, which is exactly why event_id cannot key the P6 gate. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere + * (a fixed NONZERO constant — 0 stays reserved for "unstamped"), so the + * composite key degrades to the epoch-only gate there. + */ +uint64 +cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) +{ + return hash_bytes_extended(dead_bitmap, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES, 0); +} + bool cluster_grd_recovery_in_progress(void) { @@ -1638,6 +1740,20 @@ cluster_grd_recovery_in_progress(void) return pg_atomic_read_u32(&cluster_grd_state->recovery_state) != (uint32)GRD_RECOVERY_IDLE; } +/* + * spec-2.29a: the pre-reconfig baseline epoch the WAIT_EPOCH gate compares + * against. Exposed for diagnostics + the IDLE baseline-hold unit test (the + * async pre-bump staging window must not let this re-capture a post-bump + * value; see the IDLE branch of cluster_grd_recovery_lmon_tick). + */ +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch); +} + /* spec-4.6 D4/D5 — recovery counter bump helpers for out-of-module * call sites (S4 stale mapping in cluster_lock_acquire.c; GCS block * fail-closed guard in cluster_gcs_block.c). */ @@ -1686,6 +1802,10 @@ cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) out->unaffected_holder_survived = pg_atomic_read_u64(&cluster_grd_state->unaffected_holder_survived_count); out->stale_holder_swept = pg_atomic_read_u64(&cluster_grd_state->stale_holder_swept_count); + out->cluster_gate_timeout = pg_atomic_read_u64(&cluster_grd_state->cluster_gate_timeout_count); + out->wait_epoch_escape = pg_atomic_read_u64(&cluster_grd_state->wait_epoch_escape_count); + out->pcm_dead_cleanup_entries + = pg_atomic_read_u64(&cluster_grd_state->pcm_dead_cleanup_entries); /* spec-5.16 D5 — join-direction remaster counters. */ out->join_remaster_started = pg_atomic_read_u64(&cluster_grd_state->join_remaster_started_count); @@ -1924,6 +2044,39 @@ grd_recovery_barrier_complete(uint64 gen, uint64 episode_epoch) return true; } +static void +grd_recovery_format_waiting_backend(uint64 gen, uint64 episode_epoch, char *buf, Size buflen) +{ + int beid; + pid_t self_pid = MyProcPid; + + if (buflen == 0) + return; + buf[0] = '\0'; + for (beid = 1; beid <= MaxBackends; beid++) { + PGPROC *proc = BackendIdGetProc((BackendId)beid); + uint32 registered; + uint64 acked; + uint64 acked_epoch; + + if (proc == NULL || proc->pid == 0 || proc->pid == self_pid) + continue; + registered = pg_atomic_read_u32(&proc->cluster_grd_registered_count); + if (registered == 0) + continue; + acked = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked); + acked_epoch = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked_epoch); + if (acked < gen || acked_epoch != episode_epoch) { + snprintf(buf, buflen, + "beid=%d pid=%d registered=%u acked=" UINT64_FORMAT "/" UINT64_FORMAT + " target=" UINT64_FORMAT "/" UINT64_FORMAT, + beid, (int)proc->pid, registered, acked, acked_epoch, gen, episode_epoch); + return; + } + } + snprintf(buf, buflen, "none"); +} + /* * spec-4.6 P0#3 cluster gate — announce "my local rebind barrier is * complete for `epoch`" to every declared peer (fire-and-forget; the @@ -1934,6 +2087,11 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; + /* Amendment v1.2 (R2): the DONE payload carries the sender's accepted + * dead-bitmap hash (cross-node convergence key), riding the request-id + * field pair — NOT the sender-local event_id (its dead_generation fold + * diverges across survivors' flap observation histories). */ + uint64 bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -1943,6 +2101,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); + req.holder_request_id_lo = (uint32)(bitmap_hash & 0xffffffffu); + req.holder_request_id_hi = (uint32)(bitmap_hash >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -1961,19 +2121,71 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash) { - uint64 prev; + uint64 episode_bitmap_hash; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; - /* Monotonic max: late/duplicate announcements never regress. */ - prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); - while (epoch > prev) { - if (pg_atomic_compare_exchange_u64(&cluster_grd_state->recovery_done_epoch[node], &prev, - epoch)) - break; + + /* + * The two accounting axes feed DIFFERENT consumers and must not share + * one drop gate (nightly regression: t/347 L4-iii / t/353 L5 joiner + * write wedge): + * + * 1. done_epoch[] — monotonic-max, UNCONDITIONAL (the spec-4.6 axis; + * never regress on a delayed lower-epoch frame). Consumed epoch-only + * by the spec-5.16 D3 join fence (cluster_grd_block_view_rebuilt): a + * DONE at epoch E proves the sender completed its WAIT_BARRIER at E — + * the cluster-wide rebind plus the FULL block re-declare scan against + * then-current routing — which is the fence's safety condition + * regardless of the sender's episode dead set. Crucially, a + * rejoining node never P0-accepts the JOIN episode as its own FSM + * episode (spec-5.16 D8: JOIN_COMMITTED is coordinator-side only), so + * its local episode-hash stamp stays 0/stale forever; gating this + * axis on the composite key drops every survivor DONE on the joiner, + * its join fence never lifts, and every joiner-home block access + * fail-closes 53R9L permanently. + * + * 2. done_bitmap_hash[] — composite-gated (spec-4.6a Amendment v1.2 R2). + * Read together with done_epoch[] as the (episode_epoch, + * dead_bitmap_hash) composite key by the P6 cluster gate and the + * WAIT_EPOCH coordinator witness. A stale DONE from a previous + * episode can carry the same epoch when cur==old, but its dead SET + * necessarily differs (a coordinator re-election adds the old + * coordinator to it; a re-death of the same node rides a higher epoch + * via the interposed JOIN bump), so the hash is the ABA guard: + * mismatching frames withhold the hash half and the composite + * consumers stay fail-closed. Flap-history drift changes only the + * per-instance dead_generation, never the sender's accepted dead SET, + * so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently announces a SMALLER set's hash at the + * same epoch; those frames withhold the hash half until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + + * re-announces the full set + * (grd_recovery_consume_new_event_mid_episode). Per-tick re-announce + * then converges the accounting after our P0 accept (the R4 + * pre-accept window stays closed for the composite consumers). + * + * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own + * recovery_event_bitmap_hash is 0 before the first P0 accept). This + * relies on a stamped hash never being 0: JOIN episodes hash an ALL-ZERO + * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero + * constant (asserted at the stamp site). + */ + { + uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); + + if (epoch > prev) + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) + return; + + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } /* @@ -2000,6 +2212,199 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +/* + * grd_recovery_consume_new_event_mid_episode — spec-4.6a r3-P2-2. + * + * WAIT_BARRIER / WAIT_CLUSTER guard against a NEWER local reconfig event at + * the SAME epoch. Multi-death detection skew: when two nodes die near- + * simultaneously the coordinator can fold both into ONE epoch bump with dead + * set {A,B}, while a survivor whose CSSD saw only {B} first stamps + * recovery_event_bitmap_hash = H({B}) and sails past WAIT_EPOCH on that + * bump. Its later local detection of A publishes a new event with dead set + * {A,B} but does NOT move the epoch, so the WAIT_BARRIER / WAIT_CLUSTER + * epoch guards never fire, the survivor keeps announcing DONE under H({B}), + * every peer that stamped H({A,B}) keeps dropping it (composite-key + * mismatch), and P6 — which has no timeout by design — wedges permanently. + * + * Disposition (runs in the single-writer LMON tick, after the epoch guard): + * - new event, FAIL direction, SAME dead-bitmap hash: only the sender- + * local dead_generation fold moved (flap drift re-fired the same quorum + * dead set). Absorb the event_id — the episode semantics (epoch, dead + * set) are unchanged, so re-running P1-P7 would be pure churn. + * Absorbing also keeps the IDLE dedup from re-consuming the event after + * this episode completes. + * - new event with a DIFFERENT dead-bitmap hash (grown dead set — the + * skew above — or a JOIN/FAIL interleave): abort to IDLE. The next + * tick re-consumes the current event, re-stamps + * recovery_event_bitmap_hash, re-runs recovery against the fuller dead + * set, and re-announces DONE under the new composite key — closing the + * survivor-behind wedge. The coordinator-behind direction needs no + * handling here: the coordinator's own later detection bumps the epoch + * again and the existing epoch guards abort. + * + * Returns true when the caller must return (episode aborted to IDLE). + */ +static bool +grd_recovery_consume_new_event_mid_episode(void) +{ + ReconfigEvent latest; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id == 0 + || latest.event_id == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + return false; + + if (latest.reconfig_kind == (uint8)RECONFIG_KIND_FAIL_STOP + && pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_FAIL + && cluster_grd_dead_bitmap_hash(latest.dead_bitmap) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { + pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, latest.event_id); + return false; + } + + grd_recovery_abort_to_idle(); + return true; +} + +static void +grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) +{ + va_list ap; + int n; + Size avail; + + if (buflen == 0 || *off >= (int)buflen - 1) + return; + avail = buflen - (Size)*off; + va_start(ap, fmt); + n = vsnprintf(buf + *off, avail, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((Size)n >= avail) + *off = (int)buflen - 1; + else + *off += n; +} + +static void +grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, bool is_join, + char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 done_epoch; + + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + if (is_join && join_fence_is_recipient_for(i, episode_epoch)) + continue; + done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); + if (done_epoch >= episode_epoch + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) + continue; + grd_recovery_appendf(buf, buflen, &off, + "%s%d(done=" UINT64_FORMAT "/bitmap_hash=" UINT64_FORMAT ")", + any ? "," : "", i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i])); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static void +grd_recovery_format_hw_dead(const uint64 *dead, char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + ClusterHwRemasterResult result; + + if (!grd_dead_bitmap_test(dead, i) || i == cluster_node_id) + continue; + result = cluster_hw_remaster_result(i); + grd_recovery_appendf(buf, buflen, &off, "%s%d:%s/%u", any ? "," : "", i, + cluster_hw_remaster_result_name(result), + cluster_hw_remaster_attempts(i)); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static bool +grd_recovery_adopt_observed_epoch(const uint64 *dead, uint64 cur_epoch) +{ + uint64 max_epoch = cur_epoch; + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 peer_epoch; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + peer_epoch = cluster_reconfig_get_observed_epoch(i); + if (peer_epoch > max_epoch) + max_epoch = peer_epoch; + } + if (max_epoch <= cur_epoch) + return false; + cluster_epoch_adopt_admitted(max_epoch); + return true; +} + +static void +grd_recovery_wait_cluster_watchdog(const uint64 *dead, uint64 episode_epoch) +{ + TimestampTz now; + TimestampTz deadline; + TimestampTz next_deadline; + char missing[512]; + char hw_dead[512]; + bool thread_gate; + bool hw_gate; + bool is_join; + + deadline = (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline); + now = GetCurrentTimestamp(); + if (deadline == 0 || now <= deadline) + return; + + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + grd_recovery_format_missing_survivors(dead, episode_epoch, is_join, missing, sizeof(missing)); + thread_gate = cluster_thread_recovery_gate_unfreeze(dead, (CLUSTER_MAX_NODES + 63) / 64); + hw_gate = cluster_hw_remaster_gate_unfreeze(); + grd_recovery_format_hw_dead(dead, hw_dead, sizeof(hw_dead)); + + pg_atomic_fetch_add_u64(&cluster_grd_state->cluster_gate_timeout_count, 1); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen"), + errdetail("episode_epoch=" UINT64_FORMAT ", missing_survivors=%s, " + "thread_gate=%s, hw_gate=%s, hw_dead=%s", + episode_epoch, missing, thread_gate ? "held" : "open", hw_gate ? "held" : "open", + hw_dead), + errhint("This watchdog is observational only; it never unfreezes a shard."))); + next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); +} + /* * spec-4.7 D2 — survivor block re-declare scan (Q6-A' worker-centric). @@ -2078,6 +2483,24 @@ grd_block_redeclare_scan_complete(uint64 episode_epoch) return grd_block_redeclare_epoch == episode_epoch && grd_block_redeclare_done; } +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return grd_block_redeclare_cursor; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return grd_block_redeclare_epoch; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return grd_block_redeclare_done; +} + void cluster_grd_recovery_lmon_tick(void) { @@ -2108,9 +2531,19 @@ cluster_grd_recovery_lmon_tick(void) * fires, making old_epoch already == the post-bump epoch and * wedging WAIT_EPOCH forever (cur <= old). The last stable * idle epoch is the reliable "before reconfig" value. + * + * spec-2.29a nightly-regression fix: the coordinator's own + * async pre-bump stages (fail-stop / node-remove / join + * Phase-1) bump the epoch several ticks before publishing the + * event. During that window this IDLE tick must NOT re-capture + * the already-bumped epoch as the baseline, or the P0 accept + * that follows the publish reads old == cur and wedges + * WAIT_EPOCH — the coordinator wedging itself, no IC piggyback + * needed. Hold the last stable pre-reconfig baseline instead. */ - pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, - cluster_epoch_get_current()); + if (!cluster_reconfig_has_pending_prebump_stage()) + pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, + cluster_epoch_get_current()); return; } @@ -2119,6 +2552,18 @@ cluster_grd_recovery_lmon_tick(void) * epoch (the genuine pre-reconfig baseline) — do NOT overwrite it * with evt.old_epoch here. */ pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, ev_id); + pg_atomic_write_u32(&cluster_grd_state->recovery_event_coordinator, + (uint32)evt.coordinator_node_id); + if (evt.coordinator_node_id >= 0 && evt.coordinator_node_id < CLUSTER_MAX_NODES) + pg_atomic_write_u64( + &cluster_grd_state->recovery_done_epoch_at_accept, + pg_atomic_read_u64( + &cluster_grd_state->recovery_done_epoch[evt.coordinator_node_id])); + else + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms)); for (b = 0; b < (CLUSTER_MAX_NODES + 63) / 64; b++) { uint64 word = 0; int j; @@ -2131,6 +2576,28 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } + /* Amendment v1.2 (R2) + r3-P2-2: stamp the composite convergence key's + * cross-node half. The hash is over THIS node's OWN accepted event's + * dead bitmap — there is NO cross-node ratification of the bitmap (the + * envelope propagates only the epoch). Convergence is eventual, not + * instantaneous: every survivor's CSSD deadband converges on the same + * dead set, and until it does, a survivor that accepted a SMALLER set + * at the same epoch stamps a different hash. The mid-episode + * re-consume guard (grd_recovery_consume_new_event_mid_episode) closes + * that skew: the survivor's own later detection re-stamps the full + * set's hash and re-announces DONE. dead_generation drift alone never + * changes the bitmap, so same-set re-fires hash identically. */ + pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, + cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); + /* r3-P3(a) — 0 is reserved as "unstamped/pre-accept" in + * mark_peer_done's drop test, so a stamped episode hash must never be + * 0. JOIN episodes hash an ALL-ZERO dead bitmap: the invariant is + * that hash_bytes_extended(zero[16], 0) != 0 (a fixed nonzero + * constant). For FAIL bitmaps a zero hash is a 2^-64 collision, the + * same trust level as the event_id hash; the Assert turns the + * otherwise-undebuggable "all DONEs dropped" wedge into a loud stop + * on cassert builds. */ + Assert(pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash) != 0); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2174,35 +2641,120 @@ cluster_grd_recovery_lmon_tick(void) int i; int signaled; + for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) + dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + /* - * Gate on the ACCEPTED epoch having advanced past the episode's - * old epoch: the coordinator bumped it earlier this tick; a - * non-coordinator survivor observes it via IC envelope piggyback - * a tick or two later. Running the rebind before the local - * epoch advances would mint holders the new master rejects. - * - * spec-5.16 (P0, Rule 8.A) — direction-aware. A FAIL survivor captures - * old_epoch BEFORE the coordinator bumps, so it must wait for a STRICT - * advance (cur > old) to prove it adopted the new master view. A JOIN - * observer survivor instead adopts the coordinator's JOIN epoch bump (via the - * joiner_self_tick max-peer-epoch adoption, same LMON tick) BEFORE it readmits - * the rejoiner and publishes its own observer JOIN_COMMITTED event, so by the - * time its FSM accepts that event old_epoch == cur_epoch == the join epoch. - * Requiring a strict advance there wedges the survivor in WAIT_EPOCH forever: - * it never runs the re-declare barrier, never broadcasts REDECLARE_DONE, and - * the coordinator's all-members JOIN barrier (Hardening v1.1) hangs → - * join_remaster_done never advances. For JOIN the survivor already holds the - * epoch it rebinds under (that IS the post-bump master view), so equality is - * sufficient; cur_epoch is monotonic so cur < old never occurs for JOIN. + * Gate on the ACCEPTED epoch having advanced past the episode's old epoch. + * FAIL requires proof of a post-bump epoch; JOIN keeps its existing equality + * allowance because the observer already accepted the coordinator's JOIN epoch. + * For FAIL, deadline expiry is not an escape hatch by itself: we either adopt + * a durable observed epoch or require a coordinator REDECLARE_DONE witness for + * cur==old. Without either proof, stay fail-closed in WAIT_EPOCH. */ - if (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) - == (uint32)GRD_REMASTER_DIR_JOIN - ? cur_epoch < old_epoch - : cur_epoch <= old_epoch) - return; + { + ReconfigEvent latest; + bool is_join; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id != 0 + && latest.event_id + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + grd_recovery_abort_to_idle(); + return; + } - for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) - dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + if (is_join) { + if (cur_epoch < old_epoch) + return; + } else if (cur_epoch <= old_epoch) { + TimestampTz now = GetCurrentTimestamp(); + TimestampTz wait_deadline; + TimestampTz next_deadline; + uint32 coordinator; + uint64 coord_done; + uint64 coord_done_bitmap_hash; + uint64 episode_bitmap_hash; + uint64 coord_done_at_accept; + + wait_deadline = (TimestampTz)pg_atomic_read_u64( + &cluster_grd_state->recovery_barrier_deadline); + if (wait_deadline == 0 || now <= wait_deadline) + return; + + if (grd_recovery_adopt_observed_epoch(dead, cur_epoch)) { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH adopted a durable observed epoch; " + "affected shards stay frozen until recovery completes"))); + return; + } + + coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); + episode_bitmap_hash + = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + coord_done + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) + : 0; + coord_done_bitmap_hash + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64( + &cluster_grd_state->recovery_done_bitmap_hash[coordinator]) + : 0; + coord_done_at_accept + = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); + /* + * spec-4.6a §2.3 witness, composite-key form (Amendment v1.2 + * R2): the coordinator's DONE must (a) name this episode's + * quorum dead SET (bitmap hash — the cross-node-consistent + * key; the sender-local event_id folds the per-instance + * dead_generation and diverges under flap asymmetry), (b) + * name exactly cur as the episode epoch, and (c) have been + * accounted after our P0 accept snapshot. Under strictly + * monotonic per-event epochs (c) is implied by (a)+(b) — any + * pre-accept residue carries a lower epoch — but it stays as a + * belt-and-suspenders guard against accounting bugs. + */ + if (cur_epoch == old_epoch && coord_done == cur_epoch + && coord_done_bitmap_hash == episode_bitmap_hash + && coord_done > coord_done_at_accept) { + pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " + "equal-epoch progress"), + errdetail("coordinator=%u, epoch=" UINT64_FORMAT + ", dead_bitmap_hash=" UINT64_FORMAT, + coordinator, cur_epoch, episode_bitmap_hash))); + } else { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_bitmap_hash=" UINT64_FORMAT + ", episode_bitmap_hash=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_bitmap_hash, episode_bitmap_hash, + coord_done_at_accept))); + return; + } + } + } /* * P1 freeze affected + P3 scoped sweep + P4 remaster. Direction- @@ -2306,6 +2858,7 @@ cluster_grd_recovery_lmon_tick(void) if (state == (uint32)GRD_RECOVERY_WAIT_BARRIER) { uint64 gen = pg_atomic_read_u64(&cluster_grd_state->recovery_redeclare_generation); uint64 episode_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch); + TimestampTz deadline; /* P0-1 epoch-coherence guard: a SECOND epoch bump landed * mid-episode (e.g. a third node's heartbeat flap re-fired @@ -2317,6 +2870,12 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — a newer local event at the SAME epoch (multi-death + * detection skew grew the dead set) re-consumes; same-set drift is + * absorbed without churn. */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — advance the survivor block re-declare scan one chunk * while the GES rebind barrier is still pending (worker-centric, runs * in this tick; epoch-coherent via the guard just above). */ @@ -2344,7 +2903,12 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)); grd_recovery_broadcast_done(episode_epoch); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)deadline); pg_atomic_write_u32(&cluster_grd_state->recovery_state, (uint32)GRD_RECOVERY_WAIT_CLUSTER); ereport(DEBUG1, @@ -2358,6 +2922,12 @@ cluster_grd_recovery_lmon_tick(void) if (GetCurrentTimestamp() > (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline)) { TimestampTz deadline; + char waiting_backend[256]; + bool ges_barrier_complete = grd_recovery_barrier_complete(gen, episode_epoch); + bool block_scan_complete = grd_block_redeclare_scan_complete(episode_epoch); + + grd_recovery_format_waiting_backend(gen, episode_epoch, waiting_backend, + sizeof(waiting_backend)); /* * Barrier deadline expired: fail-closed. Affected shards @@ -2376,6 +2946,12 @@ cluster_grd_recovery_lmon_tick(void) (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), errmsg("cluster GRD holder-rebuild barrier timed out; affected shards " "stay frozen"), + errdetail("gen=" UINT64_FORMAT ", episode_epoch=" UINT64_FORMAT + ", ges_barrier=%s, block_scan=%s, block_cursor=%d" + ", block_epoch=" UINT64_FORMAT ", waiting_backend=%s", + gen, episode_epoch, ges_barrier_complete ? "done" : "waiting", + block_scan_complete ? "done" : "waiting", grd_block_redeclare_cursor, + grd_block_redeclare_epoch, waiting_backend), errhint("A backend has not acked the cooperative rebind within " "cluster.grd_rebuild_timeout_ms; re-broadcasting."))); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), @@ -2401,6 +2977,13 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — same-epoch dead-set growth re-consumes here too: this + * node may already have announced DONE under the stale bitmap hash; + * the re-run re-announces under the full set's hash so peers' + * composite-key accounting converges (P6 has no timeout). */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — keep advancing the block re-declare scan after the GES * rebind barrier completed, so a large pool is fully re-declared within * the recovery window (no-op once the cursor reaches NBuffers). */ @@ -2409,6 +2992,8 @@ cluster_grd_recovery_lmon_tick(void) for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + grd_recovery_wait_cluster_watchdog(dead, episode_epoch); + /* * spec-4.11 D1 (3b-4b Part 3) — launch one per-episode online thread- * recovery worker for each in-scope dead origin (idempotent per tick; @@ -2461,8 +3046,13 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; - if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) - < episode_epoch) { + /* Amendment v1.2 (R2): composite key — the peer's DONE must name + * this episode's epoch AND the same quorum dead SET. Keying on + * event_id here wedged the gate whenever survivors' private + * dead_generation histories diverged (flap asymmetry). */ + if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { all_done = false; break; } @@ -5344,13 +5934,28 @@ cluster_grd_cleanup_on_node_dead(int32 dead_node_id) int swept = 0; int nresids; uint64 pending_x_cleared = 0; + uint64 pcm_holders_cleaned = 0; int i; pending_x_cleared = cluster_pcm_lock_clear_pending_x_for_node(dead_node_id); - if (pending_x_cleared > 0) - ereport(DEBUG1, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " - "cleared " UINT64_FORMAT " PCM pending_x entries", - dead_node_id, pending_x_cleared))); + pcm_holders_cleaned = cluster_pcm_lock_cleanup_on_node_dead(dead_node_id); + + /* + * spec-4.6a D12: dead-node PCM residue is part of the reconfig + * convergence chain -- a leftover holder/pending-X record from the dead + * node blocks post-kill DDL with lock-acquire timeouts long after the + * remaster itself finished. Surface the cleanup as a LOG summary plus an + * assertable counter (pcm/dead_cleanup_entries). + */ + if (pending_x_cleared > 0 || pcm_holders_cleaned > 0) { + if (cluster_grd_state != NULL) + pg_atomic_fetch_add_u64(&cluster_grd_state->pcm_dead_cleanup_entries, + pending_x_cleared + pcm_holders_cleaned); + ereport(LOG, (errmsg("cluster GRD dead-node cleanup for node %d: cleared " UINT64_FORMAT + " PCM pending-X waiter(s) and " UINT64_FORMAT + " PCM holder record(s) left by the dead node", + dead_node_id, pending_x_cleared, pcm_holders_cleaned))); + } if (cluster_grd_entry_htab == NULL) { ereport(DEBUG2, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " diff --git a/src/backend/cluster/cluster_grd_outbound.c b/src/backend/cluster/cluster_grd_outbound.c index dd3e99c9e4..83a453f197 100644 --- a/src/backend/cluster/cluster_grd_outbound.c +++ b/src/backend/cluster/cluster_grd_outbound.c @@ -40,6 +40,7 @@ #include "cluster/cluster_grd_outbound.h" #include "cluster/cluster_ic_rdma.h" #include "cluster/cluster_ic_router.h" +#include "cluster/cluster_lms.h" /* PGRAC: spec-7.2 D4 DATA-ring routing */ #include "cluster/cluster_ic_tier1.h" #include "cluster/cluster_lmon.h" #include "cluster/cluster_shmem.h" @@ -176,6 +177,9 @@ ring_push(uint8 msg_type, uint8 origin, uint32 dest_node_id, const void *payload cluster_grd_outbound_state->ring_head = (cluster_grd_outbound_state->ring_head + 1) % PGRAC_GES_OUTBOUND_RING_CAPACITY; cluster_grd_outbound_state->ring_count++; + /* PGRAC: spec-7.2 D1 — mark the drain family dirty inside the push + * helper so every producer (current and future) is covered. */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); return true; } @@ -207,6 +211,7 @@ reply_dirty_push(uint32 dest_node_id, const void *payload, uint16 payload_len) = (cluster_grd_outbound_state->reply_dirty_head + 1) % PGRAC_GES_REPLY_DIRTY_BUDGET; cluster_grd_outbound_state->reply_dirty_count++; cluster_grd_inc_ges_reply_deferred(); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); /* spec-7.2 D1 */ } static void @@ -239,6 +244,7 @@ cleanup_dirty_push(uint8 msg_type, uint8 origin, uint32 dest_node_id, const void = (cluster_grd_outbound_state->cleanup_dirty_head + 1) % PGRAC_GES_CLEANUP_DIRTY_BUDGET; cluster_grd_outbound_state->cleanup_dirty_count++; cluster_grd_inc_ges_cleanup_deferred(); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); /* spec-7.2 D1 */ } static void @@ -284,6 +290,20 @@ cluster_grd_outbound_enqueue_backend_msg(uint8 msg_type, uint32 dest_node_id, co Assert(cluster_grd_outbound_state != NULL); + /* + * PGRAC: spec-7.2 D4 — plane routing at the single backend staging + * entry: a msg_type registered on the DATA plane goes to the DATA + * twin ring (LMS drains + sends); everything else stays on this + * CONTROL ring (LMON). Before the plane flip every type reads + * CONTROL here, so this branch is dormant until the flip commit. + */ + { + const ClusterICMsgTypeInfo *pinfo = cluster_ic_get_msg_type_info(msg_type); + + if (pinfo != NULL && (ClusterICPlane)pinfo->plane == CLUSTER_IC_PLANE_DATA) + return cluster_lms_outbound_enqueue(msg_type, dest_node_id, payload, payload_len); + } + LWLockAcquire(cluster_grd_outbound_lock, LW_EXCLUSIVE); /* Reserved pool: BACKEND_REQUEST may consume ring slots only up to diff --git a/src/backend/cluster/cluster_grd_work_queue.c b/src/backend/cluster/cluster_grd_work_queue.c index 1f317aaa96..b3700265ba 100644 --- a/src/backend/cluster/cluster_grd_work_queue.c +++ b/src/backend/cluster/cluster_grd_work_queue.c @@ -25,6 +25,7 @@ #include "cluster/cluster_ges.h" /* GesRequestPayload (spec-5.8 D8 coupling assert) */ #include "cluster/cluster_grd_work_queue.h" +#include "cluster/cluster_lmon.h" /* PGRAC: spec-7.2 D1 enqueue wakeup */ #include "cluster/cluster_shmem.h" #include "miscadmin.h" /* IsBootstrapProcessingMode */ #include "storage/lwlock.h" @@ -119,6 +120,20 @@ cluster_grd_work_queue_enqueue(uint32 source_node_id, const void *payload, uint1 cluster_grd_work_queue_state->count++; LWLockRelease(cluster_grd_work_queue_lock); + + /* + * PGRAC: spec-7.2 D1 -- wake the drain consumer (LMON) on enqueue, + * mirroring the cluster_grd_outbound enqueue family. Without this a + * locally-mastered GES request sat in the queue until LMON woke for + * some other reason (worst case a full heartbeat interval, <= 1s), + * which made local-master lock requests slower than remote-master + * ones. Publish-before-signal: the slot is visible (released the + * LWLock above) before the wakeup fires. + * Spec: spec-7.2-ic-data-plane-decoupling.md + */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GES_WORK_QUEUE); + cluster_lmon_wakeup(); + return true; } diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..85e03bf6b8 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -226,6 +226,8 @@ int cluster_tm_convert_mode = CLUSTER_TM_CONVERT_MODE_CONVERT; /* spec-4.6 D4/D1 — failure-driven remaster tunables. */ int cluster_grd_remaster_wait_ms = 200; /* frozen-shard short wait before 53R9I */ int cluster_grd_rebuild_timeout_ms = 5000; /* holder-rebuild barrier deadline */ +int cluster_hw_remaster_retry_backoff_ms = 1000; +int cluster_hw_remaster_retry_max_attempts = 16; /* spec-5.4 D8 — SQ sequence lock tunables. */ int cluster_sequence_default_cache = 100; /* CREATE-time CACHE injection default */ @@ -307,6 +309,7 @@ int cluster_phase4_timeout = 30; * LMON main-loop tick / WaitLatch timeout in milliseconds. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; /* spec-1.12 Sprint B D8: cluster.lck_main_loop_interval (mirror). */ int cluster_lck_main_loop_interval = 1000; @@ -707,7 +710,7 @@ int cluster_gcs_reply_timeout_ms = 5000; * HC92 + HC97 — see cluster_guc.h for semantics. */ int cluster_gcs_block_retransmit_max_retries = 4; -int cluster_gcs_block_retransmit_initial_backoff_ms = 100; +int cluster_gcs_block_retransmit_initial_backoff_ms = 10; /* spec-7.2 D1 */ /* PGRAC: spec-4.7a D2/Q8 — hold-until-revoked node-level PCM cache kill-switch. * ON (default): the bufmgr acquire gate skips the remote master round-trip when @@ -720,6 +723,7 @@ bool cluster_gcs_block_local_cache = true; * remote row-lock conflict block until the holder completes; off reverts to * the spec-3.4d fail-closed (53R98) honest degradation. */ bool cluster_tx_enqueue_wait_enabled = true; +bool cluster_ic_duty_lazy = true; /* spec-7.2 D1 duty-chain on-demand gating */ int cluster_gcs_block_dedup_max_entries = 1024; /* @@ -2374,6 +2378,21 @@ cluster_init_guc(void) "request with a fresh deadline."), &cluster_grd_rebuild_timeout_ms, 5000, 100, 600000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_backoff_ms", + gettext_noop("Initial backoff before retrying a BLOCKED HW remaster worker (ms)."), + gettext_noop("Range [100, 60000]. Default 1000. Same-episode HW remaster " + "retries use exponential backoff capped at 60 seconds; the adopted " + "shards stay frozen while retrying."), + &cluster_hw_remaster_retry_backoff_ms, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_max_attempts", + gettext_noop("Maximum same-episode retries for a BLOCKED HW remaster worker."), + gettext_noop("Range [0, 1000]. Default 16. Zero disables same-episode retry. " + "Raising the value with SIGHUP lets an exhausted episode resume retrying " + "without waiting for a new reconfig episode."), + &cluster_hw_remaster_retry_max_attempts, 16, 0, 1000, PGC_SIGHUP, 0, NULL, NULL, NULL); /* spec-2.23 D11 NEW: coordinator REPORT collect deadline. */ DefineCustomIntVariable("cluster.lmd_probe_collect_timeout_ms", @@ -2573,6 +2592,15 @@ cluster_init_guc(void) &cluster_lmon_main_loop_interval, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.lmon_slow_iteration_warn_ms", + gettext_noop("LMON main-loop slow-iteration warning threshold in milliseconds."), + gettext_noop("A value greater than zero logs LMON main-loop iterations above this " + "duration; the lmon_slow_iter_count counter is maintained even when " + "logging is disabled with 0."), + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, + NULL); + DefineCustomIntVariable( "cluster.lck_main_loop_interval", gettext_noop("LCK main-loop tick interval in milliseconds."), @@ -3855,14 +3883,28 @@ cluster_init_guc(void) &cluster_tx_enqueue_wait_enabled, true, PGC_SUSET, 0, NULL, NULL, NULL); + DefineCustomBoolVariable( + "cluster.ic_duty_lazy", + gettext_noop("Run lazy-able LMON duty-chain drains on demand instead of every iteration."), + gettext_noop("When on (default), the queue-consumption duty families in the " + "LMON main loop (outbound drains / ship-ready / sinval out / " + "tt-hint / dedup TTL / backup / GES work queue) run only when " + "their producer marked them dirty or on the >= 1 Hz floor. " + "Correctness families (fence / sweeps / reconfig / recovery) " + "always run every iteration. Off restores the run-every-" + "iteration behavior (escape hatch). spec-7.2 D1. PGC_SIGHUP."), + &cluster_ic_duty_lazy, true, PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable( "cluster.gcs_block_retransmit_initial_backoff_ms", gettext_noop("Initial backoff before retry 1 (subsequent retries double)."), gettext_noop("Exponential backoff base for GCS block-ship retransmit: " "retry 1 waits this much, retry 2 doubles, etc. Default " - "100 → 100/200/400/800 ms for N=4 retries (total 1500 ms). " - "HC97. PGC_SUSET."), - &cluster_gcs_block_retransmit_initial_backoff_ms, 100, 10, 5000, PGC_SUSET, 0, NULL, NULL, + "10 → 10/20/40/80 ms for N=4 retries (total 150 ms; LAN " + "RTT scale, spec-7.2 D1). Raise on slow or congested " + "interconnects. The per-attempt reply wait itself stays " + "cluster.gcs_reply_timeout_ms. HC97. PGC_SUSET."), + &cluster_gcs_block_retransmit_initial_backoff_ms, 10, 1, 5000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("cluster.gcs_block_dedup_max_entries", diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index d1ade2a760..f2231b59b0 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -61,6 +61,7 @@ #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/fd.h" +#include "storage/ipc.h" #include "utils/wait_event.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ @@ -74,6 +75,84 @@ #include "cluster/cluster_wal_thread.h" /* node id -> thread id */ #include "cluster/storage/cluster_undo_xlog.h" /* xl_hw_reserve, XLOG_HW_RESERVE */ +/* One worker process owns exactly one dead origin / episode. */ +static int hw_worker_dead_node = -1; +static uint64 hw_worker_episode = 0; +static bool hw_worker_armed = false; + +static uint64 +hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) +{ + uint32 backoff_ms; + TimestampTz deadline; + + backoff_ms = cluster_hw_remaster_compute_backoff_ms(cluster_hw_remaster_retry_backoff_ms, + completed_retry_attempts); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), (int)backoff_ms); + return (uint64)deadline; +} + +static void +hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) +{ + /* Amendment v1.2 (R6): the worker stores next_attempt_at BEFORE the + * terminal result, and the LMON relaunch decision reads them in the + * opposite order (result first). Order the two stores so a decider + * that observes the terminal result also observes the matching backoff + * deadline — without the fence a weakly-ordered CPU could let it pair a + * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait + * (bounded self-correcting, but cheap to close outright). r3-P3(b): + * paired with the pg_read_barrier() between the result and deadline + * loads in cluster_hw_remaster_launch_workers (acquire/release pair). */ + if (res == CLUSTER_HW_REMASTER_DONE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_done(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { + uint32 attempts = cluster_hw_remaster_attempts(dead_node); + + cluster_hw_remaster_set_next_attempt_at(dead_node, + hw_remaster_next_attempt_deadline(attempts)); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + /* Amendment v1.2 (R7): leave next_attempt_at at 0 (not NO_DEADLINE) + * so the FSM's next tick takes the MARK_STRUCTURAL decision branch + * and emits the episode-once operator WARNING + errhint — covering + * the SIGHUP race where only the WORKER (not the launch-side + * precheck) discovers the structural cause. MARK_STRUCTURAL then + * stamps NO_DEADLINE, making the WARNING once-per-episode. */ + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + } +} + +static void +hw_remaster_mark_blocked_on_exit(int code pg_attribute_unused(), Datum arg) +{ + int dead_node = DatumGetInt32(arg); + + if (!hw_worker_armed) + return; + if (dead_node != hw_worker_dead_node) + return; + if (cluster_hw_remaster_launched_episode(dead_node) != hw_worker_episode) + return; + if (cluster_hw_remaster_result(dead_node) != CLUSTER_HW_REMASTER_RUNNING) + return; + + cluster_hw_bump_failclosed(); + hw_remaster_record_terminal(dead_node, CLUSTER_HW_REMASTER_BLOCKED); +} + /* ============================================================ * Minimal per-thread WAL reader over a dead origin's WAL stream. @@ -332,10 +411,24 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; if (dead_node_id < 0 || dead_node_id == cluster_node_id) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; + if (!cluster_hw_remaster_recoverable()) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: cluster.wal_threads_dir is not configured; " + "adopted shards stay fail-closed"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "restart, or restart the dead node so a JOIN rebuild can replace " + "the failure-driven remaster."))); + return CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL; + } dead_tid = cluster_wal_thread_id_for(true, dead_node_id); - if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) + if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d maps to invalid WAL thread id %u; " + "adopted shards stay fail-closed", + dead_node_id, (unsigned)dead_tid))); return CLUSTER_HW_REMASTER_BLOCKED; + } entries = (ClusterHwSnapshotEntry *)palloc(sizeof(ClusterHwSnapshotEntry) * CLUSTER_HW_AUTHORITY_MAX); @@ -384,6 +477,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_wal_state_read_slot(dead_tid, &slot) != CLUSTER_WAL_SLOT_OK || slot.highest_lsn == 0) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d WAL state slot is unusable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } validated_min = (XLogRecPtr)slot.highest_lsn; @@ -391,6 +487,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_thread_recovery_validated_end(dead_tid, lower, validated_min, &upper) != CLUSTER_THREADREC_DONE) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d validated WAL end is unavailable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } @@ -466,29 +565,26 @@ cluster_hw_remaster_worker_main(Datum main_arg) /* The bgworker framework starts the entry point with signals blocked; unblock * before any I/O so a SIGTERM during a stuck shared-storage read / shutdown is - * delivered (the default die handler FATALs, a clean fail-closed: unmarked). */ + * delivered (the default die handler FATALs into the BLOCKED exit callback). */ BackgroundWorkerUnblockSignals(); /* Capture the live reconfig episode this rebuild is for; rebuild_origin uses * it as the staleness fence before it marks any shard rebuilt. */ episode = cluster_grd_redeclare_episode_epoch(); - res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_worker_dead_node = dead_node; + hw_worker_episode = episode; + hw_worker_armed = true; + before_shmem_exit(hw_remaster_mark_blocked_on_exit, Int32GetDatum(dead_node)); - /* Observability (S5/S7): a DONE rebuilt + adopted + marked the shards; a - * BLOCKED kept them frozen (fail-closed). NOT_APPLICABLE (HW inactive) is - * neither. */ - if (res == CLUSTER_HW_REMASTER_DONE) - cluster_hw_bump_remaster_done(); - else if (res == CLUSTER_HW_REMASTER_BLOCKED) - cluster_hw_bump_remaster_blocked(); + res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_remaster_record_terminal(dead_node, res); + hw_worker_armed = false; ereport(LOG, (errmsg("cluster HW remaster worker: dead node %d -> %s", dead_node, - res == CLUSTER_HW_REMASTER_DONE ? "done" - : res == CLUSTER_HW_REMASTER_BLOCKED ? "blocked (shards kept frozen)" - : "not applicable"))); + cluster_hw_remaster_result_name(res)))); /* Returning is a clean exit(0). On BLOCKED / abnormal exit the adopted shards - * stay unmarked, so the serve gate keeps them fail-closed (8.A) and a later - * episode relaunches the rebuild -- no abnormal-exit handler is needed. */ + * stay unmarked, so the serve gate keeps them fail-closed (8.A); the terminal + * result lets the LMON FSM retry within the same episode when appropriate. */ } /* @@ -522,6 +618,7 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod { int node; int max_node; + uint64 now; if (dead == NULL || nwords <= 0) return; @@ -538,31 +635,103 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod max_node = nwords * 64; if (max_node > CLUSTER_MAX_NODES) max_node = CLUSTER_MAX_NODES; + now = (uint64)GetCurrentTimestamp(); for (node = 0; node < max_node; node++) { + ClusterHwRemasterResult result; + ClusterHwRemasterRelaunchDecision d; + uint64 launched; + uint64 next_attempt_at; + uint32 attempts; + if ((dead[node / 64] & (UINT64CONST(1) << (node % 64))) == 0) continue; if (node == cluster_node_id) continue; /* self is never its own dead origin */ - /* Idempotent: launch once per episode per dead origin. */ - if (cluster_hw_remaster_launched_episode(node) == episode_epoch) + + launched = cluster_hw_remaster_launched_episode(node); + if (!cluster_hw_remaster_recoverable() && launched != episode_epoch) { + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, 0); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL); + launched = episode_epoch; + } + + result = cluster_hw_remaster_result(node); + /* r3-P3(b) — read-side pairing for the hw_remaster_record_terminal + * write fence (deadline stored BEFORE the terminal result, with + * pg_memory_barrier between). Load the result FIRST, fence, then the + * deadline: a decider that observed a terminal result is then + * guaranteed to observe the matching backoff deadline. Without this + * the release fence is one-sided and a weakly-ordered reader could + * still pair a fresh BLOCKED with a stale (zero) deadline. attempts + * needs no fence: it is written only by this FSM (single-writer + * LMON). */ + pg_read_barrier(); + attempts = cluster_hw_remaster_attempts(node); + next_attempt_at = cluster_hw_remaster_next_attempt_at(node); + d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, + next_attempt_at, now, + cluster_hw_remaster_retry_max_attempts); + + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster is structurally blocked for dead node %d; " + "adopted shards stay fail-closed", + node), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root " + "and restart, or restart the dead node so a JOIN rebuild can " + "replace the failure-driven remaster."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + cluster_hw_bump_remaster_retry_exhausted(); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster retries exhausted for dead node %d after %u " + "attempt(s); adopted shards stay fail-closed", + node, attempts), + errhint("Fix the shared HW snapshot/WAL source, then raise " + "cluster.hw_remaster_retry_max_attempts with SIGHUP to resume " + "same-episode retrying."))); continue; - /* - * Claim BEFORE registering so a same-episode re-entry does not double- - * launch; revert on a registration failure so a later tick retries (the - * adopted shards stay fail-closed meanwhile, via the P7 gate). - */ + } + if (d.action != CLUSTER_HW_REMASTER_LAUNCH_INITIAL + && d.action != CLUSTER_HW_REMASTER_LAUNCH_RETRY) + continue; + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, d.next_attempts); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_RUNNING); if (!register_one_worker(node)) { - cluster_hw_remaster_set_launched(node, 0); + cluster_hw_remaster_set_launched( + node, d.action == CLUSTER_HW_REMASTER_LAUNCH_INITIAL ? 0 : launched); + cluster_hw_remaster_set_attempts(node, attempts); + cluster_hw_remaster_set_next_attempt_at(node, next_attempt_at); + cluster_hw_remaster_set_result(node, result); ereport(WARNING, (errmsg("could not register HW remaster worker for dead node %d", node), errhint("Background worker slots are exhausted (max_worker_processes); the " "adopted shards stay fail-closed until the rebuild can run."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_RETRY) { + cluster_hw_bump_remaster_retry(); + ereport(LOG, + (errmsg("cluster HW remaster: retrying dead node %d in episode " UINT64_FORMAT + " (attempt %u/%d)", + node, episode_epoch, d.next_attempts, + cluster_hw_remaster_retry_max_attempts))); } } } + bool cluster_hw_remaster_gate_unfreeze(void) { diff --git a/src/backend/cluster/cluster_hw_shmem.c b/src/backend/cluster/cluster_hw_shmem.c index 14bf3f7586..860241e8e9 100644 --- a/src/backend/cluster/cluster_hw_shmem.c +++ b/src/backend/cluster/cluster_hw_shmem.c @@ -100,6 +100,8 @@ typedef struct ClusterHwShared { pg_atomic_uint64 not_ready_count; /* 53RA6 serve-gate (shard adopted, unrebuilt) */ pg_atomic_uint64 remaster_done_count; /* online-remaster HW rebuild DONE (S5/S7) */ pg_atomic_uint64 remaster_blocked_count; /* online-remaster HW rebuild fail-closed (S5/S7) */ + pg_atomic_uint64 remaster_retry_count; /* same-episode retry launches */ + pg_atomic_uint64 remaster_retry_exhausted_count; /* same-episode retry cap reached */ pg_atomic_uint32 hw_rebuilt_generation[PGRAC_GRD_SHARD_COUNT]; /* §3.1b R4/R9 gate */ /* * remaster_launched_episode[node] -- the reconfig episode for which the GRD @@ -111,6 +113,9 @@ typedef struct ClusterHwShared { * race on this field. */ pg_atomic_uint64 remaster_launched_episode[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_result[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_attempts[CLUSTER_MAX_NODES]; + pg_atomic_uint64 remaster_next_attempt_at[CLUSTER_MAX_NODES]; } ClusterHwShared; /* @@ -196,12 +201,18 @@ cluster_hw_shmem_init(void) pg_atomic_init_u64(&hw_state->not_ready_count, 0); pg_atomic_init_u64(&hw_state->remaster_done_count, 0); pg_atomic_init_u64(&hw_state->remaster_blocked_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_exhausted_count, 0); /* No shard rebuilt yet; 0 matches a never-remastered shard's GRD * master_generation (0), so boot / steady state serves. */ for (s = 0; s < PGRAC_GRD_SHARD_COUNT; s++) pg_atomic_init_u32(&hw_state->hw_rebuilt_generation[s], 0); - for (s = 0; s < CLUSTER_MAX_NODES; s++) + for (s = 0; s < CLUSTER_MAX_NODES; s++) { pg_atomic_init_u64(&hw_state->remaster_launched_episode[s], 0); + pg_atomic_init_u32(&hw_state->remaster_result[s], (uint32)CLUSTER_HW_REMASTER_NONE); + pg_atomic_init_u32(&hw_state->remaster_attempts[s], 0); + pg_atomic_init_u64(&hw_state->remaster_next_attempt_at[s], 0); + } } memset(&hctl, 0, sizeof(hctl)); @@ -408,6 +419,81 @@ cluster_hw_remaster_set_launched(int node_id, uint64 episode) pg_atomic_write_u64(&hw_state->remaster_launched_episode[node_id], episode); } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return CLUSTER_HW_REMASTER_NONE; + return (ClusterHwRemasterResult)pg_atomic_read_u32(&hw_state->remaster_result[node_id]); +} + +void +cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_result[node_id], (uint32)result); +} + +uint32 +cluster_hw_remaster_attempts(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u32(&hw_state->remaster_attempts[node_id]); +} + +void +cluster_hw_remaster_set_attempts(int node_id, uint32 attempts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_attempts[node_id], attempts); +} + +uint64 +cluster_hw_remaster_next_attempt_at(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&hw_state->remaster_next_attempt_at[node_id]); +} + +void +cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u64(&hw_state->remaster_next_attempt_at[node_id], ts); +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result) +{ + switch (result) { + case CLUSTER_HW_REMASTER_NONE: + return "none"; + case CLUSTER_HW_REMASTER_RUNNING: + return "running"; + case CLUSTER_HW_REMASTER_DONE: + return "done"; + case CLUSTER_HW_REMASTER_BLOCKED: + return "blocked"; + case CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL: + return "blocked_structural"; + case CLUSTER_HW_REMASTER_NOT_APPLICABLE: + return "not_applicable"; + } + return "unknown"; +} + +bool +cluster_hw_remaster_recoverable(void) +{ + return !cluster_hw_authority_active() + || (cluster_wal_threads_dir != NULL && cluster_wal_threads_dir[0] != '\0'); +} + /* ============================================================ * Checkpoint write + recovery load (§3.1b R1/R2; PG-core hooked). @@ -747,6 +833,16 @@ cluster_hw_bump_remaster_blocked(void) { HW_BUMP(remaster_blocked_count); } +void +cluster_hw_bump_remaster_retry(void) +{ + HW_BUMP(remaster_retry_count); +} +void +cluster_hw_bump_remaster_retry_exhausted(void) +{ + HW_BUMP(remaster_retry_exhausted_count); +} uint64 cluster_hw_alloc_count(void) @@ -788,3 +884,13 @@ cluster_hw_remaster_blocked_count(void) { return HW_READ(remaster_blocked_count); } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return HW_READ(remaster_retry_count); +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return HW_READ(remaster_retry_exhausted_count); +} diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index edec2444c7..e7e96e7911 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -533,7 +533,13 @@ cluster_ic_recv_exact(int32 *out_sender_node_id, void *buf, size_t bufsize, * 6 2 envelope_version (PGRAC_IC_ENVELOPE_VERSION_V1 = 1) * 8 4 source_node_id (signed int32) * 12 24 cluster_name (NUL-terminated; truncated to 23 + NUL) - * 36 28 _pad (always zero on send;ignored on receive) + * 36 4 capabilities bitmask (smart-fusion; zero = none) + * 40 1 plane (spec-7.2: 0 = CONTROL, 1 = DATA) + * 41 2 worker_id / n_workers (spec-7.3 reserved, zero in 7.2) + * 43 1 reserved (zero) + * 44 8 conn_epoch (spec-7.2: cluster epoch at connect; + * zero from pre-7.2 senders / unenforced on CONTROL) + * 52 12 _pad tail (always zero on send;ignored on receive) * ------ ----- * total 64 * @@ -574,9 +580,20 @@ ic_le_read_uint32(const uint8 *buf) | ((uint32)buf[3] << 24); } +/* PGRAC: spec-7.2 D2 — 64-bit LE helper (HELLO conn_epoch). */ +static inline void +ic_le_write_uint64(uint8 *buf, uint64 v) +{ + int i; + + for (i = 0; i < 8; i++) + buf[i] = (uint8)((v >> (8 * i)) & 0xFF); +} + void cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version, - uint16 envelope_version, int32 source_node_id, const char *cluster_name) + uint16 envelope_version, int32 source_node_id, const char *cluster_name, + ClusterICPlane plane, uint64 conn_epoch) { size_t name_len; uint32 capabilities = 0; @@ -608,6 +625,14 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version capabilities |= PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); + + /* + * PGRAC: spec-7.2 D2 — plane byte + connection epoch ride the + * documented-zero pad (capabilities precedent; V1 stays 64B). + * Offsets 41/42 stay zero (spec-7.3 worker fields). + */ + out_buf[PGRAC_IC_HELLO_PLANE_OFFSET] = (uint8)plane; + ic_le_write_uint64(out_buf + PGRAC_IC_HELLO_CONN_EPOCH_OFFSET, conn_epoch); } bool diff --git a/src/backend/cluster/cluster_ic_rdma.c b/src/backend/cluster/cluster_ic_rdma.c index 8b2fc19ff5..787351791f 100644 --- a/src/backend/cluster/cluster_ic_rdma.c +++ b/src/backend/cluster/cluster_ic_rdma.c @@ -40,6 +40,7 @@ #include "utils/timestamp.h" #include "cluster/cluster_guc.h" #include "cluster/cluster_conf.h" +#include "cluster/cluster_epoch.h" /* PGRAC: spec-7.2 D2 HELLO conn_epoch */ #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_rdma.h" @@ -600,8 +601,12 @@ rdma_build_private_data(ClusterICRdmaPrivateData *private_data, uint8 lane_type) } private_data->lane_type = lane_type; private_data->capabilities = CLUSTER_IC_RDMA_PRIVATE_CAP_BLOCK_REPLY; + /* PGRAC: spec-7.2 D2 — RDMA lanes are CONTROL-plane owned (LMON); + * the DATA plane is TCP-only in 7.2 (RDMA data plane = spec-6.13 + * forward). conn_epoch rides along but is unenforced on CONTROL. */ cluster_ic_build_hello(private_data->hello, PGRAC_IC_HELLO_VERSION_V1, - PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, cluster_name); + PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, cluster_name, + CLUSTER_IC_PLANE_CONTROL, cluster_epoch_get_current()); } static bool diff --git a/src/backend/cluster/cluster_ic_router.c b/src/backend/cluster/cluster_ic_router.c index d8c7eddbaf..0c7f6900ad 100644 --- a/src/backend/cluster/cluster_ic_router.c +++ b/src/backend/cluster/cluster_ic_router.c @@ -204,6 +204,25 @@ cluster_ic_send_envelope(uint8 msg_type, int32 dest_node_id, const void *payload if (dest_node_id == cluster_node_id) return CLUSTER_IC_SEND_DONE; + /* + * PGRAC: spec-7.2 D3 — (3b) plane gate on the PHYSICAL send leg + * (after the self short-circuit, which stays open to every mask- + * allowed logical producer). A wire send uses this process's fds, + * so the msg_type's plane must be the plane this process owns: + * LMS emitting a CONTROL message must stage it into the CONTROL + * outbound ring for LMON instead (r4 ruling), and nothing but LMS + * may put a DATA frame on the wire. ereport — a cross-plane send + * is a coding defect, not a runtime condition to tolerate. + */ + if ((ClusterICPlane)info->plane != cluster_ic_tier1_my_plane()) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cluster_ic msg_type %u (\"%s\") is plane-%d; cannot send from " + "plane-%d owner process", + msg_type, info->name, (int)info->plane, (int)cluster_ic_tier1_my_plane()), + errhint("Stage the message into the owning plane's outbound ring instead " + "(spec-7.2 staging rule)."))); + /* (4) broadcast destination check (chunk wrap inherits broadcast_ok=true * implicit -- chunked send is always unicast or broadcast at caller level). */ @@ -317,6 +336,25 @@ cluster_ic_dispatch_envelope(const ClusterICEnvelope *env, const void *payload, if (env->dest_node_id == PGRAC_IC_BROADCAST && !info->broadcast_ok) return false; + /* + * PGRAC: spec-7.2 D3 — plane misroute gate. A msg_type is + * dispatched ONLY in its owning plane's process: a DATA frame + * arriving in LMON (or a CONTROL frame in LMS) is a wiring or + * routing defect, never "close enough" — drop the frame fail-closed + * and count it (plane_misroute_reject on this plane's tier1 + * instance). Return true: the peer connection itself is healthy + * (unlike the unregistered/forged-broadcast rejections above, which + * are peer-level failures). + */ + if ((ClusterICPlane)info->plane != cluster_ic_tier1_my_plane()) { + cluster_ic_tier1_bump_plane_misroute_reject(); + ereport(LOG, (errmsg("cluster_ic: dropping msg_type %u (\"%s\") — plane %d frame " + "dispatched in plane-%d owner process", + env->msg_type, info->name, (int)info->plane, + (int)cluster_ic_tier1_my_plane()))); + return true; + } + /* * spec-2.3 §3.5 + Q14 + R3 防御层: PG_TRY/PG_CATCH wrap. Catches * elevel == ERROR (handler violated §3.5 by raising ERROR) -- LOG diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 2ab1516045..fc3113d743 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -73,6 +73,7 @@ #include "cluster/cluster_conf.h" #include "cluster/cluster_elog.h" +#include "cluster/cluster_epoch.h" /* PGRAC: spec-7.2 D2 HELLO conn_epoch */ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_chunk.h" /* cluster_ic_chunk_reset_peer (spec-2.4) */ #include "cluster/cluster_ic.h" @@ -112,11 +113,30 @@ typedef struct ClusterICTier1Shmem { pid_t listener_pid; /* current LMON pid */ uint64 listener_incarnation; /* ++ on each LMON respawn */ uint32 magic; /* sanity check */ + /* PGRAC: spec-7.2 D3 — frames dropped by the dispatch plane gate + * (a msg_type dispatched in the wrong plane's owner process). */ + pg_atomic_uint64 plane_misroute_reject; ClusterICPeerStateShmem peers[CLUSTER_MAX_NODES]; } ClusterICTier1Shmem; #define PGRAC_IC_TIER1_SHMEM_MAGIC ((uint32)0x54494331U) /* "TIC1" */ +/* + * PGRAC: spec-7.2 D2 — per-plane shmem instances inside the single + * "pgrac cluster_ic_tier1" region ([0] = CONTROL, [1] = DATA). + * + * Tier1Shmem stays the working alias and points at THIS process's + * plane instance: process-local tier1 state (fd arrays, buffers, + * HELLO state machines) is naturally per-process, so the plane a + * process owns is implied by the process — LMON = CONTROL (the + * default), LMS = DATA (set via cluster_ic_tier1_set_my_plane at + * LmsMain entry). Every existing tier1 function keeps reading + * Tier1Shmem unchanged and lands on its own plane's peers[] / + * listener metadata / CONNECTED gate. Cross-plane observers (SQL + * views) address Tier1ShmemByPlane[] explicitly. + */ +static ClusterICTier1Shmem *Tier1ShmemByPlane[CLUSTER_IC_PLANE_N] = { NULL, NULL }; +static ClusterICPlane tier1_my_plane = CLUSTER_IC_PLANE_CONTROL; static ClusterICTier1Shmem *Tier1Shmem = NULL; /* @@ -364,6 +384,7 @@ static const char * peer_addr(int32 peer_id) { const ClusterNodeInfo *n; + const char *addr; if (peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) return NULL; @@ -372,12 +393,24 @@ peer_addr(int32 peer_id) if (n == NULL) return NULL; + /* + * PGRAC: spec-7.2 D2 — plane-selected peer address. The DATA plane + * connects to the peer's declared data_addr; a peer without one + * offers no DATA plane and is unreachable on it (NULL, no connect + * attempts — never fall back to the CONTROL address: r1-F2 no + * port guessing, and mixing planes on one socket breaks the + * per-plane single-stream ordering). + */ + addr = (tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? n->data_addr : n->interconnect_addr; + if (addr[0] == '\0') + return NULL; + if (Tier1Shmem != NULL && Tier1Shmem->peers[peer_id].interconnect_addr[0] == '\0') { - strlcpy(Tier1Shmem->peers[peer_id].interconnect_addr, n->interconnect_addr, + strlcpy(Tier1Shmem->peers[peer_id].interconnect_addr, addr, sizeof(Tier1Shmem->peers[peer_id].interconnect_addr)); Tier1Shmem->peers[peer_id].node_id = peer_id; } - return n->interconnect_addr; + return addr; } @@ -388,53 +421,109 @@ peer_addr(int32 peer_id) static Size tier1_shmem_size(void) { - return MAXALIGN(sizeof(ClusterICTier1Shmem)); + /* PGRAC: spec-7.2 D2 — one instance per plane (CONTROL + DATA). */ + return CLUSTER_IC_PLANE_N * MAXALIGN(sizeof(ClusterICTier1Shmem)); } static void tier1_shmem_init(void) { bool found; + char *base; + int plane; - Tier1Shmem = (ClusterICTier1Shmem *)ShmemInitStruct("pgrac cluster_ic_tier1", - tier1_shmem_size(), &found); + base = (char *)ShmemInitStruct("pgrac cluster_ic_tier1", tier1_shmem_size(), &found); - if (!found) { - int i; + /* PGRAC: spec-7.2 D2 — carve one instance per plane; the working + * alias follows this process's plane (CONTROL unless + * cluster_ic_tier1_set_my_plane re-aims it, i.e. in LMS). */ + for (plane = 0; plane < CLUSTER_IC_PLANE_N; plane++) + Tier1ShmemByPlane[plane] + = (ClusterICTier1Shmem *)(base + plane * MAXALIGN(sizeof(ClusterICTier1Shmem))); + Tier1Shmem = Tier1ShmemByPlane[tier1_my_plane]; - memset(Tier1Shmem, 0, tier1_shmem_size()); - Tier1Shmem->magic = PGRAC_IC_TIER1_SHMEM_MAGIC; - Tier1Shmem->listener_port = -1; - Tier1Shmem->listener_pid = 0; - Tier1Shmem->listener_incarnation = 0; - - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - Tier1Shmem->peers[i].node_id = -1; - Tier1Shmem->peers[i].state = (int32)CLUSTER_IC_PEER_DOWN; - Tier1Shmem->peers[i].last_errno = 0; - Tier1Shmem->peers[i].last_connect_at = 0; - pg_atomic_init_u64(&Tier1Shmem->peers[i].heartbeat_send_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].heartbeat_recv_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].msg_send_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].msg_recv_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].bytes_send, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].bytes_recv, 0); - /* spec-2.4 D10 NEW counters */ - pg_atomic_init_u64(&Tier1Shmem->peers[i].stale_epoch_drop_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].lamport_observe_advance_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].chunk_reassembly_timeout_count, 0); - pg_atomic_init_u32(&Tier1Shmem->peers[i].chunk_reassembly_active, 0); - /* spec-2.29 D20 + D18b: hostile-spoof + epoch piggyback counters. */ - pg_atomic_init_u64(&Tier1Shmem->peers[i].unreasonable_epoch_jump_count, 0); - pg_atomic_init_u64(&Tier1Shmem->peers[i].epoch_observe_advance_count, 0); - /* spec-2.4 v1.0.1 F3: LMON-mediated close request flag. */ - pg_atomic_init_u32(&Tier1Shmem->peers[i].close_requested, 0); + if (!found) { + memset(base, 0, tier1_shmem_size()); + + for (plane = 0; plane < CLUSTER_IC_PLANE_N; plane++) { + ClusterICTier1Shmem *s = Tier1ShmemByPlane[plane]; + int i; + + s->magic = PGRAC_IC_TIER1_SHMEM_MAGIC; + s->listener_port = -1; + s->listener_pid = 0; + s->listener_incarnation = 0; + pg_atomic_init_u64(&s->plane_misroute_reject, 0); + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + s->peers[i].node_id = -1; + s->peers[i].state = (int32)CLUSTER_IC_PEER_DOWN; + s->peers[i].last_errno = 0; + s->peers[i].last_connect_at = 0; + pg_atomic_init_u64(&s->peers[i].heartbeat_send_count, 0); + pg_atomic_init_u64(&s->peers[i].heartbeat_recv_count, 0); + pg_atomic_init_u64(&s->peers[i].msg_send_count, 0); + pg_atomic_init_u64(&s->peers[i].msg_recv_count, 0); + pg_atomic_init_u64(&s->peers[i].bytes_send, 0); + pg_atomic_init_u64(&s->peers[i].bytes_recv, 0); + /* spec-2.4 D10 NEW counters */ + pg_atomic_init_u64(&s->peers[i].stale_epoch_drop_count, 0); + pg_atomic_init_u64(&s->peers[i].lamport_observe_advance_count, 0); + pg_atomic_init_u64(&s->peers[i].chunk_reassembly_timeout_count, 0); + pg_atomic_init_u32(&s->peers[i].chunk_reassembly_active, 0); + /* spec-2.29 D20 + D18b: hostile-spoof + epoch piggyback counters. */ + pg_atomic_init_u64(&s->peers[i].unreasonable_epoch_jump_count, 0); + pg_atomic_init_u64(&s->peers[i].epoch_observe_advance_count, 0); + /* spec-2.4 v1.0.1 F3: LMON-mediated close request flag. */ + pg_atomic_init_u32(&s->peers[i].close_requested, 0); + s->peers[i].conn_epoch = 0; + } } } Assert(Tier1Shmem->magic == PGRAC_IC_TIER1_SHMEM_MAGIC); } +/* + * PGRAC: spec-7.2 D2 — re-aim the working alias at this process's plane. + * + * Called once at aux-process entry BEFORE any tier1 use (LmsMain → + * DATA). Processes that never call it stay on CONTROL, which keeps + * LMON, backends and every SQL observer on the historic instance. + */ +void +cluster_ic_tier1_set_my_plane(ClusterICPlane plane) +{ + Assert(plane >= 0 && plane < CLUSTER_IC_PLANE_N); + tier1_my_plane = plane; + if (Tier1ShmemByPlane[plane] != NULL) + Tier1Shmem = Tier1ShmemByPlane[plane]; +} + +/* PGRAC: spec-7.2 D3 — this process's plane (router plane gates). */ +ClusterICPlane +cluster_ic_tier1_my_plane(void) +{ + return tier1_my_plane; +} + +/* PGRAC: spec-7.2 D3 — dispatch plane-gate drop counter (this plane's + * instance; pre-shmem no-op keeps the router callable in unit stubs). */ +void +cluster_ic_tier1_bump_plane_misroute_reject(void) +{ + if (Tier1Shmem != NULL) + pg_atomic_fetch_add_u64(&Tier1Shmem->plane_misroute_reject, 1); +} + +uint64 +cluster_ic_tier1_get_plane_misroute_reject(ClusterICPlane plane) +{ + if (plane < 0 || plane >= CLUSTER_IC_PLANE_N || Tier1ShmemByPlane[plane] == NULL) + return 0; + return pg_atomic_read_u64(&Tier1ShmemByPlane[plane]->plane_misroute_reject); +} + static const ClusterShmemRegion cluster_ic_tier1_region = { .name = "pgrac cluster_ic_tier1", .size_fn = tier1_shmem_size, @@ -511,6 +600,20 @@ tier1_send_bytes(int32 target_node_id, const void *buf, size_t len) || Tier1Shmem->peers[target_node_id].state != (int32)CLUSTER_IC_PEER_CONNECTED) return CLUSTER_IC_SEND_WOULD_BLOCK; /* HELLO pending; caller retries */ + /* + * PGRAC: spec-7.2 D5 (INV-7.2-CONN-EPOCH sender gate) — never put a + * new epoch's DATA frame on a connection established under an older + * epoch: cross-epoch interleave on one byte stream would defeat the + * reconfig-rebuild ordering argument (8.A face). HARD_ERROR makes + * the data-plane loop close the peer; reconnect + re-HELLO rebinds + * at the current epoch (the epoch-watch in the LMS tick also force- + * closes proactively on a bump). CONTROL is exempt — its single + * LMON stream is the epoch-event carrier itself. + */ + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA + && Tier1Shmem->peers[target_node_id].conn_epoch != cluster_epoch_get_current()) + return CLUSTER_IC_SEND_HARD_ERROR; + /* * Hardening v1.0.1 F1 (spec-2.2 v1.0.1 + spec-2.3 v1.0.1 L68): * per-peer outbound buffer for partial writes. If a previous send @@ -990,10 +1093,31 @@ cluster_ic_tier1_listener_bind(void) "interconnect_addr.", cluster_node_id))); - if (!parse_host_port(self->interconnect_addr, self_host, sizeof(self_host), &self_port)) - ereport(FATAL, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("cluster_ic tier1: cannot parse interconnect_addr \"%s\"", - self->interconnect_addr))); + /* + * PGRAC: spec-7.2 D2 — plane-selected listener address. The DATA + * plane binds the node's declared data_addr; no declaration means + * this node offers no DATA plane (LOG + no listener — the caller + * treats -1 as plane-off; fail-closed enforcement arrives with + * the GCS-block plane flip). Never derived from interconnect_addr + * (r1-F2: adjacent-port collisions make offset defaults unsound). + */ + if (tier1_my_plane == CLUSTER_IC_PLANE_DATA && self->data_addr[0] == '\0') { + ereport(LOG, (errmsg("cluster_ic tier1: no data_addr declared for node %d; " + "DATA plane disabled", + cluster_node_id))); + return -1; + } + + if (!parse_host_port((tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? self->data_addr + : self->interconnect_addr, + self_host, sizeof(self_host), &self_port)) + ereport( + FATAL, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("cluster_ic tier1: cannot parse %s \"%s\"", + (tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? "data_addr" : "interconnect_addr", + (tier1_my_plane == CLUSTER_IC_PLANE_DATA) ? self->data_addr + : self->interconnect_addr))); fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (fd < 0) @@ -1221,8 +1345,11 @@ cluster_ic_tier1_finish_connect(int32 peer_id, int peer_fd) * already-sent prefix. */ self_cluster_name = (ClusterConfShmem != NULL) ? ClusterConfShmem->cluster_name : ""; + /* PGRAC: spec-7.2 D2 — HELLO carries this process's plane + the + * cluster epoch at connect time (INV-7.2-CONN-EPOCH substrate). */ cluster_ic_build_hello(tier1_hello_send_buf[peer_id], PGRAC_IC_HELLO_VERSION_V1, - PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, self_cluster_name); + PGRAC_IC_ENVELOPE_VERSION_V1, cluster_node_id, self_cluster_name, + tier1_my_plane, cluster_epoch_get_current()); tier1_hello_send_remaining[peer_id] = PGRAC_IC_HELLO_BYTES; return cluster_ic_tier1_continue_hello_send(peer_id, peer_fd); @@ -1284,6 +1411,9 @@ cluster_ic_tier1_continue_hello_send(int32 peer_id, int peer_fd) */ if (Tier1Shmem != NULL) { Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_CONNECTED; + /* PGRAC: spec-7.2 D5 — bind the connection to its epoch (dialer + * side stamped this epoch into its HELLO). */ + Tier1Shmem->peers[peer_id].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); } ereport(LOG, @@ -1338,6 +1468,28 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) return false; } + /* + * PGRAC: spec-7.2 D2 — plane match: a HELLO arriving on this + * process's listener/connection must claim the same plane (a + * CONTROL peer dialing the DATA port, or vice versa, is a wiring + * error; pre-7.2 senders read as plane 0 = CONTROL and are only + * acceptable on the CONTROL plane). On DATA the conn_epoch must + * EQUAL this node's current cluster epoch (INV-7.2-CONN-EPOCH: + * the connection is bound to the epoch it was established in — a + * dialer still on a stale epoch is refused and reconnects after + * observing the bump; fail-closed, self-healing. The initial + * epoch is legitimately 0 on both sides, so equality — not + * nonzero-ness — is the sound check; cross-version senders are + * already excluded by the version gate above). + */ + if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { + peer_record_error(peer_id, 0, "08P01", "HELLO plane mismatch (peer=%d mine=%d)", + (int)cluster_ic_hello_plane(&msg), (int)tier1_my_plane); + cluster_ic_tier1_close_peer(peer_id, "HELLO plane mismatch"); + Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_REJECTED; + return false; + } + peer_info = cluster_conf_lookup_node(msg.source_node_id); if (peer_info == NULL) { peer_record_error(peer_id, 0, "08P01", "HELLO unknown source_node_id %d", @@ -1362,6 +1514,12 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) } Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_CONNECTED; + /* PGRAC: spec-7.2 D5 — bind the connection to THIS node's current + * epoch (our own view), so the sender gate compares apples to + * apples; the peer's HELLO epoch may differ transiently during a + * cold-form / reconfig window and per-message envelope HC100 (not + * the HELLO) is the receiver-side stale-epoch guard. */ + Tier1Shmem->peers[peer_id].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(peer_id); /* cache addr in shmem for view */ cluster_sf_note_peer_hello_capabilities(peer_id, cluster_ic_hello_capabilities(&msg)); @@ -1512,6 +1670,17 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear return false; } + /* PGRAC: spec-7.2 D2 — plane match (a CONTROL peer dialing the DATA + * port, or vice versa, is a wiring error). No conn_epoch reject at + * HELLO: cross-node epoch may differ transiently during cold-form / + * reconfig, and per-message envelope HC100 is the stale-epoch guard + * (D5 §3.2 ④). */ + if (cluster_ic_hello_plane(&msg) != tier1_my_plane) { + ereport(LOG, (errmsg("cluster_ic tier1 HELLO plane mismatch (peer=%d mine=%d)", + (int)cluster_ic_hello_plane(&msg), (int)tier1_my_plane))); + return false; + } + peer_info = cluster_conf_lookup_node(msg.source_node_id); if (peer_info == NULL) { ereport(LOG, @@ -1525,6 +1694,9 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear if (Tier1Shmem != NULL) { peer_record_error(learned, 0, "", ""); /* clear any prior */ Tier1Shmem->peers[learned].state = (int32)CLUSTER_IC_PEER_CONNECTED; + /* PGRAC: spec-7.2 D5 — bind to THIS node's current epoch (our own + * view; see the named-peer path for the rationale). */ + Tier1Shmem->peers[learned].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[learned].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(learned); cluster_sf_note_peer_hello_capabilities(learned, cluster_ic_hello_capabilities(&msg)); diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..e63f85de93 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -199,6 +199,7 @@ static ClusterInjectPoint cluster_injection_points[] = { /* spec-2.6 Sprint A Step 4 D14 — 5 qvotec / quorum-lite injects. */ { .name = "cluster-qvotec-poll-pre" }, { .name = "cluster-qvotec-poll-post" }, + { .name = "cluster-qvotec-marker-service-hold" }, { .name = "cluster-voting-disk-write-fail" }, { .name = "cluster-quorum-loss-broadcast" }, { .name = "cluster-collision-detect" }, @@ -358,6 +359,20 @@ static ClusterInjectPoint cluster_injection_points[] = { * that would otherwise serve FULL/PARTIAL. */ { .name = "cluster-lms-cr-construct" }, + /* + * spec-7.2 D6 — LMS data-plane observability injections. + * + * cluster-lms-data-dispatch: + * Fires in the LMS data-plane tick just before the inbound + * envelope pump for a readable DATA peer (hit-count surface for + * the flip e2e legs). + * cluster-lms-conn-reset: + * Fires in the tick's reset scan; SKIP forces a one-shot close + * of every DATA connection (reconnect + re-HELLO converge), the + * L4-family reset/epoch legs' trigger. + */ + { .name = "cluster-lms-data-dispatch" }, + { .name = "cluster-lms-conn-reset" }, /* * spec-6.12i D-i1 — undo-TT fetch serve refusal injection. * diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..9b8e9381d8 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -183,6 +183,7 @@ cluster_lmon_shmem_init(void) memset(cluster_lmon_state, 0, sizeof(*cluster_lmon_state)); LWLockInitialize(&cluster_lmon_state->lwlock, LWTRANCHE_CLUSTER_LMON); cluster_lmon_state->status = CLUSTER_LMON_NOT_STARTED; + pg_atomic_init_u32(&cluster_lmon_state->lazy_duty_dirty, 0); } /* @@ -625,6 +626,76 @@ cluster_lmon_wakeup(void) (void)kill(pid, SIGUSR1); } +/* + * PGRAC: spec-7.2 D1 -- duty-chain on-demand gating (§3.7 classification). + * + * See the ClusterLmonDuty enum in cluster_lmon.h for the two-class + * table. The switch below has NO default case on purpose: adding an + * enum entry without classifying it here is a -Wswitch warning (CI + * warning budget catches it), and the unit truth table pins every row. + */ +bool +cluster_lmon_duty_is_lazy(ClusterLmonDuty duty) +{ + switch (duty) { + case CLUSTER_LMON_DUTY_LIVENESS: + case CLUSTER_LMON_DUTY_FENCE: + case CLUSTER_LMON_DUTY_GRD_DEAD_SWEEP: + case CLUSTER_LMON_DUTY_GRD_RECLAIM: + case CLUSTER_LMON_DUTY_GRD_STARVATION_SWEEP: + case CLUSTER_LMON_DUTY_DEADLOCK: + case CLUSTER_LMON_DUTY_GES_REPLY_WAIT_SWEEP: + case CLUSTER_LMON_DUTY_DIRECT_LAND_ABORTS: + case CLUSTER_LMON_DUTY_LMS_NATIVE_PROBE_RETRY: + case CLUSTER_LMON_DUTY_CLEAN_LEAVE: + case CLUSTER_LMON_DUTY_NODE_REMOVE: + case CLUSTER_LMON_DUTY_RECONFIG: + case CLUSTER_LMON_DUTY_GRD_RECOVERY: + case CLUSTER_LMON_DUTY_BOC_BROADCAST: + case CLUSTER_LMON_DUTY_SINVAL_RESET_ALL: + case CLUSTER_LMON_DUTY_PI_DISCARD: + return false; + case CLUSTER_LMON_DUTY_GES_WORK_QUEUE: + case CLUSTER_LMON_DUTY_SHIP_READY: + case CLUSTER_LMON_DUTY_GRD_OUTBOUND: + case CLUSTER_LMON_DUTY_SINVAL_OUT: + case CLUSTER_LMON_DUTY_SINVAL_ACK_OUT: + case CLUSTER_LMON_DUTY_TT_HINT: + case CLUSTER_LMON_DUTY_DEDUP_TTL: + case CLUSTER_LMON_DUTY_BACKUP: + return true; + case CLUSTER_LMON_DUTY_N: + break; + } + return false; +} + +void +cluster_lmon_duty_mark_dirty(ClusterLmonDuty duty) +{ + if (cluster_lmon_state == NULL) + return; /* pre-shmem: the >= 1 Hz floor covers it */ + if (!cluster_lmon_duty_is_lazy(duty)) + return; + pg_atomic_fetch_or_u32(&cluster_lmon_state->lazy_duty_dirty, + 1u << (duty - CLUSTER_LMON_DUTY_LAZY_FIRST)); +} + +bool +cluster_lmon_duty_should_run(ClusterLmonDuty duty, bool force_all) +{ + uint32 bit; + uint32 old; + + if (!cluster_lmon_duty_is_lazy(duty)) + return true; + if (!cluster_ic_duty_lazy || force_all || cluster_lmon_state == NULL) + return true; + bit = 1u << (duty - CLUSTER_LMON_DUTY_LAZY_FIRST); + old = pg_atomic_fetch_and_u32(&cluster_lmon_state->lazy_duty_dirty, ~bit); + return (old & bit) != 0; +} + TimestampTz cluster_lmon_spawned_at(void) { @@ -681,6 +752,61 @@ cluster_lmon_main_loop_iters(void) return result; } +uint64 +cluster_lmon_last_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->last_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_max_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->max_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_slow_iter_count(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->slow_iter_count; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +void +cluster_lmon_marker_complete_wakeup(void) +{ + struct Latch *latch; + + if (cluster_lmon_state == NULL) + return; + + latch = cluster_lmon_state->lmon_latch; + if (latch != NULL) + SetLatch(latch); +} + /* ============================================================ * LMON main entry (AuxiliaryProcessMain dispatch target). @@ -720,12 +846,30 @@ lmon_publish_status(ClusterLmonStatus status) cluster_lmon_state->ready_at = 0; cluster_lmon_state->last_liveness_tick_at = 0; cluster_lmon_state->main_loop_iters = 0; + cluster_lmon_state->last_iter_us = 0; + cluster_lmon_state->max_iter_us = 0; + cluster_lmon_state->slow_iter_count = 0; + cluster_lmon_state->lmon_latch = NULL; } else if (status == CLUSTER_LMON_READY) { cluster_lmon_state->ready_at = now; + cluster_lmon_state->lmon_latch = MyLatch; + } else if (status == CLUSTER_LMON_EXITED) { + cluster_lmon_state->lmon_latch = NULL; } LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_clear_latch(int code, Datum arg) +{ + if (cluster_lmon_state == NULL) + return; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->lmon_latch = NULL; + LWLockRelease(&cluster_lmon_state->lwlock); +} + static bool lmon_shutdown_requested(void) @@ -760,6 +904,44 @@ lmon_advance_liveness_tick(void) LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_record_iteration(TimestampTz iter_started_at) +{ + TimestampTz now = GetCurrentTimestamp(); + uint64 elapsed_us; + bool slow; + bool should_log = false; + static TimestampTz last_slow_log_at = 0; + + if (now <= iter_started_at) + elapsed_us = 0; + else + elapsed_us = (uint64)(now - iter_started_at); + + slow = (cluster_lmon_slow_iteration_warn_ms >= 0) + && elapsed_us >= (uint64)cluster_lmon_slow_iteration_warn_ms * 1000ULL; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->last_iter_us = elapsed_us; + if (elapsed_us > cluster_lmon_state->max_iter_us) + cluster_lmon_state->max_iter_us = elapsed_us; + if (slow) + cluster_lmon_state->slow_iter_count++; + LWLockRelease(&cluster_lmon_state->lwlock); + + if (slow && cluster_lmon_slow_iteration_warn_ms > 0 + && (last_slow_log_at == 0 + || now - last_slow_log_at >= INT64CONST(60) * INT64CONST(1000000))) { + last_slow_log_at = now; + should_log = true; + } + + if (should_log) + ereport(LOG, (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); +} + void LmonMain(void) @@ -811,6 +993,7 @@ LmonMain(void) /* Publish SPAWNING (records pid + spawned_at). */ lmon_publish_status(CLUSTER_LMON_SPAWNING); + on_shmem_exit(lmon_clear_latch, (Datum)0); /* Sprint B inject: ready-publish (test slow startup / phase 1 wait timeout). */ CLUSTER_INJECTION_POINT("cluster-lmon-ready-publish"); @@ -911,6 +1094,7 @@ LmonMain(void) int rdma_cm_fd = -1; int rdma_cq_fd = -1; TimestampTz next_heartbeat_at; + TimestampTz next_duty_floor_at; /* PGRAC: spec-7.2 D1 lazy-duty floor */ int32 self_id = cluster_node_id; int32 pi; @@ -932,13 +1116,16 @@ LmonMain(void) } next_heartbeat_at = GetCurrentTimestamp() + HEARTBEAT_INTERVAL_MS * INT64CONST(1000); + next_duty_floor_at = 0; /* PGRAC: spec-7.2 D1 — first iteration forces all duties */ for (;;) { WaitEvent ev[2 * CLUSTER_MAX_NODES + 4]; int n_events; long wait_ms; TimestampTz now; + TimestampTz iter_started_at; int32 i; + bool force_all_duties; CHECK_FOR_INTERRUPTS(); @@ -950,6 +1137,21 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); + + /* + * PGRAC: spec-7.2 D1 -- >= 1 Hz floor for the lazy duty + * families (§3.5 backstop): every lazy-able drain runs at + * least once per heartbeat interval even if its producer + * never marked it dirty. Never-lazy duties ignore this and + * run every iteration, order verbatim. Reuse the iteration + * start timestamp (stage7-p0 profiling) as "now". + */ + now = iter_started_at; + force_all_duties = (now >= next_duty_floor_at); + if (force_all_duties) + next_duty_floor_at = now + HEARTBEAT_INTERVAL_MS * INT64CONST(1000); + lmon_advance_liveness_tick(); /* @@ -1007,10 +1209,15 @@ LmonMain(void) * point but the read returns false in the skeleton path because * LMS never advances past STARTING for the ownership purpose. */ - cluster_ges_lmon_drain_work_queue(); + /* PGRAC: spec-7.2 D1 — lazy-gated (producer marks dirty at + * enqueue; >= 1 Hz floor backstops a missed mark). */ + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GES_WORK_QUEUE, force_all_duties)) + cluster_ges_lmon_drain_work_queue(); /* PGRAC: spec-6.12b — ship finished CR-server results (LMS * constructed them; only LMON owns the IC connections). */ - cluster_lms_cr_ship_ready(); + if (!cluster_gcs_block_family_on_data_plane() + && cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) + cluster_lms_cr_ship_ready(); /* * spec-5.16 (orphan-grant, Rule 8.A) — reclaim abandoned reply-wait * tombstones whose bounded TTL has elapsed. This is the documented @@ -1022,7 +1229,8 @@ LmonMain(void) * wait table full" (cap = cluster.ges_reply_wait_max_entries). */ (void)cluster_ges_reply_wait_sweep_timeout(GetCurrentTimestamp()); - cluster_grd_outbound_lmon_drain_send(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GRD_OUTBOUND, force_all_duties)) + cluster_grd_outbound_lmon_drain_send(); (void)cluster_gcs_block_lmon_drain_direct_land_aborts(); cluster_lms_native_probe_retry_tick(); @@ -1053,14 +1261,21 @@ LmonMain(void) * in LMON because LMON owns the process-local IC fds. */ cluster_scn_lmon_drain_boc_broadcast(); - cluster_sinval_drain_outbound_and_broadcast(); - cluster_sinval_drain_ack_outbound_and_send(); + /* PGRAC: spec-7.2 D1 — sinval outbound drains are lazy-gated; + * the RESET-all sentinel below stays never-lazy (order- + * sensitive protocol, §3.6). */ + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_OUT, force_all_duties)) + cluster_sinval_drain_outbound_and_broadcast(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT, force_all_duties)) + cluster_sinval_drain_ack_outbound_and_send(); cluster_sinval_broadcast_reset_all(); /* spec-3.2 D6: LMON drain cross-node TT status hint outbound. * Fire-and-forget; L172 family — only LMON owns tier1 fds. */ - cluster_tt_status_hint_drain_outbound(); - cluster_backup_lmon_tick(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_TT_HINT, force_all_duties)) + cluster_tt_status_hint_drain_outbound(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_BACKUP, force_all_duties)) + cluster_backup_lmon_tick(); /* * spec-2.34 D6 (HC93 leg a): TTL sweep of the GCS block @@ -1068,14 +1283,17 @@ LmonMain(void) * 2 × max_retransmit_window age and in-flight entries * abandoned past the same threshold. Runs in LMON context * with raw LWLock + hash_seq (per L150; no buffer pin / no - * ResourceOwner). + * ResourceOwner). PGRAC: spec-7.2 D1 — TTL/time-based, so + * floor-driven (>= 1 Hz), no producer mark. */ - cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_DEDUP_TTL, force_all_duties)) + cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); /* spec-6.12h D-h2: drain checkpoint-confirmed PI-discard write * notes and route each to the block's master (fire-and-forget; * only LMON owns the tier1 fds, L172 family). */ - cluster_gcs_block_pi_discard_drain(); + if (!cluster_gcs_block_family_on_data_plane()) + cluster_gcs_block_pi_discard_drain(); CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); @@ -1414,6 +1632,7 @@ LmonMain(void) wait_ms = 1; } + lmon_record_iteration(iter_started_at); n_events = WaitEventSetWait(wes, wait_ms, ev, lengthof(ev), WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT); @@ -1586,8 +1805,13 @@ LmonMain(void) cluster_ic_rdma_lmon_stop(); } else { /* Stub / mock / disabled mode -- preserve spec-1.11 simple loop. */ + TimestampTz next_duty_floor_at = 0; /* PGRAC: spec-7.2 D1 lazy-duty floor */ + for (;;) { int rc; + TimestampTz iter_started_at; + bool force_all_duties; + TimestampTz dnow; CHECK_FOR_INTERRUPTS(); @@ -1599,6 +1823,15 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); + + /* PGRAC: spec-7.2 D1 — >= 1 Hz floor (see the TIER_1 loop). */ + dnow = iter_started_at; + force_all_duties = (dnow >= next_duty_floor_at); + if (force_all_duties) + next_duty_floor_at + = dnow + (int64)cluster_interconnect_heartbeat_interval_ms * INT64CONST(1000); + lmon_advance_liveness_tick(); /* @@ -1654,29 +1887,40 @@ LmonMain(void) * so the outbound drain is intentionally NOT hoisted (a stub * single node has no peers to send to). */ - cluster_ges_lmon_drain_work_queue(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GES_WORK_QUEUE, force_all_duties)) + cluster_ges_lmon_drain_work_queue(); (void)cluster_gcs_block_lmon_drain_direct_land_aborts(); /* PGRAC: spec-6.12b — ship finished CR-server results (LMS * constructed them; only LMON owns the IC connections). */ - cluster_lms_cr_ship_ready(); - - cluster_sinval_drain_outbound_and_broadcast(); - cluster_sinval_drain_ack_outbound_and_send(); + if (!cluster_gcs_block_family_on_data_plane() + && cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SHIP_READY, force_all_duties)) + cluster_lms_cr_ship_ready(); + + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_OUT, force_all_duties)) + cluster_sinval_drain_outbound_and_broadcast(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT, force_all_duties)) + cluster_sinval_drain_ack_outbound_and_send(); cluster_sinval_broadcast_reset_all(); /* spec-3.2 D6: LMON drain cross-node TT status hint outbound. * Fire-and-forget; L172 family — only LMON owns tier1 fds. */ - cluster_tt_status_hint_drain_outbound(); - cluster_backup_lmon_tick(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_TT_HINT, force_all_duties)) + cluster_tt_status_hint_drain_outbound(); + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_BACKUP, force_all_duties)) + cluster_backup_lmon_tick(); - /* spec-2.34 D6 (HC93 leg a): TTL sweep GCS block dedup HTAB. */ - cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); + /* spec-2.34 D6 (HC93 leg a): TTL sweep GCS block dedup HTAB. + * PGRAC: spec-7.2 D1 — TTL/time-based, floor-driven. */ + if (cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_DEDUP_TTL, force_all_duties)) + cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); /* spec-6.12h D-h2: drain checkpoint-confirmed PI-discard notes. */ - cluster_gcs_block_pi_discard_drain(); + if (!cluster_gcs_block_family_on_data_plane()) + cluster_gcs_block_pi_discard_drain(); CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); + lmon_record_iteration(iter_started_at); rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, cluster_lmon_main_loop_interval, WAIT_EVENT_CLUSTER_BGPROC_LMON_MAIN_LOOP); diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index c3b9a39c24..590ef7eae3 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,7 +59,10 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" -#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_cssd.h" +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_write_fence.h" /* PGRAC: spec-7.2 D5 fence linkage */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" #include "cluster/cluster_grd_outbound.h" @@ -196,6 +199,8 @@ cluster_lms_shmem_init(void) * starvation counter starts at 0. */ pg_atomic_init_u64(&cluster_lms_state->lms_restart_generation, 0); pg_atomic_init_u64(&cluster_lms_state->priority_starvation_observed_count, 0); + /* spec-7.2 D6 — DATA-plane connection resets counter. */ + pg_atomic_init_u64(&cluster_lms_state->lms_conn_resets, 0); ConditionVariableInit(&cluster_lms_state->cv); } } @@ -381,6 +386,22 @@ cluster_lms_get_pid(void) return pid; } +/* + * PGRAC: spec-7.2 D4 — wake the LMS data-plane loop (mirror of + * cluster_lmon_wakeup): SIGUSR1 → lms_sigusr1_handler → SetLatch. + * Safe from any context; self-kick and pre-spawn calls no-op. + */ +void +cluster_lms_wakeup(void) +{ + pid_t pid = cluster_lms_get_pid(); + + if (pid <= 0 || pid == MyProcPid) + return; + + (void)kill(pid, SIGUSR1); +} + TimestampTz cluster_lms_get_spawned_at(void) { @@ -562,6 +583,29 @@ cluster_lms_get_priority_starvation_observed_count(void) return pg_atomic_read_u64(&cluster_lms_state->priority_starvation_observed_count); } +/* + * spec-7.2 D6 — DATA-plane connection resets counter. + * + * cluster_lms_bump_conn_resets adds the number of DATA connections a + * single reset event tore down (epoch bump in production, injection in + * the F6-1 test). Called from the LMS DATA-plane tick; a shared-memory + * atomic so pg_stat_cluster_* / the debug dump can observe reset activity. + */ +void +cluster_lms_bump_conn_resets(uint32 n) +{ + if (cluster_lms_state != NULL && n > 0) + pg_atomic_fetch_add_u64(&cluster_lms_state->lms_conn_resets, (uint64)n); +} + +uint64 +cluster_lms_get_conn_resets(void) +{ + if (cluster_lms_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_lms_state->lms_conn_resets); +} + /* ============================================================ * HC4 single ownership atomic guard. @@ -651,6 +695,14 @@ cluster_lms_wake_drain(void) * latch / ShutdownRequestPending) * ============================================================ */ +/* PGRAC: spec-7.2 D4 — SIGUSR1 wakeup handler (latch only; rule 16: + * async-signal-safe operations exclusively). */ +static void +lms_sigusr1_handler(SIGNAL_ARGS) +{ + SetLatch(MyLatch); +} + void LmsMain(void) { @@ -667,7 +719,9 @@ LmsMain(void) pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); /* No ProcSignal slot in the skeleton; see auxprocess.c early setup. */ - pqsignal(SIGUSR1, SIG_IGN); + /* PGRAC: spec-7.2 D4 — SIGUSR1 = data-plane wakeup (was SIG_IGN in + * the skeleton). Handler only sets the latch (async-signal-safe). */ + pqsignal(SIGUSR1, lms_sigusr1_handler); pqsignal(SIGUSR2, SIG_IGN); pqsignal(SIGCHLD, SIG_DFL); @@ -695,6 +749,11 @@ LmsMain(void) cluster_lms_bump_restart_generation_at_main_entry(); (void)cluster_ges_dedup_drop_stale_entries(); + /* PGRAC: spec-7.2 D2 — bring up the LMS-owned DATA-plane listener + + * mesh (false = plane off for this node; the loop below then keeps + * the historic latch-only wait and park-serve keeps working). */ + (void)cluster_lms_data_plane_startup(); + /* Transition to READY. */ LWLockAcquire(&cluster_lms_state->lwlock, LW_EXCLUSIVE); cluster_lms_state->ready_at = GetCurrentTimestamp(); @@ -734,11 +793,46 @@ LmsMain(void) * failure becomes a DENIED result; LMS never exits over a serve). */ cluster_lms_cr_drain(); - (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, - LMS_IDLE_TIMEOUT_MS, WAIT_EVENT_PG_SLEEP); - ResetLatch(MyLatch); + /* PGRAC: spec-7.2 D2 — with a live DATA plane the wait moves into + * the data-plane tick (WaitEventSet: DATA sockets + MyLatch, latch + * reset inside); the historic plain WaitLatch stays the fallback + * when the plane is off. */ + if (cluster_lms_data_plane_enabled()) { + /* PGRAC: spec-7.2 D4 — once the GCS block family flips to + * the DATA plane, LMS ships its own READY results and + * drives the PI-discard notes (the LMON tick twins go + * quiet via the same registry probe); the DATA outbound + * ring drains here either way (empty before the flip). + * D5 fence linkage: while the write-fence is enforcing and + * writes are disallowed, the IMAGE-BEARING legs (READY + * ship + PI notes) send nothing — block images must not + * leave a fenced node. The outbound ring is NOT fence + * gated: it carries only backend solicitations (REQUEST / + * FORWARD / CR + undo fetches — the master's image REPLY + * is sent from the dispatch context, never staged here), + * and a fenced node's reads must keep flowing exactly as + * they do on the ungated CONTROL ring pre-flip; gating + * them wedges post-crash rejoin behind its own catalog + * reads. Thaw wakes the latch and the held image legs + * resume (pure-read probe, no-throw, per the + * write_fence_allowed contract). */ + if (cluster_gcs_block_family_on_data_plane() + && !(cluster_write_fence_enforcing() && !cluster_write_fence_allowed())) { + cluster_lms_cr_ship_ready(); + cluster_gcs_block_pi_discard_drain(); + } + (void)cluster_lms_outbound_drain_send(); + cluster_lms_data_plane_tick(LMS_IDLE_TIMEOUT_MS); + } else { + (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + LMS_IDLE_TIMEOUT_MS, WAIT_EVENT_PG_SLEEP); + ResetLatch(MyLatch); + } } + /* PGRAC: spec-7.2 D2 — close DATA-plane fds before teardown. */ + cluster_lms_data_plane_shutdown(); + /* PGRAC: spec-6.12b — retract the wake latch before teardown. */ cluster_cr_server_publish_lms_latch(NULL); @@ -1044,6 +1138,8 @@ cluster_lms_native_probe_dispatch(uint32 slot_idx) conf_node = cluster_conf_lookup_node(candidate); if (conf_node == NULL) continue; + if (cluster_cssd_get_peer_state(candidate) == CLUSTER_CSSD_PEER_DEAD) + continue; if (!native_probe_node_bit(candidate, &peer_bit)) { pg_atomic_fetch_add_u64(&cluster_lms_state->native_probe_timeout_count, 1); slot->final_status = CLUSTER_NATIVE_PROBE_FINAL_TIMEOUT; diff --git a/src/backend/cluster/cluster_lms_data_plane.c b/src/backend/cluster/cluster_lms_data_plane.c new file mode 100644 index 0000000000..b8969f6137 --- /dev/null +++ b/src/backend/cluster/cluster_lms_data_plane.c @@ -0,0 +1,516 @@ +/*------------------------------------------------------------------------- + * + * cluster_lms_data_plane.c + * pgrac DATA-plane (LMS<->LMS) interconnect loop — spec-7.2 D2. + * + * The LMS aux process owns the DATA plane: its own tier1 listener + * (bound to the node's declared data_addr), its own active-role + * connects, its own HELLO state machines, and — after the spec-7.2 + * D3/D4 plane flip — the GCS block family dispatch and replies. + * This mirrors the LMON tier1 mesh loop (cluster_lmon.c) with the + * control-plane duties removed: + * + * - NO heartbeat send and NO heartbeat-liveness scan: the CONTROL + * plane (CSSD/LMON) is the single liveness authority (spec-7.2 + * Q-D2-1). A dead peer is discovered via CONTROL membership + * events (spec-7.2 D5 wires the forced reset), via TCP errors + * on use, or via requester timeouts (fail-closed, 53R90 family). + * - Connect-establishment timeouts still apply (a peer that never + * completes HELLO must not pin a pending slot forever). + * - Reconnect backoff mirrors the LMON cadence (one heartbeat + * interval between attempts). + * + * Process-plane discipline: LmsMain aims this process's tier1 state + * at the DATA plane (cluster_ic_tier1_set_my_plane) before any tier1 + * use, so every tier1 helper called below lands on the DATA-plane + * shmem instance and this process's private fd/buffer statics. The + * CONTROL plane in LMON is untouched. + * + * A node without a declared data_addr runs no DATA plane: startup + * returns false and LmsMain keeps its historic latch-only loop + * (park-serve construction keeps working; the plane flip in D3/D4 + * is what makes a missing DATA plane a hard error for block ships). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_lms_data_plane.c + * + * NOTES + * This is a pgrac-original file. Compiled only in --enable-cluster + * builds. Spec: spec-7.2-ic-data-plane-decoupling.md (D2c). + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "cluster/cluster_conf.h" +#include "cluster/cluster_elog.h" +#include "cluster/cluster_epoch.h" /* PGRAC: spec-7.2 D5 epoch watch */ +#include "cluster/cluster_guc.h" +#include "cluster/cluster_ic.h" +#include "cluster/cluster_ic_tier1.h" +#include "cluster/cluster_inject.h" /* PGRAC: spec-7.2 D6 injection points */ +#include "cluster/cluster_lms.h" +#include "miscadmin.h" +#include "storage/latch.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +#ifdef USE_PGRAC_CLUSTER + +/* Per-peer DATA-plane connection substate (mirrors the LMON tracker). */ +typedef enum LmsDpSubstate { + LMS_DP_DOWN = 0, + LMS_DP_CONNECT_PEND, + LMS_DP_HELLO_SENDING, + LMS_DP_HELLO_WAIT, + LMS_DP_CONNECTED +} LmsDpSubstate; + +typedef struct LmsDpPeerTrack { + int fd; + LmsDpSubstate substate; + bool is_active; /* mesh role: we dial (true) or they dial us */ + TimestampTz next_attempt_at; + TimestampTz connect_started_at; +} LmsDpPeerTrack; + +static LmsDpPeerTrack dp_track[CLUSTER_MAX_NODES]; +static int dp_pending_fds[CLUSTER_MAX_NODES]; +static int dp_listener_fd = -1; +static WaitEventSet *dp_wes = NULL; +static bool dp_wes_dirty = true; +static bool dp_enabled = false; + +/* + * cluster_lms_data_plane_startup — bind the DATA listener + init the mesh. + * + * Returns true when the DATA plane is live (listener bound); false = + * plane off for this node (no data_addr declared, cluster disabled, or + * a stub/mock interconnect tier). Callable once from LmsMain before + * the READY transition. + */ +bool +cluster_lms_data_plane_startup(void) +{ + int32 self_id = cluster_node_id; + int32 pi; + + Assert(MyBackendType == B_LMS); + + if (!cluster_enabled) + return false; + if (cluster_interconnect_tier != CLUSTER_IC_TIER_1 + && cluster_interconnect_tier != CLUSTER_IC_TIER_2 + && cluster_interconnect_tier != CLUSTER_IC_TIER_3) + return false; + + /* Aim this process's tier1 state at the DATA plane BEFORE any use. */ + cluster_ic_tier1_set_my_plane(CLUSTER_IC_PLANE_DATA); + + dp_listener_fd = cluster_ic_tier1_listener_bind(); + if (dp_listener_fd < 0) + return false; /* no data_addr — plane off (LOG emitted inside) */ + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + dp_track[pi].is_active = false; + dp_track[pi].next_attempt_at = 0; + dp_track[pi].connect_started_at = 0; + dp_pending_fds[pi] = -1; + + if (pi == self_id) + continue; + if (cluster_conf_lookup_node(pi) == NULL) + continue; + dp_track[pi].is_active + = (cluster_ic_mesh_role_for_pair(self_id, pi) == CLUSTER_IC_MESH_ACTIVE); + } + + dp_wes_dirty = true; + dp_enabled = true; + ereport(LOG, (errmsg("cluster_lms: DATA-plane listener bound (node %d)", self_id))); + return true; +} + +bool +cluster_lms_data_plane_enabled(void) +{ + return dp_enabled; +} + +/* Rebuild the WaitEventSet when the fd set changed. */ +static void +dp_rebuild_wes(void) +{ + int32 pi; + + if (dp_wes != NULL) { + FreeWaitEventSet(dp_wes); + dp_wes = NULL; + } + + dp_wes = CreateWaitEventSet(CurrentMemoryContext, 3 + 2 * CLUSTER_MAX_NODES); + AddWaitEventToSet(dp_wes, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); + AddWaitEventToSet(dp_wes, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, NULL, NULL); + if (dp_listener_fd >= 0) + AddWaitEventToSet(dp_wes, WL_SOCKET_READABLE, dp_listener_fd, NULL, (void *)(intptr_t)-1); + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + int events; + + if (dp_track[pi].fd < 0) + continue; + + switch (dp_track[pi].substate) { + case LMS_DP_CONNECT_PEND: + case LMS_DP_HELLO_SENDING: + events = WL_SOCKET_WRITEABLE; + break; + case LMS_DP_HELLO_WAIT: + events = WL_SOCKET_READABLE; + break; + case LMS_DP_CONNECTED: + events = WL_SOCKET_READABLE; + if (cluster_ic_tier1_pending_outbound(pi)) + events |= WL_SOCKET_WRITEABLE; + break; + default: + continue; + } + + AddWaitEventToSet(dp_wes, events, dp_track[pi].fd, NULL, (void *)(intptr_t)pi); + } + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_pending_fds[pi] < 0) + continue; + AddWaitEventToSet(dp_wes, WL_SOCKET_READABLE, dp_pending_fds[pi], NULL, + (void *)(intptr_t)(CLUSTER_MAX_NODES + pi)); + } + + dp_wes_dirty = false; +} + +/* + * cluster_lms_data_plane_tick — one management + wait + dispatch round. + * + * Called from the LmsMain loop in place of the historic plain + * WaitLatch: the wait now also wakes on DATA sockets. MyLatch is + * reset here, so the caller must run its latch-driven drains (CR + * park-serve etc.) BEFORE calling in, mirroring the LMON loop shape. + */ +void +cluster_lms_data_plane_tick(long timeout_ms) +{ + WaitEvent ev[2 * CLUSTER_MAX_NODES + 3]; + TimestampTz now; + int n_events; + int32 pi; + int i; + + Assert(dp_enabled); + + now = GetCurrentTimestamp(); + + /* + * PGRAC: spec-7.2 D5 (INV-7.2-CONN-EPOCH ③) — proactive connection + * reset on an epoch bump: every DATA connection is bound to the + * epoch it was established under, so a bump force-closes the mesh + * and the reconnect below re-HELLOs at the current epoch. The + * sender gate in tier1_send_bytes is the structural backstop for + * the window between the bump and this tick. + */ + CLUSTER_INJECTION_POINT("cluster-lms-conn-reset"); + { + static uint64 dp_last_epoch = 0; + static bool dp_epoch_seen = false; + static bool dp_inject_reset_done = false; + uint64 cur_epoch = cluster_epoch_get_current(); + bool epoch_bumped = dp_epoch_seen && cur_epoch != dp_last_epoch; + bool inject_reset = false; + + /* + * PGRAC: spec-7.2 D7 (F6-1) — the cluster-lms-conn-reset injection + * models a SINGLE epoch-bump reset, but its only arm path is the + * process-local injection GUC, which stays armed and re-sets + * skip_pending on every tick (CLUSTER_INJECTION_POINT above) — a + * reset storm that starves the mesh (and, on the passive side, was + * observed to trip the accept/close WES churn into a crash). Latch + * the injected reset so it fires exactly ONCE per process lifetime: + * the test then observes one clean reset -> reconnect -> converge, + * matching the real single-bump path (epoch_bumped). should_skip + * still drains the pending flag each tick (harmless once latched). + * The real epoch-bump reset is unaffected by the latch. + * + * PGRAC modifications (F6-1 review r2, Finding 3): a real epoch + * bump takes precedence over the injected reset in the SAME tick. + * We only arm inject_reset when there is no epoch bump, so the + * reason log is attributed to the epoch bump (not "F6-1 one-shot") + * and the one-shot latch is NOT consumed by an epoch-bump tick — + * the injection then still fires exactly once on a later + * injection-only tick. This preserves the pre-F6-1 short-circuit + * (epoch bump || should_skip) attribution that this reorder + * otherwise lost. Test-only path: the injection GUC is never armed + * in production, where epoch_bumped alone drives the reset. + */ + if (!epoch_bumped && cluster_injection_should_skip("cluster-lms-conn-reset") + && !dp_inject_reset_done) + inject_reset = true; + + if (!dp_epoch_seen) { + dp_last_epoch = cur_epoch; + dp_epoch_seen = true; + } else if (epoch_bumped || inject_reset) { + const char *reason = epoch_bumped ? "data-plane epoch bump reset" + : "data-plane conn-reset injection (F6-1 one-shot)"; + int n_closed = 0; + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd >= 0) { + cluster_ic_tier1_close_peer(pi, reason); + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + dp_track[pi].connect_started_at = 0; + dp_track[pi].next_attempt_at = 0; /* reconnect immediately */ + dp_wes_dirty = true; + n_closed++; + } + } + + /* + * F6-1 one-shot latch: only mark the injected reset consumed + * once it has actually torn down a live connection. This keeps + * the injection from being wasted on a tick where the mesh is + * momentarily DOWN (arm-before-connect race) — it retries until + * a peer is live, then latches so the persistent GUC does not + * storm. The real epoch-bump reset does not touch the latch. + */ + if (inject_reset && n_closed > 0) + dp_inject_reset_done = true; + + /* + * PGRAC: spec-7.2 D6 — observability. lms_conn_resets counts + * DATA-plane connections torn down by a reset (epoch bump in + * production, injection in the F6-1 test), making the reset + * path visible in pg_stat_cluster_* / the debug dump. + */ + if (n_closed > 0) + cluster_lms_bump_conn_resets((uint32)n_closed); + + dp_last_epoch = cur_epoch; + } + } + + /* Active-role reconnect for DOWN peers whose backoff elapsed. */ + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + int new_fd = -1; + + if (pi == cluster_node_id || !dp_track[pi].is_active) + continue; + if (dp_track[pi].substate != LMS_DP_DOWN) + continue; + if (dp_track[pi].next_attempt_at > now) + continue; + + dp_track[pi].next_attempt_at + = now + (int64)cluster_interconnect_heartbeat_interval_ms * INT64CONST(1000); + + /* A peer without data_addr is unreachable on this plane — + * connect_one fails fast via the NULL peer_addr; skip quietly + * (the conf may declare it later; SIGHUP reload re-checks). */ + if (cluster_ic_tier1_connect_one(pi, &new_fd) && new_fd >= 0) { + dp_track[pi].fd = new_fd; + dp_track[pi].substate = LMS_DP_CONNECT_PEND; + dp_track[pi].connect_started_at = now; + dp_wes_dirty = true; + } + } + + /* Connect-establishment timeout scan (no liveness scan on DATA: + * CONTROL is the liveness authority, Q-D2-1). */ + { + int64 connect_to_us = (int64)cluster_interconnect_connect_timeout_ms * INT64CONST(1000); + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd < 0) + continue; + if ((dp_track[pi].substate == LMS_DP_CONNECT_PEND + || dp_track[pi].substate == LMS_DP_HELLO_SENDING + || dp_track[pi].substate == LMS_DP_HELLO_WAIT) + && dp_track[pi].connect_started_at > 0 + && now > dp_track[pi].connect_started_at + connect_to_us) { + cluster_ic_tier1_close_peer(pi, "data-plane connect timeout"); + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + dp_track[pi].connect_started_at = 0; + dp_wes_dirty = true; + } + } + } + + if (dp_wes_dirty) + dp_rebuild_wes(); + + if (timeout_ms < 0) + timeout_ms = 0; + + { + /* PGRAC: spec-7.2 D6 — wait identity: SEND while any peer has a + * backpressured partial frame (we are waiting for WRITEABLE + * drainage), RECV otherwise. */ + uint32 wait_kind = WAIT_EVENT_CLUSTER_LMS_DATA_RECV; + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd >= 0 && dp_track[pi].substate == LMS_DP_CONNECTED + && cluster_ic_tier1_pending_outbound(pi)) { + wait_kind = WAIT_EVENT_CLUSTER_LMS_DATA_SEND; + break; + } + } + n_events = WaitEventSetWait(dp_wes, timeout_ms, ev, lengthof(ev), wait_kind); + } + + for (i = 0; i < n_events; i++) { + intptr_t tag = (intptr_t)ev[i].user_data; + + if (ev[i].events & WL_LATCH_SET) { + ResetLatch(MyLatch); + continue; + } + + if (tag == -1) { + /* Listener: drain all pending accepts into anon slots. */ + for (;;) { + int new_fd = -1; + int32 dummy_peer_id = -1; + int slot; + + if (!cluster_ic_tier1_accept_one(&new_fd, &dummy_peer_id)) + break; + if (new_fd < 0) + break; + + for (slot = 0; slot < CLUSTER_MAX_NODES; slot++) + if (dp_pending_fds[slot] < 0) + break; + if (slot >= CLUSTER_MAX_NODES) { + (void)close(new_fd); + continue; + } + dp_pending_fds[slot] = new_fd; + dp_wes_dirty = true; + } + } else if (tag >= 0 && tag < CLUSTER_MAX_NODES) { + int32 peer = (int32)tag; + int peer_fd = dp_track[peer].fd; + + if (peer_fd < 0) + continue; + + if (dp_track[peer].substate == LMS_DP_CONNECT_PEND + && (ev[i].events & WL_SOCKET_WRITEABLE)) { + if (cluster_ic_tier1_finish_connect(peer, peer_fd)) { + if (cluster_ic_tier1_hello_send_remaining(peer) == 0) + dp_track[peer].substate = LMS_DP_CONNECTED; + else + dp_track[peer].substate = LMS_DP_HELLO_SENDING; + dp_wes_dirty = true; + } else { + dp_track[peer].fd = -1; + dp_track[peer].substate = LMS_DP_DOWN; + dp_wes_dirty = true; + } + } else if (dp_track[peer].substate == LMS_DP_HELLO_SENDING + && (ev[i].events & WL_SOCKET_WRITEABLE)) { + if (cluster_ic_tier1_continue_hello_send(peer, peer_fd)) { + if (cluster_ic_tier1_hello_send_remaining(peer) == 0) { + dp_track[peer].substate = LMS_DP_CONNECTED; + dp_wes_dirty = true; + } + } else { + dp_track[peer].fd = -1; + dp_track[peer].substate = LMS_DP_DOWN; + dp_wes_dirty = true; + } + } else if (dp_track[peer].substate == LMS_DP_CONNECTED + && (ev[i].events & WL_SOCKET_READABLE)) { + CLUSTER_INJECTION_POINT("cluster-lms-data-dispatch"); + /* Generic envelope pump: recv + verify + dispatch. No + * DATA msg_type is registered before the D3/D4 flip, so + * pre-flip traffic is limited to HELLO/errors; post- + * flip this is the block-family dispatch entry. */ + if (!cluster_ic_tier1_recv_heartbeat_drain(peer, peer_fd)) { + cluster_ic_tier1_close_peer(peer, "data-plane recv failed"); + dp_track[peer].fd = -1; + dp_track[peer].substate = LMS_DP_DOWN; + dp_wes_dirty = true; + } + } + } else if (tag >= CLUSTER_MAX_NODES && tag < 2 * CLUSTER_MAX_NODES) { + /* Anonymous accepted fd: accumulate + verify HELLO. */ + int slot = (int)(tag - CLUSTER_MAX_NODES); + int pend_fd = dp_pending_fds[slot]; + int32 learned = -1; + + if (pend_fd < 0) + continue; + + if (cluster_ic_tier1_continue_hello_recv(slot, pend_fd, &learned)) { + if (learned >= 0) { + /* HELLO complete: bind to the learned peer. */ + if (dp_track[learned].fd >= 0 && dp_track[learned].fd != pend_fd) + cluster_ic_tier1_close_peer(learned, "data-plane duplicate connection"); + dp_track[learned].fd = pend_fd; + dp_track[learned].substate = LMS_DP_CONNECTED; + dp_pending_fds[slot] = -1; + cluster_ic_tier1_anon_hello_reset(slot); + dp_wes_dirty = true; + } + /* else: still accumulating; stay registered READABLE */ + } else { + (void)close(pend_fd); + dp_pending_fds[slot] = -1; + cluster_ic_tier1_anon_hello_reset(slot); + dp_wes_dirty = true; + } + } + } +} + +/* Close everything we own (LmsMain teardown path). */ +void +cluster_lms_data_plane_shutdown(void) +{ + int32 pi; + + if (!dp_enabled) + return; + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (dp_track[pi].fd >= 0) { + cluster_ic_tier1_close_peer(pi, "lms shutdown"); + dp_track[pi].fd = -1; + dp_track[pi].substate = LMS_DP_DOWN; + } + if (dp_pending_fds[pi] >= 0) { + (void)close(dp_pending_fds[pi]); + dp_pending_fds[pi] = -1; + } + } + if (dp_wes != NULL) { + FreeWaitEventSet(dp_wes); + dp_wes = NULL; + } + dp_enabled = false; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c new file mode 100644 index 0000000000..e7f4b9a5ec --- /dev/null +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -0,0 +1,237 @@ +/*------------------------------------------------------------------------- + * + * cluster_lms_outbound.c + * pgrac DATA-plane outbound ring — spec-7.2 D4 (Q6-B twin ring). + * + * Backend-context producers of DATA-plane messages (GCS block + * REQUEST / INVALIDATE) stage frames here; the LMS data-plane loop + * drains and sends them over its own fds. This is the DATA twin of + * the CONTROL-plane cluster_grd_outbound ring, kept to the same + * minimal single-tail-FIFO semantics (Q6-B: one consumer, one lock, + * no claim/compaction machinery — the r1-F3 argument against a + * shared dual-reader ring). Enqueue marks no LMON duty and wakes + * LMS, not LMON. + * + * Ordering note (INV-7.2-DATA-FIFO): per-peer DATA frames keep a + * single ordered stream because (a) this ring is FIFO, (b) LMS is + * its only consumer, and (c) LMS sends on per-peer sockets with the + * tier1 partial-frame buffer (WOULD_BLOCK requeues at the head + * before anything newer is sent to that peer). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_lms_outbound.c + * + * NOTES + * This is a pgrac-original file. Compiled only in --enable-cluster + * builds. Spec: spec-7.2-ic-data-plane-decoupling.md (D4). + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_gcs_block.h" /* GcsBlockRequestPayload (pre-send hook) */ +#include "cluster/cluster_ic.h" +#include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_ic_router.h" +#include "cluster/cluster_lms.h" +#include "cluster/cluster_shmem.h" +#include "miscadmin.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/elog.h" + +#ifdef USE_PGRAC_CLUSTER + +#define PGRAC_LMS_OUTBOUND_CAPACITY 256 +#define PGRAC_LMS_OUTBOUND_PAYLOAD_MAX 128 + +typedef struct ClusterLmsOutboundSlot { + uint32 dest_node_id; + uint8 msg_type; + uint16 payload_len; + uint8 payload[PGRAC_LMS_OUTBOUND_PAYLOAD_MAX]; +} ClusterLmsOutboundSlot; + +typedef struct ClusterLmsOutboundState { + uint32 head; /* next slot to fill */ + uint32 tail; /* next slot to drain */ + uint32 count; + ClusterLmsOutboundSlot ring[PGRAC_LMS_OUTBOUND_CAPACITY]; +} ClusterLmsOutboundState; + +static ClusterLmsOutboundState *cluster_lms_outbound_state = NULL; +static LWLock *cluster_lms_outbound_lock = NULL; + +static Size +cluster_lms_outbound_shmem_size(void) +{ + return MAXALIGN(sizeof(ClusterLmsOutboundState)); +} + +static void +cluster_lms_outbound_shmem_init(void) +{ + bool found; + + cluster_lms_outbound_state = (ClusterLmsOutboundState *)ShmemInitStruct( + "pgrac cluster lms data outbound", cluster_lms_outbound_shmem_size(), &found); + if (!found) + memset(cluster_lms_outbound_state, 0, sizeof(*cluster_lms_outbound_state)); + + if (!IsBootstrapProcessingMode()) + cluster_lms_outbound_lock = &(GetNamedLWLockTranche("ClusterLmsDataOutbound"))[0].lock; +} + +static const ClusterShmemRegion cluster_lms_outbound_region = { + .name = "pgrac cluster lms data outbound", + .size_fn = cluster_lms_outbound_shmem_size, + .init_fn = cluster_lms_outbound_shmem_init, + .lwlock_count = 1, + .owner_subsys = "cluster_lms_outbound", + .reserved_flags = 0, +}; + +void +cluster_lms_outbound_shmem_register(void) +{ + cluster_shmem_register_region(&cluster_lms_outbound_region); +} + +/* Named-tranche request (process_shmem_requests window; I15 pattern). */ +void +cluster_lms_outbound_request_lwlocks(void) +{ + RequestNamedLWLockTranche("ClusterLmsDataOutbound", 1); +} + +/* + * cluster_lms_outbound_enqueue — stage one DATA-plane frame for LMS. + * + * Returns false on full ring / oversized payload / pre-shmem call; + * callers treat false as WOULD_BLOCK (retry via the normal request + * retry machinery). Publish-before-signal: the slot is visible + * before the LMS wakeup fires. + */ +bool +cluster_lms_outbound_enqueue(uint8 msg_type, uint32 dest_node_id, const void *payload, + uint16 payload_len) +{ + ClusterLmsOutboundSlot *slot; + + if (cluster_lms_outbound_state == NULL || cluster_lms_outbound_lock == NULL) + return false; + if (payload_len > PGRAC_LMS_OUTBOUND_PAYLOAD_MAX) + return false; + + LWLockAcquire(cluster_lms_outbound_lock, LW_EXCLUSIVE); + if (cluster_lms_outbound_state->count >= PGRAC_LMS_OUTBOUND_CAPACITY) { + LWLockRelease(cluster_lms_outbound_lock); + return false; + } + slot = &cluster_lms_outbound_state->ring[cluster_lms_outbound_state->head]; + slot->dest_node_id = dest_node_id; + slot->msg_type = msg_type; + slot->payload_len = payload_len; + if (payload_len > 0) + memcpy(slot->payload, payload, payload_len); + cluster_lms_outbound_state->head + = (cluster_lms_outbound_state->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; + cluster_lms_outbound_state->count++; + LWLockRelease(cluster_lms_outbound_lock); + + cluster_lms_wakeup(); + return true; +} + +/* Head-requeue for WOULD_BLOCK (preserves per-peer FIFO). */ +static void +lms_outbound_requeue_head(const ClusterLmsOutboundSlot *slot) +{ + LWLockAcquire(cluster_lms_outbound_lock, LW_EXCLUSIVE); + if (cluster_lms_outbound_state->count < PGRAC_LMS_OUTBOUND_CAPACITY) { + cluster_lms_outbound_state->tail + = (cluster_lms_outbound_state->tail + PGRAC_LMS_OUTBOUND_CAPACITY - 1) + % PGRAC_LMS_OUTBOUND_CAPACITY; + cluster_lms_outbound_state->ring[cluster_lms_outbound_state->tail] = *slot; + cluster_lms_outbound_state->count++; + } + /* full → drop; fire-and-forget layers self-heal via retransmit */ + LWLockRelease(cluster_lms_outbound_lock); +} + +/* + * cluster_lms_outbound_drain_send — LMS data-plane loop: drain + send. + * + * Bounded batch per call. WOULD_BLOCK requeues at the HEAD so the + * per-peer byte stream is never reordered (INV-7.2-DATA-FIFO). The + * GCS block REQUEST pre-send hook (direct-land arm) migrates here + * with the DATA consumer (D0-② coupling item ①). + */ +int +cluster_lms_outbound_drain_send(void) +{ + int sent = 0; + + if (cluster_lms_outbound_state == NULL || cluster_lms_outbound_lock == NULL) + return 0; + + Assert(MyBackendType == B_LMS); + + while (sent < 64) { + ClusterLmsOutboundSlot slot; + ClusterICSendResult rc; + bool got = false; + + LWLockAcquire(cluster_lms_outbound_lock, LW_EXCLUSIVE); + if (cluster_lms_outbound_state->count > 0) { + slot = cluster_lms_outbound_state->ring[cluster_lms_outbound_state->tail]; + cluster_lms_outbound_state->tail + = (cluster_lms_outbound_state->tail + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; + cluster_lms_outbound_state->count--; + got = true; + } + LWLockRelease(cluster_lms_outbound_lock); + if (!got) + break; + + if (slot.msg_type == PGRAC_IC_MSG_GCS_BLOCK_REQUEST + && slot.payload_len == sizeof(GcsBlockRequestPayload)) + cluster_gcs_block_lmon_prepare_outbound_request((GcsBlockRequestPayload *)slot.payload, + (int32)slot.dest_node_id); + + rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, + slot.payload_len > 0 ? slot.payload : NULL, slot.payload_len); + if (rc == CLUSTER_IC_SEND_DONE) { + sent++; + continue; + } + if (rc == CLUSTER_IC_SEND_WOULD_BLOCK) { + lms_outbound_requeue_head(&slot); + break; + } + /* HARD_ERROR: peer down — drop; requesters retry fail-closed. */ + } + + return sent; +} + +uint32 +cluster_lms_outbound_depth(void) +{ + uint32 depth; + + if (cluster_lms_outbound_state == NULL || cluster_lms_outbound_lock == NULL) + return 0; + LWLockAcquire(cluster_lms_outbound_lock, LW_SHARED); + depth = cluster_lms_outbound_state->count; + LWLockRelease(cluster_lms_outbound_lock); + return depth; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 8825ac6828..39e31c4b72 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -57,6 +57,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" #include "cluster/cluster_membership.h" #include "cluster/cluster_node_remove.h" #include "cluster/cluster_pcm_lock.h" @@ -162,6 +163,14 @@ cluster_node_remove_get_state(ClusterNodeRemoveState *out) static uint64 nr_qvotec_last_processed_marker_seq = 0; static uint64 nr_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync nr_marker_async; +static ClusterRemovalMarker nr_marker_stage; +static int nr_marker_phase = 0; +static int32 nr_marker_node_id = -1; +static uint64 nr_marker_epoch = 0; +static uint64 nr_marker_removed_incarnation = 0; +static uint64 nr_marker_removal_event_id = 0; +static bool nr_marker_submitted = false; ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) @@ -195,6 +204,38 @@ cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) } } +bool +cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (nr_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&nr_state->marker_request_seq, + &nr_state->marker_completion_seq)) + return false; + + nr_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &nr_state->marker_request_seq, + &nr_state->marker_completion_seq, nr_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (nr_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, &nr_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out) { @@ -223,6 +264,7 @@ cluster_node_remove_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&nr_state->marker_completion_seq, nr_qvotec_inflight_marker_seq); nr_qvotec_last_processed_marker_seq = nr_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -241,24 +283,124 @@ cluster_node_remove_publish_qvotec_latch(struct Latch *latch) on_shmem_exit(nr_clear_qvotec_latch, (Datum)0); } -/* build + submit one durable removal marker; returns true iff majority-durable. */ +static ClusterMarkerAsyncKind +nr_marker_phase_kind(int phase) +{ + switch (phase) { + case CLUSTER_REMOVAL_MARKER_REMOVING: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING; + case CLUSTER_REMOVAL_MARKER_SHRUNK: + return CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK; + case CLUSTER_REMOVAL_MARKER_REMOVED: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED; + default: + return CLUSTER_MARKER_KIND_UNKNOWN; + } +} + +static void +nr_release_marker_stage(void) +{ + cluster_marker_async_release_stage(&nr_marker_async); + memset(&nr_marker_stage, 0, sizeof(nr_marker_stage)); + nr_marker_phase = 0; + nr_marker_node_id = -1; + nr_marker_epoch = 0; + nr_marker_removed_incarnation = 0; + nr_marker_removal_event_id = 0; + nr_marker_submitted = false; +} + +static bool +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, + uint64 removal_event_id) +{ + return nr_marker_async.has_staged_event && nr_marker_phase == phase + && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch + && nr_marker_removed_incarnation == removed_incarnation + && nr_marker_removal_event_id == removal_event_id; +} + +/* build + poll one durable removal marker; returns true iff majority-durable. */ static bool nr_write_marker(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, uint64 removal_event_id) { ClusterRemovalMarker m; + TimestampTz now; + uint32 result = CLUSTER_REMOVAL_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + now = GetCurrentTimestamp(); + kind = nr_marker_phase_kind(phase); + + if (nr_marker_async.has_staged_event + && !nr_marker_stage_matches(phase, node_id, remove_epoch, removed_incarnation, + removal_event_id)) { + if (!nr_marker_submitted) { + if (cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, + nr_marker_phase_kind(nr_marker_phase), + nr_marker_node_id, now)) + nr_marker_submitted = true; + return false; + } + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } else if (pr == CLUSTER_MARKER_POLL_ACKED) { + cluster_reconfig_note_marker_slow_ack(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } + return false; + } + + if (!nr_marker_async.has_staged_event) { + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; + m.version = CLUSTER_REMOVAL_MARKER_VERSION; + m.phase = (uint16)phase; + m.removed_node_id = node_id; + m.remove_epoch = remove_epoch; + m.removed_incarnation = removed_incarnation; + m.removal_event_id = removal_event_id; + cluster_removal_marker_compute_crc(&m); + + nr_marker_stage = m; + nr_marker_phase = phase; + nr_marker_node_id = node_id; + nr_marker_epoch = remove_epoch; + nr_marker_removed_incarnation = removed_incarnation; + nr_marker_removal_event_id = removal_event_id; + nr_marker_async.has_staged_event = true; + nr_marker_async.staged_expect_epoch = remove_epoch; + nr_marker_submitted = false; + } - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; - m.version = CLUSTER_REMOVAL_MARKER_VERSION; - m.phase = (uint16)phase; - m.removed_node_id = node_id; - m.remove_epoch = remove_epoch; - m.removed_incarnation = removed_incarnation; - m.removal_event_id = removal_event_id; - cluster_removal_marker_compute_crc(&m); - - return cluster_node_remove_submit_marker(&m) == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; + if (!nr_marker_submitted) { + if (!cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, kind, + node_id, now)) + return false; + nr_marker_submitted = true; + return false; + } + + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return false; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return false; + } + + cluster_reconfig_note_marker_slow_ack(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return result == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; } @@ -543,31 +685,43 @@ cluster_node_remove_drive(void) (void)baseline_dead_gen; /* contest is now signalled by out_contest, not derived */ - /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ - (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, - last_incarnation, removal_event_id); - /* - * spec-6.15 D5c (appendix B.3): durably retire the removed node's - * xid stripe slot BEFORE the removal point of no return below. - * Ordering is the identity-reuse defence: once removal commits, a - * later fresh join of the same node_id (new incarnation = a new - * durable owner) must land on a retired slot and refuse (53RB1), - * never resume the old owner's congruence class. Not durable yet - * -> stay FENCE_ARMING and retry next tick (fail-closed; same - * shape as a transient fence/quorum failure). A never-activated - * cluster returns true (nothing to retire). + * spec-2.29a review r1 P2: while the staged fence marker from a + * previous FENCE_ARMING tick is still in flight, skip the pre-work + * below — the epoch was already self-bumped at stage-entry, so + * re-running it would key a SECOND REMOVING marker to the post-bump + * epoch and re-submit the stripe retire during the very contention + * window the async stage relieves. Fall through straight to + * apply(), which polls the stage. */ - if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { - ereport(LOG, (errmsg("cluster node removal: stripe slot retire for node %d not durable " - "yet — retrying before the removal commit", - node_id))); - return; + if (!cluster_reconfig_node_removed_fence_stage_pending()) { + /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ + (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, + last_incarnation, removal_event_id); + + /* + * spec-6.15 D5c (appendix B.3): durably retire the removed node's + * xid stripe slot BEFORE the removal point of no return below. + * Ordering is the identity-reuse defence: once removal commits, a + * later fresh join of the same node_id (new incarnation = a new + * durable owner) must land on a retired slot and refuse (53RB1), + * never resume the old owner's congruence class. Not durable yet + * -> stay FENCE_ARMING and retry next tick (fail-closed; same + * shape as a transient fence/quorum failure). A never-activated + * cluster returns true (nothing to retire). + */ + if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { + ereport(LOG, + (errmsg("cluster node removal: stripe slot retire for node %d not durable " + "yet — retrying before the removal commit", + node_id))); + return; + } + /* D5d: carry the retire to WAL consumers (standby stops gap- + * filling the dead class). The voting-disk retire above is the + * correctness anchor; emission failure is tolerated (logged). */ + cluster_xid_stripe_emit_retire_wal(node_id); } - /* D5d: carry the retire to WAL consumers (standby stops gap- - * filling the dead class). The voting-disk retire above is the - * correctness anchor; emission failure is tolerated (logged). */ - cluster_xid_stripe_emit_retire_wal(node_id); /* * The commit point: guarded epoch bump + fence-arm (majority-durable, at diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 545ac33480..88ddaad4cf 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -724,6 +724,96 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) return cleared; } +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node) +{ + HASH_SEQ_STATUS scan; + struct GrdEntry *entry; + uint64 cleaned = 0; + uint32 dead_bit; + + if (cluster_pcm_htab == NULL || dead_node < 0 || dead_node >= 32) + return 0; + + dead_bit = (uint32)1u << (uint32)dead_node; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + hash_seq_init(&scan, cluster_pcm_htab); + while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool changed = false; + bool broadcast_needed = false; + bool master_holder_was_dead; + PcmState before_state; + PcmState after_state; + uint32 s_bitmap; + uint32 pi_bitmap; + + /* Cheap unlocked filter; rechecked under entry_lock below. */ + if (entry->x_holder_node != dead_node + && (pg_atomic_read_u32(&entry->s_holders_bitmap) & dead_bit) == 0 + && (pg_atomic_read_u32(&entry->pi_holders_bitmap) & dead_bit) == 0 + && (!pcm_master_holder_is_valid(entry) + || (int32)entry->master_holder.node_id != dead_node)) + continue; + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + before_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + master_holder_was_dead + = pcm_master_holder_is_valid(entry) && (int32)entry->master_holder.node_id == dead_node; + + if (entry->x_holder_node == dead_node) { + entry->x_holder_node = -1; + changed = true; + } + s_bitmap = pg_atomic_read_u32(&entry->s_holders_bitmap); + if ((s_bitmap & dead_bit) != 0) { + s_bitmap &= ~dead_bit; + pg_atomic_write_u32(&entry->s_holders_bitmap, s_bitmap); + changed = true; + } + pi_bitmap = pg_atomic_read_u32(&entry->pi_holders_bitmap); + if ((pi_bitmap & dead_bit) != 0) { + pg_atomic_write_u32(&entry->pi_holders_bitmap, pi_bitmap & ~dead_bit); + changed = true; + } + + if (entry->x_holder_node >= 0) { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); + if (master_holder_was_dead) + pcm_master_holder_set_node(entry, entry->x_holder_node); + } else if (s_bitmap != 0) { + int32 next_holder = pcm_lowest_set_bit_node(s_bitmap); + + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_S); + if (master_holder_was_dead) { + if (next_holder >= 0) + pcm_master_holder_set_node(entry, next_holder); + else + pcm_master_holder_clear(entry); + } + } else { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_N); + if (pcm_master_holder_is_valid(entry)) + pcm_master_holder_clear(entry); + } + if (master_holder_was_dead) + changed = true; + after_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + if (changed && after_state != before_state) + broadcast_needed = true; + + LWLockRelease(&entry->entry_lock.lock); + + if (broadcast_needed) + ConditionVariableBroadcast(&entry->wait_cv); + if (changed) + cleaned++; + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + + return cleaned; +} + /* ======================================================================== * PGRAC MODIFICATIONS by SqlRush — spec-5.13 D5 (clean-leave PCM release). @@ -2131,7 +2221,10 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) errmsg("block-level cache protocol state is being rebuilt " "after reconfiguration"), errhint("The block resource is recovering (survivor re-declare / " - "master rebuild after node failure); retry the transaction."))); + "master rebuild after node failure); retry the transaction. " + "If the failed node stays down, restart it to run its " + "instance recovery, or enable online thread recovery in a " + "supported scope (cluster.online_thread_recovery)."))); CHECK_FOR_INTERRUPTS(); pgstat_report_wait_start(WAIT_EVENT_GCS_BLOCK_RECOVERING); @@ -2199,6 +2292,61 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + + /* + * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after + * fail-stop. Local-master state=S with other live S holders but no local + * S bit used to fall through to the tag-only X acquire, which fail-closed + * as "no local S residency". This entry point is buffer-aware: the + * caller has already read or initialized the BufferDesc before asking for + * X, so first register a local S residency, then reuse the existing + * local S->X invalidate/upgrade path. If the invalidate cannot be proven, + * drop the temporary S claim and rethrow the same fail-closed error. + */ + if (master_state == PCM_LOCK_MODE_S && cluster_node_id >= 0 && cluster_node_id < 32) { + uint32 self_bit = (uint32)1u << (uint32)cluster_node_id; + + if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { + struct GrdEntry *entry; + bool upgraded = false; + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with + * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out + * of it — release the temporary S claim on that path too, not + * only on the false return. */ + PG_TRY(); + { + upgraded = cluster_gcs_block_local_x_upgrade(tag); + } + PG_CATCH(); + { + cluster_pcm_lock_release(tag); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!upgraded) { + cluster_pcm_lock_release(tag); + ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), + errhint("Remote S holders did not all acknowledge in time; " + "retry the statement."))); + } + + entry = pcm_find_entry(tag); + if (entry != NULL) { + pcm_entry_lock_exclusive(entry); + /* Amendment v1.2 (R9): the completed S->X upgrade replaces + * ALL of this node's S declarations for the tag (the X + * grant subsumes them), so the local S refcount is + * intentionally hard-reset rather than decremented — a + * later release of the X does not owe any S releases. */ + entry->s_holder_refcount_local = 0; + LWLockRelease(&entry->entry_lock.lock); + } + return true; + } + } } cluster_pcm_lock_acquire(tag, mode); diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 8eb25379a7..916a629399 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -94,8 +94,9 @@ #include "cluster/cluster_elog.h" /* CLUSTER_LOG (best-effort logging) */ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ -#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ -#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ +#include "cluster/cluster_inject.h" +#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ +#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D2: fence marker scan + token refresh */ @@ -883,6 +884,10 @@ qvotec_poll_once(void) * carried forward every poll like the fence marker (R12), and acked majority- * durable from the self-slot write tally. */ have_removal_submit = cluster_node_remove_qvotec_poll_pending(&removal_submit_marker); + if ((have_submit || have_leave_submit || have_join_submit || have_removal_submit) + && cluster_cr_injection_armed("cluster-qvotec-marker-service-hold", NULL)) + return; + memset(&apply_lease_request, 0, sizeof(apply_lease_request)); memset(&apply_lease_winner, 0, sizeof(apply_lease_winner)); apply_lease_winner.owner_node_id = -1; diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 1f79a6c9b9..ccaf5991c1 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -77,6 +77,7 @@ #include "cluster/cluster_clean_leave.h" /* v1.0.4 — cluster_clean_leave_in_progress (serialize) */ #include "cluster/cluster_guc.h" /* cluster_enabled, cluster_online_join */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_voting_disk_io.h" /* spec-5.15 D4 — region-3 join-marker slot I/O */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D4 — durable fence marker submit */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum */ @@ -114,6 +115,38 @@ StaticAssertDecl(CLUSTER_RECONFIG_TOUCH_KIND_COUNT == CLUSTER_TOUCH_KIND_COUNT, */ static ClusterReconfigState *ReconfigShmem = NULL; +typedef struct ClusterReconfigFenceStage { + ClusterMarkerAsync async; + ReconfigEvent event; + ClusterFenceMarker marker; + int32 node_id; + uint64 last_incarnation; + uint64 removal_event_id; + bool submitted; +} ClusterReconfigFenceStage; + +typedef struct ClusterReconfigJoinPrepareStage { + ClusterMarkerAsync async; + ReconfigEvent event; + uint8 join_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 joiner_incarnations[CLUSTER_MAX_NODES]; + int next_node; + bool submitted; +} ClusterReconfigJoinPrepareStage; + +typedef struct ClusterReconfigJoinCommitStage { + ClusterMarkerAsync async; + ClusterJoinCommitMarker marker; + int32 node_id; + uint64 admitted_incarnation; + bool submitted; +} ClusterReconfigJoinCommitStage; + +static ClusterReconfigFenceStage failstop_fence_stage; +static ClusterReconfigFenceStage node_removed_fence_stage; +static ClusterReconfigJoinPrepareStage join_prepare_stage; +static ClusterReconfigJoinCommitStage join_commit_stage; + /* ============================================================ * Shmem region lifecycle. @@ -144,6 +177,7 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->apply_counter, 0); pg_atomic_init_u64(&ReconfigShmem->dedup_skip_counter, 0); pg_atomic_init_u64(&ReconfigShmem->procsig_broadcast_count, 0); + pg_atomic_init_u32(&ReconfigShmem->prebump_sync_active, 0); /* spec-2.29a r2 t/274 */ /* spec-5.14 D6 — touched_peers observability counters. */ pg_atomic_init_u64(&ReconfigShmem->touched_abort_count, 0); pg_atomic_init_u64(&ReconfigShmem->touched_stamp_count, 0); @@ -211,6 +245,8 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->join_reject_count, 0); pg_atomic_init_u64(&ReconfigShmem->join_timeout_count, 0); pg_atomic_init_u64(&ReconfigShmem->clean_departed_cleared_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_slow_ack_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_timeout_count, 0); } /* @@ -306,6 +342,28 @@ cluster_reconfig_publish_event(const ReconfigEvent *evt) } +static void +cluster_reconfig_log_failstop_epoch_bump(const ReconfigEvent *evt) +{ + char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; + int off = 0; + int n; + + Assert(evt != NULL); + + dead[0] = '\0'; + for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { + if (evt->dead_bitmap[n / 8] & (1 << (n % 8))) + off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); + } + + ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " + "(coordinator node %d, dead node(s) {%s})", + (unsigned long long)evt->old_epoch, (unsigned long long)evt->new_epoch, + (int)evt->coordinator_node_id, dead))); +} + + /* ============================================================ * Counter accessors (Step 2 + Step 3 SRF support). * ============================================================ @@ -1032,6 +1090,42 @@ cluster_reconfig_get_clean_departed_cleared_count(void) : pg_atomic_read_u64(&ReconfigShmem->clean_departed_cleared_count); } +void +cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem == NULL || elapsed_us < 1000000ULL) + return; + + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); + ereport(LOG, (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +void +cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem != NULL) + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); + ereport(LOG, (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_slow_ack_count); +} + +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_timeout_count); +} + /* * cluster_reconfig_join_in_progress -- Hardening v1.0.4 (spec-5.13 clean-leave x * spec-5.15 online-join serialization, P1-1/P2): is a membership JOIN currently in @@ -1148,6 +1242,422 @@ cluster_reconfig_join_publish_proven(uint64 admitted_epoch) return advanced >= ((members / 2u) + 1u); } +static void +cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) +{ + cluster_marker_async_release_stage(&stage->async); + stage->submitted = false; + memset(&stage->event, 0, sizeof(stage->event)); + memset(&stage->marker, 0, sizeof(stage->marker)); + stage->node_id = -1; + stage->last_incarnation = 0; + stage->removal_event_id = 0; +} + +/* + * spec-2.29a nightly-regression fix — pre-bump staging window guard. + * + * The three pre-bump coordinator paths (fail-stop fence, node-remove, + * join Phase-1) advance the membership epoch at stage-entry but only + * publish the reconfig event once the voting-disk marker ACKs, which now + * spans several LMON ticks instead of the pre-async single-tick spin. + * Between the bump and the publish the GRD recovery IDLE tick would + * re-capture recovery_event_old_epoch as the post-bump value, so its P0 + * accept later reads old == cur and wedges WAIT_EPOCH forever (the + * spec-4.6a section 0 shape, here triggered by the coordinator on ITSELF + * — even in a 2-node cluster with no IC piggyback). While any pre-bump + * stage is live the GRD IDLE tick must hold its last stable (genuine + * pre-reconfig) baseline instead of re-capturing. join Phase-2 COMMITTED + * does not pre-bump, so it is intentionally excluded. + */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + /* spec-2.29a r2 t/274: the shmem bit covers a BACKEND-context coordinator's + * bump→publish window (LMON cannot see that process's local stage); the + * three LMON-local staged flags cover the LMON-driven paths. */ + if (ReconfigShmem != NULL && pg_atomic_read_u32(&ReconfigShmem->prebump_sync_active) != 0) + return true; + return failstop_fence_stage.async.has_staged_event + || node_removed_fence_stage.async.has_staged_event + || join_prepare_stage.async.has_staged_event; +} + +static bool +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, + int32 target_node, TimestampTz now) +{ + if (stage->submitted) + return true; + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, target_node, + now)) + return false; + stage->submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_failstop_fence_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!failstop_fence_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, now)) + return true; + + pr = cluster_write_fence_poll_marker_async(&failstop_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_log_failstop_epoch_bump(&failstop_fence_stage.event); + cluster_reconfig_broadcast_local_procsig(); + } else { + ereport(LOG, + (errmsg("cluster reconfig: fence marker did not reach a voting-disk majority " + "for epoch %llu; not publishing reconfig event (write-fenced, will retry)", + (unsigned long long)failstop_fence_stage.event.new_epoch))); + } + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; +} + +static uint64 +cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 removal_event_id, + uint64 last_incarnation) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!node_removed_fence_stage.async.has_staged_event) + return 0; + if (node_removed_fence_stage.node_id != removed_node_id + || node_removed_fence_stage.removal_event_id != removal_event_id) + return 0; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&node_removed_fence_stage, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, now)) + return 0; + + pr = cluster_write_fence_poll_marker_async(&node_removed_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return 0; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, removed_node_id, + elapsed_us); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " + "voting-disk majority for epoch %llu; not publishing removal " + "(write-fenced, will retry)", + removed_node_id, + (unsigned long long)node_removed_fence_stage.event.new_epoch))); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); + cluster_reconfig_publish_event(&node_removed_fence_stage.event); + cluster_reconfig_record_removed(removed_node_id, node_removed_fence_stage.event.new_epoch, + false); + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_shrink_to_removed(removed_node_id, last_incarnation); + LWLockRelease(&ReconfigShmem->lock); + + { + uint64 new_epoch = node_removed_fence_stage.event.new_epoch; + + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return new_epoch; + } +} + +static void +cluster_reconfig_release_join_prepare_stage(void) +{ + cluster_marker_async_release_stage(&join_prepare_stage.async); + memset(&join_prepare_stage.event, 0, sizeof(join_prepare_stage.event)); + memset(join_prepare_stage.join_bitmap, 0, sizeof(join_prepare_stage.join_bitmap)); + memset(join_prepare_stage.joiner_incarnations, 0, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; +} + +/* (spec-2.29a review r1: the former abort_join_prepare_stage — revert + * JOINING→DEAD + clear pending on PREPARE failure — was removed. Q5=A makes + * PREPARE strictly best-effort: TIMEOUT / non-ACK advances the queue and the + * staged JOIN_PENDING still publishes, so no revert path exists; reverting + * would let compute_join_bitmap re-detect the joiner and re-bump the epoch + * every deadline period — the P1-1 bump storm.) */ + +static bool +cluster_reconfig_submit_join_prepare_current(TimestampTz now) +{ + ClusterJoinCommitMarker m; + int i; + + for (i = join_prepare_stage.next_node; i < CLUSTER_MAX_NODES; i++) { + if (dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + break; + } + join_prepare_stage.next_node = i; + if (i >= CLUSTER_MAX_NODES) { + cluster_reconfig_publish_event(&join_prepare_stage.event); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + cluster_reconfig_broadcast_local_procsig(); + cluster_reconfig_release_join_prepare_stage(); + return true; + } + + if (join_prepare_stage.submitted) + return true; + + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_JCMK_MAGIC; + m.version = CLUSTER_JCMK_VERSION; + m.node_id = i; + m.phase = CLUSTER_JCMK_PHASE_PREPARE; + m.admitted_incarnation = join_prepare_stage.joiner_incarnations[i]; + m.generation = join_prepare_stage.joiner_incarnations[i]; + m.admitted_epoch = join_prepare_stage.event.new_epoch; + cluster_join_marker_compute_crc(&m); + + if (!cluster_reconfig_submit_join_marker_async(&join_prepare_stage.async, i, &m, + CLUSTER_MARKER_KIND_JOIN_PREPARE, now)) + return false; + join_prepare_stage.submitted = true; + return true; +} + +/* + * spec-2.29a review r1 P2 — node-remove driver pre-work gate. + * + * While the staged node-removed fence marker from a previous FENCE_ARMING + * tick is still in flight, the driver must not re-run its pre-work + * (REMOVING marker write / stripe retire): the epoch has already been + * self-bumped by the stage-entry, so a re-run would key a SECOND REMOVING + * marker to the post-bump epoch and add voting-disk writes during the + * exact contention window the async stage exists to relieve. + */ +bool +cluster_reconfig_node_removed_fence_stage_pending(void) +{ + return node_removed_fence_stage.async.has_staged_event; +} + +static bool +cluster_reconfig_poll_join_prepare_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + int target; + + if (!join_prepare_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_prepare_stage.submitted) { + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; + } + + target = join_prepare_stage.next_node; + pr = cluster_reconfig_poll_join_marker_async(&join_prepare_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + + /* + * spec-2.29a Q5=A (review r1 P1): PREPARE is best-effort in OUTCOME — the + * pre-async code ignored the submit result and always published + * JOIN_PENDING (only the Phase-2 COMMITTED marker is a commit point, P1-r5). + * A TIMEOUT / non-ACK PREPARE therefore advances the queue instead of + * aborting: aborting would revert the joiner JOINING→DEAD, and the next + * tick's compute_join_bitmap would re-detect it CSSD-alive and RE-BUMP the + * epoch — exactly the per-tick epoch-bump storm the P1-1 staged record + * forbids. Draining to publish keeps the joiner on a stable JOINING + * (one Phase-1 bump total, regardless of PREPARE outcomes). + */ + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + } else { + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach " + "a voting-disk majority; continuing best-effort (COMMITTED is the " + "commit point)", + target))); + } + + join_prepare_stage.submitted = false; + join_prepare_stage.next_node++; + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; +} + +static void +cluster_reconfig_release_join_commit_stage(void) +{ + cluster_marker_async_release_stage(&join_commit_stage.async); + memset(&join_commit_stage.marker, 0, sizeof(join_commit_stage.marker)); + join_commit_stage.node_id = -1; + join_commit_stage.admitted_incarnation = 0; + join_commit_stage.submitted = false; +} + +static bool +cluster_reconfig_publish_join_commit(int32 node_id, uint64 admitted_incarnation, + uint64 expected_epoch) +{ + uint64 old_epoch, new_epoch; + XLogRecPtr lsn; + ReconfigEvent evt; + uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint64 incs[CLUSTER_MAX_NODES]; + + if (cluster_epoch_get_current() + 1 != expected_epoch) + return false; + + CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + + cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); + if (new_epoch != expected_epoch) + return false; + lsn = GetXLogInsertRecPtr(); + cluster_epoch_set_changed_at_lsn((uint64)lsn); + cluster_gcs_block_on_epoch_advance(new_epoch); + cluster_sinval_reset_all_on_reconfig(); + cluster_tt_status_flush_all((uint32)new_epoch); + + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); + cluster_membership_record_admitted(node_id, admitted_incarnation); + ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); + LWLockRelease(&ReconfigShmem->lock); + + if (cluster_reconfig_is_clean_departed(node_id)) + pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); + cluster_reconfig_clear_clean_departed(node_id); + + dead_bitmap_set_bit(jb, node_id); + memset(incs, 0, sizeof(incs)); + incs[node_id] = admitted_incarnation; + + memset(&evt, 0, sizeof(evt)); + evt.event_id = cluster_reconfig_compute_event_id_v2( + RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); + evt.coordinator_node_id = cluster_node_id; + evt.old_epoch = old_epoch; + evt.new_epoch = new_epoch; + memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + evt.applied_at = GetCurrentTimestamp(); + evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); + evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; + cluster_reconfig_publish_event(&evt); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); + + return true; +} + +static bool +cluster_reconfig_poll_join_commit_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + uint64 admitted_incarnation = 0; + uint64 admitted_generation = 0; + ClusterMarkerPollResult pr; + + if (!join_commit_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_commit_stage.submitted) { + if (!cluster_reconfig_submit_join_marker_async( + &join_commit_stage.async, join_commit_stage.node_id, &join_commit_stage.marker, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, now)) + return true; + join_commit_stage.submitted = true; + return true; + } + + pr = cluster_reconfig_poll_join_marker_async(&join_commit_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " + "voting-disk majority; not committing (will retry)", + join_commit_stage.node_id))); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + if (!cluster_reconfig_get_observed_slot(join_commit_stage.node_id, &admitted_incarnation, + &admitted_generation) + || admitted_incarnation != join_commit_stage.admitted_incarnation + || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, + admitted_generation) + != CLUSTER_JOIN_ACCEPT + || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, + join_commit_stage.admitted_incarnation, + join_commit_stage.async.staged_expect_epoch)) { + pg_atomic_fetch_add_u64(&ReconfigShmem->join_reject_count, 1); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_release_join_commit_stage(); + return true; +} + /* * spec-5.15 D4 — coordinator-side join driver (called from the tick when * online_join is on and self is the min-MEMBER coordinator). Phase-1: fresh @@ -1166,6 +1676,10 @@ cluster_reconfig_drive_joins(int coordinator) if (state == NULL) return; + if (cluster_reconfig_poll_join_prepare_stage()) + return; + if (cluster_reconfig_poll_join_commit_stage()) + return; /* Phase-1 detection + a snapshot of the current pending set, under the lock * (compute_join_bitmap reads membership_state). */ @@ -1182,8 +1696,7 @@ cluster_reconfig_drive_joins(int coordinator) (void)cluster_reconfig_get_observed_slot(i, &joiner_incarnations[i], NULL); } cluster_reconfig_apply_join_as_coordinator(join_bitmap, coordinator, joiner_incarnations); - /* enter the JOIN_PENDING transition fail-closed on every local backend. */ - cluster_reconfig_broadcast_local_procsig(); + return; } /* Phase-2: commit pending joins that have converged. The just-added Phase-1 @@ -1646,6 +2159,7 @@ cluster_reconfig_lmon_tick(void) uint64 cssd_dead_generation; uint64 event_id; int i; + bool failstop_stage_handled = false; /* L20: runtime feature flag check first line. */ if (!cluster_enabled) @@ -1665,6 +2179,7 @@ cluster_reconfig_lmon_tick(void) self_id = cluster_node_id; if (self_id < 0 || self_id >= CLUSTER_MAX_NODES) return; /* defensive: bad self id, cannot participate */ + failstop_stage_handled = cluster_reconfig_poll_failstop_fence_stage(); /* * §3.1 + F11: build the raw CSSD DEAD bitmap, filtering out un-declared @@ -1886,7 +2401,7 @@ cluster_reconfig_lmon_tick(void) * stabilizing the survivor base), THEN the join edge. Each is an independent * ReconfigEvent; neither early-returns past the other. */ - if (!dead_bitmap_is_zero(dead_bitmap)) { + if (!failstop_stage_handled && !dead_bitmap_is_zero(dead_bitmap)) { CLUSTER_INJECTION_POINT("cluster-reconfig-decide-coordinator"); /* §3.2 P1.2: event_id from dead_bitmap + dead_generation snapshot. */ @@ -2024,6 +2539,16 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( CLUSTER_INJECTION_POINT("cluster-reconfig-epoch-bump-pre"); + /* + * spec-2.29a r2 t/274: mark the pre-bump→publish window BEFORE bumping the + * epoch, so a concurrent LMON GRD IDLE tick holds its WAIT_EPOCH baseline + * while this (possibly backend-context, synchronous) path bumps but has not + * yet published. Cleared at every return below. Harmless on the LMON / + * fence-off paths (they either hand off to the local staged flag or publish + * within this same call with no intervening tick). + */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 1); + /* D18: atomic CAS-loop increment. Returns pre/post snapshots. */ cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); @@ -2123,13 +2648,44 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " - "majority for epoch %llu; not publishing reconfig event " - "(write-fenced, will retry)", - (unsigned long long)new_epoch))); - return; /* fail-closed: epoch bumped, event NOT published, recovery NOT started */ + /* + * The async stage is process-local by design and is driven only by LMON + * ticks. Test-only backend callers, such as + * cluster_reconfig_inject_dead_node_test(), would otherwise stage the marker + * in their own process and return with no LMON-visible pending publish + * record. Keep those non-LMON paths on the old bounded wait: they are not + * transport-liveness actors, so this does not reintroduce the BUG-C1 LMON + * park. + */ + if (MyBackendType != B_LMON) { + ClusterFenceMarkerSubmitResult result; + + result = cluster_write_fence_submit_marker(&marker); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " + "majority for epoch %llu; not publishing reconfig event " + "(write-fenced, will retry)", + (unsigned long long)new_epoch))); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; + } + + cluster_reconfig_publish_event(&evt); + cluster_reconfig_log_failstop_epoch_bump(&evt); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; } + + failstop_fence_stage.event = evt; + failstop_fence_stage.marker = marker; + failstop_fence_stage.node_id = coordinator_node_id; + failstop_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + coordinator_node_id, GetCurrentTimestamp()); + /* LMON path: the local staged flag now covers the pending window. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; /* fail-closed until the staged marker is majority-durable */ } cluster_reconfig_publish_event(&evt); @@ -2143,21 +2699,10 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * fixed stack buffer (worst case: 128 node ids x 4 chars) keeps the LMON * tick free of allocator traffic. */ - { - char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; - int off = 0; - int n; - - dead[0] = '\0'; - for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { - if (dead_bitmap[n / 8] & (1 << (n % 8))) - off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); - } - ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " - "(coordinator node %d, dead node(s) {%s})", - (unsigned long long)old_epoch, (unsigned long long)new_epoch, - (int)coordinator_node_id, dead))); - } + cluster_reconfig_log_failstop_epoch_bump(&evt); + + /* spec-2.29a r2 t/274: fence-off path published within this call. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); } @@ -2287,6 +2832,9 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 return 0; if (removed_node_id < 0 || removed_node_id >= CLUSTER_MAX_NODES) return 0; + if (node_removed_fence_stage.async.has_staged_event) + return cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); CLUSTER_INJECTION_POINT("cluster-node-remove-shrink-committing"); @@ -2340,13 +2888,25 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 * a concurrent dead set, if any, is carried by its own fail-stop fence. */ memcpy(marker.fenced_dead_bitmap, removed_with_n, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " - "voting-disk majority for epoch %llu; not publishing removal " - "(write-fenced, will retry)", - removed_node_id, (unsigned long long)new_epoch))); - return 0; /* fail-closed: epoch bumped, removal NOT published, driver retries */ - } + node_removed_fence_stage.event.event_id + = cluster_reconfig_compute_removal_event_id(removed_with_n, removal_event_id); + node_removed_fence_stage.event.coordinator_node_id = cluster_node_id; + node_removed_fence_stage.event.old_epoch = old_epoch; + node_removed_fence_stage.event.new_epoch = new_epoch; + memcpy(node_removed_fence_stage.event.dead_bitmap, empty_dead, + CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + node_removed_fence_stage.event.applied_at = GetCurrentTimestamp(); + node_removed_fence_stage.event.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + node_removed_fence_stage.event.cssd_dead_generation = cssd_dead_generation; + node_removed_fence_stage.event.reconfig_kind = RECONFIG_KIND_NODE_REMOVED; + node_removed_fence_stage.marker = marker; + node_removed_fence_stage.node_id = removed_node_id; + node_removed_fence_stage.last_incarnation = last_incarnation; + node_removed_fence_stage.removal_event_id = removal_event_id; + node_removed_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); + return 0; /* fail-closed until the staged fence marker ACKs */ } CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); @@ -2431,6 +2991,42 @@ cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMa } } +bool +cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now) +{ + int wait_ms; + + if (ReconfigShmem == NULL || m == NULL || a == NULL) + return false; + if (target_node < 0 || target_node >= CLUSTER_MAX_NODES) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&ReconfigShmem->join_marker_request_seq, + &ReconfigShmem->join_marker_completion_seq)) + return false; + + ReconfigShmem->join_marker_target_node_id = target_node; + ReconfigShmem->join_pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit( + a, &ReconfigShmem->join_marker_request_seq, &ReconfigShmem->join_marker_completion_seq, + ReconfigShmem->join_qvotec_latch, now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (ReconfigShmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &ReconfigShmem->join_marker_completion_seq, + &ReconfigShmem->join_marker_result, now, out_result, + out_elapsed_us); +} + bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512) { @@ -2464,6 +3060,7 @@ cluster_reconfig_join_qvotec_complete(bool acked) pg_atomic_write_u64(&ReconfigShmem->join_marker_completion_seq, join_qvotec_inflight_marker_seq); join_qvotec_last_processed_marker_seq = join_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -2614,27 +3211,6 @@ cluster_reconfig_apply_join_as_coordinator( } LWLockRelease(&ReconfigShmem->lock); - /* Durable PREPARE marker per joiner (records the presented incarnation; does - * NOT seed — only COMMITTED is a basis). Best-effort: PREPARE failure does - * not block the JOIN_PENDING publish (the COMMITTED marker in Phase-2 is the - * commit point that must be majority-durable). */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - ClusterJoinCommitMarker m; - - if (!dead_bitmap_test_bit(join_bitmap, i)) - continue; - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_JCMK_MAGIC; - m.version = CLUSTER_JCMK_VERSION; - m.node_id = i; - m.phase = CLUSTER_JCMK_PHASE_PREPARE; - m.admitted_incarnation = joiner_incarnations[i]; - m.generation = joiner_incarnations[i]; /* monotonic per node (read-newest intent) */ - m.admitted_epoch = new_epoch; - cluster_join_marker_compute_crc(&m); - (void)cluster_reconfig_submit_join_marker(i, &m); - } - memset(&evt, 0, sizeof(evt)); evt.event_id = cluster_reconfig_compute_event_id_v2(RECONFIG_KIND_JOIN_PENDING, empty_dead, join_bitmap, @@ -2647,20 +3223,21 @@ cluster_reconfig_apply_join_as_coordinator( evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; evt.cssd_dead_generation = cssd_dead_generation; evt.reconfig_kind = RECONFIG_KIND_JOIN_PENDING; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + + join_prepare_stage.event = evt; + memcpy(join_prepare_stage.join_bitmap, join_bitmap, sizeof(join_prepare_stage.join_bitmap)); + memcpy(join_prepare_stage.joiner_incarnations, joiner_incarnations, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; + join_prepare_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_join_prepare_stage(); } bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) { ClusterJoinCommitMarker m; - uint64 old_epoch, new_epoch; - XLogRecPtr lsn; - ReconfigEvent evt; - uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint64 incs[CLUSTER_MAX_NODES]; if (!cluster_enabled || ReconfigShmem == NULL) return false; @@ -2704,60 +3281,17 @@ cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) m.commit_nonce = ((uint64)cluster_node_id << 56) ^ (uint64)GetCurrentTimestamp(); cluster_join_marker_compute_crc(&m); - if (cluster_reconfig_submit_join_marker(node_id, &m) != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " - "voting-disk majority; not committing (will retry)", - node_id))); + if (join_commit_stage.async.has_staged_event) return false; - } - - CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + join_commit_stage.marker = m; + join_commit_stage.node_id = node_id; + join_commit_stage.admitted_incarnation = admitted_incarnation; + join_commit_stage.async.has_staged_event = true; + join_commit_stage.async.staged_expect_epoch = m.admitted_epoch; + join_commit_stage.submitted = false; + (void)cluster_reconfig_poll_join_commit_stage(); - /* - * ② publish: bump JOIN_COMMITTED epoch + state MEMBER + last_admitted + clear - * pending + clear clean_departed[node] (INV-J10). Strict order — the publish - * (epoch + state MEMBER) precedes the joiner opening its write gate (D5). - */ - cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); - lsn = GetXLogInsertRecPtr(); - cluster_epoch_set_changed_at_lsn((uint64)lsn); - cluster_gcs_block_on_epoch_advance(new_epoch); - cluster_sinval_reset_all_on_reconfig(); - cluster_tt_status_flush_all((uint32)new_epoch); - - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); - cluster_membership_record_admitted(node_id, admitted_incarnation); - ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); - LWLockRelease(&ReconfigShmem->lock); - - /* INV-J10: clear the in-shmem clean_departed suppression so a clean-left node - * that just rejoined has its later real fail-stop honored again (the durable - * supersede across restart is resolved by the seed, RC-5). */ - if (cluster_reconfig_is_clean_departed(node_id)) - pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); - cluster_reconfig_clear_clean_departed(node_id); - - dead_bitmap_set_bit(jb, node_id); - memset(incs, 0, sizeof(incs)); - incs[node_id] = admitted_incarnation; - - memset(&evt, 0, sizeof(evt)); - evt.event_id = cluster_reconfig_compute_event_id_v2( - RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); - evt.coordinator_node_id = cluster_node_id; - evt.old_epoch = old_epoch; - evt.new_epoch = new_epoch; - memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - evt.applied_at = GetCurrentTimestamp(); - evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; - evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); - evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); - - return true; + return false; } diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index fa4b2db725..d67ad677b9 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -681,6 +681,9 @@ cluster_init_shmem_module(void) cluster_grd_outbound_shmem_register(); if (cluster_shmem_lookup_region("pgrac cluster grd work queue") == NULL) cluster_grd_work_queue_shmem_register(); + /* PGRAC: spec-7.2 D4 — DATA-plane outbound ring (Q6-B twin). */ + if (cluster_shmem_lookup_region("pgrac cluster lms data outbound") == NULL) + cluster_lms_outbound_shmem_register(); /* * spec-2.4 D2: register cluster_epoch shmem region. 64-byte cache- @@ -915,6 +918,7 @@ cluster_request_shmem(void) cluster_cr_pool_request_lwlocks(); /* spec-5.51: CR pool named LWLock tranche */ cluster_resolver_cache_request_lwlocks(); /* spec-5.55: resolver cache LWLock tranche */ cluster_ges_dedup_shmem_request(); + cluster_lms_outbound_request_lwlocks(); /* PGRAC: spec-7.2 D4 DATA ring */ /* * spec-4.5a G5: pg_xact_remote SLRU manages its own shmem (SimpleLru), @@ -1077,6 +1081,22 @@ cluster_init_shmem(void) errhint("Enable cluster.online_join on every node, or disable " "cluster.xid_striping."))); + /* + * spec-6.15b D6: shared_catalog puts catalog heap pages on one shared + * storage image, so a multi-node cluster must not allocate dense native + * xids from every node. Without striping, two nodes can reuse the seed + * native-era xid range and poison shared catalog visibility. + */ + if (cluster_enabled && cluster_shared_catalog && cluster_conf_node_count() > 1 + && !cluster_xid_striping) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", cluster_conf_node_count()), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); + /* * Stage 0.18: bind the cluster_ic vtable for the configured tier. * Stub mode allocates no shmem; tier1+ (Stage 2+) will piggyback diff --git a/src/backend/cluster/cluster_sinval.c b/src/backend/cluster/cluster_sinval.c index f96daa8850..e8f6af3df0 100644 --- a/src/backend/cluster/cluster_sinval.c +++ b/src/backend/cluster/cluster_sinval.c @@ -325,6 +325,7 @@ cluster_sinval_enqueue_batch(const SharedInvalidationMessage *msgs, int n) * tier1 TCP fds. Wake LMON immediately; its main loop drains the * outbound queue via cluster_ic_send_envelope_fanout(). */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SINVAL_OUT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); return true; } @@ -365,6 +366,7 @@ cluster_sinval_enqueue_batch_with_ack_flag(const SharedInvalidationMessage *msgs pg_atomic_write_u32(&ClusterSinvalOutbound->tail, (tail + 1) % ClusterSinvalOutbound->capacity); LWLockRelease(&ClusterSinvalOutbound->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SINVAL_OUT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); return true; } @@ -1111,6 +1113,7 @@ cluster_sinval_ack_outbound_enqueue(uint64 batch_id, int32 sender_node, uint16 s pg_atomic_write_u32(&ClusterSinvalAckOutbound->tail, next_tail); LWLockRelease(&ClusterSinvalAckOutbound->lock.lock); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } diff --git a/src/backend/cluster/cluster_tt_status_hint.c b/src/backend/cluster/cluster_tt_status_hint.c index 1bb4e32fe3..71de287b9b 100644 --- a/src/backend/cluster/cluster_tt_status_hint.c +++ b/src/backend/cluster/cluster_tt_status_hint.c @@ -292,6 +292,7 @@ cluster_tt_status_hint_emit(const ClusterTTStatusKey *key, ClusterTTStatus statu pg_atomic_fetch_add_u64(&ClusterTTHintCounters->emit_count, 1); /* L174 idempotent latch wake — LMON drain promptly. */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_TT_HINT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } @@ -345,6 +346,7 @@ cluster_tt_status_hint_emit_subcommitted(const ClusterTTStatusKey *child_key, LWLockRelease(&ClusterTTHintOutbound->lock.lock); pg_atomic_fetch_add_u64(&ClusterTTHintCounters->emit_count, 1); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_TT_HINT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } @@ -409,6 +411,7 @@ cluster_tt_status_hint_emit_multixact_overlay(const ClusterMultiXactKey *key, ui LWLockRelease(&ClusterMultiXactHintOutbound->lock.lock); pg_atomic_fetch_add_u64(&ClusterTTHintCounters->emit_count, 1); + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_TT_HINT); /* spec-7.2 D1 */ cluster_lmon_wakeup(); } diff --git a/src/backend/cluster/cluster_views.c b/src/backend/cluster/cluster_views.c index a1554dbbb2..1c85378157 100644 --- a/src/backend/cluster/cluster_views.c +++ b/src/backend/cluster/cluster_views.c @@ -276,7 +276,7 @@ static const uint32 cluster_wait_event_infos[CLUSTER_WAIT_EVENTS_COUNT] = { WAIT_EVENT_SINVAL_ACK_SEND, WAIT_EVENT_SINVAL_ACK_RECEIVE, - /* Cluster: Interconnect (9 = 7 + spec-6.13 busypoll + inline send) */ + /* Cluster: Interconnect (11 = 9 + spec-7.2 LMS data-plane recv/send) */ WAIT_EVENT_INTERCONNECT_RDMA_SEND, WAIT_EVENT_INTERCONNECT_RDMA_RECV, WAIT_EVENT_CLUSTER_IC_RDMA_POLL, @@ -286,6 +286,8 @@ static const uint32 cluster_wait_event_infos[CLUSTER_WAIT_EVENTS_COUNT] = { WAIT_EVENT_INTERCONNECT_TCP_FALLBACK, WAIT_EVENT_INTERCONNECT_TIER_SWITCH, WAIT_EVENT_INTERCONNECT_CONNECT_RETRY, + WAIT_EVENT_CLUSTER_LMS_DATA_RECV, /* PGRAC spec-7.2 D6 */ + WAIT_EVENT_CLUSTER_LMS_DATA_SEND, /* PGRAC spec-7.2 D6 */ /* Cluster: Undo (4) */ WAIT_EVENT_UNDO_REMOTE_READ, diff --git a/src/backend/cluster/cluster_wal_thread.c b/src/backend/cluster/cluster_wal_thread.c index 17edd98586..b5f7cab947 100644 --- a/src/backend/cluster/cluster_wal_thread.c +++ b/src/backend/cluster/cluster_wal_thread.c @@ -45,7 +45,9 @@ #include #include "access/xlog_internal.h" /* XLOGDIR */ +#include "cluster/cluster_conf.h" #include "cluster/cluster_guc.h" +#include "cluster/cluster_hw.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_wal_state.h" /* spec-4.2 ensure() */ @@ -385,8 +387,23 @@ cluster_wal_thread_init(void) cluster_wal_thread_shmem->dir_configured = dir_set ? 1 : 0; } - if (!dir_set) + if (!dir_set) { + if (cluster_shared_catalog && cluster_conf_has_peers()) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cluster.shared_catalog requires cluster.wal_threads_dir in a " + "multi-node cluster"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "initialise each node with pgrac-init --wal-threads-dir."))); + if (!cluster_hw_remaster_recoverable()) + ereport(WARNING, + (errmsg("cluster HW remaster is not recoverable because " + "cluster.wal_threads_dir is unset"), + errhint("A dead node's adopted HW authority cannot be rebuilt from per-thread " + "WAL until cluster.wal_threads_dir is configured and the node is " + "restarted."))); return; /* flat layout: identity stamping only (Q3-A) */ + } /* * Configuration coherence, fail-closed (L58: enumerate the diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index 2baebb13bd..8917959359 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -40,6 +40,7 @@ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_voting_disks */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_qvotec.h" /* ClusterVotingSlot (marker layout asserts) */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_voting_disk_io.h" /* D6 direct durable marker read */ @@ -769,6 +770,37 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m) } } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (cluster_write_fence_shmem == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq)) + return false; + + memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); + return cluster_marker_async_submit( + a, &cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq, cluster_write_fence_shmem->qvotec_latch, + now, (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (cluster_write_fence_shmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cluster_write_fence_shmem->marker_completion_seq, + &cluster_write_fence_shmem->marker_result, now, out_result, + out_elapsed_us); +} + /* * cluster_write_fence_clear_qvotec_latch / _publish_qvotec_latch -- qvotec publishes * its MyLatch at startup so LMON can wake it; auto-cleared at proc_exit so a stale @@ -835,6 +867,7 @@ cluster_write_fence_qvotec_complete(bool acked) pg_atomic_write_u64(&cluster_write_fence_shmem->marker_completion_seq, qvotec_inflight_marker_seq); qvotec_last_processed_marker_seq = qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c new file mode 100644 index 0000000000..6a16671c57 --- /dev/null +++ b/src/backend/cluster/cluster_xid_authority.c @@ -0,0 +1,494 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.c + * Shared XID authority: never-lowered native-era xid high-water plus + * one-way lifecycle flags, as a torn-safe file under + * cluster.shared_data_dir (spec-6.15b D1). + * + * The authority records the first xid NOT consumed by the native era + * (the pre-formation cluster.enabled=off single-writer window of the + * seed node) and two one-way flags: SEALED (a clean native-era + * shutdown published a complete high-water together with the pg_xact + * prehistory blob, so joiners may adopt) and CLUSTER_ERA (a + * cluster.enabled=on boot closed the native era forever). Writers are + * single by construction: the seed node's postmaster/checkpointer + * during the native era, and the catalog bootstrap window afterwards. + * + * File discipline mirrors cluster_oid_lease.c: temp + fsync + .bak + * roll + durable_rename on write; primary-then-.bak fail-closed read; + * ENOENT-only-absent presence probe. The pure classify layer is + * standalone-linkable for cluster_unit. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1, §4 U1-U3/U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* On-disk image size: the fixed header, exactly. */ +#define CLUSTER_XID_AUTHORITY_FILE_SIZE ((int)sizeof(ClusterXidAuthorityHeader)) + +/* ============================================================ + * Pure layer (no elog/shmem/fd; standalone-linkable for cluster_unit). + * ============================================================ */ + +/* + * cluster_xid_authority_classify -- validate an authority image buffer. + */ +ClusterXidAuthorityValidity +cluster_xid_authority_classify(const char *buf, size_t len) +{ + ClusterXidAuthorityHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidAuthorityHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_AUTHORITY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * build_path -- join cluster_shared_data_dir + relpath into dst. Returns + * false when the shared root is not configured. + */ +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * write_durable -- write buf into tmp, fsync, then durable_rename over final. + * PANICs on any I/O failure (mirrors cluster_oid_lease write_durable). + */ +static void +write_durable(const char *tmp, const char *final, const char *buf) +{ + int fd; + + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + + errno = 0; + if (write(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE) != CLUSTER_XID_AUTHORITY_FILE_SIZE) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + if (durable_rename(tmp, final, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, final))); +} + +/* + * roll_primary_to_bak -- preserve an existing primary into .bak before a new + * primary write. A missing/short primary (first write) is skipped. + */ +static void +roll_primary_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char buf[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + int fd; + int r; + + fd = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (fd < 0) + return; /* first write: no prior primary */ + + r = read(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return; /* short/odd primary: don't manufacture a .bak */ + + write_durable(baktmp, bak, buf); +} + +/* + * read_image -- read a candidate authority file and classify it. Missing or + * short is INVALID_SHORT. + */ +static ClusterXidAuthorityValidity +read_image(const char *path, char *image) +{ + int fd; + int r; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + r = read(fd, image, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + return cluster_xid_authority_classify(image, CLUSTER_XID_AUTHORITY_FILE_SIZE); +} + +/* + * copy_flags_settled -- does the on-disk copy at relpath validate AND carry + * all `must_set` flag bits with all `must_clear` bits clear? + */ +static bool +copy_flags_settled(const char *relpath, uint32 must_set, uint32 must_clear) +{ + ClusterXidAuthorityHeader hdr; + char path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (!build_path(path, sizeof(path), relpath)) + return false; + if (read_image(path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; + memcpy(&hdr, image, sizeof(hdr)); + return (hdr.flags & must_set) == must_set && (hdr.flags & must_clear) == 0; +} + +/* + * both_copies_flags_settled -- is a one-way flag transition complete in BOTH + * on-disk copies? Flag transitions install the same image as primary and + * .bak (primary first), so a crash between the two durable renames leaves a + * pre-transition .bak behind an already-transitioned primary. Nothing else + * on the restart path rewrites the pair, so without a re-assert the stale + * copy survives the whole run, and any later transient primary read failure + * resurrects the pre-transition flags through the .bak fallback -- a stale + * SEALED .bak hands joiners the previous pass's high-water and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard (review + * r3-X1, 8.A). The transition entry points therefore skip their write only + * when this returns true; a missing or invalid copy is "not settled" too, + * which restores the damaged copy for free. + */ +static bool +both_copies_flags_settled(uint32 must_set, uint32 must_clear) +{ + return copy_flags_settled(CLUSTER_XID_AUTHORITY_REL_PATH, must_set, must_clear) + && copy_flags_settled(CLUSTER_XID_AUTHORITY_BAK_REL_PATH, must_set, must_clear); +} + +/* + * cluster_xid_authority_read -- fail-closed read of the shared authority. + * Tries primary then .bak; returns false when neither is trustworthy. Never + * ereports (safe on the bootstrap early-read path). + */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return cluster_xid_authority_read_checked(out, NULL); +} + +/* + * cluster_xid_authority_read_checked -- as above, additionally reporting + * whether the image came from the .bak fallback so consumers can surface a + * LOG-once operator signal (review F8; the read itself stays silent for + * the bootstrap early-read path). + */ +bool +cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (used_bak != NULL) + *used_bak = false; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (read_image(primary_path, image) != CLUSTER_XID_AUTHORITY_VALID) { + if (read_image(bak_path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; /* fail-closed: neither trustworthy */ + if (used_bak != NULL) + *used_bak = true; + } + + memcpy(out, image, sizeof(*out)); + return true; +} + +/* + * cluster_xid_authority_present -- does any authority image (primary or .bak) + * exist on disk at all, trustworthy or not? Lets callers distinguish + * "absent: genuine first seed" from "present but corrupt: fail-closed" -- a + * corrupt authority must never be silently re-seeded (a re-seed from a + * checkpointed value could put already-consumed xids back "in the future"). + * Never ereports. + * + * Fail-closed errno discipline: only ENOENT proves absence (mirrors + * cluster_oid_authority_present; EIO/EACCES/ESTALE report present). + */ +bool +cluster_xid_authority_present(void) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + struct stat st; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (stat(primary_path, &st) == 0 || errno != ENOENT) + return true; + if (stat(bak_path, &st) == 0 || errno != ENOENT) + return true; + return false; +} + +/* + * write_header -- CRC-stamp and atomically install an authority header. + */ +static void +write_header(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + roll_primary_to_bak(primary, bak, baktmp); + write_durable(tmp, primary, buffer); +} + +/* + * write_header_both -- CRC-stamp the header and install the SAME image as + * both primary and .bak (primary first). For FLAG transitions (unseal, + * CLUSTER_ERA stamp) the usual roll-primary-to-.bak is unsafe: it would + * preserve a pre-transition image that the fail-closed read falls back to + * when the primary is later damaged -- a stale SEALED .bak hands a joiner + * the previous pass's high-water (review F1, 8.A), and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Crash + * points: both copies old (transition not yet taken: state still + * consistent), primary new + .bak old (read prefers the valid primary), + * both new. The primary-new/.bak-old window does not self-repair on its + * own: the transition entry points are re-run by every boot and re-assert + * both copies until both_copies_flags_settled holds (review r3-X1), so a + * later single-copy corruption never resurrects the pre-transition flags. + */ +static void +write_header_both(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + write_durable(tmp, primary, buffer); + write_durable(baktmp, bak, buffer); +} + +/* + * cluster_xid_authority_seed_if_absent -- read-then-seed: if the authority is + * already present (any trustworthy image) do nothing; otherwise create the + * unsealed initial image. Returns true when this call created it. + */ +bool +cluster_xid_authority_seed_if_absent(uint64 initial_native_hw) +{ + ClusterXidAuthorityHeader hdr; + + if (cluster_xid_authority_read(&hdr)) + return false; /* already seeded (join node / prior seed) */ + if (cluster_xid_authority_present()) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at seed"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "do not re-seed from a stale xid high-water.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + + memset(&hdr, 0, sizeof(hdr)); + hdr.flags = 0; + hdr.native_hw_full = initial_native_hw; + hdr.next_multi = 0; + write_header(&hdr); + return true; +} + +/* + * cluster_xid_authority_publish_native -- monotone raise of the native-era + * high-water; seal=true additionally stamps SEALED. Flags are never + * cleared and the high-water is never lowered (spec §3.1). An absent + * authority is created outright (crash between bootstrap-seed and the + * first publish); a present-but-corrupt one PANICs rather than re-seed. + */ +void +cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, bool seal) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) { + if (cluster_xid_authority_present()) + ereport(PANIC, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at publish"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "re-seeding from a stale high-water could re-issue consumed xids.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + memset(&hdr, 0, sizeof(hdr)); + } + + if (native_hw_full > hdr.native_hw_full) + hdr.native_hw_full = native_hw_full; + if (next_multi > hdr.next_multi) + hdr.next_multi = next_multi; + if (seal) + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_SEALED; + + write_header(&hdr); +} + +/* + * cluster_xid_authority_mark_cluster_era -- one-way CLUSTER_ERA stamp (first + * cluster.enabled=on boot; spec §3.1). Idempotent re-assert: re-run by + * every cluster boot, it rewrites BOTH copies until both carry the flag, so + * a crash between write_header_both's two renames (stamped primary, stale + * .bak) is repaired on the next boot instead of surviving the run (review + * r3-X1); once both copies are settled it is a no-write no-op. A + * missing/corrupt authority PANICs: the catalog bootstrap seeds it before + * any caller can reach this point, so absence here is real damage. + */ +void +cluster_xid_authority_mark_cluster_era(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at cluster-era stamp"))); + + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) != 0 + && both_copies_flags_settled(CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0)) + return; /* transition complete in both copies */ + + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; + write_header_both(&hdr); /* review F1 variant: one-way flag in BOTH copies */ +} + +/* + * cluster_xid_authority_begin_native_run -- re-open the native era for a + * follow-up cluster.enabled=off seed run (spec-6.15b §3.1 multi-pass arm). + * + * A sealed authority means "the high-water and prehistory are complete as + * of a clean native shutdown". A NEW native run invalidates that + * completeness the moment it allocates its first xid, so the seal is + * cleared up front: if this run crashes, joiners fail closed (unsealed, + * 53RB5) instead of adopting the previous pass's stale high-water -- + * false-invisible for every xid the crashed pass consumed (rule 8.A). + * The clean shutdown of this run re-publishes and re-seals monotonically. + * + * Only legal while CLUSTER_ERA is unset (the caller FATALs re-entry + * first; re-checked here defensively). Formation recipes keep native-era + * boots and first cluster boots strictly ordered; if a racing first + * cluster boot stamps CLUSTER_ERA inside this read-modify-write window, + * the stamp is re-applied by the next cluster boot (mark is re-issued + * whenever unset) and this authority is left unsealed -- which only ever + * blocks adoption, never yields a wrong answer. + * + * Idempotent re-assert (review r3-X1): re-run by every native-era boot, + * it rewrites BOTH copies until neither retains SEALED, so a crash + * between write_header_both's two renames (unsealed primary, stale + * SEALED .bak) is repaired on the next boot instead of surviving the + * run; once both copies are settled it is a no-write no-op. + */ +void +cluster_xid_authority_begin_native_run(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at native-run open"))); + + if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"))); + + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0 + && both_copies_flags_settled(0, CLUSTER_XID_AUTHORITY_FLAG_SEALED)) + return; /* open, and no copy retains SEALED */ + + hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; + write_header_both(&hdr); /* review F1: no copy may retain SEALED */ +} diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c new file mode 100644 index 0000000000..490804ccac --- /dev/null +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -0,0 +1,650 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_prehistory.c + * Native-era pg_xact prehistory: publish the seed node's CLOG page run + * [0, native_hw) into a CRC-guarded blob under cluster.shared_data_dir, + * and adopt it into a joiner's local pg_xact (spec-6.15b D2). + * + * The native era (pre-formation, cluster.enabled=off) is a single- + * writer window: the seed node's local pg_xact is the complete commit- + * status truth for every xid below the native high-water, including + * aborted ones. Publishing that page run with the shared tree lets a + * pre-seed-lineage joiner adopt first-hand truth instead of trusting + * hint bits (false-invisible / poison-stamp hazard; spec §0). This is + * a one-shot formation-time carry of pre-cluster history -- the same + * bytes a post-seed clone would have carried -- NOT a runtime mirror of + * cluster-era foreign commit bits (the rejected CLOG-overlay design; + * spec §1.4.1 boundary note). + * + * Both publish and adopt stream page-at-a-time (no large allocations): + * a CRC pass validates or stamps the whole image, then a second pass + * moves the bytes. Their callers run in single-writer windows (the + * seed's shutdown checkpoint; the joiner's postmaster-once bootstrap, + * strictly before StartupCLOG), so the two passes see stable bytes. + * Adopt fsyncs every touched segment and the pg_xact directory before + * returning (spec P2). + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_prehistory.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D2, §4 U1/U4) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* + * CLOG geometry, restated locally so the module stays standalone-linkable + * for cluster_unit: 2 status bits per xact -> 4 xacts per byte -> BLCKSZ*4 + * xacts per page (clog.c CLOG_XACTS_PER_PAGE, private to clog.c), and 32 + * pages per SLRU segment file (slru.h SLRU_PAGES_PER_SEGMENT). The unit + * truth table locks this arithmetic against drift. + */ +#define PREHISTORY_XACTS_PER_PAGE ((uint32)(BLCKSZ * 4)) +#define PREHISTORY_PAGES_PER_SEGMENT 32 + +static bool prehistory_adopted_this_boot = false; + +/* ============================================================ + * Pure layer. + * ============================================================ */ + +/* + * cluster_xid_prehistory_payload_bytes -- whole-page payload size covering + * xids [0, native_hw_full); 0 for an empty or over-cap native era (callers + * treat 0 as "nothing to carry" / refuse respectively). + */ +uint32 +cluster_xid_prehistory_payload_bytes(uint64 native_hw_full) +{ + uint64 pages; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + return 0; + + pages = (native_hw_full + PREHISTORY_XACTS_PER_PAGE - 1) / PREHISTORY_XACTS_PER_PAGE; + return (uint32)(pages * BLCKSZ); +} + +/* + * cluster_xid_prehistory_classify -- validate a full in-memory blob image + * (header + payload). Used by the unit truth table; the streaming adopt + * path performs the same checks incrementally. + */ +ClusterXidAuthorityValidity +cluster_xid_prehistory_classify(const char *buf, size_t len) +{ + ClusterXidPrehistoryHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidPrehistoryHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + if (len != sizeof(hdr) + hdr.payload_len + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(hdr.native_hw_full)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidPrehistoryHeader, crc)); + COMP_CRC32C(crc, buf + sizeof(hdr), hdr.payload_len); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Streaming file plumbing. + * ============================================================ */ + +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * read_clog_page -- read local pg_xact page `pageno` into buf. Fail-closed: + * a missing or short segment is FATAL -- the caller's window (post + * CheckPointGuts / clean shutdown) guarantees every allocated page is on + * disk, so a hole would mean fabricating "aborted" truth for real xids. + */ +static void +read_clog_page(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not open pg_xact segment \"%s\" for prehistory publish: %m", seg), + errhint("The native-era CLOG must be complete on disk; publish runs only " + "after a CLOG flush."))); + if (lseek(fd, offset, SEEK_SET) != offset || read(fd, buf, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of pg_xact segment \"%s\" page %u for prehistory publish", seg, + pageno), + errhint("The native-era CLOG must be complete on disk; a hole would fabricate " + "commit-status truth."))); + CloseTransientFile(fd); +} + +/* + * roll_blob_to_bak -- preserve an existing primary blob into .bak (streamed + * in page-sized chunks; variable length). Missing primary is skipped. + */ +static void +roll_blob_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char chunk[BLCKSZ]; + int src; + int dst; + int r; + + src = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (src < 0) + return; /* first write: no prior primary */ + + dst = OpenTransientFile(baktmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (dst < 0) { + CloseTransientFile(src); + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", baktmp))); + } + while ((r = read(src, chunk, sizeof(chunk))) > 0) { + if (write(dst, chunk, r) != r) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", baktmp))); + } + if (r < 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", primary))); + CloseTransientFile(src); + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", baktmp))); + CloseTransientFile(dst); + if (durable_rename(baktmp, bak, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", baktmp, bak))); +} + +/* ============================================================ + * Publish (seed side). + * ============================================================ */ + +/* + * cluster_xid_prehistory_publish -- snapshot local pg_xact [0, native_hw) + * into the shared blob, torn-safe. Two passes over the source pages: one + * to compute the image CRC, one to write; the caller's single-writer + * window (seed shutdown checkpoint / catalog-seed StartupXLOG) keeps the + * bytes stable between the passes. + */ +void +cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + char page[BLCKSZ]; + uint32 payload_len; + uint32 pages; + uint32 p; + int fd; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native-era xid high-water " UINT64_FORMAT + " is outside the prehistory publish range", + native_hw_full), + errhint("Native-era seed loads are capped at " UINT64_FORMAT + " xids; split the seed load or move it in-protocol.", + CLUSTER_XID_PREHISTORY_MAX_XID))); + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_PREHISTORY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + payload_len = cluster_xid_prehistory_payload_bytes(native_hw_full); + pages = payload_len / BLCKSZ; + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_PREHISTORY_MAGIC; + hdr.version = CLUSTER_XID_PREHISTORY_VERSION; + hdr.native_hw_full = native_hw_full; + hdr.payload_len = payload_len; + hdr.reserved = 0; + + /* pass 1: CRC over header fields + source pages */ + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + COMP_CRC32C(hdr.crc, page, BLCKSZ); + } + FIN_CRC32C(hdr.crc); + + /* pass 2: stream header + pages into tmp, then install torn-safe */ + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + errno = 0; + if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + errno = 0; + if (write(fd, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + } + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + roll_blob_to_bak(primary, bak, baktmp); + if (durable_rename(tmp, primary, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, primary))); +} + +/* ============================================================ + * Adopt (joiner side). + * ============================================================ */ + +/* + * verify_blob -- streaming validation of one candidate blob file against + * the sealed authority's native_hw. Returns true (and leaves *out_pages + * set) only when magic/version/hw/length/CRC all hold. + */ +static bool +verify_blob(const char *path, uint64 native_hw_full, uint32 *out_pages) +{ + ClusterXidPrehistoryHeader hdr; + char page[BLCKSZ]; + pg_crc32c crc; + uint32 pages; + uint32 p; + int fd; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return false; + if (read(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + CloseTransientFile(fd); + return false; + } + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC || hdr.version != CLUSTER_XID_PREHISTORY_VERSION + || hdr.native_hw_full != native_hw_full + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(native_hw_full)) { + CloseTransientFile(fd); + return false; + } + + pages = hdr.payload_len / BLCKSZ; + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + if (read(fd, page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(fd); + return false; + } + COMP_CRC32C(crc, page, BLCKSZ); + } + FIN_CRC32C(crc); + CloseTransientFile(fd); + + if (!EQ_CRC32C(crc, hdr.crc)) + return false; + + *out_pages = pages; + return true; +} + +/* + * cluster_xid_prehistory_adopt -- decode the shared blob into local pg_xact + * page files. Validates the whole image (primary, then .bak) BEFORE the + * first local write -- a joiner must never take half a truth. Writes are + * whole-page, sequential from page 0, fsynced per touched segment, then the + * pg_xact directory is fsynced (spec P2: complete and durable strictly + * before StartupCLOG). Idempotent: re-running overwrites the same bytes. + */ +void +cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char seg[MAXPGPATH]; + char dir[MAXPGPATH]; + char page[BLCKSZ]; + const char *src_path = NULL; + uint32 pages = 0; + uint32 p; + int src; + int dst = -1; + int cur_segno = -1; + int fd; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is missing, corrupt, or does not match the " + "sealed authority high-water " UINT64_FORMAT, + native_hw_full), + errhint("The seed node must complete a clean native-era shutdown before " + "joiners can adopt; restore \"%s\" from the shared tree's backup " + "if it was damaged.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + + /* pass 2: stream the validated pages into local pg_xact segments */ + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not re-open shared XID prehistory \"%s\": %m", src_path))); + + for (p = 0; p < pages; p++) { + int segno = (int)(p / PREHISTORY_PAGES_PER_SEGMENT); + off_t offset = (off_t)(p % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + if (read(src, page, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of shared XID prehistory \"%s\" page %u", src_path, p))); + + if (segno != cur_segno) { + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, segno); + dst = OpenTransientFile(seg, O_RDWR | O_CREAT | PG_BINARY); + if (dst < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact segment \"%s\": %m", seg))); + cur_segno = segno; + } + errno = 0; + if (lseek(dst, offset, SEEK_SET) != offset || write(dst, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not write pg_xact segment \"%s\": %m", seg))); + } + } + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + CloseTransientFile(src); + + /* P2: directory fsync makes the adopted truth durable before StartupCLOG */ + snprintf(dir, sizeof(dir), "%s/pg_xact", local_pgdata); + fd = OpenTransientFile(dir, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact directory \"%s\": %m", dir))); + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact directory \"%s\": %m", dir))); + CloseTransientFile(fd); + prehistory_adopted_this_boot = true; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return prehistory_adopted_this_boot; +} + +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +/* + * read_local_clog_page_optional -- read local pg_xact page `pageno` into + * buf. Returns false when the segment file or the page does not exist; + * PANICs on a real read error so an I/O fault is never mistaken for a + * missing page. The caller decides what a missing page means: below + * page(oldestXid) it is a legitimately truncated frozen segment, at or + * above it is an anomaly (review r3-X2). + */ +static bool +read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + int r; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) { + if (errno == ENOENT) + return false; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", seg))); + } + if (lseek(fd, offset, SEEK_SET) != offset) { + CloseTransientFile(fd); + return false; + } + errno = 0; + r = read(fd, buf, BLCKSZ); + CloseTransientFile(fd); + if (r < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", seg))); + return r == BLCKSZ; +} + +/* + * clog_slot_mask -- byte mask covering the 2-bit CLOG status slots + * [lo, hi) within one byte (0 <= lo < hi <= 4). Slot n of a byte holds + * xid bits at (n * CLOG_BITS_PER_XACT), low-order first (clog.c + * TransactionIdToBIndex). + */ +static inline unsigned char +clog_slot_mask(uint32 lo, uint32 hi) +{ + unsigned int m; + + m = ((hi >= 4) ? 0xFFu : ((1u << (hi * 2)) - 1u)) & ~((1u << (lo * 2)) - 1u); + return (unsigned char)m; +} + +/* + * clog_page_range_equal -- compare blob vs local CLOG page content for the + * page-local 2-bit slots [from_xact, to_xact) only. Sub-byte boundaries + * are masked so bits outside the range never influence the verdict. + */ +static bool +clog_page_range_equal(const char *blob_page, const char *local_page, uint32 from_xact, + uint32 to_xact) +{ + uint32 first_byte = from_xact / 4; + uint32 last_byte = (to_xact - 1) / 4; + + Assert(from_xact < to_xact && to_xact <= PREHISTORY_XACTS_PER_PAGE); + + if (first_byte == last_byte) + return ((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, to_xact - first_byte * 4)) + == 0; + + if (from_xact % 4 != 0) { + if (((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, 4)) + != 0) + return false; + first_byte++; + } + if (to_xact % 4 != 0) { + if (((unsigned char)(blob_page[last_byte] ^ local_page[last_byte]) + & clog_slot_mask(0, to_xact % 4)) + != 0) + return false; + last_byte--; + } + return first_byte > last_byte + || memcmp(blob_page + first_byte, local_page + first_byte, last_byte - first_byte + 1) + == 0; +} + +/* + * cluster_xid_prehistory_prefix_check -- see header. Streams the sealed + * blob (primary, then .bak) and byte-compares the local pg_xact prefix + * covering xids [oldest_xid_full, min(limit_xid_full, native_hw_full)) at + * per-xact (2-bit) precision. + * + * Frozen-prefix exemption (review r3-X2): SimpleLruTruncate removes whole + * pg_xact SEGMENTS once every page they cover precedes page(oldestXid) + * (slru.c SlruMayDeleteSegment), so a joiner that froze past >=1 segment + * legitimately has no file where the blob still has pages -- the old + * break-on-first-missing-local-page treated exactly that shape as a + * tail-short clone and returned CONSISTENT without ever comparing the + * surviving pages. Neither truncation nor tuple freezing rewrites the + * CLOG bytes that survive, but bits below oldestXid are dead content + * anyway (every tuple with such an xmin/xmax is frozen, so CLOG is never + * consulted for them again) and whether they still exist is an accident + * of truncation timing; lineage evidence therefore starts at oldestXid's + * own 2-bit slot, and everything below it is skipped whether the page + * survives or not. Within the comparable range [oldestXid, limit) a + * MISSING local page is an anomaly -- pg_xact always covers + * [page(oldestXid), page(nextXid)] on a well-formed node -- and returns + * UNAVAILABLE (fail-closed), never CONSISTENT. + */ +ClusterXidPrefixVerdict +cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, + uint64 limit_xid_full, uint64 oldest_xid_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char blob_page[BLCKSZ]; + char local_page[BLCKSZ]; + const char *src_path; + uint32 pages = 0; + uint32 start_page; + uint32 p; + uint64 limit; + uint64 oldest; + int src; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + limit = Min(limit_xid_full, native_hw_full); + if (limit == 0) + return CLUSTER_XID_PREFIX_CONSISTENT; + + oldest = Min(oldest_xid_full, limit); + if (oldest >= limit) + return CLUSTER_XID_PREFIX_CONSISTENT; /* everything comparable is frozen */ + start_page = (uint32)(oldest / PREHISTORY_XACTS_PER_PAGE); + + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (src >= 0) + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + + for (p = 0; p < pages; p++) { + uint64 page_first_xid = (uint64)p * PREHISTORY_XACTS_PER_PAGE; + uint64 cmp_from; + uint64 cmp_to; + + if (read(src, blob_page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + if (page_first_xid >= limit) + break; /* comparison scope ended on a page boundary */ + if (p < start_page) + continue; /* wholly frozen page: skipped, present or not */ + + cmp_from = Max(page_first_xid, oldest); + cmp_to = Min(page_first_xid + PREHISTORY_XACTS_PER_PAGE, limit); + /* oldest < limit and p >= start_page guarantee a non-empty range */ + Assert(cmp_from < cmp_to); + + if (!read_local_clog_page_optional(local_pgdata, p, local_page)) { + /* hole inside the comparable range: fail closed (r3-X2) */ + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + + if (!clog_page_range_equal(blob_page, local_page, (uint32)(cmp_from - page_first_xid), + (uint32)(cmp_to - page_first_xid))) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; + } + } + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_CONSISTENT; +} diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index b8afb18bc5..1a0f4ddccb 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -2017,8 +2017,17 @@ ServerLoop(void) * external child termination paths. cluster.lms_enabled = off * (PGC_POSTMASTER) keeps LMS un-forked and leaves the startup-time * PG-native fallback path active. + * + * PGRAC: spec-7.2 — LMS must also run in HOT STANDBY: with the + * GCS block family on the LMS-owned DATA plane, an ADG standby + * without an LMS has no DATA listener and every cross-node block + * request from or to it retransmits to exhaustion (t/335). The + * standby LMS serves the same roles as on a primary (DATA plane + * + CR/undo park-serve); grant ownership stays with LMON (HC4) + * on every node. */ - if (cluster_enabled && cluster_lms_enabled && LmsPID == 0 && pmState == PM_RUN) + if (cluster_enabled && cluster_lms_enabled && LmsPID == 0 + && (pmState == PM_RUN || pmState == PM_HOT_STANDBY)) LmsPID = StartLms(); /* diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index a1844aac56..677e4e7df2 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -1225,18 +1225,18 @@ pgstat_get_wait_cluster_interconnect(WaitEventCluster w) case WAIT_EVENT_INTERCONNECT_RDMA_RECV: event_name = "InterconnectRdmaRecv"; break; - case WAIT_EVENT_CLUSTER_IC_RDMA_POLL: - event_name = "ClusterICRdmaPoll"; - break; - case WAIT_EVENT_INTERCONNECT_RDMA_BUSYPOLL: - event_name = "InterconnectRdmaBusypoll"; - break; - case WAIT_EVENT_INTERCONNECT_RDMA_INLINE_SEND: - event_name = "InterconnectRdmaInlineSend"; - break; - case WAIT_EVENT_CLUSTER_IC_RDMA_CONNECT: - event_name = "ClusterICRdmaConnect"; - break; + case WAIT_EVENT_CLUSTER_IC_RDMA_POLL: + event_name = "ClusterICRdmaPoll"; + break; + case WAIT_EVENT_INTERCONNECT_RDMA_BUSYPOLL: + event_name = "InterconnectRdmaBusypoll"; + break; + case WAIT_EVENT_INTERCONNECT_RDMA_INLINE_SEND: + event_name = "InterconnectRdmaInlineSend"; + break; + case WAIT_EVENT_CLUSTER_IC_RDMA_CONNECT: + event_name = "ClusterICRdmaConnect"; + break; case WAIT_EVENT_INTERCONNECT_TCP_FALLBACK: event_name = "ClusterICRdmaFallback"; break; @@ -1265,6 +1265,12 @@ pgstat_get_wait_cluster_interconnect(WaitEventCluster w) case WAIT_EVENT_CLUSTER_IC_RECONNECT: event_name = "ClusterICReconnect"; break; + case WAIT_EVENT_CLUSTER_LMS_DATA_RECV: + event_name = "ClusterLmsDataRecv"; + break; + case WAIT_EVENT_CLUSTER_LMS_DATA_SEND: + event_name = "ClusterLmsDataSend"; + break; default: break; } diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be8af8e758..ac8fe1c784 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -875,6 +875,17 @@ Section: Class 53 - Insufficient Resources (pgrac extension) # the first own checkpoint). 53RB3 E ERRCODE_CLUSTER_RECOVERY_ANCHOR_UNAVAILABLE cluster_recovery_anchor_unavailable +# spec-6.15b: shared XID authority / native-era prehistory unavailable. +# Under cluster.shared_catalog=on a joiner must reconcile its transaction +# horizon against the sealed shared XID authority and adopt the native-era +# prehistory (pg_xact truth for pre-formation seed xids); a missing, +# unsealed or corrupt authority/prehistory fails closed rather than judge +# seed rows through a stale horizon or a truthless local CLOG. Also raised +# when a cluster.enabled=off boot tries to re-enter the native seed era on +# a formed cluster's shared tree. (53RB4 is claimed by the spec-7.1 lane's +# ERRCODE_CLUSTER_MXID_HALFSPACE_LIMIT; do not reuse.) +53RB5 E ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE cluster_xid_authority_unavailable + Section: Class 54 - Program Limit Exceeded # this is for wired-in limits, not resource exhaustion problems (class borrowed from DB2) diff --git a/src/include/cluster/cluster_catalog_migrate.h b/src/include/cluster/cluster_catalog_migrate.h index aa17cae1cb..18e3c640ec 100644 --- a/src/include/cluster/cluster_catalog_migrate.h +++ b/src/include/cluster/cluster_catalog_migrate.h @@ -61,6 +61,11 @@ typedef struct ClusterCatalogAuthorityMarker { pg_crc32c crc; /* over [0, offsetof(crc)) */ } ClusterCatalogAuthorityMarker; +typedef enum ClusterCatalogMigrateResult { + CLUSTER_CATALOG_MIGRATE_SEEDED = 0, + CLUSTER_CATALOG_MIGRATE_ADOPTED +} ClusterCatalogMigrateResult; + /* * cluster_catalog_migrate_tree -- establish the shared catalog relation tree * for this node. Postmaster-once, only meaningful when @@ -74,7 +79,8 @@ typedef struct ClusterCatalogAuthorityMarker { * Fail-closed: FATAL 53RB0 on a foreign/mismatched or unreadable authority. * system_identifier is the cluster-wide value from the shared pg_control. */ -extern void cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier); +extern ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, + uint64 system_identifier); /* * cluster_catalog_vet_no_unlogged -- spec-6.14 Q12 enable-time vet: FATAL diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index e49ecfc194..6a4808f793 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -65,6 +65,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ #define CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES 16 /* Default drain deadline; aligns with the feature-082 30s barrier. */ @@ -149,11 +151,17 @@ typedef struct ClusterLeaveState { int32 leaving_node_id; /* -1 if none */ uint32 _pad0; uint64 leave_epoch; /* epoch this leave is bound to (CL-I3 immutable) */ - uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound; - * an unchanged value at the epoch bump means OUR - * clean-leave committed (no real death intruded) */ + uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound + * (retained for observability; the coherence gate + * now uses leave_baseline_others_dead, spec-2.29a + * ②b — the scalar counted the leaving node's own + * expected DEAD and falsely escalated) */ uint64 barrier_deadline_us; /* fail-closed deadline (drain_timeout_ms) */ uint8 ack_bitmap[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; /* survivor acks */ + /* spec-2.29a ②b: CSSD dead set (EXCLUDING the leaving node) snapshotted when + * the leave was bound; the coherence gate escalates only if a THIRD-PARTY + * death changed this set mid-drain. */ + uint8 leave_baseline_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; pg_atomic_uint64 ges_drained_count; /* shards/grants drained (observability) */ pg_atomic_uint64 gcs_flushed_count; /* dirty/X pages flushed */ pg_atomic_uint64 shards_remastered; /* shards moved off leaving node */ @@ -198,6 +206,14 @@ typedef struct ClusterLeaveState { pg_atomic_uint32 commit_point_observed; pg_atomic_uint32 committed_durable_confirmed; pg_atomic_uint32 committed_marker_durable; + /* spec-2.29a r3 (evidence latch): the committed epoch E the coordinator's + * LEAVE_COMMITTED attested (the epoch stamped into the majority-durable + * COMMITTED marker). Written by the leaving node's confirmation handler + * under `lock` BEFORE committed_durable_confirmed is set; consumed once by + * the barrier tick when the evidence latches (so the leaver records the + * exact committed epoch instead of inferring it from its possibly-stale + * local epoch view). Guarded by `lock`. */ + uint64 committed_confirmed_epoch; /* * Hardening v1.0.2 fields. @@ -335,10 +351,33 @@ extern bool cluster_clean_leave_should_abort_writable(bool in_transaction, bool /* version-coherent leave (U3 / CL-I3 / L235): the leave is still coherent only * if neither the cluster epoch nor the CSSD dead_generation moved since the - * leave was bound; any external bump (a real death intruding) => not coherent - * => the caller must ABORTED_ESCALATE. */ + * leave was bound; any external membership change (a third-party death, or a + * third-party fail-stop that already bumped the epoch) => not coherent => the + * caller must ABORTED_ESCALATE. spec-2.29a ②b: the dead set compared here + * EXCLUDES the leaving node itself (others-dead bitmap), so the leaving node's + * own expected alive→DEAD transition never falsely escalates the leave. */ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen); + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes); + +/* spec-2.29a r3: the leaving node's barrier-tick own-commit latch — evidence + * over inference. Latches iff the durable COMMITTED marker for THIS leave + * attempt was confirmed (nonce-bound LEAVE_COMMITTED attestation); the two + * coherence observations are contract inputs the verdict must ignore (a + * third-party transient flap must neither refuse an evidenced latch — the + * t/331 C1/C4 false-escalation — nor latch anything without evidence — the + * r2 P2-1 refused-leave mis-latch wedge). */ +extern bool cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, + bool others_dead_unchanged, + bool dead_gen_unchanged); + +/* spec-2.29a r3: identity gate for a LEAVE_COMMITTED confirmation — accept it + * as marker evidence only for THIS node's CURRENT leave attempt (self- + * addressed + currently leaving + per-attempt nonce match + committed epoch + * past the bound baseline); fail-closed on any mismatch. */ +extern bool cluster_clean_leave_committed_evidence_matches( + int32 payload_leaving_node, uint64 payload_nonce, uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, uint64 current_attempt_nonce, uint64 bound_leave_epoch); /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the @@ -408,6 +447,14 @@ extern void cluster_clean_leave_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); +extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, + const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512); extern void cluster_clean_leave_qvotec_complete(bool acked); extern void cluster_clean_leave_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_conf.h b/src/include/cluster/cluster_conf.h index c37661392f..2a6ae754db 100644 --- a/src/include/cluster/cluster_conf.h +++ b/src/include/cluster/cluster_conf.h @@ -111,12 +111,25 @@ typedef struct ClusterNodeInfo { uint16 rdma_port; uint16 rdma_pkey; uint32 rdma_qkey; + + /* + * PGRAC: spec-7.2 D2 — DATA-plane (LMS<->LMS) listener address, + * host:port. Optional: empty means this node offers no DATA plane + * (the CONTROL plane on interconnect_addr is unaffected). Never + * derived from interconnect_addr by offset — adjacent-port + * collisions on single-host deployments make an implicit default + * unsound (spec-7.2 r1-F2), so it is declared explicitly or the + * data plane stays off (fail-closed once the GCS block family + * flips to the DATA plane). Appended per the layout-append rule. + */ + char data_addr[CLUSTER_NODE_ADDR_LEN]; } ClusterNodeInfo; /* - * Top-level container in shmem. Sized to ~64 KiB (accepted overhead - * given the simplicity it buys; see docs/cluster-conf-design.md §3.2). + * Top-level container in shmem. Sized to ~128 KiB (accepted overhead + * given the simplicity it buys; see docs/cluster-conf-design.md §3.2 — + * budget raised from 64 KiB when spec-7.2 D2 appended data_addr). */ typedef struct ClusterConf { uint32 magic; diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 06662971b9..2f8417a522 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1788,6 +1788,26 @@ extern uint64 cluster_gcs_get_block_storage_fallback_count(void); extern uint64 cluster_gcs_get_block_master_not_holder_count(void); extern uint64 cluster_gcs_get_block_wal_flush_before_ship_count(void); extern uint64 cluster_gcs_get_block_ship_bytes_total(void); + +/* + * PGRAC: spec-7.2 D6 — requester-side block-ship latency histogram. + * + * 16 log-scale buckets in microseconds; bucket b counts completions + * with elapsed <= bound(b), last bucket is the +inf overflow. Samples + * are recorded at the single normal-exit funnel of + * cluster_gcs_send_block_request_and_wait (GRANTED / STORAGE_FALLBACK / + * READ_IMAGE); terminal-ereport exits lose the sample. This is the + * ruler for the spec-7.2 value gate (p99 < 20ms, p50 < 5ms) and the + * 7.7/7.8 wait-closure legs. dump category 'gcs', keys + * ship_hist_us_le_ + ship_hist_us_inf. + */ +#define CLUSTER_GCS_SHIP_HIST_BUCKETS 16 +extern uint64 cluster_gcs_block_ship_hist_bound_us(int bucket); +extern uint64 cluster_gcs_block_ship_hist_count(int bucket); + +/* PGRAC: spec-7.2 D3/D4 — registry probe for the atomic plane flip. */ +extern bool cluster_gcs_block_family_on_data_plane(void); + /* PGRAC: spec-6.13 D8 — RDMA tier3/direct-land copy observability. */ extern uint64 cluster_gcs_get_scratch_copy_count(void); extern uint64 cluster_gcs_get_live_sge_send_count(void); diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..0c30376374 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -275,6 +275,8 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 recovery_event_old_epoch; pg_atomic_uint64 recovery_redeclare_generation; pg_atomic_uint64 recovery_barrier_deadline; + pg_atomic_uint32 recovery_event_coordinator; + pg_atomic_uint64 recovery_done_epoch_at_accept; /* * spec-4.6 P0-1 (Fable review) — the epoch this episode is LOCKED to. @@ -292,8 +294,21 @@ typedef struct ClusterGrdShared { /* spec-4.6 P0#3 cluster gate — per-node epoch for which that node * last announced "local rebind barrier complete" (REDECLARE_DONE). - * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ + * P6 requires done_epoch[s] >= current epoch for EVERY survivor. + * + * spec-4.6a Amendment v1.2 (R2): the cross-node convergence key is the + * COMPOSITE (episode_epoch, dead_bitmap_hash). event_id is NOT globally + * consistent (it folds the per-instance cssd_dead_generation, which + * drifts with each node's own flap observation history), so keying the + * P6 gate on it wedges the cluster whenever two survivors observed a + * different flap history. The quorum-accepted dead SET is what actually + * converges; its hash rides the DONE payload instead. event_id stays a + * purely LOCAL ABA scope (P0 accept dedup). */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_bitmap_hash[CLUSTER_MAX_NODES]; + /* hash of THIS episode's accepted dead_bitmap (LMON writes at P0 accept; + * empty-bitmap hash for JOIN episodes, identical on every node). */ + pg_atomic_uint64 recovery_event_bitmap_hash; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -311,6 +326,10 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 block_path_failclosed_count; /* D4 GCS/PCM 53R9K (L12) */ pg_atomic_uint64 unaffected_holder_survived_count; /* L13 sweep-scope guard */ pg_atomic_uint64 stale_holder_swept_count; /* P6 post-barrier sweep (L15) */ + pg_atomic_uint64 cluster_gate_timeout_count; /* WAIT_CLUSTER watchdog only */ + pg_atomic_uint64 wait_epoch_escape_count; /* proof-carrying equal-epoch advance */ + pg_atomic_uint64 + pcm_dead_cleanup_entries; /* spec-4.6a D12: dead-node PCM holder/pending-X records cleaned */ /* spec-5.1b D9 — convert state-machine observability counters. * convert_queue_full reuses the existing converts_full_count above; @@ -609,6 +628,21 @@ typedef enum ClusterGrdRecoveryState { } ClusterGrdRecoveryState; extern void cluster_grd_recovery_lmon_tick(void); +/* spec-4.6a D5/D6 — recovery state observability for pg_cluster_state. */ +extern uint32 cluster_grd_recovery_state_value(void); +extern const char *cluster_grd_recovery_state_name(uint32 state); +extern uint64 cluster_grd_recovery_last_event_id(void); +extern uint64 cluster_grd_recovery_episode_epoch_value(void); +extern uint32 cluster_grd_recovery_event_coordinator(void); +extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); +extern uint64 cluster_grd_recovery_done_bitmap_hash_for(int32 node); +extern uint64 cluster_grd_recovery_event_bitmap_hash_value(void); +/* Amendment v1.2 (R2): the cross-node DONE key — hash over the dead bitmap + * ALONE (no dead_generation fold; same kernel as the event_id hash). */ +extern uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap); +extern int cluster_grd_recovery_block_redeclare_cursor(void); +extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); +extern bool cluster_grd_recovery_block_redeclare_done(void); extern uint64 cluster_grd_redeclare_generation(void); /* spec-4.6 P0-1 — the epoch the current episode is locked to (0 = none). */ extern uint64 cluster_grd_redeclare_episode_epoch(void); @@ -625,8 +659,10 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind - * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch); + * barrier for `epoch`. Amendment v1.2 (R2): the third argument is the + * sender's dead_bitmap hash (the cross-node half of the composite + * convergence key), NOT the sender-local event_id. */ +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); @@ -650,6 +686,9 @@ typedef struct ClusterGrdRecoveryCounters { uint64 block_path_failclosed; uint64 unaffected_holder_survived; uint64 stale_holder_swept; + uint64 cluster_gate_timeout; + uint64 wait_epoch_escape; + uint64 pcm_dead_cleanup_entries; /* spec-5.16 D5 — join-direction remaster counters (distinct from the * failure-driven remaster_* above; §8 Q6-A). */ uint64 join_remaster_started; @@ -660,6 +699,7 @@ typedef struct ClusterGrdRecoveryCounters { } ClusterGrdRecoveryCounters; extern void cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out); +extern uint64 cluster_grd_recovery_event_old_epoch(void); /* spec-2.29a WAIT_EPOCH baseline */ /* spec-4.6 P0#2 — pre-remaster stale-epoch sweep SCOPED to the affected * (dead-master) shards; affected_shards is a PGRAC_GRD_SHARD_COUNT-bit diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..c7b6ce082f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -399,6 +399,8 @@ extern int cluster_ges_reply_wait_max_entries; * 5000ms; SIGHUP). */ extern int cluster_grd_remaster_wait_ms; extern int cluster_grd_rebuild_timeout_ms; +extern int cluster_hw_remaster_retry_backoff_ms; +extern int cluster_hw_remaster_retry_max_attempts; /* spec-5.4 D8: SQ sequence lock tunables. */ extern int cluster_sequence_default_cache; @@ -469,6 +471,7 @@ extern int cluster_phase4_timeout; * range: [100, 60000] (millisecond) */ extern int cluster_lmon_main_loop_interval; +extern int cluster_lmon_slow_iteration_warn_ms; /* @@ -902,9 +905,11 @@ extern int cluster_gcs_reply_timeout_ms; * * cluster.gcs_block_retransmit_initial_backoff_ms * type: int context: PGC_SUSET - * default: 100 (min 10, max 5000) - * Backoff before retry 1. Subsequent retries double: 100 → 200 → - * 400 → 800 ms (with default max_retries=4, total backoff = 1500 ms). + * default: 10 (min 1, max 5000; spec-7.2 D1 lowered from 100/10) + * Backoff before retry 1. Subsequent retries double: 10 → 20 → + * 40 → 80 ms (with default max_retries=4, total backoff = 150 ms). + * The per-attempt reply wait stays cluster.gcs_reply_timeout_ms; + * this only paces the fast-deny / timeout retry cadence (HC96). * * cluster.gcs_block_dedup_max_entries * type: int context: PGC_POSTMASTER @@ -923,6 +928,15 @@ extern int cluster_gcs_block_retransmit_initial_backoff_ms; * (default on). See cluster_guc.c for semantics. */ extern bool cluster_gcs_block_local_cache; extern bool cluster_tx_enqueue_wait_enabled; /* spec-5.2 D4 */ + +/* + * cluster.ic_duty_lazy + * type: bool context: PGC_SIGHUP default: on + * spec-7.2 D1 — gate the lazy-able (queue-consumption) LMON duty + * families behind producer dirty marks + a >= 1 Hz floor. Never-lazy + * correctness families are unaffected. Off = pre-7.2 behavior. + */ +extern bool cluster_ic_duty_lazy; extern int cluster_gcs_block_dedup_max_entries; /* PGRAC: spec-4.7 D1 — bounded wait (ms) on a RECOVERING block resource diff --git a/src/include/cluster/cluster_hw.h b/src/include/cluster/cluster_hw.h index 89656a384f..e021c1c445 100644 --- a/src/include/cluster/cluster_hw.h +++ b/src/include/cluster/cluster_hw.h @@ -312,12 +312,30 @@ extern uint32 cluster_hw_shard_rebuilt_generation(uint32 shard_id); extern void cluster_hw_mark_shard_rebuilt(uint32 shard_id, uint32 generation); /* - * Per-dead-origin online-remaster launch idempotency (S5d). The GRD FSM records - * the episode it last launched a rebuild worker for a dead origin under, and - * skips relaunching while the value equals the current episode. + * Per-dead-origin online-remaster launch state (S5d / spec-4.6a). The GRD FSM + * records the episode it last launched a rebuild worker for a dead origin under, + * and the worker records a terminal result. BLOCKED is retryable within the same + * episode; BLOCKED_STRUCTURAL is not. */ +typedef enum ClusterHwRemasterResult { + CLUSTER_HW_REMASTER_NONE = 0, + CLUSTER_HW_REMASTER_RUNNING, + CLUSTER_HW_REMASTER_DONE, + CLUSTER_HW_REMASTER_BLOCKED, + CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, + CLUSTER_HW_REMASTER_NOT_APPLICABLE, +} ClusterHwRemasterResult; + extern uint64 cluster_hw_remaster_launched_episode(int node_id); extern void cluster_hw_remaster_set_launched(int node_id, uint64 episode); +extern ClusterHwRemasterResult cluster_hw_remaster_result(int node_id); +extern void cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result); +extern uint32 cluster_hw_remaster_attempts(int node_id); +extern void cluster_hw_remaster_set_attempts(int node_id, uint32 attempts); +extern uint64 cluster_hw_remaster_next_attempt_at(int node_id); +extern void cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts); +extern const char *cluster_hw_remaster_result_name(ClusterHwRemasterResult result); +extern bool cluster_hw_remaster_recoverable(void); /* * cluster_hw_apply_hwm -- redo / remaster rebuild: create-if-absent and raise @@ -383,6 +401,8 @@ extern uint64 cluster_hw_failclosed_count(void); extern uint64 cluster_hw_not_ready_count(void); extern uint64 cluster_hw_remaster_done_count(void); extern uint64 cluster_hw_remaster_blocked_count(void); +extern uint64 cluster_hw_remaster_retry_count(void); +extern uint64 cluster_hw_remaster_retry_exhausted_count(void); extern void cluster_hw_bump_alloc(void); extern void cluster_hw_bump_authority_create(void); extern void cluster_hw_bump_reserve_wal(void); @@ -391,6 +411,8 @@ extern void cluster_hw_bump_failclosed(void); extern void cluster_hw_bump_not_ready(void); extern void cluster_hw_bump_remaster_done(void); extern void cluster_hw_bump_remaster_blocked(void); +extern void cluster_hw_bump_remaster_retry(void); +extern void cluster_hw_bump_remaster_retry_exhausted(void); #endif /* !FRONTEND */ diff --git a/src/include/cluster/cluster_hw_remaster.h b/src/include/cluster/cluster_hw_remaster.h index e1bf3ba529..e2fcc481f8 100644 --- a/src/include/cluster/cluster_hw_remaster.h +++ b/src/include/cluster/cluster_hw_remaster.h @@ -47,17 +47,91 @@ #ifndef CLUSTER_HW_REMASTER_H #define CLUSTER_HW_REMASTER_H -/* - * Result of an online-remaster HW authority rebuild for one dead origin. Only - * DONE marks the adopted shards rebuilt (opens the serve gate); BLOCKED and - * NOT_APPLICABLE never do, so a shard whose HWM is not provably rebuilt stays - * fail-closed (8.A). - */ -typedef enum ClusterHwRemasterResult { - CLUSTER_HW_REMASTER_DONE = 0, /* rebuilt + adoption snapshot durable + shards marked */ - CLUSTER_HW_REMASTER_BLOCKED, /* fail-closed: keep shards frozen (53RA6) */ - CLUSTER_HW_REMASTER_NOT_APPLICABLE, /* HW authority inactive / no adopted shard / bad input */ -} ClusterHwRemasterResult; +#include "cluster/cluster_hw.h" /* ClusterHwRemasterResult */ + +#define CLUSTER_HW_REMASTER_NO_DEADLINE PG_UINT64_MAX +#define CLUSTER_HW_REMASTER_BACKOFF_CAP_MS 60000 + +typedef enum ClusterHwRemasterLaunchAction { + CLUSTER_HW_REMASTER_LAUNCH_SKIP = 0, + CLUSTER_HW_REMASTER_LAUNCH_INITIAL, + CLUSTER_HW_REMASTER_LAUNCH_RETRY, + CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED, + CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL, +} ClusterHwRemasterLaunchAction; + +typedef struct ClusterHwRemasterRelaunchDecision { + ClusterHwRemasterLaunchAction action; + uint32 next_attempts; + uint64 next_attempt_at; + ClusterHwRemasterResult next_result; +} ClusterHwRemasterRelaunchDecision; + +static inline uint32 +cluster_hw_remaster_compute_backoff_ms(int base_ms, uint32 completed_retry_attempts) +{ + uint64 ms; + uint32 shift; + + if (base_ms < 1) + base_ms = 1; + ms = (uint64)base_ms; + shift = completed_retry_attempts > 0 ? completed_retry_attempts - 1 : 0; + while (shift-- > 0 && ms < CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms <<= 1; + if (ms > CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms = CLUSTER_HW_REMASTER_BACKOFF_CAP_MS; + return (uint32)ms; +} + +static inline ClusterHwRemasterRelaunchDecision +cluster_hw_remaster_relaunch_decide(uint64 launched_episode, uint64 current_episode, + ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int retry_max_attempts) +{ + ClusterHwRemasterRelaunchDecision d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_SKIP; + d.next_attempts = attempts; + d.next_attempt_at = next_attempt_at; + d.next_result = result; + + if (current_episode == 0) + return d; + if (launched_episode != current_episode || result == CLUSTER_HW_REMASTER_NONE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_INITIAL; + d.next_attempts = 0; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; + } + + if (result == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (result != CLUSTER_HW_REMASTER_BLOCKED) + return d; + + if (retry_max_attempts <= 0 || attempts >= (uint32)retry_max_attempts) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE && now < next_attempt_at) + return d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_RETRY; + d.next_attempts = attempts + 1; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; +} #ifndef FRONTEND @@ -81,9 +155,9 @@ extern ClusterHwRemasterResult cluster_hw_remaster_rebuild_origin(int dead_node_ /* * cluster_hw_remaster_worker_main -- dynamic-bgworker entry point (main_arg = the * dead origin node id). Captures the live reconfig episode and drives - * cluster_hw_remaster_rebuild_origin off the LMON tick. An abnormal exit simply - * leaves the shards unmarked -> the serve gate keeps them fail-closed (8.A), so - * no abnormal-exit handler is needed. + * cluster_hw_remaster_rebuild_origin off the LMON tick. A before_shmem_exit + * callback records an abnormal exit as retryable BLOCKED so the same episode can + * self-heal instead of staying pinned on the launch idempotency bit. */ extern void cluster_hw_remaster_worker_main(Datum main_arg); diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 981734a896..c8b0b859f3 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -218,6 +218,21 @@ extern const ClusterICOps *ClusterICOps_Active; */ #define PGRAC_IC_HELLO_MAGIC ((uint32)0x4F4C4C48) /* "HLLO" LE */ #define PGRAC_IC_HELLO_VERSION_V1 ((uint16)1) + +/* + * PGRAC: spec-7.2 D2 — interconnect plane. CONTROL is the historic + * LMON-owned connection (heartbeat / membership / GES / everything + * registered before spec-7.2); DATA is the LMS-owned connection the + * GCS block family migrates to. Zero MUST stay CONTROL: a pre-7.2 + * HELLO carries all-zero pad bytes and must keep parsing as a + * CONTROL-plane peer. + */ +typedef enum ClusterICPlane { + CLUSTER_IC_PLANE_CONTROL = 0, + CLUSTER_IC_PLANE_DATA = 1 +} ClusterICPlane; + +#define CLUSTER_IC_PLANE_N 2 /* * spec-2.3 D3: PGRAC_IC_ENVELOPE_VERSION_V1 is now defined in * cluster_ic_envelope.h as ((uint8)1) — the authoritative @@ -228,6 +243,17 @@ extern const ClusterICOps *ClusterICOps_Active; #define PGRAC_IC_CLUSTER_NAME_MAX 24 #define PGRAC_IC_HELLO_CAPABILITIES_OFFSET 36 #define PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 ((uint32)0x00000001U) +/* + * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero + * pad region (capabilities precedent: occupy pad bytes, do not resize V1). + * A pre-7.2 sender leaves them zero => plane CONTROL, conn_epoch 0. + * Offsets 41/42 are reserved for spec-7.3 (worker_id / n_workers) and + * stay zero in 7.2; offset 43 pads the group; 52-63 remain reserved. + */ +#define PGRAC_IC_HELLO_PLANE_OFFSET 40 +#define PGRAC_IC_HELLO_WORKER_ID_OFFSET 41 /* spec-7.3 reserved, zero in 7.2 */ +#define PGRAC_IC_HELLO_N_WORKERS_OFFSET 42 /* spec-7.3 reserved, zero in 7.2 */ +#define PGRAC_IC_HELLO_CONN_EPOCH_OFFSET 44 typedef struct ClusterICHelloMsg { uint32 magic; /* PGRAC_IC_HELLO_MAGIC */ @@ -249,6 +275,34 @@ cluster_ic_hello_capabilities(const ClusterICHelloMsg *msg) return ((uint32)p[0]) | ((uint32)p[1] << 8) | ((uint32)p[2] << 16) | ((uint32)p[3] << 24); } +/* PGRAC: spec-7.2 D2 — plane byte accessor (_pad offset 40 - 36 = 4). */ +static inline ClusterICPlane +cluster_ic_hello_plane(const ClusterICHelloMsg *msg) +{ + if (msg == NULL) + return CLUSTER_IC_PLANE_CONTROL; + return (ClusterICPlane) + msg->_pad[PGRAC_IC_HELLO_PLANE_OFFSET - PGRAC_IC_HELLO_CAPABILITIES_OFFSET]; +} + +/* PGRAC: spec-7.2 D2 — connection epoch accessor (_pad offset 44 - 36 = 8; + * LE uint64). Zero = pre-7.2 sender (or CONTROL plane, which does not + * enforce it); the DATA-plane verify path rejects zero fail-closed. */ +static inline uint64 +cluster_ic_hello_conn_epoch(const ClusterICHelloMsg *msg) +{ + const uint8 *p; + uint64 v = 0; + int i; + + if (msg == NULL) + return 0; + p = msg->_pad + (PGRAC_IC_HELLO_CONN_EPOCH_OFFSET - PGRAC_IC_HELLO_CAPABILITIES_OFFSET); + for (i = 7; i >= 0; i--) + v = (v << 8) | (uint64)p[i]; + return v; +} + /* * spec-2.2 D2 (post-codex review) -- HELLO wire encode/decode helpers. @@ -273,7 +327,8 @@ cluster_ic_hello_capabilities(const ClusterICHelloMsg *msg) */ extern void cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version, uint16 envelope_version, int32 source_node_id, - const char *cluster_name); + const char *cluster_name, ClusterICPlane plane, + uint64 conn_epoch); extern bool cluster_ic_parse_hello(const uint8 in_buf[PGRAC_IC_HELLO_BYTES], ClusterICHelloMsg *out_msg); diff --git a/src/include/cluster/cluster_ic_router.h b/src/include/cluster/cluster_ic_router.h index 5cb23bc0bd..945647ae28 100644 --- a/src/include/cluster/cluster_ic_router.h +++ b/src/include/cluster/cluster_ic_router.h @@ -98,6 +98,19 @@ | CLUSTER_IC_PRODUCER_BGWRITER | CLUSTER_IC_PRODUCER_CHECKPOINTER) /* spec-2.X may add more as new BackendType slots get assigned */ +/* + * PGRAC: spec-7.2 D3 (r3/r4 amend) — LMS producer bits for the DATA-plane + * msg_type family. _LMS_DATA pre-includes the 7.3 worker bit so the + * five GCS block registrations widen exactly once. REDECLARE stays on + * the CONTROL mask untouched (r4 ruling), and CONTROL-only msg_types + * never gain these bits — a migrated handler that must emit a CONTROL + * message stages it into the CONTROL outbound ring for LMON to send + * (direct send from LMS = producer-gate ERROR by construction). + */ +#define CLUSTER_IC_PRODUCER_LMS ((uint32)(1u << B_LMS)) +#define CLUSTER_IC_PRODUCER_LMS_WORKER ((uint32)(1u << B_LMS_WORKER)) +#define CLUSTER_IC_PRODUCER_LMS_DATA (CLUSTER_IC_PRODUCER_LMS | CLUSTER_IC_PRODUCER_LMS_WORKER) + /* ============================================================ * ClusterICMsgTypeInfo -- registration record for one msg_type. @@ -114,6 +127,18 @@ typedef struct ClusterICMsgTypeInfo { uint32 allowed_producer_mask; /* OR of CLUSTER_IC_PRODUCER_* */ bool broadcast_ok; /* dest = PGRAC_IC_BROADCAST allowed? */ void (*handler)(const ClusterICEnvelope *env, const void *payload); + + /* + * PGRAC: spec-7.2 D3 — owning plane (ClusterICPlane; uint8 keeps the + * const-initializer shape). Zero = CONTROL, so every existing + * registration is CONTROL without edits. A msg_type is sent and + * dispatched ONLY in its plane's owner process: the physical-send + * layer ereports on a cross-plane send, dispatch drops the frame + * fail-closed (plane_misroute_reject). Flip discipline (H-5 + user + * ruling): the five GCS block types turn DATA in ONE commit at the + * end of the D3+D4 sequence — no half-migrated window. + */ + uint8 plane; } ClusterICMsgTypeInfo; diff --git a/src/include/cluster/cluster_ic_tier1.h b/src/include/cluster/cluster_ic_tier1.h index 0bab9a7a23..e6a3ce349d 100644 --- a/src/include/cluster/cluster_ic_tier1.h +++ b/src/include/cluster/cluster_ic_tier1.h @@ -99,6 +99,18 @@ typedef struct ClusterICPeerStateShmem { * close. */ pg_atomic_uint32 close_requested; /* 0 = idle, 1 = pending close */ + + /* + * PGRAC: spec-7.2 D5 (INV-7.2-CONN-EPOCH) — the cluster epoch this + * connection was established under (recorded at the CONNECTED + * transition; DATA plane only, CONTROL leaves it 0/unenforced). + * The DATA physical-send gate refuses to put bytes on a connection + * whose conn_epoch != current epoch — an epoch bump structurally + * invalidates the old stream and forces close + re-HELLO, so a new + * epoch's frames can never interleave into an old epoch's byte + * stream. Written and read by the owning plane's process only. + */ + uint64 conn_epoch; } ClusterICPeerStateShmem; /* @@ -133,6 +145,20 @@ extern void cluster_ic_tier1_shmem_register(void); */ extern int cluster_ic_tier1_listener_bind(void); +/* + * PGRAC: spec-7.2 D2 — re-aim this process's tier1 working state at a + * plane ([0] = CONTROL, [1] = DATA inside the single tier1 region). + * Call once at aux-process entry BEFORE any tier1 use (LmsMain → DATA); + * processes that never call it stay on CONTROL (LMON, backends, SQL + * observers — the historic instance and contract). + */ +extern void cluster_ic_tier1_set_my_plane(ClusterICPlane plane); +extern ClusterICPlane cluster_ic_tier1_my_plane(void); + +/* PGRAC: spec-7.2 D3 — dispatch plane-gate drop counter (per plane). */ +extern void cluster_ic_tier1_bump_plane_misroute_reject(void); +extern uint64 cluster_ic_tier1_get_plane_misroute_reject(ClusterICPlane plane); + /* * Accept exactly one pending connection on the listener fd. Returns * true if accepted (sets *out_peer_fd and *out_peer_id from HELLO once diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 63efc4e045..bafbae47a2 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -98,8 +98,10 @@ #define CLUSTER_LMON_H #include "datatype/timestamp.h" +#include "port/atomics.h" #include "storage/lwlock.h" +struct Latch; /* * ClusterLmonStatus -- HC2 SSOT for LMON lifecycle state. @@ -137,9 +139,97 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 max_iter_us; /* high-water iteration wall time */ + uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ + struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ bool shutdown_requested; /* postmaster sets; LMON main loop polls + exits */ + + /* + * PGRAC: spec-7.2 D1 duty-chain on-demand gating. Dirty bitmask for + * the lazy-able (queue-consumption) duty families: producers set their + * family bit at enqueue time (fetch-or, lock-free, NOT under lwlock); + * the LMON main loop test-and-clears it to decide whether the drain + * runs this iteration. A missed bit is latency-bounded, never a + * correctness issue: the heartbeat-interval floor forces every duty + * at >= 1 Hz (cluster.ic_duty_lazy = off restores run-every-iteration). + */ + pg_atomic_uint32 lazy_duty_dirty; } ClusterLmonSharedState; +/* + * ClusterLmonDuty -- spec-7.2 D1 classification table (§3.7). + * + * EVERY tick/drain call in the LMON main-loop duty chain is enumerated + * here and classified by cluster_lmon_duty_is_lazy(): + * + * never-lazy (correctness families): fence / sweeps / reconfig / + * recovery / epoch-quorum / order-writ members (I47: dead-sweep + * MUST run BEFORE reconfig; clean-leave MUST run BEFORE reconfig). + * These run every iteration, order and cadence verbatim. + * + * lazy-able (queue-consumption families): drains whose producers + * mark_dirty at enqueue time (or that are purely TTL/time-based and + * ride the >= 1 Hz floor). Skipping an iteration only delays the + * drain, bounded by the floor. + * + * A NEW duty added to the chain without an enum entry is never-lazy by + * construction (a plain unconditional call). To make it lazy it must + * be enumerated + classified here, and the unit truth table + * (test_cluster_lmon.c) re-pinned -- that is the anti-drift gate. + */ +typedef enum ClusterLmonDuty { + /* never-lazy (correctness families) */ + CLUSTER_LMON_DUTY_LIVENESS = 0, + CLUSTER_LMON_DUTY_FENCE, + CLUSTER_LMON_DUTY_GRD_DEAD_SWEEP, /* I47: MUST run BEFORE reconfig */ + CLUSTER_LMON_DUTY_GRD_RECLAIM, + CLUSTER_LMON_DUTY_GRD_STARVATION_SWEEP, + CLUSTER_LMON_DUTY_DEADLOCK, + CLUSTER_LMON_DUTY_GES_REPLY_WAIT_SWEEP, /* 5.16 rule-8.A backstop */ + CLUSTER_LMON_DUTY_DIRECT_LAND_ABORTS, + CLUSTER_LMON_DUTY_LMS_NATIVE_PROBE_RETRY, + CLUSTER_LMON_DUTY_CLEAN_LEAVE, /* CL-I13: MUST run BEFORE reconfig */ + CLUSTER_LMON_DUTY_NODE_REMOVE, /* INV-LF1: before reconfig */ + CLUSTER_LMON_DUTY_RECONFIG, + CLUSTER_LMON_DUTY_GRD_RECOVERY, /* AFTER reconfig tick (spec-4.6) */ + CLUSTER_LMON_DUTY_BOC_BROADCAST, + CLUSTER_LMON_DUTY_SINVAL_RESET_ALL, /* order-sensitive protocol */ + CLUSTER_LMON_DUTY_PI_DISCARD, + /* lazy-able (queue-consumption families) -- keep contiguous; the + * dirty bit is (duty - CLUSTER_LMON_DUTY_LAZY_FIRST) */ + CLUSTER_LMON_DUTY_GES_WORK_QUEUE, + CLUSTER_LMON_DUTY_SHIP_READY, + CLUSTER_LMON_DUTY_GRD_OUTBOUND, + CLUSTER_LMON_DUTY_SINVAL_OUT, + CLUSTER_LMON_DUTY_SINVAL_ACK_OUT, + CLUSTER_LMON_DUTY_TT_HINT, + CLUSTER_LMON_DUTY_DEDUP_TTL, /* TTL/time-based: floor-driven, no producer */ + CLUSTER_LMON_DUTY_BACKUP, + CLUSTER_LMON_DUTY_N +} ClusterLmonDuty; + +#define CLUSTER_LMON_DUTY_LAZY_FIRST CLUSTER_LMON_DUTY_GES_WORK_QUEUE + +StaticAssertDecl(CLUSTER_LMON_DUTY_N - CLUSTER_LMON_DUTY_LAZY_FIRST <= 32, + "lazy duty families must fit the uint32 dirty bitmask"); + +/* + * Duty-gating API (spec-7.2 D1). + * + * cluster_lmon_duty_is_lazy pure classification (unit-pinned). + * cluster_lmon_duty_mark_dirty producer side, any process; lock-free + * fetch-or; no-op for never-lazy duties + * and pre-shmem calls. Call BEFORE the + * cluster_lmon_wakeup() kick. + * cluster_lmon_duty_should_run LMON loop side; true for never-lazy, + * lazy=off, or force_all (>= 1 Hz floor); + * otherwise test-and-clears the bit. + */ +extern bool cluster_lmon_duty_is_lazy(ClusterLmonDuty duty); +extern void cluster_lmon_duty_mark_dirty(ClusterLmonDuty duty); +extern bool cluster_lmon_duty_should_run(ClusterLmonDuty duty, bool force_all); + /* * Public API. @@ -188,6 +278,10 @@ extern TimestampTz cluster_lmon_spawned_at(void); extern TimestampTz cluster_lmon_ready_at(void); extern TimestampTz cluster_lmon_last_liveness_tick_at(void); extern int64 cluster_lmon_main_loop_iters(void); +extern uint64 cluster_lmon_last_iter_us(void); +extern uint64 cluster_lmon_max_iter_us(void); +extern uint64 cluster_lmon_slow_iter_count(void); +extern void cluster_lmon_marker_complete_wakeup(void); /* * Status enum -> canonical lowercase string ("not_started", "spawning", diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index 99c0e47f55..afe960df71 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -257,6 +257,15 @@ typedef struct ClusterLmsSharedState { * BOOST = 11 awaits spec-2.28+ with PG core lock manager * `LockWaitQueueInsertAtHead`改造 + integrated receiver. */ pg_atomic_uint64 priority_starvation_observed_count; + + /* spec-7.2 D6 — DATA-plane connection resets observability counter. + * + * Bumped by the LMS DATA-plane tick (cluster_lms_data_plane.c) once + * per connection torn down by a proactive reset: an epoch bump in + * production (INV-7.2-CONN-EPOCH ③) or the cluster-lms-conn-reset + * injection in the F6-1 fault test. Monotone; cluster-wide reset + * activity is the sum across nodes. */ + pg_atomic_uint64 lms_conn_resets; } ClusterLmsSharedState; @@ -294,6 +303,32 @@ extern void cluster_lms_request_shutdown(void); */ extern void LmsMain(void) pg_attribute_noreturn(); +/* + * PGRAC: spec-7.2 D2 — LMS-owned DATA-plane interconnect loop + * (cluster_lms_data_plane.c). startup binds the data_addr listener and + * aims this process's tier1 state at the DATA plane (false = plane off: + * no data_addr / cluster off / stub tier); tick runs one management + + * WaitEventSet round (sockets + MyLatch — replaces the historic plain + * WaitLatch when the plane is live); shutdown closes owned fds. + */ +extern bool cluster_lms_data_plane_startup(void); +extern bool cluster_lms_data_plane_enabled(void); +extern void cluster_lms_data_plane_tick(long timeout_ms); +extern void cluster_lms_data_plane_shutdown(void); + +/* + * PGRAC: spec-7.2 D4 — LMS wakeup (mirror of cluster_lmon_wakeup) + the + * DATA-plane outbound ring (cluster_lms_outbound.c; Q6-B twin ring: + * backends stage DATA frames, LMS drains and sends on its own fds). + */ +extern void cluster_lms_wakeup(void); +extern void cluster_lms_outbound_shmem_register(void); +extern void cluster_lms_outbound_request_lwlocks(void); +extern bool cluster_lms_outbound_enqueue(uint8 msg_type, uint32 dest_node_id, const void *payload, + uint16 payload_len); +extern int cluster_lms_outbound_drain_send(void); +extern uint32 cluster_lms_outbound_depth(void); + /* * Read-only accessors for SQL view + diagnostics. */ @@ -333,6 +368,10 @@ extern void cluster_lms_bump_restart_generation_at_main_entry(void); extern void cluster_lms_inc_priority_starvation_observed(void); extern uint64 cluster_lms_get_priority_starvation_observed_count(void); +/* spec-7.2 D6 — DATA-plane connection resets observability counter. */ +extern void cluster_lms_bump_conn_resets(uint32 n); +extern uint64 cluster_lms_get_conn_resets(void); + /* * State enum -> canonical lowercase string ("disabled", "not_started", * "starting", "ready", "draining", "stopped"). Out-of-range returns diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h new file mode 100644 index 0000000000..56cd836bf5 --- /dev/null +++ b/src/include/cluster/cluster_marker_async.h @@ -0,0 +1,218 @@ +/*------------------------------------------------------------------------- + * + * cluster_marker_async.h + * Small process-local async FSM for qvotec marker submit mailboxes. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_marker_async.h + * + * NOTES + * The mailbox remains the existing request_seq/completion_seq/result + * slot. This wrapper changes only the waiting shape: submit publishes + * a request and returns; the owning LMON tick polls completion without + * sleeping (spec-2.29a — the pre-async 2ms pg_usleep spin inside the + * LMON tick starved the CSSD heartbeat relay and caused false-DEAD + * storms during cold formation, BUG-C1). + * + * The staged has_staged_event flag is the P1-1 bump-once contract: a + * pre-bump caller (fail-stop fence / node-remove / join Phase-1) must + * not re-enter its epoch-bump path while a stage is live. + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_MARKER_ASYNC_H +#define CLUSTER_MARKER_ASYNC_H + +#include "c.h" + +#include + +#include "datatype/timestamp.h" +#include "port/atomics.h" +#include "storage/latch.h" + +typedef enum ClusterMarkerAsyncState { + CLUSTER_MARKER_ASYNC_IDLE = 0, + CLUSTER_MARKER_ASYNC_SUBMITTED +} ClusterMarkerAsyncState; + +typedef enum ClusterMarkerPollResult { + CLUSTER_MARKER_POLL_IDLE = 0, + CLUSTER_MARKER_POLL_PENDING, + CLUSTER_MARKER_POLL_ACKED, + CLUSTER_MARKER_POLL_TIMEOUT +} ClusterMarkerPollResult; + +typedef enum ClusterMarkerAsyncKind { + CLUSTER_MARKER_KIND_UNKNOWN = 0, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + CLUSTER_MARKER_KIND_JOIN_PREPARE, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED +} ClusterMarkerAsyncKind; + +typedef struct ClusterMarkerAsync { + ClusterMarkerAsyncState state; + uint64 inflight_seq; + uint64 deadline_us; + TimestampTz submitted_at; + ClusterMarkerAsyncKind kind; + int32 target_node; + + /* + * Caller-owned staged publish/commit state. While true, the owner must not + * re-enter the pre-bump path; it either submits/polls this record, publishes + * after ACK, or releases it after failure/timeout. + */ + bool has_staged_event; + uint64 staged_expect_epoch; +} ClusterMarkerAsync; + +static inline const char * +cluster_marker_async_kind_name(ClusterMarkerAsyncKind kind) +{ + switch (kind) { + case CLUSTER_MARKER_KIND_FENCE_FAILSTOP: + return "fence_failstop"; + case CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED: + return "fence_node_removed"; + case CLUSTER_MARKER_KIND_JOIN_PREPARE: + return "join_prepare"; + case CLUSTER_MARKER_KIND_JOIN_COMMITTED: + return "join_committed"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING: + return "clean_leave_committing"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED: + return "clean_leave_committed"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING: + return "node_remove_removing"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK: + return "node_remove_shrunk"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED: + return "node_remove_removed"; + case CLUSTER_MARKER_KIND_UNKNOWN: + default: + return "unknown"; + } +} + +static inline void +cluster_marker_async_init(ClusterMarkerAsync *a) +{ + memset(a, 0, sizeof(*a)); + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->target_node = -1; +} + +static inline bool +cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) +{ + return a != NULL && a->state == CLUSTER_MARKER_ASYNC_SUBMITTED; +} + +static inline bool +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq) +{ + return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); +} + +static inline bool +cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, + TimestampTz now, uint64 timeout_us, ClusterMarkerAsyncKind kind, + int32 target_node) +{ + uint64 seq; + + Assert(a != NULL); + Assert(request_seq != NULL); + Assert(completion_seq != NULL); + + if (a->state == CLUSTER_MARKER_ASYNC_SUBMITTED) + return true; + if (cluster_marker_async_mailbox_busy(request_seq, completion_seq)) + return false; + + pg_write_barrier(); + seq = pg_atomic_add_fetch_u64(request_seq, 1); + if (qvotec_latch != NULL) + SetLatch(qvotec_latch); + + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->inflight_seq = seq; + a->deadline_us = (uint64)now + timeout_us; + a->submitted_at = now; + a->kind = kind; + a->target_node = target_node; + return true; +} + +static inline ClusterMarkerPollResult +cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + uint64 elapsed; + + Assert(a != NULL); + Assert(completion_seq != NULL); + Assert(result_slot != NULL); + + if (out_result != NULL) + *out_result = 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (a->state != CLUSTER_MARKER_ASYNC_SUBMITTED) + return CLUSTER_MARKER_POLL_IDLE; + + if (pg_atomic_read_u64(completion_seq) == a->inflight_seq) { + pg_read_barrier(); + if (out_result != NULL) + *out_result = pg_atomic_read_u32(result_slot); + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_ACKED; + } + + if ((uint64)now >= a->deadline_us) { + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_TIMEOUT; + } + + return CLUSTER_MARKER_POLL_PENDING; +} + +static inline void +cluster_marker_async_release_stage(ClusterMarkerAsync *a) +{ + if (a == NULL) + return; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->inflight_seq = 0; + a->deadline_us = 0; + a->submitted_at = 0; + a->kind = CLUSTER_MARKER_KIND_UNKNOWN; + a->target_node = -1; + a->has_staged_event = false; + a->staged_expect_epoch = 0; +} + +#endif /* CLUSTER_MARKER_ASYNC_H */ diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index ce6bbd7b19..ef722e3f62 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -74,6 +74,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + #include "cluster/cluster_write_fence.h" /* CLUSTER_FENCE_MARKER_BYTES (marker offset) */ /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ @@ -415,6 +417,14 @@ extern void cluster_node_remove_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); +extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, + const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out); extern void cluster_node_remove_qvotec_complete(bool acked); extern void cluster_node_remove_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index d88d82dc8f..3fb909557f 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -415,6 +415,12 @@ extern void cluster_pcm_lock_clear_pending_x(BufferTag tag); extern int32 cluster_pcm_lock_query_pending_x_requester(BufferTag tag); extern uint64 cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node); +/* PGRAC: spec-4.6a BUG-C2 — failure-path PCM holder cleanup. Removes a + * DEAD node from X/S/PI holder records in this master's PCM directory and + * demotes entries to N when no live holder remains. Idempotent under repeated + * dead-sweep ticks. */ +extern uint64 cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node); + /* PGRAC: spec-5.13 D5 (clean-leave PCM release) — a leaving node clears its * OWN holder records (X / S / PI) from the local PCM directory after the GCS * flush seam has persisted all dirty X blocks (CL-I5). Demotes an entry's diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index e34c3a1942..1aa106f690 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -56,7 +56,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" -#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ struct Latch; /* spec-5.15 D4 — join-marker qvotec mailbox latch (pointer only) */ @@ -225,6 +226,16 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 apply_counter; /* total events observed */ pg_atomic_uint64 dedup_skip_counter; /* duplicate event_id skipped */ pg_atomic_uint64 procsig_broadcast_count; /* PROCSIG broadcast tally */ + /* spec-2.29a r2 t/274: a backend-context coordinator (e.g. the fail-stop + * inject test) advances the epoch then SYNCHRONOUSLY waits for the fence + * marker before publishing. During that (seconds-long) wait the epoch is + * bumped but the reconfig event is not yet published, so a concurrently + * running LMON GRD IDLE tick would re-capture the post-bump epoch as its + * WAIT_EPOCH baseline and wedge (old==cur). The LMON-process-local staged + * flags cannot see a backend's pre-bump window, so publish it in shmem: + * set before the bump, cleared after publish / on failure. The GRD IDLE + * baseline-hold guard reads this OR the local stage. */ + pg_atomic_uint32 prebump_sync_active; /* spec-5.14 D6 — touched_peers fail-stop observability (Q8 A': these * live in this existing region; the region count is unchanged). */ @@ -316,6 +327,8 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 join_reject_count; pg_atomic_uint64 join_timeout_count; pg_atomic_uint64 clean_departed_cleared_count; + pg_atomic_uint64 marker_slow_ack_count; + pg_atomic_uint64 marker_timeout_count; /* * spec-5.15 D1 — per declared node, the freshest voting-slot incarnation + @@ -406,6 +419,12 @@ extern void cluster_reconfig_shmem_register(void); */ extern void cluster_reconfig_get_last_event(ReconfigEvent *out); +/* spec-2.29a: true while a pre-bump coordinator stage (fail-stop fence / + * node-remove / join Phase-1) has bumped the epoch but not yet published its + * reconfig event -- the GRD recovery IDLE tick must hold its baseline instead + * of re-capturing the post-bump epoch (else WAIT_EPOCH wedges). */ +extern bool cluster_reconfig_has_pending_prebump_stage(void); + /* ============================================================ * Coordinator path APIs (Step 2 D2 wiring). @@ -580,10 +599,24 @@ extern bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarn */ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); +extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now); +extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512); extern void cluster_reconfig_join_qvotec_complete(bool acked); extern void cluster_reconfig_publish_join_qvotec_latch(struct Latch *latch); +extern void cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern void cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern uint64 cluster_reconfig_get_marker_slow_ack_count(void); +extern uint64 cluster_reconfig_get_marker_timeout_count(void); + /* ============================================================ * spec-5.15 D5 — joiner-side write gate + admission (INV-J9 / §2.4 / §2.7). * ============================================================ @@ -756,5 +789,11 @@ extern uint64 cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_n uint64 last_incarnation, bool *out_contest); +/* spec-2.29a review r1 P2: true while the staged node-removed fence marker is + * in flight -- the node-remove driver skips its FENCE_ARMING pre-work + * (REMOVING marker / stripe retire) instead of re-keying it to the live + * (already self-bumped) epoch. */ +extern bool cluster_reconfig_node_removed_fence_stage_pending(void); + #endif /* CLUSTER_RECONFIG_H */ diff --git a/src/include/cluster/cluster_views.h b/src/include/cluster/cluster_views.h index 46b702f4df..49f8b1f1f9 100644 --- a/src/include/cluster/cluster_views.h +++ b/src/include/cluster/cluster_views.h @@ -51,7 +51,7 @@ * internal table in cluster_views.c stays in sync with the enum. */ #define CLUSTER_WAIT_EVENTS_COUNT \ - 118 /* spec-6.13 D8: RDMA busypoll + inline wait events included */ + 120 /* spec-7.2 D6: +2 LMS data-plane; spec-6.13 D8 RDMA included */ /* diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index 7543664e72..ac5a965882 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -39,6 +39,8 @@ #ifndef CLUSTER_WRITE_FENCE_H #define CLUSTER_WRITE_FENCE_H +#include "cluster/cluster_marker_async.h" + /* * cluster_write_fence_decide -- the PURE cooperative write-fence judge (spec-4.12 * D3). A shared-storage write is allowed iff enforcement is on AND the local @@ -517,6 +519,14 @@ extern void cluster_write_fence_note_baseline_stale(void); */ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); +extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern void cluster_write_fence_publish_qvotec_latch(struct Latch *latch); extern bool cluster_write_fence_qvotec_poll_pending(ClusterFenceMarker *out); extern void cluster_write_fence_qvotec_complete(bool acked); diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h new file mode 100644 index 0000000000..0e73859d46 --- /dev/null +++ b/src/include/cluster/cluster_xid_authority.h @@ -0,0 +1,266 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.h + * Shared XID authority + native-era prehistory (spec-6.15b D1/D2). + * + * Under cluster.shared_catalog the pre-formation ("native era", + * cluster.enabled=off single-writer) xid consumption of the seed node is + * invisible to every other node: a joiner's per-node recovery anchor was + * seeded from its pre-seed clone-era control file, so its transaction + * horizon (nextXid) stays below the seed's committed xids and MVCC + * silently judges seed rows "in the future" (false-invisible), while its + * local pg_xact carries no commit bits for them (poison-hint hazard on + * the shared catalog pages). + * + * Two small durable files under cluster.shared_data_dir close this: + * + * global/pgrac_xid_authority -- never-lowered native-era xid + * high-water (first xid NOT consumed by the native era) plus two + * one-way lifecycle flags: SEALED (a clean native-era shutdown + * published a complete high-water + prehistory; joiners may adopt) + * and CLUSTER_ERA (a cluster.enabled=on boot happened; the native + * era is closed forever and may not be re-entered). + * + * global/pgrac_xid_prehistory -- the raw local pg_xact page run + * covering [0, native_hw), CRC-guarded. A joiner whose own + * nextXid proves it is a pre-seed lineage adopts these bytes into + * its local pg_xact before StartupCLOG, giving it first-hand + * commit-status truth for every native-era xid (no hint-bit + * trust, no CLOG overlay of cluster-era foreign bits -- see the + * spec's no-clog-overlay boundary note). + * + * Torn-safe file discipline mirrors cluster_oid_lease.c: temp + fsync + + * .bak roll + durable_rename on write; primary-then-.bak fail-closed + * read; ENOENT-only-absent presence probe. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_xid_authority.h + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_XID_AUTHORITY_H +#define CLUSTER_XID_AUTHORITY_H + +#include "port/pg_crc32c.h" + +/* ============================================================ + * On-disk authority image (spec-6.15b §2). + * ============================================================ */ + +#define CLUSTER_XID_AUTHORITY_MAGIC 0x0141D617 +#define CLUSTER_XID_AUTHORITY_VERSION 1 + +/* A clean native-era shutdown published a complete hw + prehistory. + * Cleared again -- only while CLUSTER_ERA is unset -- when a follow-up + * native-era run boots, so a crash of that run leaves joiners + * fail-closed instead of adopting the previous pass's stale hw. */ +#define CLUSTER_XID_AUTHORITY_FLAG_SEALED 0x0001 +/* A cluster.enabled=on boot closed the native era forever (one-way). */ +#define CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA 0x0002 + +typedef struct ClusterXidAuthorityHeader { + uint32 magic; + uint32 version; + uint32 flags; + uint32 reserved; /* zero */ + uint64 native_hw_full; /* FullTransactionId value: first xid NOT + * consumed by the native era; never lowered */ + uint64 next_multi; /* native-era nextMulti at publish; vet-only + * (spec Q5: >FirstMultiXactId refuses seed) */ + pg_crc32c crc; /* over all preceding bytes */ +} ClusterXidAuthorityHeader; + +/* On-disk ABI locked at compile time (review F7; mirrors the recovery + * anchor's precedent). The unit truth table re-checks these at runtime. */ +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, flags) == 8, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, native_hw_full) == 16, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, next_multi) == 24, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, crc) == 32, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidAuthorityHeader) == 40, + "xid authority header size is on-disk ABI"); + +/* Relative paths under cluster.shared_data_dir. */ +#define CLUSTER_XID_AUTHORITY_REL_PATH "global/pgrac_xid_authority" +#define CLUSTER_XID_AUTHORITY_BAK_REL_PATH "global/pgrac_xid_authority.bak" +#define CLUSTER_XID_AUTHORITY_TMP_REL_PATH "global/pgrac_xid_authority.tmp" +#define CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH "global/pgrac_xid_authority.bak.tmp" + +typedef enum ClusterXidAuthorityValidity { + CLUSTER_XID_AUTHORITY_VALID = 0, + CLUSTER_XID_AUTHORITY_INVALID_SHORT, + CLUSTER_XID_AUTHORITY_INVALID_MAGIC, + CLUSTER_XID_AUTHORITY_INVALID_CRC, +} ClusterXidAuthorityValidity; + +/* ============================================================ + * On-disk prehistory image (spec-6.15b D2). + * ============================================================ */ + +#define CLUSTER_XID_PREHISTORY_MAGIC 0x0142D617 +#define CLUSTER_XID_PREHISTORY_VERSION 1 + +/* + * Refusal cap on the native era (spec §3.4): 8M xids = 256 CLOG pages = + * 2MB of payload. A seed load that consumes more must be split or moved + * in-protocol; the cap keeps the blob bounded and the adopt O(small). + */ +#define CLUSTER_XID_PREHISTORY_MAX_XID UINT64CONST(8388608) + +typedef struct ClusterXidPrehistoryHeader { + uint32 magic; + uint32 version; + uint64 native_hw_full; /* payload covers xids [0, native_hw_full) */ + uint32 payload_len; /* raw pg_xact bytes following this header */ + uint32 reserved; /* zero */ + pg_crc32c crc; /* over header fields above + payload */ +} ClusterXidPrehistoryHeader; + +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, native_hw_full) == 8, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, payload_len) == 16, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, crc) == 24, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidPrehistoryHeader) == 32, + "xid prehistory header size is on-disk ABI"); + +#define CLUSTER_XID_PREHISTORY_REL_PATH "global/pgrac_xid_prehistory" +#define CLUSTER_XID_PREHISTORY_BAK_REL_PATH "global/pgrac_xid_prehistory.bak" +#define CLUSTER_XID_PREHISTORY_TMP_REL_PATH "global/pgrac_xid_prehistory.tmp" +#define CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH "global/pgrac_xid_prehistory.bak.tmp" + +/* ============================================================ + * Pure layer (standalone-linkable; exercised by cluster_unit). + * ============================================================ */ + +extern ClusterXidAuthorityValidity cluster_xid_authority_classify(const char *buf, size_t len); + +/* + * cluster_xid_prehistory_payload_bytes -- whole-CLOG-page payload size + * covering xids [0, native_hw_full). 0 when native_hw_full is 0 (nothing + * to carry); the per-page constant mirrors CLOG_XACTS_PER_PAGE without + * dragging clog.c internals into the unit build. + */ +extern uint32 cluster_xid_prehistory_payload_bytes(uint64 native_hw_full); + +extern ClusterXidAuthorityValidity cluster_xid_prehistory_classify(const char *buf, size_t len); + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * Fail-closed read: primary then .bak; false when neither validates. + * Never ereports (safe on the bootstrap early-read path). + */ +extern bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out); + +/* As read(), reporting .bak fallback for a consumer-side LOG (review F8). */ +extern bool cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak); + +/* + * ENOENT-only-absent presence probe (spec-6.14 §3.6 posture): any stat() + * failure other than ENOENT reports present, so a transiently unreadable + * authority is never re-seeded over. + */ +extern bool cluster_xid_authority_present(void); + +/* + * Seed the authority (unsealed) when neither image exists. Returns true + * when this call created it. A trustworthy existing image is a no-op. + */ +extern bool cluster_xid_authority_seed_if_absent(uint64 initial_native_hw); + +/* + * Monotone raise of native_hw_full + set SEALED (never lowers, never + * clears flags). seal=false publishes a crash-safe interim high-water + * during the native era without opening adoption. PANICs on I/O error. + */ +extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, + bool seal); + +/* + * One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). Idempotent + * re-assert: rewrites BOTH copies until both carry the flag (repairing a + * torn previous stamp); no-write no-op once both are stamped. + */ +extern void cluster_xid_authority_mark_cluster_era(void); + +/* + * A follow-up native-era (cluster.enabled=off) boot on a sealed authority + * re-opens the era: clears SEALED so a crash of this run never exposes the + * previous pass's stale high-water to joiners. Caller vets CLUSTER_ERA + * first (re-entry is FATAL). Idempotent re-assert: rewrites BOTH copies + * until neither retains SEALED (repairing a torn previous unseal); + * no-write no-op once both are open. + */ +extern void cluster_xid_authority_begin_native_run(void); + +/* ============================================================ + * Prehistory publish / adopt (spec-6.15b D2; P2: adopt is complete and + * fsynced strictly before StartupCLOG -- both run under the postmaster / + * shutdown-checkpoint single-writer windows, file-level only). + * ============================================================ */ + +/* + * Seed side: snapshot local pg_xact pages [0, native_hw_full) into the + * shared prehistory file (torn-safe). PANICs on I/O error; FATALs when + * the native era exceeds CLUSTER_XID_PREHISTORY_MAX_XID. + */ +extern void cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full); + +/* + * Joiner side: decode the shared prehistory into local pg_xact page files, + * pg_fsync each touched segment and the pg_xact directory before returning. + * Idempotent (same-byte overwrite). FATALs (53RB5) on a missing/corrupt + * blob or a native_hw mismatch against the sealed authority. + */ +extern void cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full); +extern bool cluster_xid_prehistory_was_adopted(void); + +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +typedef enum ClusterXidPrefixVerdict { + CLUSTER_XID_PREFIX_CONSISTENT = 0, /* local prefix matches the blob */ + CLUSTER_XID_PREFIX_DIVERGED, /* local pg_xact contradicts the blob */ + CLUSTER_XID_PREFIX_UNAVAILABLE, /* no trustworthy blob to compare */ +} ClusterXidPrefixVerdict; + +/* + * Compare the local pg_xact bytes covering xids [oldest_xid_full, + * limit_xid_full) with the sealed prehistory blob at 2-bit (per-xact) + * precision. Bits below oldest_xid_full are frozen truth that CLOG never + * consults again and that SimpleLruTruncate may already have removed + * (whole segments); they are exempt whether the pages survive or not. A + * local page missing INSIDE the comparable range is fail-closed + * UNAVAILABLE, never CONSISTENT: pg_xact covers [oldestXid, nextXid) on a + * well-formed node, so a hole is an anomaly, and a front-truncated + * divergent clone must not pass as a "shorter clone" (review r3-X2). + * Callers FATAL on DIVERGED/UNAVAILABLE: a joiner whose own native-era + * history contradicts the seed's is not a pre-seed lineage, and neither + * skipping (trusting local bits) nor adopting (overwriting the joiner's + * own outcomes) is sound for it. + */ +extern ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, + uint64 native_hw_full, + uint64 limit_xid_full, + uint64 oldest_xid_full); + +#endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index c67502ae2c..1897b3af0b 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -468,6 +468,9 @@ typedef enum { WAIT_EVENT_CLUSTER_IC_TCP_SEND, WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT, WAIT_EVENT_CLUSTER_IC_RECONNECT, + /* PGRAC: spec-7.2 D6 -- LMS data-plane loop waits. */ + WAIT_EVENT_CLUSTER_LMS_DATA_RECV, + WAIT_EVENT_CLUSTER_LMS_DATA_SEND, /* Cluster: Undo (8 events) -- AD-010; spec-3.9 adds CR_CONSTRUCT; * spec-3.11 adds TT_DURABLE_IO; spec-3.18 D7 adds BUF_FLUSH + EXTENT_CLAIM */ diff --git a/src/test/cluster_regress/expected/cluster_smoke.out b/src/test/cluster_regress/expected/cluster_smoke.out index 062304fd1c..2d7ea5527d 100644 --- a/src/test/cluster_regress/expected/cluster_smoke.out +++ b/src/test/cluster_regress/expected/cluster_smoke.out @@ -76,14 +76,14 @@ SELECT attname, format_type(atttypid, atttypmod) (7 rows) -- ---------- --- 3. Cluster wait events: 118 rows (anchored by +-- 3. Cluster wait events: 120 rows (anchored by -- CLUSTER_WAIT_EVENTS_COUNT, spec-0.11 + StaticAssertDecl -- in cluster_views.c; spec-6.2 D10 +4 authority waits). -- ---------- SELECT count(*) FROM pg_stat_cluster_wait_events; count ------- - 118 + 120 (1 row) -- ---------- @@ -107,7 +107,7 @@ SELECT count(DISTINCT type) FROM pg_stat_cluster_wait_events; SELECT count(*) FROM pg_stat_gcluster_wait_events; count ------- - 118 + 120 (1 row) -- ---------- diff --git a/src/test/cluster_regress/sql/cluster_smoke.sql b/src/test/cluster_regress/sql/cluster_smoke.sql index 04fd94fa1c..866c81b441 100644 --- a/src/test/cluster_regress/sql/cluster_smoke.sql +++ b/src/test/cluster_regress/sql/cluster_smoke.sql @@ -44,7 +44,7 @@ SELECT attname, format_type(atttypid, atttypmod) -- ---------- --- 3. Cluster wait events: 118 rows (anchored by +-- 3. Cluster wait events: 120 rows (anchored by -- CLUSTER_WAIT_EVENTS_COUNT, spec-0.11 + StaticAssertDecl -- in cluster_views.c; spec-6.2 D10 +4 authority waits). -- ---------- diff --git a/src/test/cluster_tap/t/010_views.pl b/src/test/cluster_tap/t/010_views.pl index 524beb3e71..81182cab24 100644 --- a/src/test/cluster_tap/t/010_views.pl +++ b/src/test/cluster_tap/t/010_views.pl @@ -50,8 +50,8 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); is($node->safe_psql( 'postgres', @@ -88,7 +88,7 @@ 'Cluster: Reconfig' => 8, # spec-5.18 D12: +ReconfigNodeRemoveCleanupWait 'Cluster: Recovery' => 7, # spec-4.12 D6: +ClusterWriteFenceVerify 'Cluster: Sinval' => 6, - 'Cluster: Interconnect' => 9, # spec-6.13: +busypoll + inline send waits + 'Cluster: Interconnect' => 11, # spec-6.13: +busypoll + inline send waits; spec-7.2 D6: +LmsDataRecv/Send 'Cluster: Undo' => 4, 'Cluster: ADG' => 4, ); diff --git a/src/test/cluster_tap/t/011_gviews.pl b/src/test/cluster_tap/t/011_gviews.pl index 61dda90bdc..e7cbbdbb3d 100644 --- a/src/test/cluster_tap/t/011_gviews.pl +++ b/src/test/cluster_tap/t/011_gviews.pl @@ -15,7 +15,7 @@ # # What this test verifies: # - The global view exists and is queryable. -# - It returns exactly 118 rows (1 node x 118 cluster wait events). +# - It returns exactly 120 rows (1 node x 118 cluster wait events). # - It exposes exactly 1 distinct node_id at 0.17 (placeholder). # - The single node_id matches the cluster.node_id GUC. # - Per-class row counts match docs/wait-events-design.md §2.1. @@ -62,8 +62,8 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_gcluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', @@ -102,7 +102,7 @@ 'Cluster: Reconfig' => 8, # spec-5.18 D12: +ReconfigNodeRemoveCleanupWait 'Cluster: Recovery' => 7, # spec-4.12 D6: +ClusterWriteFenceVerify 'Cluster: Sinval' => 6, - 'Cluster: Interconnect' => 9, # spec-6.13: +busypoll + inline send waits + 'Cluster: Interconnect' => 11, # spec-6.13: +busypoll + inline send waits; spec-7.2 D6: +LmsDataRecv/Send 'Cluster: Undo' => 4, 'Cluster: ADG' => 4, ); diff --git a/src/test/cluster_tap/t/012_ic.pl b/src/test/cluster_tap/t/012_ic.pl index f894be28ad..dae334be93 100644 --- a/src/test/cluster_tap/t/012_ic.pl +++ b/src/test/cluster_tap/t/012_ic.pl @@ -102,13 +102,13 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_gcluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); # ---------- diff --git a/src/test/cluster_tap/t/013_conf.pl b/src/test/cluster_tap/t/013_conf.pl index a9200432d6..1e2001a5f5 100644 --- a/src/test/cluster_tap/t/013_conf.pl +++ b/src/test/cluster_tap/t/013_conf.pl @@ -113,13 +113,13 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_gcluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', q{SHOW "cluster.interconnect_tier"}), 'stub', @@ -272,6 +272,42 @@ 'startup failure log includes line number context'); +# ---------- +# spec-7.2 D2: data_addr node key — valid host:port parses and the +# server starts (the DATA plane consumer lands with the LMS loop; here +# we only pin parse acceptance); malformed value is a startup FATAL. +# ---------- +unlink $conf_path; +PostgreSQL::Test::Utils::append_to_file($conf_path, <<'EOC'); +[node.0] +interconnect_addr = 10.0.0.10:6432 +data_addr = 10.0.0.10:6532 +EOC + +$node->adjust_conf('postgresql.conf', 'cluster.node_id', '0'); +unlink $node->logfile; +$node->start; +is($node->safe_psql('postgres', 'SELECT 1'), '1', + 'spec-7.2 data_addr host:port accepted at startup'); +$node->stop; + +unlink $conf_path; +PostgreSQL::Test::Utils::append_to_file($conf_path, <<'EOC'); +[node.0] +interconnect_addr = 10.0.0.10:6432 +data_addr = not-an-addr +EOC + +unlink $node->logfile; +$start_failed = !$node->start(fail_ok => 1); +ok($start_failed, + 'spec-7.2 malformed data_addr refuses startup'); + +$log = PostgreSQL::Test::Utils::slurp_file($node->logfile); +like($log, qr/data_addr must be empty or in host:port form/, + 'spec-7.2 malformed data_addr log names the key'); + + # ---------- # Recovery: remove pgrac.conf, restore cluster.node_id default, server # should come back up on the single-node fallback path. diff --git a/src/test/cluster_tap/t/014_ic_mock.pl b/src/test/cluster_tap/t/014_ic_mock.pl index 08f3cfbdda..dc848c4545 100644 --- a/src/test/cluster_tap/t/014_ic_mock.pl +++ b/src/test/cluster_tap/t/014_ic_mock.pl @@ -171,8 +171,8 @@ is( $node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..d749f086da 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', + 'pg_stat_cluster_injections returns 161 rows (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '161 injection point names match the full registry (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -189,8 +189,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); # ---------- # Test 11 (Hardening v1.0.1 / codex review P2-2): SQL SRF rejects diff --git a/src/test/cluster_tap/t/016_perfmon.pl b/src/test/cluster_tap/t/016_perfmon.pl index cf0c4bbfb1..a7e3a98abe 100644 --- a/src/test/cluster_tap/t/016_perfmon.pl +++ b/src/test/cluster_tap/t/016_perfmon.pl @@ -154,8 +154,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..739b6a0e49 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -63,8 +63,8 @@ 'postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'all 55 categories appear (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'all 56 categories appear (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); # ---------- @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '158', - 'all 158 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', + 'all 161 injection points have a .fault_type entry under inject category (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '158', - 'all 158 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', + 'all 161 injection points have a .hits entry under inject category (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -157,8 +157,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index 676dfc9e1a..60db597af5 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L9 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', + 'L9 total injection registry size is 161 (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..76e50ecdba 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -109,9 +109,9 @@ # between "pgrac cluster cr relgen" and "pgrac cluster cr tuple stats"). # spec-6.12h D-h3a: +1 "pgrac cluster pi shadow" (PI ship-SCN shadow table; # sorts between "pgrac cluster oid lease,pgrac cluster pcm grd" and "pgrac cluster qvotec"). -my $expected_region_count = $has_visibility_inject ? '79' : '78'; +my $expected_region_count = $has_visibility_inject ? '80' : '79'; my $expected_regions = - 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo record cursor'; + 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lms data outbound,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo record cursor'; $expected_regions .= ',pgrac cluster visibility inject' if $has_visibility_inject; # spec-4.12 D7: cooperative write-fence region; always registered. Sorts after @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '161', + 'L15 total injection registry size is 161 (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); # ---------- @@ -309,8 +309,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L17 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L17 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..39f6a34abc 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -56,7 +56,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound # ---------- @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '161', + 'L11 pg_stat_cluster_injections is 161 (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- @@ -199,8 +199,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L12 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L12 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..1354ed353e 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -71,7 +71,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound # ---------- @@ -204,14 +204,14 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '161', + 'L12a pg_stat_cluster_injections is 161 (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L12b pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L12b pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..f7ddfec0fa 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -61,7 +61,7 @@ # admission reason counters; +1 "pgrac cluster clean_leave" (spec-5.13); +1 # "pgrac cluster cr relgen" (spec-5.56 D4); +1 "pgrac cluster cr tuple stats" # (spec-5.54 D5); full list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound # ---------- @@ -137,8 +137,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L9 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ---------- @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '161', + 'L11 pg_stat_cluster_injections is 161 (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- @@ -168,8 +168,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12 pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12 pg_cluster_state has 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -232,7 +232,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..a01a9639c1 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,15 @@ # ---------- -# L1: pg_cluster_state.pcm category has 22 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5) +# L1: pg_cluster_state.pcm category has 23 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5 +# + spec-4.6a D12 dead_cleanup_entries) # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '22', - 'L1 pg_cluster_state.pcm category has 22 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5)'); + '23', + 'L1 pg_cluster_state.pcm category has 23 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5 + spec-4.6a D12)'); # ---------- @@ -162,8 +163,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L6a pg_stat_cluster_injections has 158 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', + 'L6a pg_stat_cluster_injections has 161 entries (spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', @@ -189,8 +190,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L7b pg_cluster_state has 55 distinct categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L7b pg_cluster_state has 56 distinct categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -209,7 +210,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..55de4798c9 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -146,8 +146,8 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'E1 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'E1 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); ok($node->safe_psql('postgres', q{SELECT count(*) > 0 FROM pg_stat_cluster_wait_events WHERE type='Cluster: GES'}) @@ -159,7 +159,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', 'E4 pg_stat_gcluster_wait_events returns 118 rows (single-node, spec-6.13 RDMA wait surface)'); + '120', 'E4 pg_stat_gcluster_wait_events returns 120 rows (single-node, spec-6.13 RDMA wait surface)'); # ============================================================ @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', 'M1 158 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', 'M1 161 injection points (spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '316', - 'M5 inject category has 158×2 = 316 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + ) eq '322', + 'M5 inject category has 161×2 = 322 sub-keys (.fault_type + .hits; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); @@ -395,8 +395,8 @@ is($node->safe_psql('postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'O2 pg_cluster_state has all 55 categories (spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'O2 pg_cluster_state has all 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE value IS NULL}), diff --git a/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl b/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl index 997da11f84..823cbd5e76 100644 --- a/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl +++ b/src/test/cluster_tap/t/078_ic_tier1_partial_send_recv.pl @@ -66,6 +66,8 @@ sub build_hello # Declare 2 nodes so we can spoof "peer 1" connecting to "peer 0" (us). my $self_ic_port = PostgreSQL::Test::Cluster::get_free_port(); my $spoof_ic_port = PostgreSQL::Test::Cluster::get_free_port(); +my $self_data_port = PostgreSQL::Test::Cluster::get_free_port(); +my $spoof_data_port = PostgreSQL::Test::Cluster::get_free_port(); my $cluster_name = 'pgrac-078'; my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index cebe221c82..dfcbea09c6 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '22', - 'L1 pg_cluster_state pcm category has 22 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter)'); +is($pcm_category_rows, '23', + 'L1 pg_cluster_state pcm category has 23 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter + spec-4.6a D12 dead_cleanup_entries)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( @@ -81,7 +81,7 @@ # L6 — wait event count baseline through spec-6.1. my $wait_event_count = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_stat_cluster_wait_events"); -is($wait_event_count, '118', +is($wait_event_count, '120', 'L6 wait event baseline 118 (spec-6.13 RDMA wait surface)'); # L7 — no PCM wire opcode smoke (no SQL-visible PCM wire opcode enum surface) diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index 84faab2ede..9f508ee72c 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -7,10 +7,10 @@ # any wire send (HC72), so wire path coverage is effectively limited # to SQL-visible surface invariants: # -# L1 fresh cluster startup: pg_cluster_state.gcs has 67 keys +# L1 fresh cluster startup: pg_cluster_state.gcs has 85 keys # L2 api_state = "active" after postmaster phase 1 init # L3 WAIT_EVENT_GCS_REPLY_WAIT registered in pg_stat_cluster_wait_events -# L4 CLUSTER_WAIT_EVENTS_COUNT == 118 (cumulative through spec-6.13) +# L4 CLUSTER_WAIT_EVENTS_COUNT == 120 (cumulative through spec-7.2) # L5 msg_type registry surface visible: pg_cluster_ic_msg_types has # gcs_request + gcs_reply rows # L6 workload (SELECT/UPDATE/VACUUM) does NOT inc send_request_count @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 67 keys. +# L1 — pg_cluster_state.gcs surface has 85 keys (spec-7.2 D6 hist). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L1 pg_cluster_state.gcs category has 67 keys (spec-2.37 D12)'); + '85', + 'L1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6)'); # L2 — api_state = "active" after postmaster phase 1 init. @@ -87,11 +87,11 @@ sub gcs_value { 'L3 ClusterGcsReplyWait wait event registered (spec-2.32 D7)'); -# L4 — CLUSTER_WAIT_EVENTS_COUNT == 118 (spec-6.13). +# L4 — CLUSTER_WAIT_EVENTS_COUNT == 120 (spec-7.2 D6). my $total_wait_events = $node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'); -is($total_wait_events, '118', - 'L4 wait_events count 118 (spec-6.13 RDMA wait surface)'); +is($total_wait_events, '120', + 'L4 wait_events count 120 (spec-7.2 D6 LMS data-plane wait surface)'); # L6 — Production workload does NOT trigger wire path (HC72 short-circuit). diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 9b5015e4e9..323f16e364 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -8,7 +8,7 @@ # # L1 ClusterPair startup — both postmasters healthy + tier1 connected # L2 fresh baseline gcs counters on both nodes (block_* counters = 0) -# L3 pg_cluster_state.gcs category has 67 keys (cumulative through +# L3 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through # spec-6.13 direct-land observability) # L4 4 NEW wait events registered in pg_stat_cluster_wait_events: # ClusterGCSBlockShipWait, ClusterGCSBlockRequestDispatch, @@ -113,19 +113,19 @@ sub gcs_int_value { # ============================================================ -# L3: pg_cluster_state.gcs category has 67 keys +# L3: pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) # (cumulative GCS surface through spec-6.13 direct-land observability). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); # ============================================================ @@ -152,7 +152,7 @@ sub gcs_int_value { is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', + '120', 'L5 total cluster wait event count = 118 (spec-6.13 RDMA wait surface)'); diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 6b489f8c64..95ad463b15 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,13 +9,13 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a) +# L3 pg_cluster_state.gcs has 85 keys (cumulative through spec-7.2 D6 hist) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) -# L5 CLUSTER_WAIT_EVENTS_COUNT = 85 (was 83 spec-2.33) +# L5 CLUSTER_WAIT_EVENTS_COUNT = 120 (spec-7.2 +2) # L6 3 NEW GUC visible + defaults + contexts: # cluster.gcs_block_retransmit_max_retries PGC_SUSET 4 -# cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 100 +# cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 10 (spec-7.2 D1) # cluster.gcs_block_dedup_max_entries PGC_POSTMASTER 1024 # L7 single-shot ship workload — retransmit_attempt_count=0 # L8 inject `cluster-gcs-block-drop-reply-before-send:skip:1` → @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs category has 85 keys (cumulative through spec-7.2 D6 hist). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys'); # ============================================================ @@ -144,8 +144,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L5 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L5 pg_stat_cluster_wait_events returns 120 rows (spec-7.2 LMS data-plane waits)'); # ============================================================ @@ -153,7 +153,7 @@ sub gcs_int # ============================================================ for my $row ( [ 'cluster.gcs_block_retransmit_max_retries', '4', 'superuser' ], - [ 'cluster.gcs_block_retransmit_initial_backoff_ms', '100', 'superuser' ], + [ 'cluster.gcs_block_retransmit_initial_backoff_ms', '10', 'superuser' ], [ 'cluster.gcs_block_dedup_max_entries', '1024', 'postmaster' ], ) { diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index 085bf47bd0..c04dc7dcb2 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 7 NEW counters all 0 + catversion >= 202605420 -# L3 pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.13) +# L3 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) # L4 cross-node forward path: node A read first → master_holder = A; # force same tag on node B via test-only injection → master # chooses forward path → A direct-ships to B → block_forward_sent @@ -108,18 +108,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); # ============================================================ @@ -211,8 +211,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L9 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); done_testing(); diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 2a479ae004..9319413f6a 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -12,8 +12,8 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 -# L3 pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a) -# L4 catversion lower-bound >= 202605430; wait event count == 118 +# L3 pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) +# L4 catversion lower-bound >= 202605430; wait event count == 120 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader # sees starvation_denied_pending_x_count tick @@ -106,18 +106,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node0 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L3 node1 pg_cluster_state.gcs category has 67 keys'); + '85', + 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); # ============================================================ @@ -132,8 +132,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L4 wait event count == 118 (spec-6.13 RDMA wait surface)'); + '120', + 'L4 wait event count == 120 (spec-7.2 D6 LMS data-plane wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index 8f0e94d343..64f60a9614 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -125,13 +125,13 @@ sub gcs_int is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - "L4 node$i wait event count == 118 (spec-6.13 RDMA wait surface)"); + '120', + "L4 node$i wait event count == 120 (spec-7.2 D6 LMS data-plane wait surface)"); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - "L4 node$i pg_cluster_state.gcs has 67 keys"); + '85', + "L4 node$i pg_cluster_state.gcs has 85 keys"); } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 5c3f5504ed..b3fbdf32e3 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -20,7 +20,7 @@ # L7 SQLSTATE 53R93 ERRCODE_CLUSTER_LOST_WRITE_DETECTED literal- # encodable in PG SQL (catalog 形式 verification) # L8 GUC switch back to 'error' SHOW returns 'error' -# L9 pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.14a) +# L9 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) # L10 Reply status enum value 12 (DENIED_LOST_WRITE) is新增的 # 最大 value (baseline workload must not trigger lost-write) # L11 spec-2.41 D / P1-C — behavioral lost-write inject: a @@ -101,14 +101,14 @@ sub gcs_int is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L2 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L2 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', - 'L2 pg_cluster_state.gcs category has 67 keys (cumulative through spec-6.14a)'); + '85', + 'L2 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); # ============================================================ @@ -198,7 +198,7 @@ sub gcs_int is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '67', + '85', 'L9 node1 pg_cluster_state.gcs has 67 keys (cross-node parity)'); diff --git a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl index aea5668348..81639615bd 100644 --- a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl +++ b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl @@ -93,8 +93,8 @@ sub sinval_int # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L3 wait event count == 118 (spec-6.13 RDMA wait surface)'); + '120', + 'L3 wait event count == 120 (spec-7.2 D6 LMS data-plane wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl index 89e2273a7f..978a8259c2 100644 --- a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl +++ b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl @@ -13,7 +13,7 @@ # L1 catversion bump 202605450 → 202605460 # L2 pg_cluster_state sinval category has 16 keys (D8 +3 fanout + D9 +3 ack + D1 applied) # L3 3 NEW GUC defaults (sinval_ack_mode=peer_enqueued / timeout 5000 / slots 256) -# L4 pg_stat_cluster_wait_events count == 118 (cumulative through spec-6.13) +# L4 pg_stat_cluster_wait_events count == 120 (cumulative through spec-7.2) # L5 ClusterSinvalAckWait / ClusterSinvalAckSend / ClusterSinvalAckReceive visible # L6 53R95 ERRCODE_CLUSTER_SINVAL_ACK_TIMEOUT encodable # L7 node0 CREATE TABLE → node1 SELECT visible within 5s ack timeout @@ -77,12 +77,12 @@ 'L3c cluster.sinval_ack_wait_slots default 256'); # ============================================================ -# L4: pg_stat_cluster_wait_events count == 118. +# L4: pg_stat_cluster_wait_events count == 120. # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L4 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L4 pg_stat_cluster_wait_events returns 120 rows (spec-7.2 D6 LMS data-plane wait surface)'); # ============================================================ # L5: 3 NEW ack wait events visible. diff --git a/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl b/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl index 5b033f5f43..8f8186b321 100644 --- a/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl +++ b/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl @@ -215,8 +215,8 @@ sub tt_int # additions to sinval surface). is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '120', + 'L9 pg_stat_cluster_wait_events returns 120 rows (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl index f12abe0f14..9bc0e1fccf 100644 --- a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl +++ b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl @@ -325,8 +325,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); my $tt_categories = $pair->node0->safe_psql('postgres', q{ SELECT string_agg(c, ',' ORDER BY c) diff --git a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl index 4e65595eff..56d8cd35f9 100644 --- a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl +++ b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl @@ -296,8 +296,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L14a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L14a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl index 80447a5161..8f4f1c5ce7 100644 --- a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl +++ b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl @@ -287,7 +287,7 @@ # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', 'L13a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', 'L13a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl index 3c66d90c53..1e0847375a 100644 --- a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl +++ b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl @@ -41,9 +41,10 @@ # L5 (D3 flipped) the rebound holders release against their # current-epoch masters and both resources are immediately # re-acquirable. Pinned gap was: release silently swallowed. -# L7 (D5 flipped) grd_recovery dump category: 13 counters, key -# roster per spec-4.6 §2.4, episode trail asserted -# (remaster_started/done, shards_remastered=2048, +# L7 (D5 flipped) grd_recovery dump category: 31 keys (spec-4.6 +# §2.4 base 13 + spec-5.16 D5 join-direction 5 + spec-4.6a +# liveness/episode surface 13), full key roster, episode trail +# asserted (remaster_started/done, shards_remastered=2048, # holders_redeclared>=1, rebuild_timeout=0). # # Discipline notes: @@ -395,22 +396,29 @@ sub poll_query_until_timeout # ---------- -# L7 (FLIPPED by D5; spec-5.16 D5 +5 join-direction counters): grd_recovery -# dump category exposes 18 counters and the recovery episode left the -# expected trail. +# L7 (FLIPPED by D5; spec-5.16 D5 +5 join-direction counters; spec-4.6a +# +13 episode/liveness keys): grd_recovery dump category exposes 31 +# keys and the recovery episode left the expected trail. # ---------- is($pair->node1->safe_psql('postgres', q{SELECT count(*) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - '18', 'L7 grd_recovery dump category exposes 18 counters (D5 + spec-5.16 join)'); + '31', + 'L7 grd_recovery dump category exposes 31 keys (spec-4.6 D5 + spec-5.16 join + spec-4.6a)'); is($pair->node1->safe_psql('postgres', q{ SELECT string_agg(key, ',' ORDER BY key) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - 'block_path_failclosed,converts_requeued,holders_rebound,holders_redeclared,' - . 'rebuild_timeout,remaster_done,remaster_failed,remaster_started,' - . 'shards_remastered,stale_holder_swept,stale_request_drop,' - . 'unaffected_holder_survived,waiters_requeued', - 'L7 grd_recovery key roster matches spec-4.6 §2.4'); + 'block_path_failclosed,block_redeclare_cursor,block_redeclare_done,' + . 'block_redeclare_epoch,cluster_gate_timeout,converts_requeued,' + . 'done_self_bitmap_hash,done_self_epoch,episode_epoch,event_coordinator,' + . 'event_old_epoch,holders_rebound,holders_redeclared,' + . 'join_block_recovering_failclosed,join_block_views_rebuilt,' + . 'join_remaster_done,join_remaster_started,join_shards_remastered,' + . 'last_event_id,rebuild_timeout,remaster_done,remaster_failed,' + . 'remaster_started,shards_remastered,stale_holder_swept,' + . 'stale_request_drop,state,state_enum_value,unaffected_holder_survived,' + . 'wait_epoch_escape,waiters_requeued', + 'L7 grd_recovery key roster (spec-4.6 §2.4 + spec-5.16 D5 + spec-4.6a)'); is($pair->node1->safe_psql('postgres', q{SELECT value::bigint >= 1 FROM cluster_dump_state() diff --git a/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl b/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl index f695cd636b..a4f14e62b3 100644 --- a/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl +++ b/src/test/cluster_tap/t/288_cf_enqueue_concurrent.pl @@ -79,6 +79,8 @@ sub cf_counter my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---- node0 init -> backup -> node1 init_from_backup (shared sysid) ---- my $node0 = PostgreSQL::Test::Cluster->new('cf_cc_node0'); @@ -131,9 +133,11 @@ sub cf_counter [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); diff --git a/src/test/cluster_tap/t/289_cf_bootstrap.pl b/src/test/cluster_tap/t/289_cf_bootstrap.pl index 43924f15eb..632c0c93ab 100644 --- a/src/test/cluster_tap/t/289_cf_bootstrap.pl +++ b/src/test/cluster_tap/t/289_cf_bootstrap.pl @@ -103,6 +103,8 @@ # ---- IC ports ---- my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (shared sysid). @@ -168,9 +170,11 @@ [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); @@ -446,6 +450,8 @@ sub poison_authority_from_node0 mkdir "$neg_shared/global" or die "mkdir neg global: $!"; my $neg_ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $neg_ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $neg_data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $neg_data_port1 = PostgreSQL::Test::Cluster::get_free_port(); my $neg = PostgreSQL::Test::Cluster->new('cf_boot_neg'); $neg->init; @@ -469,9 +475,11 @@ sub poison_authority_from_node0 [node.0] interconnect_addr = 127.0.0.1:$neg_ic0 +data_addr = 127.0.0.1:$neg_data_port0 [node.1] interconnect_addr = 127.0.0.1:$neg_ic1 +data_addr = 127.0.0.1:$neg_data_port1 EOC my $nret = $neg->start(fail_ok => 1); diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 9c156e1e53..2447312952 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -30,13 +30,18 @@ # retryable transient, NOT an HW fault -- so this leg retries and SKIPs # (never fails) when only that orthogonal recovery is outstanding. # -# Negative (corrupt the dead master's snapshot -> fail-closed): +# Negative/self-heal extensions (spec-4.6a): # L6 a fresh pair; corrupt node0's HW snapshot on shared storage, then # kill node0. The survivor's rebuild must FAIL CLOSED # (hw.remaster_blocked_count advances) -- it never trusts a corrupt # snapshot, never auto-creates at 0, never reads FileSize as the -# authority. The affected shards stay frozen (P7 gate), so no extend -# can silently re-hand a block. +# authority. +# L7 hide node0's snapshot, kill node0, observe BLOCKED, then restore +# the same file. The same reconfig episode must retry and converge +# to DONE without waiting for a later episode. +# L8 keep the snapshot corrupt with retry_max_attempts=2. The survivor +# must report retry_exhausted once and keep fail-closed; DONE must not +# advance. # # Harness: ClusterPair shared_data + wal_threads_root + 3 voting disks. # Mirrors the kill/reconfig pattern of t/249 / t/274. @@ -181,17 +186,22 @@ sub retry_sql { my ($node, $sql) = @_; my $last = ''; + my $last_retryable = 0; for my $attempt (1 .. 30) { my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); - return (1, $out) if defined $rc && $rc == 0; - $last = $err // ''; + my $combined = join("\n", grep { defined $_ && $_ ne '' } ($out, $err)); + return (1, $out, 0) + if defined $rc && $rc == 0 && $combined !~ /ERROR:/; + + $last = $combined; + $last_retryable = + $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; # retry only the orthogonal post-reconfig settling transients - last - unless $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; + last unless $last_retryable; usleep(500_000); } - return (0, $last); + return (0, $last, $last_retryable); } # L5a (the robust HW proof): the rebuilt authority is SERVEABLE -- the HW serve @@ -210,22 +220,229 @@ sub retry_sql # that is NOT an HW fault. Retry, and SKIP (never fail) when only that orthogonal # recovery is outstanding; a 53RA6 from the HW gate would NOT be retried and would # fail the test. -my ($ext_ok, $ext_res) = +my ($ext_ok, $ext_res, $ext_retryable) = retry_sql($n1, 'INSERT INTO r1 SELECT g,g,g,g FROM generate_series(4001,4500) g'); SKIP: { - skip - "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", - 1 - unless $ext_ok; - my ($cnt_ok, $cnt) = retry_sql($n1, 'SELECT count(*) FROM r1'); - is($cnt, '4500', - 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') - or diag("count result: $cnt"); + if (!$ext_ok) + { + skip + "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", + 1 + if $ext_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("extend result: $ext_res"); + } + else + { + my ($cnt_ok, $cnt, $cnt_retryable) = retry_sql($n1, 'SELECT count(*) FROM r1'); + if (!$cnt_ok) + { + skip + "post-remaster count gated by orthogonal visibility recovery (not HW): $cnt", + 1 + if $cnt_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("count result: $cnt"); + } + else + { + is($cnt, '4500', + 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') + or diag("count result: $cnt"); + } + } } $pair->stop_pair; +# ====================================================================== +# SELF-HEAL: first attempt BLOCKED, same episode retries after repair. +# ====================================================================== + +my $heal = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_selfheal', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +$heal->start_pair; +my $h0 = $heal->node0; +my $h1 = $heal->node1; + +ok(poll_until($h1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L7: self-heal pair in quorum'); +poll_until( + $h1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($h0, $h1, "s$_", 4000) for (1 .. 6); +$h0->safe_psql('postgres', 'CHECKPOINT'); +$h1->safe_psql('postgres', 'CHECKPOINT'); + +my $hsnap = $heal->shared_data_root . "/global/pg_hw_snapshot.0"; +my $hhold = "$hsnap.hold"; +ok(-f $hsnap, 'L7: node0 HW snapshot exists before self-heal injection'); +rename($hsnap, $hhold) or die "rename $hsnap -> $hhold: $!"; + +my $h_done_before = hw_counter($h1, 'remaster_done_count'); +my $h_blocked_before = hw_counter($h1, 'remaster_blocked_count'); +my $h_retry_before = hw_counter($h1, 'remaster_retry_count'); +$heal->kill_node9(0); +ok( poll_until( + $h1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30), + 'L7: self-heal pair declared node0 dead'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_blocked_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_blocked_count'", + 'true', 40), + 'L7: first HW rebuild attempt failed closed while the snapshot was hidden'); + +rename($hhold, $hsnap) or die "rename $hhold -> $hsnap: $!"; +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_retry_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_count'", + 'true', 40), + 'L7: same reconfig episode launched a HW remaster retry'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_done_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_done_count'", + 'true', 60), + 'L7: restored snapshot let the same-episode retry converge to DONE'); +is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, + 'L7: self-heal path did not exhaust the retry budget'); + +# spec-4.6a section 4 (D8, Amendment v1.2 R1): after the same-episode DONE the +# episode has converged (P7 ran: DONE counter + no exhaustion asserted above), +# but in this out-of-scope deployment (online_thread_recovery off) EVERY tag +# whose static master is the dead node stays fail-closed until that node +# returns — including brand-new blocks, which have no materialization proof +# either (the cold-block door cannot distinguish never-existed from +# not-yet-replayed; main's t/293 L5b documents the same posture). The +# survivor write is therefore the honest two-outcome contract: success (all +# touched tags survivor-mastered) or the explicit bounded 53R9L — never a +# hang and never an unbounded wedge. +{ + my ($ext_ok, $ext_res) = (0, ''); + for my $try (1 .. 10) + { + $ext_res = eval { + $h1->safe_psql('postgres', + 'CREATE TABLE IF NOT EXISTS s1_heal (a int, b int, c int, d int); ' + . 'INSERT INTO s1_heal SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 1; + } ? 'ok' : ($@ // 'error'); + if ($ext_res eq 'ok') { $ext_ok = 1; last; } + sleep 1; + } + if ($ext_ok) + { + ok(1, 'L7: survivor new-table extend succeeded after same-episode DONE (P7 unfreeze)'); + } + else + { + like($ext_res, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + 'L7: survivor extend fail-closed with an explicit SQLSTATE (dead-master tag, out-of-scope posture)'); + } +} + +$heal->stop_pair; + +# ====================================================================== +# NEGATIVE: corrupt snapshot persists -> retries exhaust and stay closed. +# ====================================================================== + +my $exh = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_exhausted', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', + ]); +$exh->start_pair; +my $e0 = $exh->node0; +my $e1 = $exh->node1; + +ok(poll_until($e1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L8: exhausted pair in quorum'); +poll_until( + $e1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($e0, $e1, "x$_", 4000) for (1 .. 6); +$e0->safe_psql('postgres', 'CHECKPOINT'); +$e1->safe_psql('postgres', 'CHECKPOINT'); + +my $esnap = $exh->shared_data_root . "/global/pg_hw_snapshot.0"; +ok(-f $esnap, 'L8: node0 HW snapshot exists before exhaustion injection'); +open(my $efh, '>', $esnap) or die "open $esnap: $!"; +binmode $efh; +print $efh ("\xDE\xAD\xBE\xEF" x 64); +close $efh; + +my $e_done_before = hw_counter($e1, 'remaster_done_count'); +my $e_exh_before = hw_counter($e1, 'remaster_retry_exhausted_count'); +$exh->kill_node9(0); +poll_until( + $e1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30); +ok( poll_until( + $e1, + "SELECT (value::bigint > $e_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L8: persistent corrupt snapshot exhausted the same-episode retry budget'); +is(hw_counter($e1, 'remaster_done_count'), $e_done_before, + 'L8: DONE did not advance after retry exhaustion on corrupt snapshot'); + +# spec-4.6a section 4 (D8/L408): after exhaustion the WAIT_CLUSTER watchdog +# keeps sounding (counter + WARNING line) while shards stay frozen. +ok( poll_until( + $e1, + q{SELECT (value::bigint > 0)::text FROM pg_cluster_state } + . q{WHERE category='grd_recovery' AND key='cluster_gate_timeout'}, + 'true', 30), + 'L8: WAIT_CLUSTER watchdog counter advances while exhausted episode stays frozen'); +{ + my $log = PostgreSQL::Test::Utils::slurp_file($e1->logfile); + like( + $log, + qr/cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen/, + 'L8: watchdog WARNING appears in the survivor log'); +} + +$exh->stop_pair; + # ====================================================================== # NEGATIVE: corrupt the dead master's snapshot -> rebuild fails closed. # ====================================================================== @@ -242,6 +459,9 @@ sub retry_sql 'cluster.ges_retransmit_max_attempts = 0', 'cluster.cssd_heartbeat_interval_ms = 1000', 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', ]); $neg->start_pair; my $m0 = $neg->node0; @@ -277,6 +497,7 @@ sub retry_sql } my $blocked_before = hw_counter($m1, 'remaster_blocked_count'); +my $neg_exh_before = hw_counter($m1, 'remaster_retry_exhausted_count'); $neg->kill_node9(0); poll_until( $m1, @@ -291,6 +512,12 @@ sub retry_sql . "WHERE category='hw' AND key='remaster_blocked_count'", 'true', 40), 'L6: HW rebuild FAILED CLOSED on the corrupt snapshot (remaster_blocked advanced; no auto-create-0)'); +ok( poll_until( + $m1, + "SELECT (value::bigint > $neg_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L6: corrupt snapshot eventually exhausts bounded same-episode retries'); $neg->stop_pair; diff --git a/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl b/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl index 79e31609f1..504ac321d5 100644 --- a/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl +++ b/src/test/cluster_tap/t/334_ic_rdma_soft_roce.pl @@ -83,6 +83,8 @@ sub command_exists my $rdma_port0 = PostgreSQL::Test::Cluster::get_free_port(); my $rdma_port1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); for my $node ($pair->node0, $pair->node1) { @@ -100,6 +102,7 @@ sub write_rdma_pgrac_conf [node.0] interconnect_addr = 127.0.0.1:@{[$pair->ic_port(0)]} +data_addr = 127.0.0.1:$data_port0 rdma_addr = $rdma_host:$rdma_port0 rdma_gid = rxe rdma_port = 1 @@ -108,6 +111,7 @@ sub write_rdma_pgrac_conf [node.1] interconnect_addr = 127.0.0.1:@{[$pair->ic_port(1)]} +data_addr = 127.0.0.1:$data_port1 rdma_addr = $rdma_host:$rdma_port1 rdma_gid = rxe rdma_port = 1 diff --git a/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl b/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl index 56240b21f5..2e1c4a0a89 100644 --- a/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl +++ b/src/test/cluster_tap/t/335_adg_two_thread_rfs_apply.pl @@ -98,15 +98,19 @@ sub write_pair_conf my ($node0, $node1, $cluster_name) = @_; my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); my $conf = <data_dir . '/pgrac.conf', $conf); diff --git a/src/test/cluster_tap/t/336_cluster_backup_pitr.pl b/src/test/cluster_tap/t/336_cluster_backup_pitr.pl index e48af1e510..660ddb5f0f 100644 --- a/src/test/cluster_tap/t/336_cluster_backup_pitr.pl +++ b/src/test/cluster_tap/t/336_cluster_backup_pitr.pl @@ -440,6 +440,8 @@ sub configure_pair_restore my ($restore_node, $shared_dir, $target_scn) = @_; my $restore_ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $restore_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $restore_data_port0 = PostgreSQL::Test::Cluster::get_free_port(); + my $restore_data_port1 = PostgreSQL::Test::Cluster::get_free_port(); my $target_conf = defined($target_scn) ? "cluster.recovery_target_scn = '$target_scn'\n" : ""; @@ -452,9 +454,11 @@ sub configure_pair_restore [node.0] interconnect_addr = 127.0.0.1:$restore_ic0 +data_addr = 127.0.0.1:$restore_data_port0 [node.1] interconnect_addr = 127.0.0.1:$restore_ic1 +data_addr = 127.0.0.1:$restore_data_port1 EOC $restore_node->append_conf('postgresql.conf', "port = " . $restore_node->port . "\n" diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..6db3cccd8b 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -94,6 +94,8 @@ my $shared_root = PostgreSQL::Test::Utils::tempdir(); mkdir "$shared_root/global" or die "mkdir shared global: $!"; +my $wal_root = PostgreSQL::Test::Utils::tempdir(); + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); my @disks; for my $i (0 .. 2) @@ -109,12 +111,14 @@ my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). # ---------- my $node0 = PostgreSQL::Test::Cluster->new('sc_ddl_node0'); -$node0->init(allows_streaming => 1); +$node0->init(allows_streaming => 1, extra => [ '-X', "$wal_root/thread_1" ]); $node0->start; $node0->backup('scb'); $node0->stop; @@ -122,6 +126,24 @@ my $node1 = PostgreSQL::Test::Cluster->new('sc_ddl_node1'); $node1->init_from_backup($node0, 'scb'); +# Relocate node1's backup-copied WAL into its shared thread dir. shared_catalog +# multi-node formation is fail-fast without cluster.wal_threads_dir, and the WAL +# thread validator requires pg_wal to resolve to the configured thread_N dir. +{ + my $pgwal = $node1->data_dir . '/pg_wal'; + my $wal2 = "$wal_root/thread_2"; + mkdir $wal2 or die "mkdir $wal2: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal2/$e") or die "rename $pgwal/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal2, $pgwal) or die "symlink $pgwal -> $wal2: $!"; +} + # Config common to the shared-catalog feature (both nodes, both phases). my $sc_common = <append_conf('postgresql.conf', $cluster_conf); @@ -183,9 +208,11 @@ [node.0] interconnect_addr = 127.0.0.1:$ic0 +data_addr = 127.0.0.1:$data_port0 [node.1] interconnect_addr = 127.0.0.1:$ic1 +data_addr = 127.0.0.1:$data_port1 EOC PostgreSQL::Test::Utils::append_to_file($node0->data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); @@ -200,6 +227,24 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time, via the pg_authid read). First-contact probes therefore +# retry; every semantic assert below stays strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + # node1 must actually be up. my $n1_up = 0; for (1 .. 60) @@ -212,7 +257,7 @@ # ---------- # L1: both alive + IC-connected on the shared-sysid cluster. # ---------- -my $a0 = $node0->safe_psql('postgres', 'SELECT 1'); +my ($n0_up, $a0) = psql_retry_ok($node0, 'SELECT 1'); is($a0, '1', 'L1: node0 is up on the shared-catalog 2-node cluster'); is($n1_up, 1, 'L1: node1 is up on the shared-catalog 2-node cluster'); @@ -220,10 +265,10 @@ my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L1: node0 sees node1 heartbeats (IC connected)'); @@ -233,9 +278,10 @@ # node1 ever running the DDL. This is the Stage 2 exit debt # (spec-2.0:135) closure. # ---------- -$node0->safe_psql('postgres', +my ($ddl_ok) = psql_retry_ok($node0, 'CREATE TABLE a1_shared_cat (id int PRIMARY KEY, note text);' . "INSERT INTO a1_shared_cat VALUES (1, 'from-node0');"); +die 'first cross-node DDL never succeeded' unless $ddl_ok; # Poll node1 up to 1s (10 x 100ms). "retry is safe" per the GCS remaster # transient; a settled shared catalog resolves well within the budget. @@ -819,4 +865,23 @@ sub oid_authority_hw qr/cluster\.shared_catalog is off but the shared tree holds/, 'L7: the refusal is the designed off-flip vet FATAL (never a stale serve)'); +# ---------- +# L8 (spec-4.6a D11 level-2 fail-fast): a multi-node shared_catalog=on boot +# without cluster.wal_threads_dir is refused at startup -- that shape cannot +# rebuild a dead node's HW authority from per-thread WAL, so a node failure +# would be permanently unrecoverable (the silent-BLOCKED wedge of +# spec-4.6a section 0). Flip shared_catalog back on and blank the WAL +# threads root; the boot must fail on the D11 vet, not come up degraded. +# ---------- +$node1->append_conf('postgresql.conf', "cluster.shared_catalog = on +"); +$node1->append_conf('postgresql.conf', "cluster.wal_threads_dir = '' +"); +my $nowalret = $node1->start(fail_ok => 1); +is($nowalret, 0, 'L8: multi-node shared_catalog boot without wal_threads_dir fails'); +my $nowallog = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($nowallog, + qr/cluster\.shared_catalog requires cluster\.wal_threads_dir in a multi-node cluster/, + 'L8: the refusal is the spec-4.6a D11 startup FATAL with the config errhint'); + done_testing(); diff --git a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl index 301827c967..e6571dacd2 100644 --- a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl +++ b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl @@ -44,24 +44,14 @@ # L4 neither node's log carries the pre-D9 "unsupported side effect" # fail-closed marker. # -# KNOWN-BLOCKED (two-layer, investigated 2026-07-06 after the 6.12/6.15 -# rebase). This test deterministically runs WITHOUT cluster.xid_striping -# so it brings up cleanly and reproduces Layer 1 -- turning striping on -# (which is what would make the serving legs' xids resolvable) drags in -# Layer 2, whose non-convergence makes bring-up itself flaky. +# KNOWN-BLOCKED (membership layer, investigated 2026-07-06 after the +# 6.12/6.15 rebase; D6 in spec-6.15b now requires striping for every +# shared_catalog multi-node boot). With cluster.xid_striping=on the xid +# collision layer is gone and merged recovery classifies the +# RM_CLUSTER_XID_STRIPE JOIN/RETIRE notes as GLOBAL. The remaining +# blocker is: # -# Layer 1 (xid collision) -- what striping WOULD fix: on this striped-off -# config the two threads share bare xids (pg_waldump-proven: the same xid -# on both threads writing the same catalog block), so post-merge TT -# lookups for the other thread's xids are permanently UNKNOWN and every -# cross-thread catalog read fail-closes 53R97 (honest, never false- -# visible). With cluster.xid_striping=on the collision is gone AND merged -# recovery classifies the RM_CLUSTER_XID_STRIPE JOIN/RETIRE notes as -# GLOBAL (see cluster_recovery_record_class, added this session), so both -# nodes boot and complete cold merged recovery -- verified in a striping- -# on diagnostic run. But that run then exposes: -# -# Layer 2 (the real blocker, root cause pinned 2026-07-06 via debug1): +# Layer 2 (root cause pinned 2026-07-06 via debug1): # the L2 up-check and L3 truth-matrix serving legs stay RED because of a # MEMBERSHIP COLD-FORMATION gap, not the snap-back fence (the fence # RECOVERING / "block master is recovering" errors are all downstream @@ -143,6 +133,8 @@ my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). @@ -211,6 +203,8 @@ # ---------- my $cluster_conf = <data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); @@ -251,6 +247,23 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time). First-contact probes retry; semantic asserts stay strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + my $n1_up = 0; for (1 .. 60) { @@ -262,16 +275,17 @@ # ---------- # L0: both alive + IC-connected. # ---------- -is($node0->safe_psql('postgres', 'SELECT 1'), '1', 'L0: node0 is up'); +my ($n0_up_rc, $n0_up_out) = psql_retry_ok($node0, 'SELECT 1'); +is($n0_up_out, '1', 'L0: node0 is up'); is($n1_up, 1, 'L0: node1 is up'); my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L0: node0 sees node1 heartbeats (IC connected)'); @@ -287,15 +301,17 @@ # CREATE t_crash_uncommitted (+ row, NO commit) -> 8.A leg # -> node1's cold merge must divert all of these. # ---------- -$node1->safe_psql('postgres', +my ($survivor_ok) = psql_retry_ok($node1, "CREATE TABLE t_survivor (id int PRIMARY KEY, note text);" . "INSERT INTO t_survivor VALUES (1, 'alive');"); +die 'node1 first DDL never succeeded' unless $survivor_ok; -$node0->safe_psql('postgres', +my ($keep_ok) = psql_retry_ok($node0, "CREATE TABLE t_keep (id int PRIMARY KEY, note text);" . "INSERT INTO t_keep VALUES (1, 'keep');" . "CREATE TABLE t_dropme (id int PRIMARY KEY, note text);" . "INSERT INTO t_dropme VALUES (1, 'doomed');"); +die 'node0 first DDL never succeeded' unless $keep_ok; # Shared-tree relfile of t_dropme, captured BEFORE the drop. my $drop_relpath = $node0->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl b/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl index 2391157fc1..cd340fa646 100644 --- a/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl +++ b/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl @@ -101,6 +101,8 @@ my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); # ---------- # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid), @@ -164,6 +166,8 @@ # ---------- my $cluster_conf = <data_dir . '/pgrac.conf', $pgrac_conf); PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); @@ -216,7 +222,17 @@ # ---------- # R0: both alive; one warm cross-node catalog table; baseline relfilenumber. # ---------- -is($node0->safe_psql('postgres', 'SELECT 1'), '1', 'R0: node0 is up'); +# Immediately after online_join + striping activation, a backend's bootstrap +# catalog read can transiently hit a peer whose block view is still rebuilding. +# Poll both nodes with the same tolerant shape used by t/339. +my $n0_up = 0; +for (1 .. 60) +{ + my ($rc) = $node0->psql('postgres', 'SELECT 1'); + if (defined $rc && $rc == 0) { $n0_up = 1; last; } + usleep(500_000); +} +is($n0_up, 1, 'R0: node0 is up'); is($n1_up, 1, 'R0: node1 is up'); $node0->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl b/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl index 1fb21179be..49fd3e518a 100644 --- a/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl +++ b/src/test/cluster_tap/t/353_cluster_6_12d_space_lease.pl @@ -117,6 +117,15 @@ sub poll_until 'd612_lease', quorum_voting_disks => 3, shared_data => 1, + # The L5 reborn leg takes a FRESH HW space lease, which needs the survivor + # to complete HW remaster of the dead node's shards. HW remaster reads the + # dead node's per-thread WAL, so on a shared_data multi-node pair it is only + # recoverable when cluster.wal_threads_dir is set + # (cluster_hw_remaster_recoverable); without it the remaster is marked + # BLOCKED_STRUCTURAL and the reborn node's fresh-lease access stays + # fail-closed. Every HW-remaster-dependent kill test opts into this + # (t/247/248/274/293). Spec: spec-4.6a-grd-recovery-liveness.md (D4). + wal_threads_root => 1, extra_conf => [ 'autovacuum = off', 'jit = off', diff --git a/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl b/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl new file mode 100644 index 0000000000..cbe507003f --- /dev/null +++ b/src/test/cluster_tap/t/358_data_plane_flip_e2e.pl @@ -0,0 +1,225 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 358_data_plane_flip_e2e.pl +# spec-7.2 D3+D4 atomic plane flip — bidirectional e2e + staging +# survival (the flip-commit acceptance legs, user-ruled hard +# constraints on the flip): +# +# L1 flip fact: pg_cluster_state gcs/block_family_plane = 1 on +# both nodes; DATA-plane listeners bound (log evidence) +# L2 forward transfer: node0 seeds + X-holds a shared table; +# node1's UPDATE ships the block over the LMS DATA plane +# (statement succeeds, requester block_request_count grows, +# plane_misroute_reject stays 0 on both nodes) +# L3 reverse transfer: node1 now holds; node0's UPDATE ships +# back (bidirectional evidence) +# L4 staging survival (D0-①b case): node1 READS an X-held block +# -- the holder-side LMS FORWARD path downgrades X->S and its +# GCS_REQUEST notify must reach LMON via the CONTROL ring, not +# a cross-plane direct send; no plane-gate ERROR may appear in +# either node's log +# L5 hygiene: zero plane misroutes + zero "cannot send from +# plane" / "plane mismatch" strings across the whole run +# +# Spec: spec-7.2-ic-data-plane-decoupling.md (flip commit) +# +# IDENTIFICATION +# src/test/cluster_tap/t/358_data_plane_flip_e2e.pl +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Time::HiRes qw(usleep); +use Test::More; + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'dpflip', + quorum_voting_disks => 3, + shared_data => 1, + storage_backend => 'block_device', + extra_conf => [ + 'autovacuum = off', + 'fsync = off', + 'shared_buffers = 64MB', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.online_join = on', + 'cluster.xid_striping = on', + 'cluster.crossnode_runtime_visibility = on', + 'cluster.crossnode_cr_data_plane = on', + 'cluster.block_self_contained = on', + ]); +$pair->start_pair; +usleep(2_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'CONTROL peers up 0->1'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'CONTROL peers up 1->0'); +my ($node0, $node1) = ($pair->node0, $pair->node1); + +sub gcs_int +{ + my ($node, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='gcs' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + +# Poll until a write statement succeeds on $node — the online-join +# admission gate rejects writable transactions until the join settles +# (t/347 pattern). +sub poll_write_ok +{ + my ($node, $sql, $timeout_s, $label) = @_; + $timeout_s //= 60; + my $deadline = time + $timeout_s; + while (time < $deadline) + { + my $ok = eval { $node->safe_psql('postgres', $sql); 1 } // 0; + return 1 if $ok; + usleep(500_000); + } + diag("poll_write_ok timeout ($label)"); + return 0; +} + +# ============================================================ +# L1 — flip fact + DATA listeners. +# ============================================================ +is(gcs_int($node0, 'block_family_plane'), 1, 'L1 node0 block family on DATA plane'); +is(gcs_int($node1, 'block_family_plane'), 1, 'L1 node1 block family on DATA plane'); + +my $log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +my $log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($log0, qr/DATA-plane listener bound/, 'L1 node0 DATA listener bound'); +like($log1, qr/DATA-plane listener bound/, 'L1 node1 DATA listener bound'); + +# ============================================================ +# shared table (relfilenode coincidence on the shared device). +# ============================================================ +ok(poll_write_ok($node0, 'CREATE TABLE flip_gate0 (x int)', 90, 'node0 write gate'), + 'node0 write gate open (join admitted)'); +ok(poll_write_ok($node1, 'CREATE TABLE flip_gate1 (x int)', 90, 'node1 write gate'), + 'node1 write gate open (join admitted)'); + +# The write-gate polling burned an uneven number of OIDs per node. Align +# the two OID counters (t/347 pattern): measure the relfilenode delta and +# burn single OIDs on the lagging node until an identical CREATE lands on +# the same relfilenode on both. +my ($p0, $p1) = ('', ''); +for my $attempt (1 .. 8) +{ + $node0->safe_psql('postgres', + 'CREATE TABLE flip_t (aid int, bal int) WITH (fillfactor = 50)'); + $node1->safe_psql('postgres', + 'CREATE TABLE flip_t (aid int, bal int) WITH (fillfactor = 50)'); + $p0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('flip_t')}); + $p1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('flip_t')}); + last if $p0 eq $p1; + my ($n0) = $p0 =~ /(\d+)$/; + my ($n1) = $p1 =~ /(\d+)$/; + my ($lag, $burn) = $n0 < $n1 ? ($node0, $n1 - $n0) : ($node1, $n0 - $n1); + $lag->safe_psql('postgres', + "SELECT lo_unlink(lo_create(0)) FROM generate_series(1, $burn)"); + $node0->safe_psql('postgres', 'DROP TABLE flip_t'); + $node1->safe_psql('postgres', 'DROP TABLE flip_t'); +} +is($p0, $p1, 'shared-table coincidence'); +$node0->safe_psql('postgres', + 'INSERT INTO flip_t SELECT g, 0 FROM generate_series(1, 200) g'); +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node1->safe_psql('postgres', 'CHECKPOINT'); + +sub timed_update_retry +{ + my ($node, $sql) = @_; + for my $attempt (1 .. 20) + { + return 1 if eval { $node->safe_psql('postgres', $sql); 1 }; + usleep(500_000); + } + return 0; +} + +# ============================================================ +# L2 — forward transfer (node0 holds, node1 ships in). +# +# Wire-request counters are asserted AFTER the L3 swap below: a single +# row's block may be mastered by the requester itself (HC72 master==self +# short-circuits the wire REQUEST and the transfer rides FORWARD), so a +# per-leg counter assert is master-placement dependent. The full-table +# X swaps guarantee every block is acquired by both nodes, so each node +# must send wire REQUESTs for its remote-mastered half. +# ============================================================ +my $req1_before = gcs_int($node1, 'block_request_count'); +my $req0_before = gcs_int($node0, 'block_request_count'); + +ok(timed_update_retry($node0, 'UPDATE flip_t SET bal = bal + 1'), + 'L2 setup: node0 X-holds the table'); +usleep(500_000); + +ok(timed_update_retry($node1, 'UPDATE flip_t SET bal = bal + 1 WHERE aid = 7'), + 'L2 node1 write of node0-held block succeeds over the DATA plane'); +is(gcs_int($node0, 'plane_misroute_reject'), 0, 'L2 node0 zero plane misroutes'); +is(gcs_int($node1, 'plane_misroute_reject'), 0, 'L2 node1 zero plane misroutes'); + +# ============================================================ +# L3 — reverse transfer (node1 now holds, node0 ships back). +# ============================================================ +ok(timed_update_retry($node1, 'UPDATE flip_t SET bal = bal + 1'), + 'L3 setup: node1 takes the table X (reverse ship)'); +usleep(500_000); + +ok(timed_update_retry($node0, 'UPDATE flip_t SET bal = bal + 1 WHERE aid = 11'), + 'L3 node0 write of node1-held block succeeds (bidirectional)'); +cmp_ok(gcs_int($node1, 'block_request_count'), '>', $req1_before, + 'L3 node1 issued wire block requests across the swap'); +cmp_ok(gcs_int($node0, 'block_request_count'), '>', $req0_before, + 'L3 node0 issued wire block requests across the swap'); + +# ============================================================ +# L4 — staging survival read-back: node0 X-holds the aid=11 block from +# L3; node1 scans the whole table across it (remote X-held blocks are +# served read-image, spec-5.2). The D0-①b staging case itself (holder- +# side FORWARD/BAST chain emitting GCS_REQUEST from LMS context via the +# CONTROL ring) was exercised by every X transfer in the L2/L3 swap — +# its survival proof is the transfers succeeding with zero plane +# misroutes (L2/L3) and zero plane-gate errors in either log (L5). +# +# No full-table re-hold here: the L2/L3 swap leaves foreign recycled- +# slot ITL residue behind, and resolving THAT is the #119 co-sample +# lane (a full-table write on node0 walls on the fail-closed retryable +# verdict until then; the read-back needs no new X). +# ============================================================ +my $sum = ''; +for my $attempt (1 .. 10) +{ + $sum = eval { $node1->safe_psql('postgres', 'SELECT count(*) FROM flip_t') }; + last if defined($sum) && $sum eq '200'; + usleep(300_000); +} +is($sum, '200', 'L4 node1 reads the whole table across X-held blocks'); + +# ============================================================ +# L5 — hygiene: no plane-gate errors anywhere in the run. +# ============================================================ +$pair->stop_pair; +$log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +$log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +unlike($log0, qr/cannot send from plane|plane mismatch|dropping msg_type/, + 'L5 node0 log free of plane-gate errors'); +unlike($log1, qr/cannot send from plane|plane mismatch|dropping msg_type/, + 'L5 node1 log free of plane-gate errors'); + +done_testing(); diff --git a/src/test/cluster_tap/t/360_lms_data_plane_faults.pl b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl new file mode 100644 index 0000000000..3c78f09b49 --- /dev/null +++ b/src/test/cluster_tap/t/360_lms_data_plane_faults.pl @@ -0,0 +1,441 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 360_lms_data_plane_faults.pl +# spec-7.2 D7 — LMS DATA-plane value gate + fault-leg coverage. +# Complements t/358 (flip core L1-L5) with the quantitative value +# gate and documents the fault legs that are KNOWN-BLOCKED on a +# pre-existing substrate wall. +# +# REACHABLE legs (real assertions): +# L1 flip fact: block_family_plane = 1 on both nodes; DATA-plane +# listeners bound (log evidence). +# L2 value gate: ping-pong the shared table X between the nodes, +# then the requester ship-latency histogram must show +# p50 < 5ms and p99 < 20ms (裁决④ loopback口径). Actual +# percentiles are diag'd. +# L3 hygiene: zero plane misroutes + no plane-gate errors. +# L5 conn-reset injection (F6-1, un-SKIPPED): arm the one-shot +# cluster-lms-conn-reset injection, observe exactly one DATA-mesh +# reset in node0's log (L5.1), verify the mesh reconnects (L5.2) +# and that the reset is one-shot (L5.3), then prove block transfer +# CONVERGES after the reset (L5.4/L5.5, spec §4 line 193 "重连收敛"): +# the lms_conn_resets D6 counter recorded the reset, and a FRESH +# post-reset coincidence table (new, un-recycled xids that never hit +# the #119 wall) is X-swapped cross-node with wire block REQUESTs on +# both sides. The reset is now latched one-shot (cluster_lms_data_ +# plane.c) so the persistent GUC arm no longer storms or crashes the +# passive LMS. +# +# KNOWN-BLOCKED / deferred legs (honest SKIP; see docs/stage7-substrate- +# findings.md). The stage7 P0 substrate (spec-6.15b/4.6a/2.29a) was +# evaluated and does NOT unblock either of these: +# L4 kill -9 LMS x3 crash recovery — F1-8: a SIGKILL of the LMS +# triggers a node crash-restart, but block_device crash- +# recovery replays a relation create/extend that needs the +# raw_layout_lock via GES (cluster_lock_acquire_seven_step), +# and GES is unavailable before the node re-joins the cluster +# -> recovery FATAL "could not acquire raw layout lock". This +# is a recovery-vs-membership ordering gap in the spec-6.0a +# block_device backend (raw_layout_lock, cluster_shared_fs_ +# block_device.c:570-600), orthogonal to the spec-7.2 DATA plane +# and untouched by the P0 substrate -- needs the F1 recovery-vs- +# membership fix (separate spec). +# L6 data-dispatch injection reachability — F6-2: the +# cluster-lms-data-dispatch point sits on the LMS async recv +# pump's CONNECTED&READABLE branch, which the actual block +# REPLY recv does not traverse (hits stay 0). Observability +# point-placement polish, not a correctness gap (block-over-DATA +# is proven by t/358 + the L2 value gate); needs repositioning. +# +# Spec: spec-7.2-ic-data-plane-decoupling.md §4 / D7 +# +# IDENTIFICATION +# src/test/cluster_tap/t/360_lms_data_plane_faults.pl +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Time::HiRes qw(usleep); +use Test::More; + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'dpfault', + quorum_voting_disks => 3, + shared_data => 1, + storage_backend => 'block_device', + extra_conf => [ + 'autovacuum = off', + 'fsync = off', + 'shared_buffers = 64MB', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.online_join = on', + 'cluster.xid_striping = on', + 'cluster.crossnode_runtime_visibility = on', + 'cluster.crossnode_cr_data_plane = on', + 'cluster.block_self_contained = on', + ]); +$pair->start_pair; +usleep(2_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'CONTROL peers up 0->1'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'CONTROL peers up 1->0'); +my ($node0, $node1) = ($pair->node0, $pair->node1); + +sub gcs_int +{ + my ($node, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='gcs' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + +sub lms_int +{ + my ($node, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='lms' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + +sub poll_write_ok +{ + my ($node, $sql, $timeout_s, $label) = @_; + $timeout_s //= 60; + my $deadline = time + $timeout_s; + while (time < $deadline) + { + return 1 if eval { $node->safe_psql('postgres', $sql); 1 }; + usleep(500_000); + } + diag("poll_write_ok timeout ($label)"); + return 0; +} + +sub timed_update_retry +{ + my ($node, $sql, $tries) = @_; + $tries //= 30; + for my $attempt (1 .. $tries) + { + return 1 if eval { $node->safe_psql('postgres', $sql); 1 }; + usleep(500_000); + } + return 0; +} + +# ============================================================ +# L1 — flip fact + DATA listeners (reachable). +# ============================================================ +is(gcs_int($node0, 'block_family_plane'), 1, 'L1 node0 block family on DATA plane'); +is(gcs_int($node1, 'block_family_plane'), 1, 'L1 node1 block family on DATA plane'); + +my $log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +my $log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($log0, qr/DATA-plane listener bound/, 'L1 node0 DATA listener bound'); +like($log1, qr/DATA-plane listener bound/, 'L1 node1 DATA listener bound'); + +# ============================================================ +# shared table (relfilenode coincidence, t/347 OID-align pattern). +# ============================================================ +ok(poll_write_ok($node0, 'CREATE TABLE fg0 (x int)', 90, 'node0 gate'), + 'node0 write gate open (join admitted)'); +ok(poll_write_ok($node1, 'CREATE TABLE fg1 (x int)', 90, 'node1 gate'), + 'node1 write gate open (join admitted)'); + +my ($p0, $p1) = ('', ''); +for my $attempt (1 .. 8) +{ + $node0->safe_psql('postgres', + 'CREATE TABLE fault_t (aid int, bal int) WITH (fillfactor = 50)'); + $node1->safe_psql('postgres', + 'CREATE TABLE fault_t (aid int, bal int) WITH (fillfactor = 50)'); + $p0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('fault_t')}); + $p1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('fault_t')}); + last if $p0 eq $p1; + my ($n0) = $p0 =~ /(\d+)$/; + my ($n1) = $p1 =~ /(\d+)$/; + my ($lag, $burn) = $n0 < $n1 ? ($node0, $n1 - $n0) : ($node1, $n0 - $n1); + $lag->safe_psql('postgres', + "SELECT lo_unlink(lo_create(0)) FROM generate_series(1, $burn)"); + $node0->safe_psql('postgres', 'DROP TABLE fault_t'); + $node1->safe_psql('postgres', 'DROP TABLE fault_t'); +} +is($p0, $p1, 'shared-table coincidence'); +$node0->safe_psql('postgres', + 'INSERT INTO fault_t SELECT g, 0 FROM generate_series(1, 200) g'); +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node1->safe_psql('postgres', 'CHECKPOINT'); + +# ============================================================ +# L2 — value gate: ping-pong the table X, then p50 < 5ms, p99 < 20ms. +# ============================================================ +my $swaps = 0; +for my $r (1 .. 20) +{ + last unless timed_update_retry($node0, 'UPDATE fault_t SET bal = bal + 1', 20); + usleep(100_000); + last unless timed_update_retry($node1, 'UPDATE fault_t SET bal = bal + 1', 20); + usleep(100_000); + $swaps++; +} +ok($swaps > 0, "L2 completed $swaps X ping-pong swaps over the DATA plane"); + +my @bounds_us = (500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, + 200000, 500000, 1000000, 2000000, 5000000, 10000000, 30000000); +my @counts; +my $total = 0; +for my $b (0 .. $#bounds_us) +{ + my $c = gcs_int($node0, "ship_hist_us_le_$bounds_us[$b]") + + gcs_int($node1, "ship_hist_us_le_$bounds_us[$b]"); + $counts[$b] = $c; + $total += $c; +} +my $inf = gcs_int($node0, 'ship_hist_us_inf') + gcs_int($node1, 'ship_hist_us_inf'); +$total += $inf; + +sub pct_bound +{ + my ($pct, $tot, $counts_ref, $bounds_ref) = @_; + my $target = $pct * $tot; + my $cum = 0; + for my $b (0 .. $#$bounds_ref) + { + $cum += $counts_ref->[$b]; + return $bounds_ref->[$b] if $cum >= $target; + } + return undef; # +inf overflow +} + +my $p50 = pct_bound(0.50, $total, \@counts, \@bounds_us); +my $p99 = pct_bound(0.99, $total, \@counts, \@bounds_us); +diag(sprintf('L2 value gate: ships=%d p50<=%s us p99<=%s us (+inf=%d)', + $total, defined($p50) ? $p50 : 'inf', defined($p99) ? $p99 : 'inf', $inf)); + +ok($total > 0, "L2 requester histogram recorded ship samples ($total)"); +ok(defined($p50) && $p50 <= 5000, + sprintf('L2 value gate p50 < 5ms (p50 <= %s us)', defined($p50) ? $p50 : 'inf')); +ok(defined($p99) && $p99 <= 20000, + sprintf('L2 value gate p99 < 20ms (p99 <= %s us)', defined($p99) ? $p99 : 'inf')); + +# ============================================================ +# L3 — hygiene: zero plane misroutes. +# ============================================================ +is(gcs_int($node0, 'plane_misroute_reject'), 0, 'L3 node0 zero plane misroutes'); +is(gcs_int($node1, 'plane_misroute_reject'), 0, 'L3 node1 zero plane misroutes'); + +# ============================================================ +# L4 — KNOWN-BLOCKED: kill -9 LMS x3 crash recovery. +# (F1-8: block_device crash-recovery raw_layout_lock needs GES, which is +# unavailable before the crash-restarting node re-joins -> recovery FATAL. +# Orthogonal to the spec-7.2 DATA plane; see docs/stage7-substrate- +# findings.md. The SIGKILL is NOT issued here -- it would crash the node +# into an unrecoverable state.) +# ============================================================ +SKIP: { + skip 'F1-8 KNOWN-BLOCKED: block_device crash-recovery raw_layout_lock ' + . 'depends on GES, unavailable before a crash-restarting node re-joins ' + . '(recovery FATAL "could not acquire raw layout lock"); recovery-vs-' + . 'membership substrate gap, orthogonal to the spec-7.2 DATA plane', 3; + ok(0, 'L4.1 LMS crash-restart recovers + DATA listener rebinds'); + ok(0, 'L4.2 CONTROL membership reconnects after LMS crash'); + ok(0, 'L4.3 cross-node block transfer recovers after LMS crash (x3)'); +} + +# ============================================================ +# L5 — F6-1 (UN-SKIPPED): conn-reset injection resets the DATA mesh once, +# then the mesh reconnects and block transfer converges. +# +# The cluster-lms-conn-reset injection is now one-shot (spec-7.2 D7 F6-1, +# cluster_lms_data_plane.c): although the process-local injection GUC stays +# armed, a static latch fires the injected reset exactly ONCE per LMS +# process — modelling a single epoch-bump reset instead of a per-tick storm. +# So arming the persistent GUC no longer storms the mesh or crashes the +# passive LMS, and the real single-bump path (cur_epoch != dp_last_epoch) is +# unaffected by the latch. +# ============================================================ +my $reset_re = qr/conn-reset injection \(F6-1 one-shot\)/; +my $conn_re = qr/state CONNECTED/; + +sub count_re { my ($f, $re) = @_; return () = PostgreSQL::Test::Utils::slurp_file($f) =~ /$re/g; } + +my $resets_before = count_re($node0->logfile, $reset_re); +my $conn0_before = count_re($node0->logfile, $conn_re); +my $conn1_before = count_re($node1->logfile, $conn_re); + +# Arm the one-shot conn-reset injection on node0's LMS via the GUC + reload +# (the LMS re-reads cluster.injection_points on SIGHUP). append_conf adds the +# setting (the conf has no prior injection_points line to adjust); a later +# appended line wins over any earlier one. +$node0->append_conf('postgresql.conf', + "cluster.injection_points = 'cluster-lms-conn-reset:skip'"); +$node0->reload; + +# Wait for the LMS tick to fire the single reset (heartbeat-interval driven). +my $reset_now = $resets_before; +for my $i (1 .. 60) +{ + usleep(500_000); + $reset_now = count_re($node0->logfile, $reset_re); + last if $reset_now > $resets_before; +} +ok($reset_now > $resets_before, 'L5.1 conn-reset injection reset the DATA mesh once'); + +# Disarm (the SKIP arm survives GUC reloads, but the one-shot latch already +# prevents any further injected reset, so this is belt-and-suspenders). +$node0->append_conf('postgresql.conf', "cluster.injection_points = ''"); +$node0->reload; + +# L5.2 — the mesh RECONNECTS after the injected reset. Proven by log evidence: +# both nodes emit a fresh "state CONNECTED" AFTER the reset -- node0 re-dials +# (active role) and node1 re-accepts (passive role, after its recv sees the +# drop). Once CONNECTED the tier1 link is ready for block transfer; L5.4/L5.5 +# below prove that convergence directly with a real cross-node transfer. +# +# We do NOT re-probe convergence on fault_t itself: the L2 value-gate ping- +# pong recycled fault_t's rows' xmax into reused TT slots, so any access to +# THOSE recycled rows fail-closes with "cluster TT status unknown" -- the pre- +# existing #119 recycled-slot wall (spec-7.1a / gap-㉖). That wall is PER- +# recycled-slot: it walls only the rows whose xids the ping-pong recycled, and +# does NOT wall a fresh table's new, un-recycled xids. It is orthogonal to the +# DATA-plane decoupling and is NOT caused by the reset. So the convergence +# probe (L5.4/L5.5) uses a FRESH coincidence table whose new tuples never hit +# #119 -- the direct "block transfer recovers" proof the reconnect log evidence +# alone did not give. +my $reconnect_deadline = time + 30; +my ($conn0_after, $conn1_after) = ($conn0_before, $conn1_before); +while (time < $reconnect_deadline) +{ + $conn0_after = count_re($node0->logfile, $conn_re); + $conn1_after = count_re($node1->logfile, $conn_re); + last if $conn0_after > $conn0_before && $conn1_after > $conn1_before; + usleep(500_000); +} +ok($conn0_after > $conn0_before && $conn1_after > $conn1_before, + "L5.2 DATA mesh reconnects after the reset " + . "(node0 CONNECTED $conn0_before->$conn0_after, node1 $conn1_before->$conn1_after)"); + +# One-shot guarantee: the persistent GUC arm produced a single reset event +# (one close-log per peer), NOT a per-tick storm. In this 2-node pair that +# is exactly one injected close-log line. +my $reset_final = () = PostgreSQL::Test::Utils::slurp_file($node0->logfile) =~ /$reset_re/g; +my $injected = $reset_final - $resets_before; +ok($injected == 1, + "L5.3 conn-reset injection is one-shot (fired $injected time; no per-tick storm)"); + +# ============================================================ +# L5.4 / L5.5 — F6-1 CONVERGENCE (spec §4 line 193 "重连收敛"): after the reset +# + reconnect, a real cross-node block transfer converges. +# +# The reconnect log evidence (L5.2) shows the link is back; this leg proves the +# DATA plane actually SHIPS BLOCKS again. We use a FRESH coincidence table +# (fault2_t) created AFTER the reset: its tuples carry new, un-recycled xids, so +# they never hit the #119 recycled-slot wall that walls fault_t (that wall is +# per-recycled-slot, not table-wide -- see the L5.2 note). The nodes then swap +# X on it exactly as t/358 L2/L3 do; convergence = both writes succeed AND both +# requesters' block_request_count grow (real wire REQUESTs after the reset) AND +# zero plane misroutes. The lms_conn_resets D6 counter also confirms the reset +# was observable. +# ============================================================ +cmp_ok(lms_int($node0, 'lms_conn_resets'), '>', 0, + 'L5.4 node0 lms_conn_resets D6 counter recorded the DATA-plane reset'); + +# Fresh coincidence table on new xids (t/347 OID-align pattern; independent of +# the reset -- just re-aligns the OID counters the run has since drifted). +my ($q0, $q1) = ('', ''); +for my $attempt (1 .. 8) +{ + $node0->safe_psql('postgres', + 'CREATE TABLE fault2_t (aid int, bal int) WITH (fillfactor = 50)'); + $node1->safe_psql('postgres', + 'CREATE TABLE fault2_t (aid int, bal int) WITH (fillfactor = 50)'); + $q0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('fault2_t')}); + $q1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('fault2_t')}); + last if $q0 eq $q1; + my ($m0) = $q0 =~ /(\d+)$/; + my ($m1) = $q1 =~ /(\d+)$/; + my ($lag, $burn) = $m0 < $m1 ? ($node0, $m1 - $m0) : ($node1, $m0 - $m1); + $lag->safe_psql('postgres', + "SELECT lo_unlink(lo_create(0)) FROM generate_series(1, $burn)"); + $node0->safe_psql('postgres', 'DROP TABLE fault2_t'); + $node1->safe_psql('postgres', 'DROP TABLE fault2_t'); +} +is($q0, $q1, 'L5.4 post-reset shared-table coincidence (fresh un-recycled xids)'); +$node0->safe_psql('postgres', + 'INSERT INTO fault2_t SELECT g, 0 FROM generate_series(1, 200) g'); +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node1->safe_psql('postgres', 'CHECKPOINT'); + +# Cross-node X swap: node0 holds -> node1 ships in, node1 holds -> node0 ships +# back. The full swap guarantees each node issues wire block REQUESTs for its +# remote-mastered half (t/358 L2/L3 shape; a single-row leg alone is master- +# placement dependent per HC72 master==self). +my $req0_before = gcs_int($node0, 'block_request_count'); +my $req1_before = gcs_int($node1, 'block_request_count'); + +ok(timed_update_retry($node0, 'UPDATE fault2_t SET bal = bal + 1', 20), + 'L5.5 setup: node0 X-holds the fresh post-reset table'); +usleep(500_000); +ok(timed_update_retry($node1, 'UPDATE fault2_t SET bal = bal + 1 WHERE aid = 7', 20), + 'L5.5 node1 ships a node0-held block over the reconnected DATA plane'); +usleep(500_000); +ok(timed_update_retry($node1, 'UPDATE fault2_t SET bal = bal + 1', 20), + 'L5.5 setup: node1 takes the table X (reverse ship)'); +usleep(500_000); +ok(timed_update_retry($node0, 'UPDATE fault2_t SET bal = bal + 1 WHERE aid = 11', 20), + 'L5.5 node0 ships a node1-held block back (bidirectional convergence)'); + +cmp_ok(gcs_int($node1, 'block_request_count'), '>', $req1_before, + 'L5.5 node1 issued wire block requests after the reset (transfer converged)'); +cmp_ok(gcs_int($node0, 'block_request_count'), '>', $req0_before, + 'L5.5 node0 issued wire block requests after the reset (transfer converged)'); +is(gcs_int($node0, 'plane_misroute_reject'), 0, 'L5.5 node0 zero plane misroutes post-reset'); +is(gcs_int($node1, 'plane_misroute_reject'), 0, 'L5.5 node1 zero plane misroutes post-reset'); + +# ============================================================ +# L6 — HONEST SKIP: data-dispatch injection-point reachability (F6-2). +# (The cluster-lms-data-dispatch injection point sits on the LMS async recv +# pump's CONNECTED&READABLE branch, which the actual block REPLY recv does +# not traverse -- hits stay 0 even armed at startup. This is an OBSERVABILITY +# point-placement issue, NOT a correctness gap: that block traffic really +# flows over the DATA plane is already proven independently by t/358 (flip +# fact: block_family_plane=1 + DATA listeners) and by the L2 value gate above +# (100s of real X-transfers, p99 <= 500us over the DATA connection). The +# stage7 P0 substrate does not touch this; un-SKIPping needs repositioning +# the injection point onto the true block recv landing site -- deferred as a +# spec-7.2 observability follow-up. See docs/stage7-substrate-findings.md F6-2.) +# ============================================================ +SKIP: { + skip 'F6-2 SKIP (observability polish, not correctness): ' + . 'cluster-lms-data-dispatch sits off the actual block REPLY recv path ' + . '(hits stay 0 even armed at startup); DATA-plane block flow is already ' + . 'proven by t/358 + the L2 value gate; needs injection-point ' + . 'repositioning', 1; + ok(0, 'L6 cluster-lms-data-dispatch fires on the live block recv path'); +} + +# ============================================================ +# hygiene tail: no plane-gate errors anywhere in the run. +# ============================================================ +$pair->stop_pair; +$log0 = PostgreSQL::Test::Utils::slurp_file($node0->logfile); +$log1 = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +unlike($log0, qr/cannot send from plane|plane mismatch/, + 'L3 node0 log free of plane-gate errors'); +unlike($log1, qr/cannot send from plane|plane mismatch/, + 'L3 node1 log free of plane-gate errors'); + +done_testing(); diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl new file mode 100644 index 0000000000..37f99b898f --- /dev/null +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -0,0 +1,576 @@ +#------------------------------------------------------------------------- +# +# 361_xid_authority_native_era_2node.pl +# spec-6.15b BUG-A1/A2 -- shared_catalog native-era XID authority. +# +# node1 is a cold pre-seed clone of node0: it has the same sysid and a +# pre-seed pg_control/pg_xact horizon, but it does not contain node0's +# cluster.enabled=off seed transactions. node0 then seeds shared_catalog, +# consumes native xids, cleanly shuts down, and publishes the sealed XID +# authority plus pg_xact prehistory. On first cluster boot node1 must use +# its per-node recovery anchor's pre-seed nextXid as the adoption proof, +# copy the native-era prehistory into local pg_xact before StartupCLOG, and +# max-merge nextXid with the sealed authority before ShmemVariableCache is +# seeded. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared catalog requires --enable-cluster'; +} + +sub make_shared_root +{ + my $root = PostgreSQL::Test::Utils::tempdir(); + mkdir "$root/global" or die "mkdir $root/global: $!"; + return $root; +} + +sub make_voting_disks +{ + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); + my @disks; + for my $i (0 .. 2) + { + my $p = "$disk_dir/disk$i"; + open(my $fh, '>', $p) or die "open $p: $!"; + binmode $fh; + print $fh ("\0" x (128 * 512)); + close $fh; + push @disks, $p; + } + return join(',', @disks); +} + +sub cold_clone_data_dir +{ + my ($src, $dst) = @_; + + PostgreSQL::Test::RecursiveCopy::copypath( + $src->data_dir, + $dst->data_dir, + filterfn => sub { + my $rel = shift; + return ($rel ne 'log' && $rel ne 'postmaster.pid'); + }); + chmod(0700, $dst->data_dir) or die "chmod clone data dir: $!"; + + # The cold copy carries node0's port; append node1's port so the last + # setting wins, matching init_from_backup's post-copy rewrite. + $dst->append_conf('postgresql.conf', 'port = ' . $dst->port . "\n"); +} + +sub append_pgrac_conf +{ + my ($node, $name, $ic0, $ic1) = @_; + my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); +} + +sub start_background +{ + my ($node) = @_; + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $node->data_dir, + '-l', $node->logfile, '-o', '--cluster-name=' . $node->name, 'start'); +} + +sub wait_sql_eq +{ + my ($node, $sql, $want, $name, $attempts) = @_; + $attempts //= 80; + my $got = ''; + + for (1 .. $attempts) + { + my ($rc, $out) = $node->psql('postgres', $sql); + if (defined $rc && $rc == 0 && defined $out) + { + $got = $out; + last if $got eq $want; + } + usleep(250_000); + } + is($got, $want, $name); +} + +sub read_xid_authority +{ + my ($shared_root) = @_; + my $path = "$shared_root/global/pgrac_xid_authority"; + open(my $fh, '<:raw', $path) or die "open $path: $!"; + read($fh, my $buf, 40) or die "read $path: $!"; + close $fh; + my ($magic, $version, $flags, $reserved, $native_hw, $next_multi, $crc) = + unpack('L< L< L< L< Q< Q< L<', $buf); + return { + magic => $magic, + version => $version, + flags => $flags, + native_hw => $native_hw, + next_multi => $next_multi, + crc => $crc, + }; +} + +sub corrupt_file +{ + my ($path) = @_; + open(my $fh, '>:raw', $path) or die "open $path: $!"; + print $fh "not-a-valid-xid-authority"; + close $fh; +} + +sub append_common_shared_catalog_conf +{ + my ($node, $shared_root) = @_; + $node->append_conf('postgresql.conf', <append_conf('postgresql.conf', <new('sxid_no_stripe'); + + $n->init; + append_common_shared_catalog_conf($n, $shared_root); + $n->append_conf('postgresql.conf', <start(fail_ok => 1); + ok($start_failed, 'D6: shared_catalog multi-node refuses cluster.xid_striping=off'); + like(PostgreSQL::Test::Utils::slurp_file($n->logfile), + qr/requires cluster\.xid_striping/, + 'D6: startup log names the xid_striping requirement'); +} + +# ------------------------------------------------------------------------- +# Main green path: pre-seed clone adopts node0 native-era pg_xact truth. +# ------------------------------------------------------------------------- +my $shared_root = make_shared_root(); +my $disks_csv = make_voting_disks(); +my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); +my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); + +my $node0 = PostgreSQL::Test::Cluster->new('sxid_node0'); +$node0->init(allows_streaming => 1); +$node0->start; +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node0->stop; + +my $node1 = PostgreSQL::Test::Cluster->new('sxid_node1'); +cold_clone_data_dir($node0, $node1); +# F3: real scram TCP login leg needs a host auth line AND a TCP listener on +# the joiner (test nodes default to unix sockets only). +PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pg_hba.conf', + "host all sxid_login 127.0.0.1/32 scram-sha-256\n"); +$node1->append_conf('postgresql.conf', "listen_addresses = '127.0.0.1'\n"); + +append_common_shared_catalog_conf($node0, $shared_root); +$node0->append_conf('postgresql.conf', <start; +ok(-e "$shared_root/global/pgrac_catalog_authority", + 'G1: node0 seeded the shared catalog authority marker'); +ok(-e "$shared_root/global/pg_control", + 'G1: node0 seeded the shared pg_control authority'); +ok(-e "$shared_root/global/pgrac_xid_authority", + 'G1: node0 seeded the shared XID authority before native-era load'); + +my $abort_xid = $node0->safe_psql('postgres', q{ +BEGIN; +SELECT txid_current(); +CREATE SCHEMA sxid_abort_shadow; +ROLLBACK; +}); + +$node0->safe_psql('postgres', q{ +CREATE SCHEMA sxid; +CREATE ROLE sxid_login LOGIN PASSWORD 'sxidpass'; +}); + +my $seed_schema_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_namespace WHERE nspname = 'sxid'}); +my $seed_role_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_authid WHERE rolname = 'sxid_login'}); + +is($node0->safe_psql('postgres', "SELECT txid_status($seed_schema_xid)"), + 'committed', 'G1: seed schema catalog xid is committed on the seed'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G1: native-era aborted xid is aborted on the seed'); + +$node0->stop; + +my $auth = read_xid_authority($shared_root); +ok(($auth->{flags} & 0x0001) != 0, 'G2: clean seed shutdown sealed the XID authority'); +cmp_ok($auth->{native_hw}, '>', $seed_schema_xid, + 'G2: native high-water is above the seed schema catalog xid'); +cmp_ok($auth->{native_hw}, '>', $seed_role_xid, + 'G2: native high-water is above the seed role xid'); +ok(-e "$shared_root/global/pgrac_xid_prehistory", + 'G2: clean seed shutdown published XID prehistory'); + +append_strict_two_node_conf($node0, $disks_csv, undef); +$node0->append_conf('postgresql.conf', "cluster.node_id = 0\n"); + +append_common_shared_catalog_conf($node1, $shared_root); +append_strict_two_node_conf($node1, $disks_csv, undef); +$node1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + +append_pgrac_conf($node0, 'sxid', $ic0, $ic1); +append_pgrac_conf($node1, 'sxid', $ic0, $ic1); + +start_background($node1); +$node0->start; +$node1->_update_pid(1); + +wait_sql_eq($node1, 'SELECT 1', '1', 'G3: node1 is up after XID prehistory adopt'); +wait_sql_eq($node0, + "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " + . "FROM pg_cluster_ic_peers WHERE node_id = 1", + 't', 'G3: node0 sees node1 heartbeats'); + +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}, + '1', 'G4: node1 sees the native-era seed schema catalog row'); +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}, + '1', 'G4: node1 sees the seed role catalog row'); +wait_sql_eq($node1, + q{SET ROLE sxid_login; SELECT current_user}, + 'sxid_login', 'G4: node1 can resolve and SET ROLE to the seed role'); +# F3: the seed role's scram password must survive adoption -- a real TCP +# login exercises rolpassword visibility at connection time. +$node1->connect_ok( + 'host=127.0.0.1 port=' . $node1->port + . ' user=sxid_login password=sxidpass dbname=postgres', + 'G4: seed role logs in over scram TCP on node1'); + +wait_sql_eq($node1, "SELECT txid_status($seed_schema_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed schema xid'); +wait_sql_eq($node1, "SELECT txid_status($seed_role_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed role xid'); +wait_sql_eq($node1, "SELECT txid_status($abort_xid)", 'aborted', + 'G5: node1 txid_status sees the native-era aborted xid'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G5: node0 still sees the aborted xid after node1 reads'); +# F4: the poison-stamp proof must re-read the shared HEAP rows on node0 -- +# a wrong HEAP_XMIN_INVALID hint written by node1 lives in the shared +# catalog block, not in node0's CLOG. +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}), + '1', 'G5: node0 still sees the seed schema row after node1 scans (no poison stamp)'); +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}), + '1', 'G5: node0 still sees the seed role row after node1 scans (no poison stamp)'); + +my $state_hw = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_native_hw'}); +my $state_sealed = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_sealed'}); +my $state_adopted = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_prehistory_adopted'}); + +is($state_hw, "$auth->{native_hw}", 'G6: pg_cluster_state exposes authority native high-water'); +is($state_sealed, 't', 'G6: pg_cluster_state exposes sealed XID authority'); +is($state_adopted, 't', 'G6: pg_cluster_state exposes node1 prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/adopted XID prehistory through native high-water/, + 'G6: node1 log records XID prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/merged nextXid with XID authority native high-water/, + 'G6: node1 log records StartupXLOG nextXid max-merge'); + +$node1->stop; +$node0->stop; + +# ------------------------------------------------------------------------- +# Negative lifecycle legs on the formed shared tree. +# ------------------------------------------------------------------------- +$node0->append_conf('postgresql.conf', <start(fail_ok => 1); +ok($reentry_failed, 'N1: cluster.enabled=off cannot re-enter native seed era after formation'); +like(PostgreSQL::Test::Utils::slurp_file($node0->logfile), + qr/native seed era cannot be re-entered/, + 'N1: startup log names native-era re-entry refusal'); + +corrupt_file("$shared_root/global/pgrac_xid_authority"); +corrupt_file("$shared_root/global/pgrac_xid_authority.bak"); +# F9: pin the 53RB5 SQLSTATE contract on one leg (startup FATALs surface in +# the server log only, so %e in log_line_prefix carries the SQLSTATE; the +# other negative legs deliberately match message text). +# %e must precede %q: the refusal FATAL comes from the postmaster (a +# non-session process), and %q stops expansion there. +$node1->append_conf('postgresql.conf', + "log_line_prefix = '%m [%p] %e %q%a '\n"); +my $corrupt_failed = !$node1->start(fail_ok => 1); +ok($corrupt_failed, 'N2: corrupt shared XID authority fails closed'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/shared XID authority is unavailable|present but corrupt|cluster_xid_authority_unavailable/, + 'N2: startup log names corrupt/unavailable XID authority'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/53RB5/, 'N2: the refusal carries SQLSTATE 53RB5'); + +# ------------------------------------------------------------------------- +# Unsealed authority negative: seed startup wrote the authority, but the seed +# crashed before clean shutdown could publish prehistory and seal it. +# ------------------------------------------------------------------------- +{ + my $u_shared = make_shared_root(); + my $u_disks = make_voting_disks(); + my $u_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $u_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $u0 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node0'); + my $u1 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node1'); + + $u0->init(allows_streaming => 1); + $u0->start; + $u0->safe_psql('postgres', 'CHECKPOINT'); + $u0->stop; + cold_clone_data_dir($u0, $u1); + + append_common_shared_catalog_conf($u0, $u_shared); + $u0->append_conf('postgresql.conf', <start; + ok(-e "$u_shared/global/pgrac_xid_authority", + 'N3: unsealed fixture created the authority during seed startup'); + $u0->stop('immediate'); + + append_common_shared_catalog_conf($u1, $u_shared); + append_strict_two_node_conf($u1, $u_disks, undef); + $u1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($u1, 'sxid_unsealed', $u_ic0, $u_ic1); + + my $unsealed_failed = !$u1->start(fail_ok => 1); + ok($unsealed_failed, 'N3: joiner refuses an unsealed XID authority'); + like(PostgreSQL::Test::Utils::slurp_file($u1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N3: startup log names the unsealed authority'); +} + +# ------------------------------------------------------------------------- +# Multi-pass seed negative: a SECOND native-era run unseals the authority at +# boot, so crashing that run leaves joiners fail-closed instead of silently +# adopting the first pass's stale high-water (spec-6.15b §3.1 multi-pass arm). +# ------------------------------------------------------------------------- +{ + my $m_shared = make_shared_root(); + my $m_disks = make_voting_disks(); + my $m_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $m_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $m0 = PostgreSQL::Test::Cluster->new('sxid_multipass_node0'); + my $m1 = PostgreSQL::Test::Cluster->new('sxid_multipass_node1'); + + $m0->init(allows_streaming => 1); + $m0->start; + $m0->safe_psql('postgres', 'CHECKPOINT'); + $m0->stop; + cold_clone_data_dir($m0, $m1); + + append_common_shared_catalog_conf($m0, $m_shared); + $m0->append_conf('postgresql.conf', < sealed + $m0->start; + $m0->safe_psql('postgres', 'CREATE SCHEMA sxid_pass1;'); + $m0->stop; + my $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) != 0, + 'N4: first native pass sealed the XID authority'); + + # pass 2: booting a follow-up native run re-opens (unseals) the era + $m0->start; + $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) == 0, + 'N4: follow-up native pass unsealed the XID authority at boot'); + like(PostgreSQL::Test::Utils::slurp_file($m0->logfile), + qr/re-opened native seed era/, + 'N4: seed log names the re-opened native era'); + + # crash pass 2: the stale pass-1 high-water must NOT become adoptable + $m0->stop('immediate'); + append_common_shared_catalog_conf($m1, $m_shared); + append_strict_two_node_conf($m1, $m_disks, undef); + $m1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($m1, 'sxid_multipass', $m_ic0, $m_ic1); + + my $multipass_failed = !$m1->start(fail_ok => 1); + ok($multipass_failed, + 'N4: joiner refuses adoption after a crashed follow-up native pass'); + like(PostgreSQL::Test::Utils::slurp_file($m1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N4: startup log fails closed on the unsealed authority'); +} + +# ------------------------------------------------------------------------- +# Divergent-lineage negative (review F2 / spec Q6 amendment): a same-sysid +# clone that ran STANDALONE after cloning holds its own outcomes in the +# native xid range; neither skipping nor adopting is sound -> FATAL 53RB5. +# ------------------------------------------------------------------------- +{ + my $d_shared = make_shared_root(); + my $d_disks = make_voting_disks(); + my $d_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $d_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $d0 = PostgreSQL::Test::Cluster->new('sxid_diverge_node0'); + my $d1 = PostgreSQL::Test::Cluster->new('sxid_diverge_node1'); + + $d0->init(allows_streaming => 1); + $d0->start; + $d0->safe_psql('postgres', 'CHECKPOINT'); + $d0->stop; + cold_clone_data_dir($d0, $d1); + + # d1 diverges: a plain standalone run committing 30 transactions writes + # d1's own outcomes into the xid range the seed is about to consume. + $d1->start; + $d1->safe_psql('postgres', + join('', map { "BEGIN; CREATE TABLE d_t$_ (i int); COMMIT;\n" } 1 .. 30)); + $d1->stop; + + # d0 seeds: one aborted xid guarantees a byte-level contradiction with + # d1's straight-committed overlap. + append_common_shared_catalog_conf($d0, $d_shared); + $d0->append_conf('postgresql.conf', <start; + $d0->safe_psql('postgres', q{ +BEGIN; +CREATE SCHEMA d_abort_shadow; +ROLLBACK; +}); + $d0->safe_psql('postgres', 'CREATE SCHEMA d_seed;'); + $d0->stop; + + append_common_shared_catalog_conf($d1, $d_shared); + append_strict_two_node_conf($d1, $d_disks, undef); + $d1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($d1, 'sxid_diverge', $d_ic0, $d_ic1); + + my $diverge_failed = !$d1->start(fail_ok => 1); + ok($diverge_failed, 'N5: divergent-lineage joiner is refused'); + like(PostgreSQL::Test::Utils::slurp_file($d1->logfile), + qr/divergent native-era lineage/, + 'N5: startup log names the divergent lineage'); +} + +# ------------------------------------------------------------------------- +# L9 dormant leg: with cluster.shared_catalog=off nothing of the XID +# authority machinery may touch the shared tree. +# ------------------------------------------------------------------------- +{ + my $o_shared = make_shared_root(); + my $o0 = PostgreSQL::Test::Cluster->new('sxid_off_node0'); + + $o0->init(allows_streaming => 1); + $o0->append_conf('postgresql.conf', <start; + $o0->safe_psql('postgres', 'CREATE SCHEMA off_mode_probe;'); + $o0->stop; + ok(!-e "$o_shared/global/pgrac_xid_authority", + 'L9: shared_catalog=off never creates the XID authority'); + ok(!-e "$o_shared/global/pgrac_xid_prehistory", + 'L9: shared_catalog=off never creates the XID prehistory'); +} + +done_testing(); diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl new file mode 100644 index 0000000000..75747a223c --- /dev/null +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -0,0 +1,522 @@ +#------------------------------------------------------------------------- +# +# 362_shared_catalog_4node_kill_selfheal.pl +# spec-4.6a D9 -- 4-node shared_catalog fail-stop self-heal. +# +# Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) +# so a killed node's HW authority is recoverable from its per-thread WAL. +# Amendment v1.2 (R1): in this out-of-scope deployment (4 nodes, online +# thread recovery off) the dead master's PRE-EXISTING writes stay +# fail-closed until the failed node restarts — its unproven blocks (bounded +# 53R9L) and equally its transaction verdicts (TT status unknown for its +# xids, the visibility door). Reads and DDL are asserted on the honest +# two-outcome contract (success or explicit SQLSTATE, never a hang); +# new-object DDL/writes prove P7 unfreeze. L4h (r3-P1-1) is the serve-gate +# LOCK: a fresh backend's wide catalog scan on a cold survivor MUST fail +# 53R9L (no success arm) while the dead node is down. +# After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means +# the test harness is misconfigured, and lack of convergence is a real FAIL. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterQuad; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared_catalog 4-node kill self-heal requires --enable-cluster'; +} + +sub poll_until +{ + my ($node, $query, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) }; + return 1 if defined $last && $last eq $want; + usleep(300_000); + } + diag("poll_until timed out: '$query' last='" . ($last // '(undef)') + . "' want='$want'"); + return 0; +} + +sub retry_sql +{ + my ($node, $sql, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); + return (1, $out // '') if defined $rc && $rc == 0; + $last = $err // ''; + usleep(500_000); + } + return (0, $last); +} + +sub state_counter +{ + my ($node, $cat, $key) = @_; + my $v = eval { + $node->safe_psql('postgres', + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'") + }; + return defined $v && $v ne '' ? $v + 0 : 0; +} + +sub sum_hw +{ + my ($quad, $key, @idx) = @_; + my $sum = 0; + for my $i (@idx) + { + $sum += state_counter($quad->node($i), 'hw', $key); + } + return $sum; +} + +# Amendment v1.2 (R1): after the kill, a NEW backend's parse may cold-read a +# dead-master catalog page and honestly fail closed (53R9L) until the failed +# node returns. The convergence observability therefore rides PRE-WARMED +# long-lived psql sessions (catcache hot; pg_cluster_* reads touch shmem, not +# dead-master blocks), while fresh-connection legs assert the two-outcome +# contract. bg_query returns (1, value) or (0, error-ish) without dying. +sub bg_query +{ + my ($bg, $sql) = @_; + my $out = eval { $bg->query($sql) }; + return (0, $@ // 'query failed') if $@; + return (1, $out // ''); +} + +sub bg_counter +{ + my ($bg, $cat, $key) = @_; + my ($ok, $v) = bg_query($bg, + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'"); + return ($ok && defined $v && $v ne '') ? $v + 0 : 0; +} + +sub bg_sum +{ + my ($bgs, $cat, $key, @idx) = @_; + my $sum = 0; + $sum += bg_counter($bgs->{$_}, $cat, $key) for @idx; + return $sum; +} + +sub bg_poll_until +{ + my ($bg, $sql, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($ok, $out) = bg_query($bg, $sql); + $last = $out; + return 1 if $ok && defined $out && $out eq $want; + usleep(500_000); + } + diag("bg_poll_until timed out: '$sql' last='$last' want='$want'"); + return 0; +} + +sub logs_contain +{ + my ($quad, $pattern, @idx) = @_; + for my $i (@idx) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + } + return 0; +} + +sub log_until +{ + my ($quad, $idx, $pattern, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($idx)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + usleep(300_000); + } + diag("log_until timed out on node$idx waiting for /$pattern/"); + return 0; +} + +sub startup_env_blocker +{ + my ($quad) = @_; + my $log = ''; + if ($quad) + { + for my $i (0 .. 3) + { + $log .= eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + } + } + return $log =~ /could not create shared memory segment|No space left on device|No space left|SHMMNI|semget/i; +} + +my $quad = eval { + PostgreSQL::Test::ClusterQuad->new_quad( + 'sc4_selfheal', + shared_catalog => 1, + quorum_voting_disks => 3, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.lms_enabled = on', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +}; +if (!$quad) +{ + my $err = $@ || 'unknown constructor failure'; + plan skip_all => "L1 shared_catalog ClusterQuad formation hit local substrate blocker: $err" + if $err =~ /could not create shared memory|No space left|SHMMNI|semget/i; + BAIL_OUT("ClusterQuad(shared_catalog) constructor failed: $err"); +} + +my $started = 1; +for my $i (1 .. 3) +{ + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $quad->node($i)->data_dir, + '-l', $quad->node($i)->logfile, + '-o', "--cluster-name=sc4_selfheal_node$i", 'start'); +} +unless ($quad->node(0)->start(fail_ok => 1)) +{ + $started = 0; +} +for my $i (1 .. 3) +{ + $quad->node($i)->_update_pid(-1); +} +if (!$started) +{ + if (startup_env_blocker($quad)) + { + eval { $quad->stop_quad; }; + plan skip_all => 'L1 shared_catalog ClusterQuad formation hit local shared-memory/resource limit'; + } + my $fatal = logs_contain($quad, qr/cluster\.shared_catalog requires cluster\.wal_threads_dir|structurally blocked|BLOCKED_STRUCTURAL/i, 0 .. 3); + eval { $quad->stop_quad; }; + ok(!$fatal, 'L1: shared_catalog quad must not hit wal_threads_dir/BLOCKED_STRUCTURAL configuration failure'); + BAIL_OUT('ClusterQuad(shared_catalog) failed to start for a non-environment reason'); +} + +# L1: all four members formed with quorum and per-thread WAL configured. +for my $i (0 .. 3) +{ + ok(poll_until($quad->node($i), 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 60), + "L1: node$i reached quorum"); + is(state_counter($quad->node($i), 'hw', 'remaster_recoverable'), 1, + "L1: node$i reports HW remaster recoverable"); +} +for my $to (1 .. 3) +{ + ok($quad->wait_for_peer_state(0, $to, 'connected', 60), + "L1: node0 sees node$to connected"); +} + +my $n0 = $quad->node0; +my $dead = 3; +my @survivors = (0, 1, 2); + +# Seed shared-catalog DDL and force node3 to own real HW state before death. +for my $t (1 .. 2) +{ + my ($ddl_ok, $ddl_res) = retry_sql($n0, + "CREATE TABLE sc4_hw_$t (id int, pad int)", 60); + ok($ddl_ok, "L1: coordinator created shared_catalog table sc4_hw_$t") + or diag($ddl_res); + my ($ok, $res) = retry_sql($quad->node($dead), + "INSERT INTO sc4_hw_$t SELECT g, g FROM generate_series(1,5000) g", 60); + ok($ok, "L1: node$dead extended shared_catalog table sc4_hw_$t before kill") + or diag($res); +} +for my $i (0 .. 3) +{ + retry_sql($quad->node($i), 'CHECKPOINT', 30); +} + +# Amendment v1.2 (R1): pre-warmed observability sessions (see helper note). +my %bg; +for my $i (@survivors) +{ + $bg{$i} = $quad->node($i)->background_psql('postgres', on_error_stop => 0); + bg_query($bg{$i}, "SELECT value FROM pg_cluster_state WHERE category='hw' AND key='remaster_done_count'"); + bg_query($bg{$i}, "SELECT count(*) FROM pg_cluster_grd_shards WHERE master_node_id=$dead"); + bg_query($bg{$i}, 'SELECT txid_current()'); +} + +my $done_before = bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors); +my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); +my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); +my %grd_done_before; +$grd_done_before{$_} = bg_counter($bg{$_}, 'grd_recovery', 'remaster_done') + for @survivors; + +# L2: kill node3 and require every survivor to see the real DEAD edge. +# Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, +# ordinary catalog reads may legitimately return 53R9I until the barrier opens. +$quad->kill_node9($dead); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/cssd: peer $dead transitioned .* DEAD/, 45), + "L2: survivor node$i declared node$dead dead"); +} + +# L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +# The map read is hard-asserted on the warm COORDINATOR session only: the view +# scan on a cold survivor deterministically cold-reads a dead-master catalog +# page out of the small test buffer pool and honestly fails closed (53R9L) +# until node3 returns — the map itself is a deterministic recompute barriered +# by P6, and each cold survivor is instead hard-asserted on its own HW +# remaster worker DONE log line below. +ok(bg_poll_until($bg{0}, + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + 'L3: coordinator remastered all GRD shards off node3 (warm session)'); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/HW remaster worker: dead node $dead -> done/, 90), + "L3: survivor node$i HW remaster worker reported done"); +} +my $done_converged = 0; +my $deadline = time() + 90; +while (time() < $deadline) +{ + if (bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors) > $done_before) + { + $done_converged = 1; + last; + } + usleep(500_000); +} +ok($done_converged, 'L3: HW remaster DONE advanced on at least one survivor after kill'); +ok(!logs_contain($quad, qr/blocked_structural|structurally blocked|BLOCKED_STRUCTURAL/i, @survivors), + 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); + +# L6: if a transient BLOCKED occurred, a same-episode retry must also have run. +my $blocked_after = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +if ($blocked_after > $blocked_before) +{ + ok(bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors) > $retry_before, + 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); +} +else +{ + pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); +} + +# L4h (r3-P1-1 hard leg): with the serve gate in place, an out-of-scope +# formation (4 nodes / online_thread_recovery off) NEVER publishes the dead +# node's materialization proof, so EVERY node3-mastered page stays RECOVERING +# until node3 returns — and node3 is never restarted in this test. A wide +# catalog scan from a FRESH backend on a cold survivor therefore MUST fail +# closed with 53R9L: pg_class + pg_attribute span dozens of pages hashed ~1/4 +# to node3, and a fresh backend's full scan cold-reads pages no prior session +# pulled into the 16MB test pool. Deliberately NO success arm — a NORMAL +# serve of a dead-master page here is the exact 8.A regression this leg locks +# (reverting the serve-gate fix turns this leg red while the two-outcome legs +# stay green). Gate on the survivor's OWN episode reaching IDLE first so the +# failure surface is deterministically the post-episode materialization proof +# (53R9L), never the in-episode shard freeze (53R9I). The scan touches no +# node3-written rows (node3 ran only heap INSERTs, no DDL), so the TT +# visibility door cannot fire first: 53R9L is the ONLY legal outcome. +{ + my $cold = 2; + + # Compare in Perl, not SQL: the warm session's catcache covers exactly the + # bg_counter query shape it ran pre-kill; a cast/operator added in SQL + # would itself cold-read a dead-master catalog page and trip 53R9L. + my $idle = 0; + my $idle_deadline = time() + 90; + while (time() < $idle_deadline) + { + if (bg_counter($bg{$cold}, 'grd_recovery', 'remaster_done') + > $grd_done_before{$cold}) + { + $idle = 1; + last; + } + usleep(500_000); + } + ok($idle, "L4h: survivor node$cold GRD recovery episode reached IDLE (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($cold)->psql('postgres', + 'SELECT count(*) FROM pg_class c JOIN pg_attribute a ON a.attrelid = c.oid', + timeout => 60); + my $elapsed = time() - $t0; + isnt(defined $rc ? $rc : 0, 0, + "L4h: fresh wide catalog scan on survivor node$cold MUST fail while node$dead is down"); + like($err // '', + qr/block-level cache protocol state is being rebuilt after reconfiguration/, + 'L4h: the failure is the explicit 53R9L dead-master fail-closed gate'); + cmp_ok($elapsed, '<', 60, 'L4h: fail-closed scan returned bounded (no hang)'); + diag("L4h: scan failed closed as required: " . ($err // '(no stderr)')) + if defined $err && $err ne ''; +} + +# L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an +# out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a +# BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a +# hang. sc4_hw_* spans ~20+ pages, so some page's static master is node3 with +# overwhelming probability; the redo-coverage proof cannot be published without +# online thread recovery, so the scan hits 53R9L (resource recovering). If by +# hash luck no scanned page is node3-mastered the scan succeeds — both are the +# honest two-outcome contract; only a hang/timeout or an unexpected error fails. +for my $t (1 .. 2) +{ + my $t0 = time(); + my ($rc, $out, $err) = $quad->node(1)->psql('postgres', + "SELECT count(*) FROM sc4_hw_$t", timeout => 60); + my $elapsed = time() - $t0; + if (defined $rc && $rc == 0) + { + ok(1, "L4a: dead-master table sc4_hw_$t read succeeded (no node$dead-mastered page scanned)"); + diag("L4a: sc4_hw_$t count=$out"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + "L4a: dead-master table sc4_hw_$t read fail-closed with an explicit SQLSTATE"); + } + cmp_ok($elapsed, '<', 60, "L4a: sc4_hw_$t read returned bounded (no hang)"); +} + +# L4: survivor SQL recovers after fail-stop — hard-asserted on the pre-warmed +# sessions; a FRESH connection is the honest two-outcome contract (v1.2 R1): +# success, or an explicit fail-closed SQLSTATE on a dead-master catalog page — +# never a hang. +for my $i (@survivors) +{ + my $got = 0; + my $dl = time() + 60; + while (time() < $dl) + { + my ($ok, $out) = bg_query($bg{$i}, 'SELECT txid_current()'); + if ($ok && defined $out && $out =~ /^\d+$/) { $got = 1; last; } + usleep(500_000); + } + ok($got, "L4: survivor node$i accepts txid_current after reconfig (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($i)->psql('postgres', 'SELECT txid_current()', + timeout => 60); + if (defined $rc && $rc == 0) + { + ok(1, "L4: fresh connection txid_current succeeded on node$i"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, + "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); + } + cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); +} +my ($ddl_rc, $ddl_out, $ddl_err) = $n0->psql('postgres', + 'CREATE TABLE sc4_after_kill (id int)', timeout => 60); +my $ddl_ok = defined $ddl_rc && $ddl_rc == 0; +if ($ddl_ok) +{ + ok(1, 'L4: coordinator survivor DDL succeeded after node kill'); +} +else +{ + # write-fenced (53R51, spec-4.12): a transient lease/epoch fence on the + # writer right after the reconfig — an explicit, retryable fail-closed + # rejection, squarely inside the honest two-outcome contract. + like($ddl_err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, + 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); + diag("L4: DDL fail-closed honestly: $ddl_err"); +} +for my $i (1, 2) +{ + if ($ddl_ok) + { + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); + } + else + { + ok(1, "L4: survivor node$i visibility leg not applicable (DDL fail-closed honestly)"); + } +} + +# L4 (spec-4.6a D12, Amendment v1.2 R3): node3 held real PCM/HW state at kill +# time (it extended sc4_hw_1/2 with 5000 rows each), so the dead-node residue +# cleanup MUST account a positive delta across the survivors — a >= 0 +# assertion would stay green with D12 entirely disabled. +{ + my $cleanup_total = 0; + for my $i (@survivors) + { + my ($ok, $n) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='pcm' AND key='dead_cleanup_entries'"); + ok($ok && defined $n && $n eq '1', + "L4: survivor node$i exposes the pcm/dead_cleanup_entries key"); + $cleanup_total += bg_counter($bg{$i}, 'pcm', 'dead_cleanup_entries'); + } + cmp_ok($cleanup_total - $cleanup_before, '>', 0, + 'L4: dead-node PCM residue cleanup accounted a positive delta after kill'); + diag("L4: pcm/dead_cleanup_entries delta across survivors = " + . ($cleanup_total - $cleanup_before)); +} + +# L5: watchdog counters are allowed to be zero, but the KEYS must exist +# (state_counter reads a missing key as 0, so presence is the real assertion). +for my $i (@survivors) +{ + my ($ok1, $n1) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='cluster_gate_timeout'"); + my ($ok2, $n2) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='wait_epoch_escape'"); + ok($ok1 && $ok2 && defined $n1 && $n1 eq '1' && defined $n2 && $n2 eq '1', + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counter keys"); +} + +$bg{$_}->quit for @survivors; + +$quad->stop_quad; +done_testing(); diff --git a/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl new file mode 100644 index 0000000000..005c0d4a4b --- /dev/null +++ b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl @@ -0,0 +1,191 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 363_cluster_2_29a_marker_async_liveness.pl +# BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. +# +# The hold injection freezes qvotec marker completion until released. Before +# BUG-C1, LMON synchronously waited inside marker submit, stopped sending HELLOs, +# and peers could false-DEAD it during a long marker write. This TAP holds the +# marker service for >=90s while an online join wants a PREPARE marker and checks: +# L1 hold injection is registered. +# L2 a true leave/restart creates a join marker backlog. +# L3 during a 95s hold, LMON keeps iterating and both live nodes keep each +# other CSSD-alive (no false-DEAD). +# L4 marker timeout/backlog is observable and does not publish admission. +# L5 releasing the hold lets the join converge. +# L6 an actual stopped node still becomes DEAD (no true-DEAD regression). +# +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterPair; +use Test::More; +use Time::HiRes qw(usleep); + +sub poll_until +{ + my ($node, $query, $expected, $timeout_s, $label) = @_; + $expected //= 't'; + $timeout_s //= 30; + my $deadline = time + $timeout_s; + my $last = ''; + while (time < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) } // ''; + return 1 if defined $last && $last eq $expected; + usleep(200_000); + } + diag("poll_until timeout ($label): last='$last' expected='$expected'"); + return 0; +} + +sub state_int +{ + my ($node, $category, $key) = @_; + my $v = $node->safe_psql('postgres', + "SELECT COALESCE((SELECT value::bigint FROM pg_cluster_state " + . "WHERE category = '$category' AND key = '$key'), 0)"); + return $v + 0; +} + +sub set_injection +{ + my ($node, $value) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$value'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(700_000); + return; +} + +my @conf = ( + 'autovacuum = off', + 'jit = off', + 'cluster.online_join = on', + 'cluster.join_convergence_timeout_ms = 180000', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 8', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.lmon_slow_iteration_warn_ms = 0', +); + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'marker_async_liveness', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [@conf]); +$pair->start_pair; +usleep(3_000_000); + +my $node0 = $pair->node0; +my $node1 = $pair->node1; + +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_cluster_injections + WHERE name = 'cluster-qvotec-marker-service-hold'}), + '1', + 'L1 hold injection point is registered'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 node1 sees node0 connected'); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 30, 'node1 initially member'), + 'L1 node1 initially MEMBER'); + +my $epoch0 = $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->stop; +ok(poll_until($node0, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead'), + 'L2 node0 detects the stopped node1 as dead'); +ok(poll_until($node0, + q{SELECT state <> 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'node1 demoted from member'), + 'L2 node1 is demoted from MEMBER before rejoin'); + +set_injection($node0, 'cluster-qvotec-marker-service-hold'); +my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); +my $epoch_hold_start = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->start; +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 CSSD alive after restart'), + 'L2 node1 is live again while marker service is held'); + +sleep 95; + +my $lmon_iters_after = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_after = state_int($node0, 'reconfig', 'marker_timeout_count'); + +cmp_ok($lmon_iters_after, '>', $lmon_iters_before + 20, + 'L3 node0 LMON keeps iterating during >=90s qvotec marker hold'); +ok(poll_until($node1, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 5, 'node1 still sees node0 alive after hold'), + 'L3 node1 still sees node0 CSSD-alive after the hold (no false-DEAD)'); +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 5, 'node0 still sees node1 alive after hold'), + 'L3 node0 still sees node1 CSSD-alive after the hold'); + +cmp_ok($timeouts_after, '>', $timeouts_before, + 'L4 marker timeout/backlog counter advances while qvotec marker service is held'); +isnt($node0->safe_psql('postgres', + q{SELECT state FROM pg_cluster_membership WHERE node_id = 1}), + 'member', + 'L4 held/timeout marker does not publish node1 admission'); + +# P1-1 bump-once (spec-2.29a review r1): the whole >=95s hold must cost at most +# ONE join Phase-1 epoch bump (staged record; PREPARE timeouts drain best-effort +# and never revert the joiner, so compute_join_bitmap cannot re-detect + re-bump +# per deadline period). Pre-fix abort+revert behavior re-bumped ~once per 3.5s +# deadline (~27 bumps in this window); a <=2 bound separates cleanly without +# being flaky against one incidental episode. +my $epoch_hold_end = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; +cmp_ok($epoch_hold_end - $epoch_hold_start, '<=', 2, + 'L4 epoch advance across the hold window is bounded (no per-deadline bump storm)'); + +set_injection($node0, ''); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 joins after hold release'), + 'L5 node0 admits node1 after hold release'); +ok(poll_until($node1, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 self member after hold release'), + 'L5 node1 self-state reaches MEMBER after hold release'); +cmp_ok($node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0, + '>', $epoch0, + 'L5 membership epoch advances after rejoin'); + +$node1->stop; +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead after final stop'), + 'L6 true DEAD detection still works after async marker liveness fix'); +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'membership dead after final stop'), + 'L6 membership fail-stop path still demotes a truly stopped peer'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..ae08788210 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -41,7 +41,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ - test_cluster_epoch test_cluster_fence test_cluster_reconfig \ + test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory \ test_cluster_tt_slot_allocator test_cluster_itl_reader_real_triple \ test_cluster_itl_cleanout test_cluster_visibility_inject test_cluster_itl_cleanout_perf \ @@ -63,8 +63,10 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_sequence \ test_cluster_shared_catalog \ test_cluster_oid_lease \ + test_cluster_xid_authority \ test_cluster_relmap_authority \ test_cluster_hw \ + test_cluster_hw_remaster_retry \ test_cluster_dl \ test_cluster_extend_gate \ test_cluster_ir \ @@ -178,7 +180,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -443,6 +445,21 @@ test_cluster_oid_lease: test_cluster_oid_lease.c unit_test.h \ $(CLUSTER_OID_LEASE_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-6.15b D1/D2: test_cluster_xid_authority links cluster_xid_authority.o +# (the shared XID authority + native-era prehistory: torn-safe file I/O, +# seal/flag monotonicity, prehistory publish/adopt byte round trip). fd.c +# openers, pg_fsync, durable_rename and the ereport machinery are stubbed +# locally; CRC32C comes from libpgport. The startup wiring (bootstrap adopt, +# StartupXLOG horizon merge, shutdown-checkpoint publish) is TAP territory. +CLUSTER_XID_AUTHORITY_O = $(top_builddir)/src/backend/cluster/cluster_xid_authority.o \ + $(top_builddir)/src/backend/cluster/cluster_xid_prehistory.o +test_cluster_xid_authority: test_cluster_xid_authority.c unit_test.h \ + $(CLUSTER_XID_AUTHORITY_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_XID_AUTHORITY_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + + # spec-6.14 D5: test_cluster_relmap_authority links cluster_relmap_authority.o # (the two-phase pending/committed authority substrate). fd.c + ereport # stubbed locally; CRC32C from libpgport. The write-path activation (relmapper diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 540cff5b7d..90151fc3cb 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -181,6 +181,17 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): linker-only exception-stack stubs (the + * linked cluster_pcm_lock.o now wraps the S->X upgrade in PG_TRY). */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index b703798b38..04363fd736 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -159,14 +159,104 @@ UT_TEST(test_phase_fsm) UT_TEST(test_version_coherent) { - /* nothing moved -> coherent */ - UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, 3, 3)); - /* epoch bumped by an external death -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 3)); - /* dead_generation moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, 3, 4)); - /* both moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 4)); + /* spec-2.29a ②b: coherence = epoch unchanged AND the others-dead bitmap + * (dead set EXCLUDING the leaving node) unchanged. The reflexive-case + * matrix from the spec §②b 8.A argument, exercised on the pure predicate. */ + uint8 od_none[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + uint8 od_third[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + const int n = CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES; + + od_third[0] = 0x04; /* a THIRD-PARTY node (e.g. node 2) became DEAD */ + + /* (i) nothing moved (leaving node's own DEAD is already excluded upstream, + * so its transition does not appear here) -> coherent */ + UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, od_none, od_none, n)); + /* (iii) epoch bumped by an external fail-stop -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_none, n)); + /* (ii) a third-party death entered the others-dead set, epoch not yet + * bumped (the P1-b window) -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, od_third, n)); + /* (iv) both moved -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_third, n)); + /* fail-closed on a missing view */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, NULL, od_none, n)); + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, NULL, n)); +} + + +/* ============================================================ + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r3, evidence + * over inference) + * ============================================================ */ + +UT_TEST(test_own_commit_latched) +{ + /* The latch verdict is EVIDENCE-only: the first argument is "the durable + * COMMITTED marker for THIS leave attempt was confirmed" (the coordinator's + * nonce-bound LEAVE_COMMITTED attestation). The two coherence observations + * (others-dead bitmap / scalar dead_generation) stay in the signature as + * contract inputs the verdict must IGNORE — a third-party transient + * false-DEAD flap on the leaver's local CSSD view advances the (monotone) + * dead_generation and can transiently disturb the bitmap, and neither may + * refuse a latch that marker evidence backs (the t/331 C1/C4 false- + * escalation), nor may any combination latch without evidence (the r2 P2-1 + * refused-leave mis-latch hang). */ + + /* (a) evidence present, no flap noise -> latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); + + /* (d) PINNING LEG (t/331 C1/C4 regression): a third-party flap advanced the + * scalar dead_generation (it never rebounds) while the bitmap rebounded to + * its bound value — WITH marker evidence the leave still latches; the flap + * must not false-escalate a committed leave. */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, false)); + + /* (d) flap currently visible in the others-dead bitmap too -> still latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, true)); + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, false)); + + /* (c) no evidence -> never latch, whatever the coherence observations say + * (a refused leave never produces a COMMITTED marker, so the barrier + * deadline escalation stays armed and bounds the wait — the r2 P2-1 + * mis-latch wedge stays closed). */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, false)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, false)); +} + + +/* ============================================================ + * U3c — LEAVE_COMMITTED evidence identity gate (spec-2.29a r3) + * ============================================================ */ + +UT_TEST(test_committed_evidence_matches) +{ + /* args: payload_leaving_node, payload_nonce, payload_epoch, + * self_node, current_leaving_node, current_attempt_nonce, + * bound_leave_epoch */ + + /* (a) confirmation for THIS node's CURRENT attempt, committed epoch past + * the bound baseline -> evidence accepted */ + UT_ASSERT(cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 1, 42, 7)); + + /* (b) stale identity: a LEAVE_COMMITTED (and through it a COMMITTED + * marker) from a PREVIOUS leave attempt of the same node — nonce differs + * -> fail-closed, no latch */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 41, 8, 1, 1, 42, 7)); + + /* (b) misrouted: confirmation addressed to a different leaving node */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(2, 42, 8, 1, 1, 42, 7)); + + /* (b) this node is not currently leaving (idle: leaving_node_id == -1, + * or tracking someone else's leave) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, -1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 2, 42, 7)); + + /* (b) committed epoch not past the bound baseline (attested epoch must be + * the guarded-CAS successor of the baseline this leave bound) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 7, 1, 1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 0, 1, 1, 42, 7)); } @@ -382,10 +472,12 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(8); + UT_PLAN(10); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); + UT_RUN(test_own_commit_latched); + UT_RUN(test_committed_evidence_matches); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); diff --git a/src/test/cluster_unit/test_cluster_conf.c b/src/test/cluster_unit/test_cluster_conf.c index a202eff583..9397d008be 100644 --- a/src/test/cluster_unit/test_cluster_conf.c +++ b/src/test/cluster_unit/test_cluster_conf.c @@ -80,6 +80,10 @@ bool cluster_allow_single_node = true; /* spec-2.1 D1; storage stub matches defa */ bool cluster_enabled = true; +/* spec-7.2 flip: post_validate consults the interconnect tier for the + * multi-node data_addr gate; stub = stub tier (gate exempt). */ +int cluster_interconnect_tier = 0; + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -247,12 +251,25 @@ UT_TEST(test_rdma_node_field_budgets) { UT_ASSERT_EQ(CLUSTER_NODE_RDMA_ADDR_LEN, 64); UT_ASSERT_EQ(CLUSTER_NODE_RDMA_GID_LEN, 40); - UT_ASSERT(sizeof(ClusterConf) <= 65536); + UT_ASSERT(sizeof(ClusterConf) <= 131072); +} + +/* spec-7.2 D2: budget 64 -> 128 KiB when data_addr joined the node entry + * (shmem-only anchor against accidental bloat, not a wire contract). */ +UT_TEST(test_conf_size_under_128k) +{ + UT_ASSERT(sizeof(ClusterConf) <= 131072); } -UT_TEST(test_conf_size_under_64k) +/* spec-7.2 D2: data_addr appended to ClusterNodeInfo — budget + the + * append-only layout rule (alignment preserved). */ +UT_TEST(test_data_addr_field_budget) { - UT_ASSERT(sizeof(ClusterConf) <= 65536); + ClusterNodeInfo n; + + UT_ASSERT_EQ(sizeof(n.data_addr), CLUSTER_NODE_ADDR_LEN); + UT_ASSERT_EQ(sizeof(ClusterNodeInfo) % 8, 0); + UT_ASSERT(sizeof(ClusterConf) <= 131072); } @@ -388,13 +405,14 @@ UT_TEST(test_cluster_enabled_linkable) int main(void) { - UT_PLAN(20); + UT_PLAN(21); UT_RUN(test_max_nodes_constant); UT_RUN(test_conf_magic_constant); UT_RUN(test_node_role_int32_sized); UT_RUN(test_node_info_alignment); UT_RUN(test_rdma_node_field_budgets); - UT_RUN(test_conf_size_under_64k); + UT_RUN(test_conf_size_under_128k); + UT_RUN(test_data_addr_field_budget); UT_RUN(test_role_to_string_primary); UT_RUN(test_role_to_string_standby_arbiter); UT_RUN(test_role_from_string_valid); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..c91d184974 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -45,6 +45,7 @@ #include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b catalog counter stubs */ #include "cluster/cluster_debug.h" +#include "cluster/cluster_grd.h" /* ClusterGrdRecoveryCounters */ #include "cluster/cluster_hang.h" /* spec-5.11: ClusterHangDumpData for dump_hang stubs */ #include "cluster/cluster_hang_resolve.h" /* spec-5.12: ClusterHangResolveCounters for dump stubs */ #include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ @@ -53,6 +54,7 @@ #include "cluster/cluster_xnode_lever.h" /* spec-6.12 lever counter stub */ #include "cluster/cluster_hw_lease.h" /* spec-6.12d lease counter stub */ #include "cluster/cluster_relmap_authority.h" /* spec-6.14 D5 header-read stub */ +#include "cluster/cluster_xid_authority.h" /* spec-6.15b XID authority dump stubs */ #undef printf #undef fprintf @@ -342,6 +344,21 @@ cluster_relmap_authority_read_header(bool shared_map, Oid dbid, ClusterRelmapAut return false; } +/* spec-6.15b D7 stubs: dump_catalog reads the XID authority observation + * keys; this standalone debug unit does not link the file-I/O authority + * objects. */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return false; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return false; +} + /* spec-4.12 D7 + spec-4.12b D6 stubs: dump_write_fence (cluster_debug.o) reads 8 * counters now, and cluster_startup_phase.o (linked here) references the rejoin * self-fence gate. cluster_write_fence.o is not linked -- provide stubs returning @@ -564,6 +581,9 @@ uint64 cluster_hw_failclosed_count(void); uint64 cluster_hw_not_ready_count(void); uint64 cluster_hw_remaster_done_count(void); uint64 cluster_hw_remaster_blocked_count(void); +uint64 cluster_hw_remaster_retry_count(void); +uint64 cluster_hw_remaster_retry_exhausted_count(void); +bool cluster_hw_remaster_recoverable(void); uint64 cluster_hw_alloc_count(void) { @@ -604,6 +624,21 @@ cluster_hw_remaster_blocked_count(void) { return 0; } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return 0; +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return 0; +} +bool +cluster_hw_remaster_recoverable(void) +{ + return true; +} /* spec-5.7 D4 dump_dl stubs (cluster_dl.c not linked in this binary). */ uint64 cluster_dl_lease_count(void); @@ -940,6 +975,30 @@ cluster_gcs_get_block_request_count(void) { return 0; } +/* spec-7.2 D6 stubs: ship-latency histogram accessors. */ +uint64 +cluster_gcs_block_ship_hist_bound_us(int bucket) +{ + return (bucket == 15) ? UINT64_MAX : (uint64)(bucket + 1) * 1000; +} +uint64 +cluster_gcs_block_ship_hist_count(int bucket) +{ + (void)bucket; + return 0; +} +/* spec-7.2 flip stubs: plane facts. */ +bool +cluster_gcs_block_family_on_data_plane(void) +{ + return false; +} +uint64 +cluster_ic_tier1_get_plane_misroute_reject(ClusterICPlane plane) +{ + (void)plane; + return 0; +} uint64 cluster_gcs_get_block_reply_count(void) { @@ -2638,6 +2697,21 @@ cluster_lmon_main_loop_iters(void) { return 0; } +uint64 +cluster_lmon_last_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_max_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_slow_iter_count(void) +{ + return 0; +} int cluster_lmon_status(void) { @@ -3288,15 +3362,77 @@ cluster_grd_deadlock_chunk_oo_buffer_overflow_count(void) return 0; } -/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter - * snapshot. Zero-fill; layout = 13 × uint64 (must track the - * ClusterGrdRecoveryCounters struct in cluster_grd.h). */ -struct ClusterGrdRecoveryCounters; -void cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out); +uint32 +cluster_grd_recovery_state_value(void) +{ + return (uint32)GRD_RECOVERY_IDLE; +} + +const char * +cluster_grd_recovery_state_name(uint32 state pg_attribute_unused()) +{ + return "idle"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_bitmap_hash_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return 0; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return true; +} + +/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter snapshot. */ void -cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out) +cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) { - memset(out, 0, 13 * sizeof(uint64)); + memset(out, 0, sizeof(*out)); } uint32 @@ -3377,6 +3513,16 @@ cluster_reconfig_get_clean_departed_cleared_count(void) { return 0; } +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return 0; +} +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return 0; +} void cluster_touched_peers_self_hex(char *buf, Size buflen) { @@ -3557,6 +3703,12 @@ cluster_lms_get_priority_starvation_observed_count(void) { return 0; } +/* spec-7.2 D6 stub — DATA-plane connection resets observability counter. */ +uint64 +cluster_lms_get_conn_resets(void) +{ + return 0; +} const char * cluster_lms_state_to_string(int s pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 444accf8f0..8729fbd744 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -70,6 +70,7 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_thread_recovery.h" #include "common/hashfn.h" #include "storage/buf_internals.h" #include "storage/lwlock.h" diff --git a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c index 095c16b2de..74a3b54fdb 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c @@ -196,7 +196,7 @@ UT_TEST(test_new_wait_events_distinct) } -UT_TEST(test_cluster_wait_events_count_118) +UT_TEST(test_cluster_wait_events_count_120) { /* spec-2.34 D7: 83 → 85 (+ 2 reliability wait events). * spec-2.36 D8: 85 → 88 (+ 3 CF 3-way wait events). @@ -207,7 +207,7 @@ UT_TEST(test_cluster_wait_events_count_118) * spec-4.7 D1: 98 → 99 (+ 1 GCS block RECOVERING short-wait). * spec-4.11 D5: 99 → 100 (+ 1 online thread recovery short-wait). * spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -313,7 +313,7 @@ main(void) UT_RUN(test_retry_total_backoff_default_1500ms); UT_RUN(test_lwtranche_distinct); UT_RUN(test_new_wait_events_distinct); - UT_RUN(test_cluster_wait_events_count_118); + UT_RUN(test_cluster_wait_events_count_120); UT_RUN(test_dedup_full_status_distinct_from_master_not_holder); UT_RUN(test_block_data_size_equals_blcksz); UT_RUN(test_dedup_entry_collision_field_layout); diff --git a/src/test/cluster_unit/test_cluster_ges.c b/src/test/cluster_unit/test_cluster_ges.c index 1df0a6b7fc..dd2a47185a 100644 --- a/src/test/cluster_unit/test_cluster_ges.c +++ b/src/test/cluster_unit/test_cluster_ges.c @@ -296,7 +296,8 @@ cluster_cancel_token_consume(void) * completion. Standalone fixture has no recovery shmem; no-op. */ void cluster_grd_recovery_mark_peer_done(int32 node pg_attribute_unused(), - uint64 epoch pg_attribute_unused()) + uint64 epoch pg_attribute_unused(), + uint64 event_id pg_attribute_unused()) {} ClusterGrdEntryResult diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..78827886d2 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -55,6 +55,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -85,6 +86,15 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; the GRD fixture has no PCM shmem — no-op (the cleanup + * logic itself is unit-tested in test_cluster_pcm_lock.c). */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -316,6 +326,13 @@ cluster_epoch_get_current(void) return ut_mock_epoch; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch) +{ + if (admitted_epoch > ut_mock_epoch) + ut_mock_epoch = admitted_epoch; +} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -377,10 +394,30 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, return end; } +/* spec-4.6a r2-P1-1: controllable last-event mock so the recovery FSM's P0 + * accept + WAIT_EPOCH event-scoped witness can be unit-driven. Default + * zeroed = pre-existing inert behavior. */ +static ReconfigEvent ut_mock_last_event; void cluster_reconfig_get_last_event(ReconfigEvent *out) { - memset(out, 0, sizeof(*out)); + *out = ut_mock_last_event; +} + +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + +/* spec-2.29a nightly-regression fix: controllable pre-bump-stage flag so the + * GRD IDLE baseline-hold can be unit-driven (default false = pre-fix + * re-capture behavior; existing tests are unaffected). */ +static bool ut_mock_pending_prebump = false; +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return ut_mock_pending_prebump; } /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the @@ -419,6 +456,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { @@ -432,10 +487,12 @@ SendProcSignal(int pid pg_attribute_unused(), int reason pg_attribute_unused(), return 0; } +/* spec-4.6a r2-P1-1: controllable clock (default 0 = pre-existing value). */ +static int64 ut_mock_now = 0; int64 GetCurrentTimestamp(void) { - return 0; + return ut_mock_now; } void @@ -3701,6 +3758,30 @@ ut_jr_setup_3node(void) cluster_grd_master_map_init(); } +/* spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE accounting converges on the + * composite key (episode_epoch, dead_bitmap_hash), so a test that feeds DONEs + * must first drive a P0 accept to stamp the episode's bitmap hash. Accept an + * empty-dead-bitmap FAIL event (hash identical to a JOIN episode's, no fence + * side effects; the FSM parks in WAIT_EPOCH awaiting a strict bump, which the + * fence assertions never touch). Returns the stamped hash for the callers' + * mark_peer_done payloads. */ +static uint64 +ut_jr_stamp_episode_bitmap_hash(uint64 event_id) +{ + bool saved_enabled = cluster_enabled; + + cluster_enabled = true; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle tick: baseline capture */ + ut_mock_last_event.event_id = event_id; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + cluster_grd_recovery_lmon_tick(); /* P0 accept stamps the bitmap hash */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_enabled = saved_enabled; + return cluster_grd_recovery_event_bitmap_hash_value(); +} + /* U1 — recompute moves the joiner's home shards back from the survivor. */ UT_TEST(test_jr_u1_recompute_moves_joiner_home) { @@ -3890,9 +3971,12 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(901); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* the rejoiner's home block */ @@ -3903,15 +3987,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3922,10 +4006,13 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(902); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); @@ -3934,15 +4021,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5); - cluster_grd_recovery_mark_peer_done(1, 5); - cluster_grd_recovery_mark_peer_done(2, 5); + cluster_grd_recovery_mark_peer_done(0, 5, done_hash); + cluster_grd_recovery_mark_peer_done(1, 5, done_hash); + cluster_grd_recovery_mark_peer_done(2, 5, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3955,10 +4042,12 @@ UT_TEST(test_jr_u15_master_side_gate_decision) int n1[1] = { 1 }; BufferTag tag; bool deny; + uint64 done_hash; memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(903); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; @@ -3967,9 +4056,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4012,17 +4101,20 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) int n2[1] = { 2 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); /* declared {0,1,2}, all members */ /* --- Episode 1: node 1 rejoins and completes (all survivors re-declare). --- */ ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(904); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4035,15 +4127,327 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20); + cluster_grd_recovery_mark_peer_done(0, 20, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20); + cluster_grd_recovery_mark_peer_done(1, 20, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } +/* ============================================================ + * spec-4.6a Amendment v1.2 (R2) — composite-key REDECLARE_DONE proof. + * + * The cross-node convergence key is (episode_epoch, dead_bitmap_hash). + * A DONE naming a DIFFERENT dead set (the old-event ABA shape) must be + * dropped at the accounting layer and never satisfy the WAIT_EPOCH + * coordinator witness. A DONE naming the SAME dead set must be + * accounted even though the sender's local event_id differs (the + * dead_generation-drift shape that wedged the event_id key: R2), and + * then satisfies the witness — the only equal-epoch escape + * (proof-carrying, never timeout-only). + * ============================================================ */ + +UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) +{ + ClusterGrdRecoveryCounters before; + ClusterGrdRecoveryCounters after; + uint8 old_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 episode_hash; + uint64 old_event_hash; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline. ut_mock_epoch already + * holds the post-bump value, reproducing the mis-captured-baseline wedge + * shape (cur == old) from spec-4.6a section 0. */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Stage the fail-stop event: dead node 2, coordinator 0. The local + * event_id (77) folds this node's own dead_generation; a peer that saw a + * different flap history computes a DIFFERENT id for the same episode. */ + ut_mock_last_event.event_id = 77; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); + episode_hash = cluster_grd_recovery_event_bitmap_hash_value(); + UT_ASSERT_EQ(episode_hash, cluster_grd_dead_bitmap_hash(ut_mock_last_event.dead_bitmap)); + + /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 + * instead of node 2 — e.g. the previous episode before a coordinator + * re-election) withholds the composite hash half. The epoch axis + * accounts unconditionally (the spec-5.16 join-fence liveness feed — + * see cluster_grd_recovery_mark_peer_done); the composite consumers + * stay blocked, pinned by the no-escape tick below. */ + memset(old_dead, 0, sizeof(old_dead)); + old_dead[0] = 0x08; + old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); + cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + + /* Deadline expiry with ONLY the stale-set DONE accounted: the witness + * must NOT fire (its hash conjunct is unsatisfied), and the FSM stays + * fail-closed in WAIT_EPOCH — the epoch axis alone can never unlock the + * composite escape. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_EPOCH); + + /* Generation-drift shape (the R2 wedge): the peer's local event_id + * differs, but its DONE names the SAME quorum dead set -> the composite + * key matches and the DONE is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, episode_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), episode_hash); + + /* Witness advance: deadline expired + cur==old + coordinator DONE naming + * THIS dead set at exactly cur, accounted after the accept snapshot. The + * FSM takes the composite-key equal-epoch escape exactly once. (The + * no-proof tick above re-armed the deadline to +rebuild_timeout, so jump + * well past it.) */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)130 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 — survivor-behind multi-death detection skew. The coordinator + * folded {1,2} into ONE epoch bump; this survivor accepted only {1} first and + * stamped H({1}) at that epoch. Its later local detection of node 2 publishes + * a new event with the GROWN set at the SAME epoch (no further bump), which + * pre-fix was never re-consumed past WAIT_EPOCH: DONE hashes could never match + * and P6 (no timeout) wedged permanently. The mid-episode guard must abort to + * IDLE, re-consume, re-stamp the full set's hash, re-run the remaster against + * the fuller dead set, and let full-set DONEs account. */ +UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint8 grown[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 h_small; + uint64 h_grown; + int i; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline (old = 10). */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Event 1: this survivor's CSSD has seen only node 1 dead so far. */ + ut_mock_last_event.event_id = 601; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x02; /* {1} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> parks in WAIT_EPOCH (cur==old) */ + h_small = cluster_grd_recovery_event_bitmap_hash_value(); + + /* The coordinator's single folded bump arrives via piggyback: this node + * sails past WAIT_EPOCH with the SMALL set stamped. */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER, episode epoch 11 */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); + + /* The wedge shape: a full-set DONE from the coordinator withholds the + * composite hash half while this node's stamp is behind (the epoch axis + * accounts unconditionally for the join-fence liveness feed; P6 still + * blocks on the missing hash). */ + memset(grown, 0, sizeof(grown)); + grown[0] = 0x06; /* {1,2} */ + h_grown = cluster_grd_dead_bitmap_hash(grown); + UT_ASSERT_NE(h_grown, h_small); + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); + + /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 602; + ut_mock_last_event.dead_bitmap[0] = 0x06; + cluster_grd_recovery_lmon_tick(); /* WAIT_BARRIER guard -> abort to IDLE */ + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed + 1); + + /* Re-consume: re-stamp to the FULL set's hash, re-run the remaster against + * the fuller set (nothing may stay mastered by 1 OR 2), land back in + * WAIT_BARRIER under the same epoch. */ + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_grown); + for (i = 0; i < PGRAC_GRD_SHARD_COUNT; i++) + UT_ASSERT_EQ(cluster_grd_shard_master(i), (int32)0); + + /* The coordinator's per-tick full-set DONE re-announce now accounts. */ + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), h_grown); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 no-regression leg — dead_generation drift re-fires the SAME dead + * set under the same epoch (new event_id, identical bitmap). The guard must + * ABSORB it: no abort-to-IDLE churn, stamp unchanged, and the event_id is + * consumed so the post-episode IDLE dedup does not re-run the episode. */ +UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint64 h_set; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle baseline (old = 10) */ + + ut_mock_last_event.event_id = 611; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; /* {2} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + h_set = cluster_grd_recovery_event_bitmap_hash_value(); + + /* Same-set re-fire (drift): absorbed, no episode churn. The tick may + * legitimately advance the episode (WAIT_BARRIER -> WAIT_CLUSTER with the + * trivial unit barrier) — the assertion is that it never falls back to + * IDLE and the stamp/counters do not move. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 612; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed); /* no abort */ + UT_ASSERT_NE(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_set); + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 612); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* Joiner-side DONE-accounting liveness (nightly t/347 L4-iii / t/353 L5 + * regression pin). A rejoining node never P0-accepts the JOIN episode as + * its own FSM episode (spec-5.16 D8: JOIN_COMMITTED is published + * coordinator-side only), so its local episode-hash stamp stays 0 forever. + * The survivors' per-tick REDECLARE_DONE re-announces are that node's ONLY + * feed for the spec-5.16 D3 join fence (cluster_grd_block_view_rebuilt); + * gating the done_epoch axis on the composite key drops every frame on the + * joiner, the fence never lifts, and every joiner-home block access + * fail-closes 53R9L until the caller's retry budget burns out. The epoch + * axis must account unconditionally — a DONE at epoch E proves the sender + * completed its WAIT_BARRIER at E (cluster-wide rebind + full block + * re-declare scan against then-current routing), which is the fence's + * safety condition regardless of the sender's episode dead set — while the + * composite hash half stays withheld pre-accept (P6 gate and WAIT_EPOCH + * witness unchanged). */ +UT_TEST(test_recovery_idle_joiner_accounts_done_epoch_for_fence) +{ + uint8 join1[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint8 zero_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + int n1[1] = { 1 }; + BufferTag tag; + uint64 join_hash; + + memset(&tag, 0, sizeof(tag)); + ut_jr_setup_3node(); /* FSM IDLE, episode hash stamp 0: never accepted */ + ut_mock_epoch = 10; + ut_jr_build_bitmap(join1, n1, 1); + cluster_grd_arm_join_pcm_fence(join1); + ut_mock_static_master = 1; /* the rejoiner's home block */ + + /* The real JOIN-episode DONE payload: hash of the all-zero dead bitmap + * (nonzero by the r3-P3(a) invariant). */ + memset(zero_bitmap, 0, sizeof(zero_bitmap)); + join_hash = cluster_grd_dead_bitmap_hash(zero_bitmap); + UT_ASSERT_NE(join_hash, 0); + + UT_ASSERT(cluster_grd_join_remaster_active_for_shard(tag)); + UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); + + /* Survivor DONEs arrive while this node's FSM is IDLE and unstamped + * (node 1 is this episode's recipient — skipped by the barrier). */ + cluster_grd_recovery_mark_peer_done(1, 10, join_hash); + cluster_grd_recovery_mark_peer_done(0, 10, join_hash); + cluster_grd_recovery_mark_peer_done(2, 10, join_hash); + + /* Epoch axis accounted -> the join fence lifts (the wedge fix)... */ + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(2), 10); + UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); + + /* ...while the composite hash half stays withheld pre-accept (v1.2 R2: + * P6 / witness consumers remain fail-closed on this node). */ + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(2), 0); +} + + +/* ============================================================ + * spec-2.29a nightly-regression fix — GRD IDLE baseline hold while a + * pre-bump coordinator stage is in flight. + * + * The async fence/join/node-remove marker staging bumps the epoch + * several ticks before publishing. While that window is open the GRD + * IDLE tick must NOT re-capture the post-bump epoch as its WAIT_EPOCH + * baseline, or the P0 accept that follows reads old == cur and wedges. + * ============================================================ */ + +UT_TEST(test_recovery_idle_holds_baseline_during_prebump_stage) +{ + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_pending_prebump = false; + + /* Idle tick with epoch 5 captures baseline 5 (no staging; get_last_event + * returns event_id 0 so the tick stays IDLE and re-captures). */ + ut_mock_epoch = 5; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* A pre-bump stage goes live and the coordinator bumps the epoch to 6 + * before publishing. The IDLE tick must HOLD baseline 5. */ + ut_mock_pending_prebump = true; + ut_mock_epoch = 6; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* Marker ACKs, stage clears; a subsequent idle tick resumes tracking. */ + ut_mock_pending_prebump = false; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 6); + + cluster_enabled = false; +} + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4054,8 +4458,10 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * spec-6.3a:+6 lifecycle; 5.8 D1b:+4 (U2a-d); D1c:+2 (U3a-b); * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); - * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + * +1 (U17 cross-episode fence Hardening); + * spec-4.6a r3-P2-2:+2 (same-epoch dead-set growth re-stamp + no-churn); + * spec-2.29a:+1 (idle baseline hold during pre-bump stage). */ + UT_PLAN(84); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4564,15 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); + + /* spec-4.6a r3-P2-2 — same-epoch multi-death detection-skew closure. */ + UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); + UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + UT_RUN(test_recovery_idle_holds_baseline_during_prebump_stage); + + /* spec-4.6a nightly regression — joiner-side DONE accounting liveness. */ + UT_RUN(test_recovery_idle_joiner_accounts_done_epoch_for_fence); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..e2bb7352b8 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -61,6 +61,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -91,6 +92,14 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; this fixture has no PCM shmem — no-op. */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -298,6 +307,10 @@ cluster_epoch_get_current(void) return 0; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch pg_attribute_unused()) +{} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -355,6 +368,19 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + +/* spec-2.29a: no pre-bump staging in the starvation fixture (no-op false). */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return false; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -391,6 +417,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_hw_remaster_retry.c b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c new file mode 100644 index 0000000000..456c5979f8 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_hw_remaster_retry.c + * Unit tests for spec-4.6a same-episode HW remaster retry decisions. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_hw_remaster_retry.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_hw_remaster.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +static ClusterHwRemasterRelaunchDecision +decide(uint64 launched, uint64 episode, ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int max_attempts) +{ + return cluster_hw_remaster_relaunch_decide(launched, episode, result, attempts, next_attempt_at, + now, max_attempts); +} + +UT_TEST(test_new_episode_initial_launch_resets_state) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(10, 11, CLUSTER_HW_REMASTER_BLOCKED, 7, 1234, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_running_done_and_not_applicable_do_not_relaunch) +{ + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_RUNNING, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_DONE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_NOT_APPLICABLE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_structural_blocked_warns_once_and_never_retries) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, 0, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, CLUSTER_HW_REMASTER_NO_DEADLINE, + 3000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_blocked_waits_until_backoff_deadline) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 4999, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 5000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 1); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_cap_exhausts_once_and_sighup_raise_recovers) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, 0, 5000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 6000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + /* Operator raised cluster.hw_remaster_retry_max_attempts via SIGHUP. */ + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 7000, 3); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 3); +} + +UT_TEST(test_zero_max_attempts_disables_retry) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 0, 5000, 0); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); +} + +/* spec-4.6a section 2.1 truth-table row 8: latched == episode but the result + * slot reads NONE (registration failed and was reverted) -> take the + * first-launch row again; no attempt is charged. */ +UT_TEST(test_none_result_registration_failed_falls_back_to_initial) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_NONE, 3, 999, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_backoff_exponential_cap) +{ + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 0), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 1), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 2), 2000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 7), 60000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(100, 20), 60000); +} + + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_new_episode_initial_launch_resets_state); + UT_RUN(test_running_done_and_not_applicable_do_not_relaunch); + UT_RUN(test_structural_blocked_warns_once_and_never_retries); + UT_RUN(test_blocked_waits_until_backoff_deadline); + UT_RUN(test_cap_exhausts_once_and_sighup_raise_recovers); + UT_RUN(test_zero_max_attempts_disables_retry); + UT_RUN(test_none_result_registration_failed_falls_back_to_initial); + UT_RUN(test_backoff_exponential_cap); + UT_DONE(); + + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 6a08b4104f..76d9a357d2 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -519,7 +519,7 @@ UT_TEST(test_hello_wire_roundtrip) bool ok; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 42, - "pgrac-test"); + "pgrac-test", CLUSTER_IC_PLANE_CONTROL, 0); ok = cluster_ic_parse_hello(wire, &parsed); UT_ASSERT(ok); @@ -554,7 +554,7 @@ UT_TEST(test_hello_wire_reference_bytes) * unintended endian flips, and uninitialized memory leakage. */ cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, - 0x01020304, "AB"); + 0x01020304, "AB", CLUSTER_IC_PLANE_CONTROL, 0); /* magic */ UT_ASSERT_EQ(wire[0], 0x48); @@ -577,11 +577,47 @@ UT_TEST(test_hello_wire_reference_bytes) UT_ASSERT_EQ(wire[13], 'B'); for (i = 14; i < 36; i++) UT_ASSERT_EQ(wire[i], 0); - /* _pad must be all zero */ + /* _pad must be all zero: a CONTROL-plane HELLO with conn_epoch 0 is + * byte-identical to the pre-7.2 wire (spec-7.2 D2 compat pin). */ for (i = 36; i < PGRAC_IC_HELLO_BYTES; i++) UT_ASSERT_EQ(wire[i], 0); } +/* + * spec-7.2 D2 — DATA-plane HELLO reference bytes: plane byte at offset + * 40, conn_epoch LE at 44-51, spec-7.3 worker bytes (41/42) still zero, + * tail 52-63 still zero. Plus accessor roundtrip. + */ +UT_TEST(test_hello_wire_data_plane_bytes) +{ + uint8 wire[PGRAC_IC_HELLO_BYTES]; + ClusterICHelloMsg parsed; + int i; + + cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 3, "AB", + CLUSTER_IC_PLANE_DATA, UINT64CONST(0x1122334455667788)); + + UT_ASSERT_EQ(wire[PGRAC_IC_HELLO_PLANE_OFFSET], 1); + UT_ASSERT_EQ(wire[PGRAC_IC_HELLO_WORKER_ID_OFFSET], 0); + UT_ASSERT_EQ(wire[PGRAC_IC_HELLO_N_WORKERS_OFFSET], 0); + UT_ASSERT_EQ(wire[43], 0); + /* conn_epoch = 0x1122334455667788 little-endian */ + UT_ASSERT_EQ(wire[44], 0x88); + UT_ASSERT_EQ(wire[45], 0x77); + UT_ASSERT_EQ(wire[46], 0x66); + UT_ASSERT_EQ(wire[47], 0x55); + UT_ASSERT_EQ(wire[48], 0x44); + UT_ASSERT_EQ(wire[49], 0x33); + UT_ASSERT_EQ(wire[50], 0x22); + UT_ASSERT_EQ(wire[51], 0x11); + for (i = 52; i < PGRAC_IC_HELLO_BYTES; i++) + UT_ASSERT_EQ(wire[i], 0); + + UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); + UT_ASSERT_EQ(cluster_ic_hello_plane(&parsed), CLUSTER_IC_PLANE_DATA); + UT_ASSERT(cluster_ic_hello_conn_epoch(&parsed) == UINT64CONST(0x1122334455667788)); +} + UT_TEST(test_hello_smart_fusion_capability_gate) { uint8 wire[PGRAC_IC_HELLO_BYTES]; @@ -591,7 +627,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, - "sf-off"); + "sf-off", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), 0); @@ -599,7 +635,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_interconnect_tier = CLUSTER_IC_TIER_2; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, - "sf-tier-mismatch"); + "sf-tier-mismatch", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), 0); @@ -607,7 +643,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, - "sf-tier-match"); + "sf-tier-match", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2); @@ -621,7 +657,8 @@ UT_TEST(test_hello_parse_rejects_bad_magic) uint8 wire[PGRAC_IC_HELLO_BYTES]; ClusterICHelloMsg parsed; - cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "x"); + cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "x", + CLUSTER_IC_PLANE_CONTROL, 0); /* Corrupt magic */ wire[0] = 0xDE; wire[1] = 0xAD; @@ -638,7 +675,7 @@ UT_TEST(test_hello_build_truncates_long_name) const char *long_name = "this-cluster-name-is-way-longer-than-the-fixed-cap-on-purpose"; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 7, - long_name); + long_name, CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); /* @@ -654,7 +691,7 @@ UT_TEST(test_hello_build_truncates_long_name) int main(void) { - UT_PLAN(20); /* spec-2.3 D3: 6 ClusterMsgHeader/msg_send/recv tests deleted */ + UT_PLAN(21); /* spec-2.3 D3: 6 ClusterMsgHeader/msg_send/recv tests deleted */ UT_RUN(test_ic_send_bytes_linkable); UT_RUN(test_ic_recv_bytes_linkable); UT_RUN(test_ic_init_linkable); @@ -674,6 +711,7 @@ main(void) /* HELLO wire encode/decode + reference bytes (post-codex review) */ UT_RUN(test_hello_wire_roundtrip); UT_RUN(test_hello_wire_reference_bytes); + UT_RUN(test_hello_wire_data_plane_bytes); /* spec-7.2 D2 */ UT_RUN(test_hello_smart_fusion_capability_gate); UT_RUN(test_hello_parse_rejects_bad_magic); UT_RUN(test_hello_build_truncates_long_name); diff --git a/src/test/cluster_unit/test_cluster_ic_router.c b/src/test/cluster_unit/test_cluster_ic_router.c index 4edca9b0a7..32ddbd7680 100644 --- a/src/test/cluster_unit/test_cluster_ic_router.c +++ b/src/test/cluster_unit/test_cluster_ic_router.c @@ -314,6 +314,22 @@ void cluster_ic_tier1_bump_epoch_observe_advance(int32 peer_id pg_attribute_unused()) {} +/* spec-7.2 D3 stubs: plane gates (this harness models the CONTROL-plane + * owner; the plane-gate truth table is pinned in the test body). */ +static ClusterICPlane router_test_my_plane = CLUSTER_IC_PLANE_CONTROL; +static uint64 router_test_misroute_count = 0; + +ClusterICPlane +cluster_ic_tier1_my_plane(void) +{ + return router_test_my_plane; +} +void +cluster_ic_tier1_bump_plane_misroute_reject(void) +{ + router_test_misroute_count++; +} + /* MyBackendType -- writable from tests to flip producer scope */ extern BackendType MyBackendType; BackendType MyBackendType = B_INVALID; diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..598ca2b3d8 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -178,6 +178,7 @@ cluster_postmaster_start_lmon(void) * MyLatch + cluster_inject framework. Stubs cover them all -- * runtime LmonMain is not exercised. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; #include "cluster/cluster_inject.h" int cluster_injection_armed_count = 0; @@ -199,6 +200,16 @@ int cluster_interconnect_heartbeat_interval_ms = 1000; int cluster_interconnect_connect_timeout_ms = 5000; int cluster_interconnect_recv_timeout_ms = 30000; +/* spec-7.2 D1 GUC (cluster_lmon_duty_should_run references it). */ +bool cluster_ic_duty_lazy = true; + +/* spec-7.2 D4 stub: plane-flip registry probe (skeleton = CONTROL). */ +bool +cluster_gcs_block_family_on_data_plane(void) +{ + return false; +} + #include "cluster/cluster_ic_tier1.h" int @@ -590,6 +601,9 @@ pg_usleep(long microsec pg_attribute_unused()) /* Sprint B: Latch / WaitLatch / ResetLatch stubs (LmonMain runtime * is not invoked at unit-test level). */ struct Latch *MyLatch = NULL; +void +SetLatch(struct Latch *latch pg_attribute_unused()) +{} int WaitLatch(struct Latch *latch pg_attribute_unused(), int wakeEvents pg_attribute_unused(), long timeout pg_attribute_unused(), uint32 wait_event_info pg_attribute_unused()) @@ -600,6 +614,11 @@ void ResetLatch(struct Latch *latch pg_attribute_unused()) {} +#include "storage/ipc.h" +void +on_shmem_exit(pg_on_exit_callback function pg_attribute_unused(), Datum arg pg_attribute_unused()) +{} + /* cluster_lmon.c references MyBackendType (set by LmonMain). */ #include "miscadmin.h" BackendType MyBackendType = B_INVALID; @@ -804,9 +823,65 @@ UT_TEST(test_lmon_public_symbols_linkable) UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_size); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_init); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_register); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_last_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_max_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_slow_iter_count); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_marker_complete_wakeup); UT_ASSERT_NOT_NULL((void *)LmonMain); } +UT_TEST(test_lmon_iteration_counters_null_safe) +{ + UT_ASSERT_EQ((unsigned long long)cluster_lmon_last_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_max_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_slow_iter_count(), 0ULL); + cluster_lmon_marker_complete_wakeup(); +} + +/* + * spec-7.2 D1 -- duty classification truth table (§3.7). + * + * Pins EVERY ClusterLmonDuty row to its never-lazy / lazy-able class. + * Re-classifying a duty (or adding one) MUST update this table + * deliberately -- that is the anti-drift gate against a future spec + * silently sliding a correctness family into the lazy set. + */ +UT_TEST(test_lmon_duty_lazy_truth_table) +{ + /* never-lazy (correctness families) */ + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_LIVENESS)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_FENCE)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_DEAD_SWEEP)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_RECLAIM)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_STARVATION_SWEEP)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_DEADLOCK)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GES_REPLY_WAIT_SWEEP)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_DIRECT_LAND_ABORTS)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_LMS_NATIVE_PROBE_RETRY)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_CLEAN_LEAVE)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_NODE_REMOVE)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_RECONFIG)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_RECOVERY)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_BOC_BROADCAST)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SINVAL_RESET_ALL)); + UT_ASSERT(!cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_PI_DISCARD)); + /* lazy-able (queue-consumption families) */ + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GES_WORK_QUEUE)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SHIP_READY)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_GRD_OUTBOUND)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SINVAL_OUT)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_SINVAL_ACK_OUT)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_TT_HINT)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_DEDUP_TTL)); + UT_ASSERT(cluster_lmon_duty_is_lazy(CLUSTER_LMON_DUTY_BACKUP)); + /* enum shape: lazy block is contiguous + fits the uint32 bitmask */ + UT_ASSERT_EQ(CLUSTER_LMON_DUTY_N - CLUSTER_LMON_DUTY_LAZY_FIRST, 8); + /* pre-shmem safety: mark is a no-op, should_run fails open (true) */ + cluster_lmon_duty_mark_dirty(CLUSTER_LMON_DUTY_GRD_OUTBOUND); + UT_ASSERT(cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_GRD_OUTBOUND, false)); + UT_ASSERT(cluster_lmon_duty_should_run(CLUSTER_LMON_DUTY_FENCE, false)); +} + /* ============================================================ * Test runner @@ -815,12 +890,14 @@ UT_TEST(test_lmon_public_symbols_linkable) int main(void) { - UT_PLAN(5); + UT_PLAN(7); UT_RUN(test_lmon_status_enum_values_frozen); UT_RUN(test_lmon_shared_state_size_under_4kb); UT_RUN(test_lmon_status_to_string_lookup); UT_RUN(test_lmon_status_unknown_returns_unknown); UT_RUN(test_lmon_public_symbols_linkable); + UT_RUN(test_lmon_iteration_counters_null_safe); + UT_RUN(test_lmon_duty_lazy_truth_table); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_lock_acquire.c b/src/test/cluster_unit/test_cluster_lock_acquire.c index 5636b8fff8..a5ca5a719c 100644 --- a/src/test/cluster_unit/test_cluster_lock_acquire.c +++ b/src/test/cluster_unit/test_cluster_lock_acquire.c @@ -365,6 +365,47 @@ cluster_grd_lookup_master(const ClusterResId *resid pg_attribute_unused()) return -1; } +/* spec-4.6a: the S4-reject diagnostic references the CSSD peer-state view; + * fixture has no CSSD shmem — stub DEAD-free defaults. */ +#include "cluster/cluster_cssd.h" +ClusterCssdPeerState +cluster_cssd_get_peer_state(int32 peer_id pg_attribute_unused()) +{ + return CLUSTER_CSSD_PEER_ALIVE; +} +const char * +cluster_cssd_peer_state_to_string(ClusterCssdPeerState s pg_attribute_unused()) +{ + return "alive"; +} + +/* spec-4.6a: the same diagnostic is the first ereport in this .o — swallow + * elog machinery (mirror test_cluster_grd.c pattern). */ +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} + /* spec-4.6 P0 regression harness — test-controlled S3/S4 outcomes. * stub_reserve_result default NOT_READY keeps the legacy tests on the * pre-reservation-fail path; the S4 default-deny test flips it to OK diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c new file mode 100644 index 0000000000..b90fa463a2 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -0,0 +1,246 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_marker_async.c + * Unit tests for the process-local qvotec marker async FSM. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_marker_async.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_marker_async.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +static int ut_set_latch_count = 0; + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +void +SetLatch(Latch *latch pg_attribute_unused()) +{ + ut_set_latch_count++; +} + +static void +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot) +{ + cluster_marker_async_init(a); + pg_atomic_init_u64(request_seq, 0); + pg_atomic_init_u64(completion_seq, 0); + pg_atomic_init_u32(result_slot, 0); + ut_set_latch_count = 0; +} + +UT_TEST(test_init_defaults_idle) +{ + ClusterMarkerAsync a; + + cluster_marker_async_init(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); +} + +UT_TEST(test_submit_publishes_one_request) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 7); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(pg_atomic_read_u64(&completion_seq), 0); + UT_ASSERT_EQ(ut_set_latch_count, 1); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.deadline_us, 6000); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_FAILSTOP); + UT_ASSERT_EQ(a.target_node, 7); +} + +UT_TEST(test_poll_pending_before_completion) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 99; + uint64 elapsed = 99; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_PREPARE, 2)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 1500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_PENDING); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(result, 0); + UT_ASSERT_EQ(elapsed, 0); +} + +UT_TEST(test_poll_ack_returns_result_and_elapsed) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 0; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, 3)); + pg_atomic_write_u32(&result_slot, 42); + pg_atomic_write_u64(&completion_seq, 1); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_ACKED); + UT_ASSERT_EQ(result, 42); + UT_ASSERT_EQ(elapsed, 1500); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_poll_timeout_releases_submit_state) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 1000, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, 4)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2000, NULL, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_TIMEOUT); + UT_ASSERT_EQ(elapsed, 1000); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_mailbox_busy_does_not_publish) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + pg_atomic_write_u64(&request_seq, 2); + pg_atomic_write_u64(&completion_seq, 1); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, 5); + + UT_ASSERT(!ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 2); + UT_ASSERT_EQ(ut_set_latch_count, 0); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_reentrant_submit_does_not_bump_again) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, 6)); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1200, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 99); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED); + UT_ASSERT_EQ(a.target_node, 6); +} + +UT_TEST(test_release_stage_clears_staged_record) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + a.has_staged_event = true; + a.staged_expect_epoch = 123; + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED, 8)); + + cluster_marker_async_release_stage(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); + UT_ASSERT_EQ(a.staged_expect_epoch, 0); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_init_defaults_idle); + UT_RUN(test_submit_publishes_one_request); + UT_RUN(test_poll_pending_before_completion); + UT_RUN(test_poll_ack_returns_result_and_elapsed); + UT_RUN(test_poll_timeout_releases_submit_state); + UT_RUN(test_mailbox_busy_does_not_publish); + UT_RUN(test_reentrant_submit_does_not_bump_again); + UT_RUN(test_release_stage_clears_staged_record); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c0ff3b6710..08ba568886 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -100,6 +100,9 @@ static Size fake_pcm_keysize = 0; static Size fake_pcm_entrysize = 0; static LWLock *fake_lwlock_held = NULL; static LWLockMode fake_lwlock_mode = LW_EXCLUSIVE; +static LWLock *fake_lwlock_stack[16]; +static LWLockMode fake_lwlock_mode_stack[16]; +static int fake_lwlock_depth = 0; static uint32 fake_init_wait_event_seen = 0; static uint32 fake_lwlock_wait_event_seen = 0; @@ -121,6 +124,7 @@ static struct { static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; +static bool fake_local_x_upgrade_result = false; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -130,6 +134,19 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): the S->X upgrade is now wrapped in + * PG_TRY/PG_CATCH, which references the exception stack + re-throw. The + * unit's errfinish mock longjmps to its own buffer (never through + * PG_exception_stack), so these exist for the linker only. */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { @@ -155,6 +172,10 @@ reset_fake_pcm_runtime(int max_entries) fake_pcm_keysize = 0; fake_pcm_entrysize = 0; fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + fake_lwlock_depth = 0; + memset(fake_lwlock_stack, 0, sizeof(fake_lwlock_stack)); + memset(fake_lwlock_mode_stack, 0, sizeof(fake_lwlock_mode_stack)); fake_init_wait_event_seen = 0; fake_lwlock_wait_event_seen = 0; ut_wait_event_info_storage = 0; @@ -167,6 +188,7 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; cluster_pcm_grd_max_entries = max_entries; @@ -191,9 +213,20 @@ bool cluster_read_scache = false; bool cluster_gcs_block_local_x_upgrade(BufferTag tag); bool -cluster_gcs_block_local_x_upgrade(BufferTag tag pg_attribute_unused()) +cluster_gcs_block_local_x_upgrade(BufferTag tag) { - return false; + uint32 holders_bm; + int n; + + if (!fake_local_x_upgrade_result) + return false; + holders_bm = cluster_pcm_lock_query_s_holders_bitmap(tag); + if (cluster_node_id >= 0 && cluster_node_id < 32) + holders_bm &= ~((uint32)1u << (uint32)cluster_node_id); + for (n = 0; n < 32; n++) + if ((holders_bm & ((uint32)1u << n)) != 0) + (void)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, n); + return cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_X_UPGRADE, cluster_node_id); } /* spec-4.7a D4 — stub CSSD peer liveness for the other-live-holder gate. @@ -348,6 +381,10 @@ LWLockInitialize(LWLock *lock pg_attribute_unused(), int tranche_id pg_attribute bool LWLockAcquire(LWLock *lock, LWLockMode mode) { + Assert(fake_lwlock_depth < (int)lengthof(fake_lwlock_stack)); + fake_lwlock_stack[fake_lwlock_depth] = lock; + fake_lwlock_mode_stack[fake_lwlock_depth] = mode; + fake_lwlock_depth++; fake_lwlock_held = lock; fake_lwlock_mode = mode; fake_lwlock_wait_event_seen = ut_wait_event_info_storage; @@ -357,14 +394,28 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) void LWLockRelease(LWLock *lock) { - Assert(fake_lwlock_held == lock); - fake_lwlock_held = NULL; + Assert(fake_lwlock_depth > 0); + Assert(fake_lwlock_stack[fake_lwlock_depth - 1] == lock); + fake_lwlock_depth--; + fake_lwlock_stack[fake_lwlock_depth] = NULL; + if (fake_lwlock_depth > 0) { + fake_lwlock_held = fake_lwlock_stack[fake_lwlock_depth - 1]; + fake_lwlock_mode = fake_lwlock_mode_stack[fake_lwlock_depth - 1]; + } else { + fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + } } bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode) { - return fake_lwlock_held == lock && fake_lwlock_mode == mode; + int i; + + for (i = fake_lwlock_depth - 1; i >= 0; i--) + if (fake_lwlock_stack[i] == lock && fake_lwlock_mode_stack[i] == mode) + return true; + return false; } /* ---------- @@ -1280,6 +1331,79 @@ UT_TEST(test_pcm_d3_not_double_x) } } +UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) +{ + BufferTag tag = make_tag(96); + BufferDesc buf; + bool save = cluster_gcs_block_local_cache; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + cluster_gcs_block_local_cache = true; + fake_local_x_upgrade_result = true; + + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); + UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); + UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); + + cluster_gcs_block_local_cache = save; +} + +UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) +{ + BufferTag stag = make_tag(94); + BufferTag xtag = make_tag(95); + uint32 s_bitmap; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + + /* Dead node 2 was the first S holder, so master_holder points at it. */ + cluster_node_id = 2; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 2); + UT_ASSERT(!cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + + cluster_node_id = 2; + cluster_pcm_lock_acquire(xtag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_X); + + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)2); + + s_bitmap = cluster_pcm_lock_query_s_holders_bitmap(stag); + UT_ASSERT_EQ((int)(s_bitmap & (1u << 2)), 0); + UT_ASSERT((s_bitmap & (1u << 1)) != 0); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(stag), (int)PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 1); + + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_N); + UT_ASSERT(cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + UT_ASSERT((fake_cv_broadcast_count) >= (1)); + + /* Idempotent: a repeated dead-sweep pass has nothing left to clean. */ + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)0); + + /* spec-4.6a D12 (r2-P1-3) pending-X form: a dead requester's parked X + * intent is cleared by the companion HC124 sweep the same dead-sweep hook + * drives, and the sweep is idempotent too. */ + cluster_pcm_lock_set_pending_x(stag, 2, 1234); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), 2); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)1); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), -1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)0); +} + /* spec-5.2a D2 (U1): clean-page X-transfer arm is one-shot. arm(true) sets * the backend-local flag; consume() reads-and-clears it (the acquire path * calls consume() once so the eligibility can never leak into a SUBSEQUENT @@ -1313,7 +1437,7 @@ UT_TEST(test_clean_page_xfer_arm_is_one_shot) int main(void) { - UT_PLAN(39); + UT_PLAN(41); UT_RUN(test_pcm_lock_mode_constant_aliases_match_pcm_state); UT_RUN(test_pcm_lock_transition_count_is_9); UT_RUN(test_pcm_lock_transition_enum_values_are_1_to_9); @@ -1352,6 +1476,8 @@ main(void) UT_RUN(test_pcm_d1_recovering_gate_fail_closed); UT_RUN(test_pcm_d2_rebuild_from_redeclare); UT_RUN(test_pcm_d3_not_double_x); + UT_RUN(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades); + UT_RUN(test_pcm_dead_node_cleanup_drops_holder_records); UT_RUN(test_clean_page_xfer_arm_is_one_shot); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..7d075852dd 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,6 +207,9 @@ UT_TEST(test_gate_raw_compare_traps) } int +/* cppcheck-suppress constParameter + * Reason: main() keeps the standard test harness signature used by the + * other cluster_unit binaries; argv is intentionally unused. */ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ diff --git a/src/test/cluster_unit/test_cluster_qvotec.c b/src/test/cluster_unit/test_cluster_qvotec.c index 62b2099932..3400d68962 100644 --- a/src/test/cluster_unit/test_cluster_qvotec.c +++ b/src/test/cluster_unit/test_cluster_qvotec.c @@ -212,6 +212,14 @@ void cluster_elog_init(void) {} +#include "cluster/cluster_inject.h" +bool +cluster_cr_injection_armed(const char *name pg_attribute_unused(), + uint64 *out_param pg_attribute_unused()) +{ + return false; +} + /* Step 3 D7 stubs: signal/ps_display/procsignal symbols not linked * here (cluster_qvotec.c references them for ClusterQvotecMain; * unit test never invokes Main, just address-takes for T-6). */ diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 573406761c..6d6fcbfd0f 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -61,6 +61,14 @@ UT_DEFINE_GLOBALS(); +/* spec-2.29a: cluster_reconfig.c gates the async marker stage on + * MyBackendType == B_LMON. The fixture exercises the pre-2.29a bounded-wait + * semantics (write_fence submit stub returns FAILED synchronously), so run as + * a non-LMON backend; the async FSM itself is covered by + * test_cluster_marker_async.c. */ +#include "miscadmin.h" +BackendType MyBackendType = B_INVALID; + /* ============================================================ * Stubs — link cluster_reconfig.o + cluster_epoch.o standalone. @@ -357,6 +365,48 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +/* spec-2.29a review r1 P1-c: controllable async-marker stubs so the tick-level + * P1-1 invariants (bump-once while PENDING, node-remove zero-false-contest, + * publish deferred to ACK) can be hard-asserted. Defaults preserve the + * pre-existing inert behavior (submit rejected, poll idle). */ +static bool ut_fence_async_submit_ok = false; +static ClusterMarkerPollResult ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; +static uint32 ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +static int ut_fence_async_submit_calls = 0; +static int ut_fence_async_poll_calls = 0; + +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m pg_attribute_unused(), + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now pg_attribute_unused()) +{ + ut_fence_async_submit_calls++; + if (!ut_fence_async_submit_ok) + return false; + /* mirror the real wrapper's FSM effect so is_submitted() sees SUBMITTED */ + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->kind = kind; + a->target_node = target_node; + return true; +} +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now pg_attribute_unused(), + uint32 *out_result, uint64 *out_elapsed_us) +{ + ut_fence_async_poll_calls++; + if (out_result != NULL) + *out_result = ut_fence_async_poll_result; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_ACKED + || ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_TIMEOUT) + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return ut_fence_async_poll_pr; +} +void +cluster_lmon_marker_complete_wakeup(void) +{} /* spec-3.1 D7 stub: cluster_reconfig_apply_epoch_bump_as_coordinator * calls cluster_tt_status_flush_all. Fixture has no TT overlay shmem; @@ -962,6 +1012,142 @@ UT_TEST(test_reconfig_lmon_tick_survivor_does_not_advance_epoch) } +/* ============================================================ + * T-reconfig-2.29a — P1-1 staged-record hard assertions (spec-2.29a + * review r1 P1-c). + * + * (a) fail-stop fence stage: while the marker is PENDING across ticks + * the epoch is bumped EXACTLY ONCE and nothing publishes; the + * staged event publishes only on ACKED+ACK. + * (b) node-remove stage: a tick re-entry while the fence marker is + * PENDING returns 0 with *out_contest == false (no false contest + * against our own already-advanced baseline) and no re-bump. + * ============================================================ */ + +UT_TEST(test_reconfig_failstop_fence_stage_bump_once_while_pending) +{ + uint64 epoch_after_bump; + uint64 apply_before; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD; + ut_dead_generation = 1; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + apply_before = cluster_reconfig_get_apply_counter(); + + /* tick #1: stage-entry — bump once, submit, DO NOT publish. */ + cluster_reconfig_lmon_tick(); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT(epoch_after_bump > 0); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* ticks #2/#3: marker PENDING — the bump path must NOT re-enter + * (P1-1 invariant 0: zero re-bump, zero publish, zero side-effects). */ + cluster_reconfig_lmon_tick(); + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* tick #4: ACKED + ACK — the STAGED event publishes; still no re-bump. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)(apply_before + 1)); + + /* restore fixture defaults for the neighboring tests */ + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + +UT_TEST(test_reconfig_node_remove_stage_no_false_contest) +{ + uint64 baseline; + uint64 epoch_after_bump; + uint64 ret; + bool contest; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + baseline = cluster_epoch_get_current(); + + /* call #1: guarded CAS advances baseline→baseline+1, stages the fence + * marker, returns 0 (not yet committed), contest MUST be false. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT_EQ((unsigned long long)epoch_after_bump, (unsigned long long)(baseline + 1)); + + /* call #2 (tick re-entry with the SAME stale baseline while PENDING): + * without the staged record this would re-run the baseline CAS against + * our own advanced epoch and report a FALSE contest. P1-1: it must + * poll the stage instead — 0, contest==false, no second bump. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + /* call #3: ACKED + ACK — removal publishes at the STAGED epoch. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, (unsigned long long)epoch_after_bump); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + /* ============================================================ * T-reconfig-8 — ProcessInterrupts I6 guard (D4). * @@ -1575,6 +1761,8 @@ main(void) /* T-reconfig-3 — lmon_tick dedup. */ UT_RUN(test_reconfig_lmon_tick_dedups_same_event_id); UT_RUN(test_reconfig_lmon_tick_refires_on_dead_gen_bump); + UT_RUN(test_reconfig_failstop_fence_stage_bump_once_while_pending); + UT_RUN(test_reconfig_node_remove_stage_no_false_contest); /* T-reconfig-4 / 4b — Q2 A'' + I2 + L20 + F11 + empty-dead-set gates. */ UT_RUN(test_reconfig_lmon_tick_skips_when_not_in_quorum); diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index d5bfd4ca76..a5f87916c9 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -772,6 +772,13 @@ cluster_grd_outbound_shmem_register(void) void cluster_grd_work_queue_shmem_register(void) {} +/* spec-7.2 D4 stubs: DATA-plane outbound ring registration + tranche. */ +void +cluster_lms_outbound_shmem_register(void) +{} +void +cluster_lms_outbound_request_lwlocks(void) +{} /* spec-2.16 cluster_shmem.c also calls RequestNamedLWLockTranche for the * 2 NEW Step 2 tranches. PG built-in stub already declared in this diff --git a/src/test/cluster_unit/test_cluster_stage2_acceptance.c b/src/test/cluster_unit/test_cluster_stage2_acceptance.c index 8a6390e67f..71a34d3593 100644 --- a/src/test/cluster_unit/test_cluster_stage2_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage2_acceptance.c @@ -210,14 +210,14 @@ UT_TEST(test_stage2_fault_inject_point_names) /* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 118 ===== */ -UT_TEST(test_stage2_wait_events_count_snapshot_118) +UT_TEST(test_stage2_wait_events_count_snapshot_120) { /* spec-2.39 D13 ship value. Future spec adding wait events MUST * update this snapshot (update-required contract per spec v0.2 F5 * — current state, not "==93 forever"). spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering). spec-4.11 D5: 99 → 100 * (+ ClusterThreadRecovery). spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -277,7 +277,7 @@ main(void) UT_RUN(test_stage2_msg_types_cumulative_registration); UT_RUN(test_stage2_capability_counter_symbols_linkable); UT_RUN(test_stage2_fault_inject_point_names); - UT_RUN(test_stage2_wait_events_count_snapshot_118); + UT_RUN(test_stage2_wait_events_count_snapshot_120); UT_RUN(test_stage2_sqlstate_53r60_through_95_encodable); UT_RUN(test_stage2_guc_enum_snapshot); UT_RUN(test_stage2_ic_msg_reserved_0_sentinel); diff --git a/src/test/cluster_unit/test_cluster_stage3_acceptance.c b/src/test/cluster_unit/test_cluster_stage3_acceptance.c index 448d2b6c59..7cc2ed5299 100644 --- a/src/test/cluster_unit/test_cluster_stage3_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage3_acceptance.c @@ -375,14 +375,14 @@ UT_TEST(test_stage3_sqlstate_mvcc_surface_encodable) /* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 118 ===== */ -UT_TEST(test_stage3_wait_events_count_snapshot_118) +UT_TEST(test_stage3_wait_events_count_snapshot_120) { /* spec-4.2 D5 value (95 + 2 wal-state registry I/O). Update-required contract: a future spec * adding a wait event MUST bump this snapshot (it is current state, not * "==93 forever"). spec-4.6 D4: 97 → 98; spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering); spec-4.11 D5: 99 → 100 - * (+ ClusterThreadRecovery). spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * (+ ClusterThreadRecovery). spec-6.13 D8 snapshot: 118. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -475,7 +475,7 @@ main(void) UT_RUN(test_undo_4_8ab_redo_determinism_converges); UT_RUN(test_stage3_capability_dump_category_names); UT_RUN(test_stage3_sqlstate_mvcc_surface_encodable); - UT_RUN(test_stage3_wait_events_count_snapshot_118); + UT_RUN(test_stage3_wait_events_count_snapshot_120); UT_RUN(test_stage3_tt_enum_values_locked); UT_RUN(test_stage3_retention_active_retains_invariant); UT_RUN(test_stage3_bind_opcode_reserved); diff --git a/src/test/cluster_unit/test_cluster_stage4_acceptance.c b/src/test/cluster_unit/test_cluster_stage4_acceptance.c index 3e0617f786..130708cc8d 100644 --- a/src/test/cluster_unit/test_cluster_stage4_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage4_acceptance.c @@ -203,13 +203,13 @@ UT_TEST(test_stage4_sqlstate_recovery_fence_surface_encodable) /* ===== L5 — wait-events count snapshot ===== */ -UT_TEST(test_stage4_wait_events_count_snapshot_118) +UT_TEST(test_stage4_wait_events_count_snapshot_120) { /* Current Stage 4 surface value; spec-6.2 adds the latest 4 Smart Fusion * authority waits and spec-6.13 adds 2 RDMA tier3 waits. update-required * contract: a future spec adding cluster wait events MUST bump this snapshot * (and the dump/test baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -312,7 +312,7 @@ main(void) UT_RUN(test_stage4_undo_opcodes_preserved_and_info_mask_clear); UT_RUN(test_stage4_recovery_dump_category_names); UT_RUN(test_stage4_sqlstate_recovery_fence_surface_encodable); - UT_RUN(test_stage4_wait_events_count_snapshot_118); + UT_RUN(test_stage4_wait_events_count_snapshot_120); UT_RUN(test_stage4_write_fence_enums_locked); UT_RUN(test_stage4_thread_recovery_scope_enum_complete); UT_RUN(test_stage4_undo_writeback_boundary_enum_complete); diff --git a/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c index 181c36b0f3..11f18a489e 100644 --- a/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c @@ -182,13 +182,14 @@ UT_TEST(test_stage5_5_cross_instance_coordinator_enums_locked) /* ===== L6 — wait-events count snapshot ===== */ -UT_TEST(test_stage5_5_wait_events_count_snapshot_118) +UT_TEST(test_stage5_5_wait_events_count_snapshot_120) { /* The whole CR read-path band (5.51-5.57) adds NO new wait events — it reuses * the spec-3.9 ClusterCRConstruct event. spec-6.0a adds 7 block_device * wait events after that band; spec-6.1 adds 2 RDMA wait events; spec-6.2 - * adds 4 Smart Fusion authority waits; spec-6.13 adds 2 RDMA tier3 waits. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * adds 4 Smart Fusion authority waits; spec-6.13 adds 2 RDMA tier3 waits; + * spec-7.2 D6 adds 2 LMS DATA-plane waits. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } @@ -200,6 +201,6 @@ main(void) UT_RUN(test_stage5_5_cr_dump_category_names); UT_RUN(test_stage5_5_admission_policy_enum_locked); UT_RUN(test_stage5_5_cross_instance_coordinator_enums_locked); - UT_RUN(test_stage5_5_wait_events_count_snapshot_118); + UT_RUN(test_stage5_5_wait_events_count_snapshot_120); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c index 63750488ed..b5e83402dc 100644 --- a/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c @@ -191,7 +191,7 @@ UT_TEST(test_beta_wait_events_count) /* Current Stage 5 beta surface value. update-required contract: a future * spec adding cluster wait events MUST bump this snapshot (and the dump/test * baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ } diff --git a/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c index c672b41494..f998e5ea74 100644 --- a/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c @@ -192,7 +192,7 @@ UT_TEST(test_stage5_wait_events_count_and_multinode_set) * authority waits and spec-6.13 adds 2 RDMA tier3 waits. update-required * contract: a future spec adding cluster wait events MUST bump this snapshot * (and the dump/test baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 120); /* spec-7.2 D6 +2 */ /* The multi-node write-path wait events MG-B aggregates for the M2 share * must all be present and pairwise distinct (a reorder/removal would change diff --git a/src/test/cluster_unit/test_cluster_views.c b/src/test/cluster_unit/test_cluster_views.c index 00027881db..60eee45139 100644 --- a/src/test/cluster_unit/test_cluster_views.c +++ b/src/test/cluster_unit/test_cluster_views.c @@ -207,7 +207,7 @@ cluster_shmem_iter_regions(int *idx pg_attribute_unused(), UT_DEFINE_GLOBALS(); -UT_TEST(test_cluster_wait_events_count_is_118) +UT_TEST(test_cluster_wait_events_count_is_120) { /* * Cumulative registration roster: 61 prior + 3 added by spec-2.6 D11 @@ -239,8 +239,8 @@ UT_TEST(test_cluster_wait_events_count_is_118) * enum in wait_event.h and CLUSTER_WAIT_EVENTS_COUNT must move * together, and this test number must be bumped in lockstep. */ - /* spec-6.13 D8: RDMA tier3 wait events included -> 118. */ - UT_ASSERT_EQ(CLUSTER_WAIT_EVENTS_COUNT, 118); + /* spec-7.2 D6: +2 LMS data-plane -> 120. */ + UT_ASSERT_EQ(CLUSTER_WAIT_EVENTS_COUNT, 120); } @@ -285,7 +285,7 @@ int main(void) { UT_PLAN(5); - UT_RUN(test_cluster_wait_events_count_is_118); + UT_RUN(test_cluster_wait_events_count_is_120); UT_RUN(test_srf_symbol_linkable); UT_RUN(test_adg_srf_symbol_linkable); UT_RUN(test_first_event_is_ges_enqueue_acquire); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c new file mode 100644 index 0000000000..19b98855fa --- /dev/null +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -0,0 +1,890 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_xid_authority.c + * Runtime unit tests for the shared XID authority + native-era + * prehistory module (spec-6.15b D1/D2): on-disk layout lock, the pure + * classification and payload-sizing logic, the torn-safe read/write + * round trip with .bak fallback, seal/flag monotonicity, and a real + * prehistory publish -> adopt byte round trip between two fake PGDATA + * pg_xact trees. + * + * Like test_cluster_oid_lease.c this exercises the module for REAL + * against temp directories: fd.c openers map onto open(2)/close(2), + * pg_fsync onto fsync(2), durable_rename onto rename(2). The errstart + * stub aborts on elevel >= ERROR, so every FATAL/PANIC leg (corrupt + * authority at adopt, cap refusal, era re-entry) is TAP territory + * (t/361), not unit territory; reaching an abort here is a real bug. + * + * Covers (spec §4): + * U1 classify truth table (short / magic / CRC / valid), both files + * U2 publish monotonicity (native_hw never lowered) + * U3 flags one-way (SEALED / CLUSTER_ERA never cleared) + * U4 prehistory payload sizing + publish/adopt byte round trip + * U5 header layout locked by offset assertions + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2, §4 U1-U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#include +#include +#include + +#include "cluster/cluster_xid_authority.h" +#include "port/pg_crc32c.h" +#include "storage/fd.h" +#include "utils/elog.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* Global read by cluster_xid_authority.o's file I/O. */ +char *cluster_shared_data_dir = NULL; + +/* ---- Assert + ereport machinery (aborts on ERROR; the read paths never + * ereport, the write paths only PANIC on real I/O failure). ---- */ +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + if (elevel >= ERROR) { + printf("# unexpected ereport(elevel=%d) -- aborting\n", elevel); + abort(); + } + return false; +} +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{} +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} +int +errcode_for_file_access(void) +{ + return 0; +} +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errdetail(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errhint(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +/* ---- functional fd.c stubs: map onto the kernel ---- */ +int +OpenTransientFile(const char *fileName, int fileFlags) +{ + return open(fileName, fileFlags, 0600); +} +int +CloseTransientFile(int fd) +{ + return close(fd); +} +int +pg_fsync(int fd) +{ + return fsync(fd); +} +int +durable_rename(const char *oldfile, const char *newfile, int elevel pg_attribute_unused()) +{ + return (rename(oldfile, newfile) != 0) ? -1 : 0; +} + +/* ---- temp dir scaffolding ---- */ +static char test_root[MAXPGPATH]; + +static void +setup_shared_dir(void) +{ + char globaldir[MAXPGPATH]; + + snprintf(test_root, sizeof(test_root), "/tmp/pgrac_xid_auth_ut_%d", (int)getpid()); + mkdir(test_root, 0700); + snprintf(globaldir, sizeof(globaldir), "%s/global", test_root); + mkdir(globaldir, 0700); + cluster_shared_data_dir = test_root; +} + +static void +unlink_files(void) +{ + const char *rels[] = { CLUSTER_XID_AUTHORITY_REL_PATH, CLUSTER_XID_AUTHORITY_BAK_REL_PATH, + CLUSTER_XID_PREHISTORY_REL_PATH, CLUSTER_XID_PREHISTORY_BAK_REL_PATH }; + char p[MAXPGPATH]; + int i; + + for (i = 0; i < (int)lengthof(rels); i++) { + snprintf(p, sizeof(p), "%s/%s", test_root, rels[i]); + unlink(p); + } +} + +/* Build a fake PGDATA with a pg_xact/0000 of n_pages, filled with byte. */ +static void +make_fake_pgdata(char *pgdata, size_t len, const char *tag, int n_pages, unsigned char byte) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd; + int i; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + snprintf(seg, sizeof(seg), "%s/0000", dir); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + memset(page, byte, sizeof(page)); + for (i = 0; i < n_pages; i++) + (void)!write(fd, page, sizeof(page)); + close(fd); +} + +/* Build a fake PGDATA whose pg_xact spans SLRU segments: n_pages whole CLOG + * pages, 32 per segment file (0000, 0001, ...), page p filled with byte + * (p & 0xFF) so cross-segment ordering is byte-verifiable. */ +static void +make_fake_pgdata_multiseg(char *pgdata, size_t len, const char *tag, int n_pages) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd = -1; + int cur_seg = -1; + int p; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + for (p = 0; p < n_pages; p++) { + int segno = p / 32; + + if (segno != cur_seg) { + if (fd >= 0) + close(fd); + snprintf(seg, sizeof(seg), "%s/%04X", dir, segno); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + cur_seg = segno; + } + memset(page, p & 0xFF, sizeof(page)); + (void)!write(fd, page, sizeof(page)); + } + if (fd >= 0) + close(fd); +} + +/* ============================================================ + * U5: on-disk layout locked + * ============================================================ */ + +UT_TEST(test_layout_offsets_locked) +{ + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, flags), 8); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, reserved), 12); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, native_hw_full), 16); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, next_multi), 24); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, crc), 32); + + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, native_hw_full), 8); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, payload_len), 16); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, reserved), 20); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, crc), 24); +} + +/* ============================================================ + * U1: classify truth table + * ============================================================ */ + +UT_TEST(test_classify_short_magic_crc_valid) +{ + ClusterXidAuthorityHeader hdr; + char buf[sizeof(ClusterXidAuthorityHeader)]; + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_authority_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, 3), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = 0xDEADBEEF; + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr.version = CLUSTER_XID_AUTHORITY_VERSION; + hdr.native_hw_full = 816; + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr.crc); + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), CLUSTER_XID_AUTHORITY_VALID); + + buf[16] ^= 0x01; /* flip a native_hw bit -> CRC mismatch */ + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_CRC); +} + +/* ============================================================ + * U4: prehistory payload sizing + * ============================================================ */ + +UT_TEST(test_payload_bytes_boundaries) +{ + const uint32 xids_per_page = BLCKSZ * 4; /* CLOG: 2 bits/xact */ + + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(0), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(1), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(816), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page + 1), (uint32)(2 * BLCKSZ)); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID), + (uint32)(CLUSTER_XID_PREHISTORY_MAX_XID / 4)); + /* over the refusal cap: 0 = "no valid payload" (publish FATALs) */ + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID + 1), 0); +} + +/* ============================================================ + * torn-safe authority round trips + * ============================================================ */ + +UT_TEST(test_seed_if_absent_creates_unsealed) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0); + + /* second seed is a no-op and keeps the existing value */ + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(123456), false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_publish_monotone_and_seal) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + + /* interim (unsealed) publish raises the high-water */ + cluster_xid_authority_publish_native(810, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 810); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* sealing publish raises further and stamps SEALED */ + cluster_xid_authority_publish_native(816, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); + + /* a lower publish never lowers, and never clears SEALED */ + cluster_xid_authority_publish_native(500, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + +UT_TEST(test_mark_cluster_era_one_way) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + cluster_xid_authority_mark_cluster_era(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + /* later publishes keep both flags (never cleared) */ + cluster_xid_authority_publish_native(900, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_primary_corrupt_falls_back_to_bak) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + /* second write rolls the 791 image into .bak */ + cluster_xid_authority_publish_native(816, 1, true); + + /* corrupt the primary in place */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + /* falls back to the rolled 791 image */ + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_present_distinguishes_corrupt_from_absent) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + + /* a lone corrupt primary: present=true, read=false (fail-closed) */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR | O_CREAT | O_TRUNC, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), false); +} + +/* ============================================================ + * U4: prehistory publish -> adopt byte round trip + * ============================================================ */ + +UT_TEST(test_prehistory_publish_adopt_round_trip) +{ + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char seed_page[BLCKSZ]; + char join_page[BLCKSZ]; + int fd; + + unlink_files(); + + /* seed: one clog page of 0xAA; joiner: same-size page of zeros */ + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "join", 1, 0x00); + + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), false); + cluster_xid_prehistory_adopt(pgdata_join, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), true); + + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_seed); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, seed_page, BLCKSZ), BLCKSZ); + close(fd); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); + + /* adopt is idempotent: run again, bytes unchanged */ + cluster_xid_prehistory_adopt(pgdata_join, 816); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); +} + +UT_TEST(test_prehistory_multi_segment_round_trip) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; /* segment 0000 full (32 pages) + 0001 (1 page) */ + const uint64 hw = (uint64)n_pages * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char want[BLCKSZ]; + char got[BLCKSZ]; + int p; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "seedms", n_pages); + /* short pre-seed clone: only one zeroed page in segment 0000 */ + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "joinms", 1); + + cluster_xid_prehistory_publish(pgdata_seed, hw); + cluster_xid_prehistory_adopt(pgdata_join, hw); + + for (p = 0; p < n_pages; p++) { + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", pgdata_join, p / 32); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, (off_t)(p % 32) * BLCKSZ, SEEK_SET), (off_t)(p % 32) * BLCKSZ); + UT_ASSERT_EQ(read(fd, got, BLCKSZ), BLCKSZ); + close(fd); + memset(want, p & 0xFF, sizeof(want)); + UT_ASSERT_EQ(memcmp(got, want, BLCKSZ), 0); + } +} + +UT_TEST(test_unseal_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1: after begin_native_run, NO on-disk copy may still say SEALED -- + * a corrupt primary falling back to a stale SEALED .bak would hand a + * joiner the previous pass's high-water (8.A false-invisible). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_begin_native_run(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + /* read failing entirely (no trustworthy copy) is also acceptable: + * joiners fail closed on unavailable exactly like on unsealed. */ +} + +UT_TEST(test_mark_cluster_era_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1 variant: after mark_cluster_era, no on-disk copy may lack the + * one-way CLUSTER_ERA flag -- a stale .bak without it would let an + * enabled=off boot re-enter the native era on a formed tree. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_mark_cluster_era(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +/* + * write_raw_image / read_raw_image -- install / fetch one fixed-size + * authority image at `path` (raw open/write, no CRC games: the images + * moved around here are byte copies of real, valid on-disk images). + */ +static void +write_raw_image(const char *path, const char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(write(fd, image, len), (long long)len); + close(fd); +} + +static void +read_raw_image(const char *path, char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(read(fd, image, len), (long long)len); + close(fd); +} + +UT_TEST(test_unseal_torn_crash_repairs_stale_sealed_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1: a crash BETWEEN write_header_both's two renames leaves + * primary=UNSEALED(new) / .bak=SEALED(old high-water). Every native-era + * boot re-runs the transition, which must re-assert BOTH copies; before + * the fix it early-returned on the already-unsealed primary and the + * stale SEALED .bak survived the whole run -- any later primary read + * failure resurrected the previous pass's high-water (8.A). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* capture the pre-transition (SEALED) primary image */ + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + /* complete the transition, then re-install the old SEALED image as + * .bak, simulating the crash window between the two durable renames */ + cluster_xid_authority_begin_native_run(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next native-era boot re-runs the transition */ + cluster_xid_authority_begin_native_run(); + + /* the .bak copy itself must have been repaired to an unsealed image */ + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* a later primary read failure must fall back to the REPAIRED image, + * not resurrect the previous pass's sealed high-water */ + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); +} + +UT_TEST(test_mark_cluster_era_torn_crash_repairs_stale_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1 variant: the same torn-crash window across the + * CLUSTER_ERA stamp leaves a .bak without the one-way flag; a .bak + * fallback would then let an enabled=off boot re-enter the native era + * on a formed tree. The stamp is re-run by every cluster boot and + * must repair the lagging copy. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + cluster_xid_authority_mark_cluster_era(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next cluster boot re-runs the stamp */ + cluster_xid_authority_mark_cluster_era(); + + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_prefix_check_divergence_truth_table) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + int fd; + + /* F2: identical prefix -> CONSISTENT (both trees one page of 0xAA) */ + unlink_files(); + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "f2seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "f2join", 1, 0xAA); + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* flip one status byte INSIDE the limit -> DIVERGED */ + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 100, SEEK_SET), 100); /* byte 100 = xids 400..403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_DIVERGED); + + /* the same flipped byte OUTSIDE the limit -> CONSISTENT (adopt arm: + * bytes at/after own_next belong to the seed alone) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400, 0), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* partial-byte boundary: xid 400's 2-bit slot enters scope at limit 401 */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401, 0), + CLUSTER_XID_PREFIX_DIVERGED); + + /* missing local page inside the comparable range -> fail-closed + * UNAVAILABLE, never CONSISTENT: with oldestXid=0 nothing below the + * hole is frozen, so "shorter clone" is not a possible innocent + * explanation (review r3-X2) */ + unlink(seg); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_UNAVAILABLE); + + /* no trustworthy blob -> UNAVAILABLE */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_UNAVAILABLE); + + /* multi-segment divergence: 33-page trees, flip a byte in segment 0001 */ + { + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "f2seedms", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "f2joinms", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), + CLUSTER_XID_PREFIX_CONSISTENT); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "Z", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), + CLUSTER_XID_PREFIX_DIVERGED); + } +} + +UT_TEST(test_prefix_check_front_truncation) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + const uint64 page32_first = 32 * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + unsigned char b; + int fd; + + /* review r3-X2 (a): front truncation is no alibi. SimpleLruTruncate + * removed the joiner's segment 0000 (oldestXid frozen past it), but the + * surviving page 32 diverges inside the comparable range -> DIVERGED. + * Pre-fix, break-on-first-missing-local-page returned CONSISTENT and + * the adopt arm then overwrote the joiner's own live-xid outcomes. */ + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "ftseed", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "ftjoin", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + UT_ASSERT_EQ(unlink(seg), 0); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); /* byte 600 = slots 2400..2403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first), + CLUSTER_XID_PREFIX_DIVERGED); + + /* (b) frozen exemption: restore byte 600 (page fill 0x20), then flip + * ONLY slot 2400's two low bits (0x20 -> 0x21). With oldestXid at slot + * 2401 the divergence sits strictly below the boundary, mid-byte: the + * lead mask must exclude it -> CONSISTENT (missing segment 0000 below + * page(oldestXid) is equally exempt). */ + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); + b = 0x21; + (void)!write(fd, &b, 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2401), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* same bytes with oldestXid AT the divergent slot -> DIVERGED (the + * boundary slot itself is comparable) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_DIVERGED); + + /* review r3-X2 (c): a hole AT/ABOVE page(oldestXid) inside the + * comparable range is an anomaly -> UNAVAILABLE (fail-closed), never + * CONSISTENT */ + UT_ASSERT_EQ(unlink(seg), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_UNAVAILABLE); +} + +UT_TEST(test_prehistory_classify_corrupt) +{ + char buf[64]; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + /* publish a real blob, then flip a payload bit -> CRC catches it */ + { + char pgdata_seed[MAXPGPATH]; + char p[MAXPGPATH]; + char *image; + struct stat st; + int fd; + int r; + + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed2", 1, 0x55); + cluster_xid_prehistory_publish(pgdata_seed, 816); + + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_PREHISTORY_REL_PATH); + UT_ASSERT_EQ(stat(p, &st), 0); + image = malloc(st.st_size); + fd = open(p, O_RDONLY, 0); + r = (int)read(fd, image, st.st_size); + close(fd); + UT_ASSERT_EQ(r, (int)st.st_size); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_VALID); + + image[sizeof(ClusterXidPrehistoryHeader) + 100] ^= 0x01; + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_INVALID_CRC); + free(image); + } + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); +} + +UT_TEST(test_begin_native_run_unseals_before_cluster_era) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* a follow-up native run clears SEALED but keeps the high-water */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* idempotent while open */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* the run's clean shutdown re-publishes and re-seals monotonically */ + cluster_xid_authority_publish_native(900, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + +int +main(void) +{ + setup_shared_dir(); + + UT_PLAN(18); + UT_RUN(test_layout_offsets_locked); + UT_RUN(test_classify_short_magic_crc_valid); + UT_RUN(test_payload_bytes_boundaries); + UT_RUN(test_seed_if_absent_creates_unsealed); + UT_RUN(test_publish_monotone_and_seal); + UT_RUN(test_mark_cluster_era_one_way); + UT_RUN(test_begin_native_run_unseals_before_cluster_era); + UT_RUN(test_unseal_survives_primary_corruption_via_bak); + UT_RUN(test_mark_cluster_era_survives_primary_corruption_via_bak); + UT_RUN(test_unseal_torn_crash_repairs_stale_sealed_bak); + UT_RUN(test_mark_cluster_era_torn_crash_repairs_stale_bak); + UT_RUN(test_primary_corrupt_falls_back_to_bak); + UT_RUN(test_present_distinguishes_corrupt_from_absent); + UT_RUN(test_prehistory_publish_adopt_round_trip); + UT_RUN(test_prehistory_multi_segment_round_trip); + UT_RUN(test_prefix_check_divergence_truth_table); + UT_RUN(test_prefix_check_front_truncation); + UT_RUN(test_prehistory_classify_corrupt); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/perl/PostgreSQL/Test/ClusterPair.pm b/src/test/perl/PostgreSQL/Test/ClusterPair.pm index 74c1e0c34d..1f84f527f5 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterPair.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterPair.pm @@ -75,12 +75,16 @@ sub new_pair { my ($class, $cluster_name, %opts) = @_; - # Allocate 4 distinct free ports. get_free_port() reserves the port + # Allocate 6 distinct free ports. get_free_port() reserves the port # until the test exits, so successive calls return distinct values. + # spec-7.2 D2: every node also gets an explicit DATA-plane port + # (data_addr) -- allocator-provided, never offset-derived (r1-F2). my $pg_port_0 = PostgreSQL::Test::Cluster::get_free_port(); my $pg_port_1 = PostgreSQL::Test::Cluster::get_free_port(); my $ic_port_0 = PostgreSQL::Test::Cluster::get_free_port(); my $ic_port_1 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port_0 = PostgreSQL::Test::Cluster::get_free_port(); + my $data_port_1 = PostgreSQL::Test::Cluster::get_free_port(); my $node0 = PostgreSQL::Test::Cluster->new("${cluster_name}_node0", port => $pg_port_0); @@ -241,9 +245,11 @@ name = $cluster_name [node.0] interconnect_addr = 127.0.0.1:$ic_port_0 +data_addr = 127.0.0.1:$data_port_0 [node.1] interconnect_addr = 127.0.0.1:$ic_port_1 +data_addr = 127.0.0.1:$data_port_1 EOC # node1's pgrac.conf -- same except optionally a different cluster_name @@ -255,9 +261,11 @@ name = $alt_name [node.0] interconnect_addr = 127.0.0.1:$ic_port_0 +data_addr = 127.0.0.1:$data_port_0 [node.1] interconnect_addr = 127.0.0.1:$ic_port_1 +data_addr = 127.0.0.1:$data_port_1 EOC PostgreSQL::Test::Utils::append_to_file( @@ -271,6 +279,7 @@ EOC cluster_name => $cluster_name, pg_ports => [ $pg_port_0, $pg_port_1 ], ic_ports => [ $ic_port_0, $ic_port_1 ], + data_ports => [ $data_port_0, $data_port_1 ], voting_disk_paths => \@voting_disk_paths, wal_threads_root => $wal_threads_root, shared_data_root => $shared_data_root, diff --git a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm index 3b3a2bb29f..07242fc2ca 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm @@ -55,6 +55,29 @@ use PostgreSQL::Test::Utils; our $NODES = 4; + +# Relocate an init_from_backup node's copied pg_wal into its shared WAL thread +# directory, matching the layout initdb -X produces for the seed node. +sub _relocate_backup_pg_wal +{ + my ($node, $wal_threads_root, $thread_id) = @_; + my $pgwal = $node->data_dir . '/pg_wal'; + my $wal_thread = "$wal_threads_root/thread_$thread_id"; + + mkdir $wal_thread or die "mkdir $wal_thread: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal_thread/$e") + or die "rename $pgwal/$e -> $wal_thread/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal_thread, $pgwal) or die "symlink $pgwal -> $wal_thread: $!"; + return; +} + #----------------------------------------------------------------------- # new_quad($class, $cluster_name, %opts) # @@ -66,14 +89,19 @@ our $NODES = 4; # voting-disk files (QVOTEC reaches quorum OK) # wal_threads_root : 1 -> shared per-thread WAL root # shared_data : 1 -> shared data root (cluster_fs backend) +# shared_catalog : 1 -> t/337-style shared-catalog formation; +# implies shared_data + wal_threads_root #----------------------------------------------------------------------- sub new_quad { my ($class, $cluster_name, %opts) = @_; - # Allocate 8 distinct free ports (4 PG + 4 IC). + # Allocate 12 distinct free ports (4 PG + 4 IC + 4 DATA). + # spec-7.2 D2: DATA-plane ports are allocator-provided, never + # offset-derived (r1-F2). my @pg_ports; my @ic_ports; + my @data_ports; for (0 .. $NODES - 1) { push @pg_ports, PostgreSQL::Test::Cluster::get_free_port(); @@ -82,6 +110,10 @@ sub new_quad { push @ic_ports, PostgreSQL::Test::Cluster::get_free_port(); } + for (0 .. $NODES - 1) + { + push @data_ports, PostgreSQL::Test::Cluster::get_free_port(); + } my @nodes; for my $i (0 .. $NODES - 1) @@ -91,6 +123,12 @@ sub new_quad "${cluster_name}_node$i", port => $pg_ports[$i]); } + if ($opts{shared_catalog}) + { + $opts{shared_data} = 1; + $opts{wal_threads_root} = 1; + } + # spec-4.6: strict-mode opt-in (mirror ClusterTriple) -- pre-allocate N # shared voting-disk files so QVOTEC reaches quorum_state=OK and the GES # inbound validation (in_quorum, check 4) accepts cross-node traffic. @@ -127,23 +165,75 @@ sub new_quad if ($opts{shared_data}) { $shared_data_root = PostgreSQL::Test::Utils::tempdir(); + if ($opts{shared_catalog}) + { + mkdir "$shared_data_root/global" + or die "mkdir $shared_data_root/global: $!"; + } + } + + if ($opts{shared_catalog}) + { + my $sc_common = <init(allows_streaming => 1, + extra => [ '-X', "$wal_threads_root/thread_1" ]); + $nodes[0]->start; + $nodes[0]->backup('clusterquad_scb'); + $nodes[0]->stop; + + for my $i (1 .. $NODES - 1) + { + $nodes[$i]->init_from_backup($nodes[0], 'clusterquad_scb'); + _relocate_backup_pg_wal($nodes[$i], $wal_threads_root, $i + 1); + } + + # Seed the shared catalog/controlfile/OID authorities in a single-node era, + # then append the real cluster config below. Last GUC value wins. + $nodes[0]->append_conf('postgresql.conf', $sc_common); + $nodes[0]->append_conf('postgresql.conf', <start; + die "shared_catalog seed did not create catalog authority" + unless -e "$shared_data_root/global/pgrac_catalog_authority"; + $nodes[0]->stop; + } + else + { + my $wal_node_index = 0; + for my $node (@nodes) + { + if (defined $wal_threads_root) + { + my $thread_id = $wal_node_index + 1; + $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); + } + else + { + $node->init; + } + $wal_node_index++; + } } - my $wal_node_index = 0; for my $node (@nodes) { if (defined $wal_threads_root) { - my $thread_id = $wal_node_index + 1; - $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); $node->append_conf('postgresql.conf', "cluster.wal_threads_dir = '$wal_threads_root'\n"); } - else - { - $node->init; - } - $wal_node_index++; if (defined $shared_data_root) { @@ -153,6 +243,15 @@ sub new_quad "cluster.shared_data_dir = '$shared_data_root'\n"); $node->append_conf('postgresql.conf', "cluster.smgr_user_relations = on\n"); + if ($opts{shared_catalog}) + { + $node->append_conf('postgresql.conf', + "cluster.controlfile_shared_authority = on\n"); + $node->append_conf('postgresql.conf', + "cluster.shared_catalog = on\n"); + $node->append_conf('postgresql.conf', + "cluster.merged_recovery = on\n"); + } } # Enable cluster + tier1, same baseline as ClusterTriple (spec-2.2). @@ -194,8 +293,9 @@ sub new_quad my $peers_block = ""; for my $i (0 .. $NODES - 1) { - $peers_block .= - "[node.$i]\ninterconnect_addr = 127.0.0.1:$ic_ports[$i]\n\n"; + $peers_block .= "[node.$i]\n" + . "interconnect_addr = 127.0.0.1:$ic_ports[$i]\n" + . "data_addr = 127.0.0.1:$data_ports[$i]\n\n"; } my $pgrac_conf_body = < $cluster_name, pg_ports => \@pg_ports, ic_ports => \@ic_ports, + data_ports => \@data_ports, voting_disk_paths => \@voting_disk_paths, wal_threads_root => $wal_threads_root, shared_data_root => $shared_data_root, diff --git a/src/test/perl/PostgreSQL/Test/ClusterTriple.pm b/src/test/perl/PostgreSQL/Test/ClusterTriple.pm index 1268e1f524..4b625e1792 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterTriple.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterTriple.pm @@ -57,6 +57,13 @@ sub new_triple PostgreSQL::Test::Cluster::get_free_port(), PostgreSQL::Test::Cluster::get_free_port(), ); + # spec-7.2 D2: explicit DATA-plane ports (allocator-provided, never + # offset-derived -- r1-F2). + my @data_ports = ( + PostgreSQL::Test::Cluster::get_free_port(), + PostgreSQL::Test::Cluster::get_free_port(), + PostgreSQL::Test::Cluster::get_free_port(), + ); my @nodes; for my $i (0 .. 2) @@ -176,8 +183,9 @@ sub new_triple my $peers_block = ""; for my $i (0 .. 2) { - $peers_block .= - "[node.$i]\ninterconnect_addr = 127.0.0.1:$ic_ports[$i]\n\n"; + $peers_block .= "[node.$i]\n" + . "interconnect_addr = 127.0.0.1:$ic_ports[$i]\n" + . "data_addr = 127.0.0.1:$data_ports[$i]\n\n"; } my $pgrac_conf_body = < $cluster_name, pg_ports => \@pg_ports, ic_ports => \@ic_ports, + data_ports => \@data_ports, voting_disk_paths => \@voting_disk_paths, wal_threads_root => $wal_threads_root, shared_data_root => $shared_data_root,