From 14a90239431eabe6dba7648d6bac379d8db7fd1b Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 22 Jun 2026 06:38:34 -0400 Subject: [PATCH] perf(lock): stripe the locker-hash shared latch across cache lines The lock-get hot path resolves its locker under a single shared latch (mtx_lockers). After that latch was made shared (PR #28) the residual scaling wall is the one reader-count word inside the underlying pthread_rwlock: every lock_get/lock_put atomically RMWs the same counter, so all cores contend on a single cache line and throughput plateaus. Replace the single locker latch with an array of LOCK_LOCKER_STRIPES (64) shared latches. The read-only hot path (__lock_get_api -> getlocker lookup) read-locks just the one stripe selected by the locker id, so the per-operation reader-count atomics spread across 64 cache lines. Writers (locker create/free, free-list refill, and the deadlock-detector, failchk, and stat locker-list walks) lock all stripes exclusive, which is correct because two lockers in different stripes can hash to the same bucket chain; locking all stripes excludes every reader regardless of bucket. Stripe 0 is always taken first / released last for a total, deadlock-free order, and doubles as the representative latch for wait-time statistics. This subsumes both the "shared-latch reference-count cache line" and the "locker-hash partitioning" items: they are the same contended line. lock_bench on a 12-core/24-HT Xeon (base = master 8f207cf1e), ops/sec: distinct threads: 1 8 12 24 master 1.26M 6.23M 6.31M 6.12M (plateaus ~6M) striped 1.27M 9.04M 12.29M 15.45M (2.52x @24t) shared threads: 1 8 12 24 master 1.38M 6.34M 6.75M 6.16M striped 1.49M 8.87M 12.14M 15.50M (2.52x @24t) Striped tracks the ~16M no-lock ceiling and shows no single-thread regression. Correctness verified with the TCL suite: lock001, lock002, lock003 (multi-process, 5 processes against the shared region), txn001, txn002, ssi001, ssi002 all pass. --- src/dbinc/lock.h | 61 +++++++++++++++++++++++++++++++++++++----- src/lock/lock.c | 4 +-- src/lock/lock_region.c | 20 +++++++------- src/lock/lock_stat.c | 6 +++-- 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/src/dbinc/lock.h b/src/dbinc/lock.h index 1090779ef..685fc0600 100644 --- a/src/dbinc/lock.h +++ b/src/dbinc/lock.h @@ -15,6 +15,16 @@ extern "C" { #define DB_LOCK_DEFAULT_N 1000 /* Default # of locks in region. */ +/* + * Number of striped shared latches guarding the locker hash table. A power + * of two so the stripe index is a mask of the locker id. Readers (the + * lock-get hot path) take one stripe shared; writers take all stripes + * exclusive. Sized to comfortably exceed common core counts so the + * per-operation reader-count atomics rarely share a cache line. + */ +#define LOCK_LOCKER_STRIPES 64 +#define LOCK_LOCKER_STRIPE(id) ((id) & (LOCK_LOCKER_STRIPES - 1)) + /* * The locker id space is divided between the transaction manager and the lock * manager. Lock IDs start at 1 and go to DB_LOCK_MAXID. Txn IDs start at @@ -70,7 +80,16 @@ typedef struct __db_lockregion { /* SHARED */ u_int32_t detect; /* run dd on every conflict */ db_timespec next_timeout; /* next time to expire a lock */ db_mutex_t mtx_dd; /* mutex for lock object dd list. */ - db_mutex_t mtx_lockers; /* mutex for locker allocation. */ + /* + * Striped shared latches over the locker hash table. The lock-get + * hot path read-locks one stripe (chosen by locker id) so the + * per-operation reader-count atomics spread across LOCK_LOCKER_STRIPES + * cache lines instead of contending on a single rwlock counter. + * Writers (locker create/free and the deadlock-detector, failchk, and + * stat list walks) lock all stripes exclusive. Stripe 0 doubles as the + * representative mutex for wait-time statistics. + */ + db_mutex_t mtx_locker_stripe[LOCK_LOCKER_STRIPES]; SH_TAILQ_HEAD(__dobj) dd_objs; /* objects with waiters */ /* free locker header */ roff_t locker_mem_off; /* block memory for lockers */ @@ -327,12 +346,40 @@ struct __db_lock { /* SHARED */ MUTEX_LOCK(env, (region)->mtx_dd) #define UNLOCK_DD(env, region) \ MUTEX_UNLOCK(env, (region)->mtx_dd) -#define LOCK_LOCKERS(env, region) \ - MUTEX_LOCK(env, (region)->mtx_lockers) -#define RDLOCK_LOCKERS(env, region) \ - MUTEX_READLOCK(env, (region)->mtx_lockers) -#define UNLOCK_LOCKERS(env, region) \ - MUTEX_UNLOCK(env, (region)->mtx_lockers) +/* + * The locker hash table is guarded by a set of striped shared latches + * (mtx_locker_stripe[]). The lock-get hot path only reads a single bucket + * chain, so it read-locks the one stripe selected by the locker id; this + * spreads the per-operation reader-count atomics across LOCK_LOCKER_STRIPES + * cache lines instead of serializing on one rwlock counter. + * + * Any writer (locker create/free, free-list refill, and the global + * locker-list walks in the deadlock detector, failchk, and stat) may touch + * an arbitrary bucket chain, and two lockers in different stripes can hash + * to the same bucket. Writers therefore lock ALL stripes exclusive, which + * excludes every reader regardless of which bucket it is walking. Stripe 0 + * is always acquired first/released last so the ordering is total and + * deadlock-free. On any error the embedded MUTEX_* return propagates, after + * unwinding the stripes already taken. + * + * LOCK_LOCKERS/UNLOCK_LOCKERS keep their original meaning (mutual exclusion + * of the whole locker table); RDLOCK_LOCKER/RDUNLOCK_LOCKER are the new + * single-stripe shared fast path keyed by locker id. + */ +#define RDLOCK_LOCKER(env, region, id) \ + MUTEX_READLOCK(env, (region)->mtx_locker_stripe[LOCK_LOCKER_STRIPE(id)]) +#define RDUNLOCK_LOCKER(env, region, id) \ + MUTEX_UNLOCK(env, (region)->mtx_locker_stripe[LOCK_LOCKER_STRIPE(id)]) +#define LOCK_LOCKERS(env, region) do { \ + u_int32_t __s; \ + for (__s = 0; __s < LOCK_LOCKER_STRIPES; __s++) \ + MUTEX_LOCK(env, (region)->mtx_locker_stripe[__s]); \ +} while (0) +#define UNLOCK_LOCKERS(env, region) do { \ + u_int32_t __s; \ + for (__s = LOCK_LOCKER_STRIPES; __s-- > 0; ) \ + MUTEX_UNLOCK(env, (region)->mtx_locker_stripe[__s]); \ +} while (0) /* * __lock_locker_hash -- diff --git a/src/lock/lock.c b/src/lock/lock.c index 7bd16e870..2f95f4259 100644 --- a/src/lock/lock.c +++ b/src/lock/lock.c @@ -596,9 +596,9 @@ __lock_get_api(env, locker, flags, obj, lock_mode, lock) region = env->lk_handle->reginfo.primary; - RDLOCK_LOCKERS(env, region); + RDLOCK_LOCKER(env, region, locker); ret = __lock_getlocker_int(env->lk_handle, locker, 0, &sh_locker); - UNLOCK_LOCKERS(env, region); + RDUNLOCK_LOCKER(env, region, locker); LOCK_SYSTEM_LOCK(env->lk_handle, region); if (ret == 0) ret = __lock_get_internal(env->lk_handle, diff --git a/src/lock/lock_region.c b/src/lock/lock_region.c index 29695cdfb..6e792a1b2 100644 --- a/src/lock/lock_region.c +++ b/src/lock/lock_region.c @@ -254,16 +254,18 @@ __lock_region_init(env, lt) return (ret); /* - * The locker mutex is a SHARED latch: the hot lock-get path looks up - * an existing locker (a read-only hash walk) and takes it shared, so - * many cores can resolve their locker concurrently; locker create, - * free, the deadlock detector's locker-list walk, failchk, and stat - * take it exclusive. This removes the per-operation global - * serialization on lock_get/lock_put. + * The locker hash table is guarded by a set of striped SHARED latches. + * The hot lock-get path looks up an existing locker (a read-only hash + * walk) and takes one stripe shared, so many cores resolve their + * locker concurrently without contending on a single rwlock reader + * count; locker create, free, the deadlock detector's locker-list + * walk, failchk, and stat take all stripes exclusive. This removes the + * per-operation global serialization on lock_get/lock_put. */ - if ((ret = __mutex_alloc( - env, MTX_LOCK_REGION, DB_MUTEX_SHARED, ®ion->mtx_lockers)) != 0) - return (ret); + for (i = 0; i < LOCK_LOCKER_STRIPES; i++) + if ((ret = __mutex_alloc(env, MTX_LOCK_REGION, + DB_MUTEX_SHARED, ®ion->mtx_locker_stripe[i])) != 0) + return (ret); /* Allocate room for the locker hash table and initialize it. */ if ((ret = __env_alloc(<->reginfo, diff --git a/src/lock/lock_stat.c b/src/lock/lock_stat.c index f75ef204e..de5987f0b 100644 --- a/src/lock/lock_stat.c +++ b/src/lock/lock_stat.c @@ -164,7 +164,7 @@ __lock_stat(env, statp, flags) &stats->st_region_wait, &stats->st_region_nowait); __mutex_set_wait_info(env, region->mtx_dd, &stats->st_objs_wait, &stats->st_objs_nowait); - __mutex_set_wait_info(env, region->mtx_lockers, + __mutex_set_wait_info(env, region->mtx_locker_stripe[0], &stats->st_lockers_wait, &stats->st_lockers_nowait); stats->st_regsize = lt->reginfo.rp->size; if (LF_ISSET(DB_STAT_CLEAR)) { @@ -173,7 +173,9 @@ __lock_stat(env, statp, flags) if (!LF_ISSET(DB_STAT_SUBSYSTEM)) { __mutex_clear(env, region->mtx_region); __mutex_clear(env, region->mtx_dd); - __mutex_clear(env, region->mtx_lockers); + for (i = 0; i < LOCK_LOCKER_STRIPES; i++) + __mutex_clear(env, + region->mtx_locker_stripe[i]); for (i = 0; i < region->part_t_size; i++) __mutex_clear(env, lt->part_array[i].mtx_part); }