From ea5b43fc1241e67f6812e2f376ce3214a9ff0bd9 Mon Sep 17 00:00:00 2001 From: Phoebe Goldman Date: Tue, 26 May 2026 13:08:52 -0400 Subject: [PATCH 1/4] V8 heap metrics: track per instance, with a unique integer ID Prior to this commit, the `V8HeapMetrics` were tracked only for the "main" instance of a database, i.e. the reducer worker. This meant that we had little to no visibility into memory usage by procedures. In this commit, we add a label `instance_id` to all of those metrics, a `u64` ID unique (scoped to the database) to the V8 instance. The ID is set to 0 for the main worker, and drawn from an `AtomicU64` counter starting at 1 for the procedure workers. As part of this change, I've made it so that `V8HeapMetrics::drop` does `remove_label_values`, and `V8HeapMetrics::observe` uses `set` rather than inc/dec by a delta. The previous code appears to have been in an odd middle ground, where the metrics were used only for the main worker, but were treated in `observe` and `drop` as if they might be a concurrent aggregation of multiple workers' values. Relatedly, `remove_database_gauges` (in `crates/core/src/host/host_controller.rs`) no longer needs to clean up the database gauges. It couldn't if it wanted to, either, 'cause it won't know the set of instance IDs to remove. --- crates/core/src/host/host_controller.rs | 24 +-- crates/core/src/host/v8/mod.rs | 206 +++++++++++++++--------- crates/core/src/worker_metrics/mod.rs | 34 ++-- 3 files changed, 150 insertions(+), 114 deletions(-) diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index fbed6f33ebb..d9c287d4841 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1411,26 +1411,8 @@ where .data_size_blob_store_bytes_used_by_blobs .remove_label_values(db); let _ = WORKER_METRICS.wasm_memory_bytes.remove_label_values(db); - let worker_kind = crate::host::v8::V8_WORKER_KIND_MAIN; - let _ = WORKER_METRICS - .v8_total_heap_size_bytes - .remove_label_values(db, worker_kind); - let _ = WORKER_METRICS - .v8_total_physical_size_bytes - .remove_label_values(db, worker_kind); - let _ = WORKER_METRICS - .v8_used_global_handles_size_bytes - .remove_label_values(db, worker_kind); - let _ = WORKER_METRICS - .v8_used_heap_size_bytes - .remove_label_values(db, worker_kind); - let _ = WORKER_METRICS - .v8_heap_size_limit_bytes - .remove_label_values(db, worker_kind); - let _ = WORKER_METRICS - .v8_external_memory_bytes - .remove_label_values(db, worker_kind); - let _ = WORKER_METRICS.v8_native_contexts.remove_label_values(db, worker_kind); - let _ = WORKER_METRICS.v8_detached_contexts.remove_label_values(db, worker_kind); + + // N.b.: the V8 heap metrics that are handled by `V8HeapMetrics` will be cleaned up by that type's `drop` method. + let _ = WORKER_METRICS.v8_request_queue_length.remove_label_values(db); } diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index db5b64bf5fc..8fae748111c 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -109,6 +109,7 @@ use std::cell::Cell; use std::num::NonZeroUsize; use std::os::raw::c_void; use std::panic::{self, AssertUnwindSafe}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, LazyLock}; use std::time::Instant; use tokio::sync::{mpsc, oneshot}; @@ -169,14 +170,22 @@ impl V8Runtime { static V8_RUNTIME_GLOBAL: LazyLock = LazyLock::new(V8RuntimeInner::init); const REDUCER_ARGS_BUFFER_SIZE: usize = 4_096; const JS_PROCEDURE_INSTANCE_QUEUE_CAPACITY: usize = 1; -pub(crate) const V8_WORKER_KIND_MAIN: &str = "main"; #[derive(Copy, Clone)] -enum JsWorkerKind { +pub(crate) enum JsWorkerKind { Main, Procedure, } +impl AsRef for JsWorkerKind { + fn as_ref(&self) -> &str { + match self { + Self::Main => "main", + Self::Procedure => "procedure", + } + } +} + impl JsWorkerKind { const fn checks_heap(self) -> bool { matches!(self, Self::Main) @@ -258,6 +267,7 @@ impl V8RuntimeInner { procedure_instance_pool_size: config.procedure_instance_pool_size, heap_policy: config.heap_policy, metrics, + instance_id_source: Arc::new(AtomicU64::new(FIRST_PROCEDURE_WORKER_INSTANCE_ID)), }; Ok(ModuleWithInstance::Js { module, init_inst }) @@ -273,6 +283,11 @@ pub struct JsModule { procedure_instance_pool_size: NonZeroUsize, heap_policy: V8HeapPolicyConfig, metrics: InstanceManagerMetrics, + + /// Source for instance IDs, which are used as metrics labels. + /// + /// When creating a new instance, [`AtomicU64::fetch_add`] from this with [`Ordering::Relaxed`]. + instance_id_source: Arc, } impl JsModule { @@ -304,6 +319,9 @@ impl JsModule { let heap_policy = self.heap_policy; let metrics = self.metrics.clone(); + // Relaxed ordering fine: this is a source of unique IDs, not a synch primitive. + let instance_id = self.instance_id_source.fetch_add(1, Ordering::Relaxed); + // This has to be done in a blocking context because of `blocking_recv`. let (_, instance) = spawn_procedure_instance_worker( program, @@ -312,6 +330,7 @@ impl JsModule { core_pinner, heap_policy, metrics, + instance_id, ) .await .expect("`spawn_procedure_instance_worker` should succeed when passed `ModuleCommon`"); @@ -947,6 +966,13 @@ fn handle_detached_worker_request( } struct V8HeapMetrics { + /// So we can `remove_label_values` during `drop`. + database_identity: Identity, + /// So we can `remove_label_values` during `drop`. + worker_kind: JsWorkerKind, + /// So we can `remove_label_values` during `drop`. + instance_id: u64, + total_heap_size_bytes: IntGauge, total_physical_size_bytes: IntGauge, used_global_handles_size_bytes: IntGauge, @@ -955,7 +981,6 @@ struct V8HeapMetrics { external_memory_bytes: IntGauge, native_contexts: IntGauge, detached_contexts: IntGauge, - last_observed: V8HeapSnapshot, } #[derive(Clone, Copy, Default)] @@ -986,79 +1011,104 @@ impl V8HeapSnapshot { } impl V8HeapMetrics { - fn new(database_identity: &Identity) -> Self { + fn new(database_identity: &Identity, worker_kind: JsWorkerKind, instance_id: u64) -> Self { Self { - total_heap_size_bytes: WORKER_METRICS - .v8_total_heap_size_bytes - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - total_physical_size_bytes: WORKER_METRICS - .v8_total_physical_size_bytes - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - used_global_handles_size_bytes: WORKER_METRICS - .v8_used_global_handles_size_bytes - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - used_heap_size_bytes: WORKER_METRICS - .v8_used_heap_size_bytes - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - heap_size_limit_bytes: WORKER_METRICS - .v8_heap_size_limit_bytes - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - external_memory_bytes: WORKER_METRICS - .v8_external_memory_bytes - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - native_contexts: WORKER_METRICS - .v8_native_contexts - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - detached_contexts: WORKER_METRICS - .v8_detached_contexts - .with_label_values(database_identity, V8_WORKER_KIND_MAIN), - last_observed: V8HeapSnapshot::default(), + database_identity: *database_identity, + worker_kind, + instance_id, + total_heap_size_bytes: WORKER_METRICS.v8_total_heap_size_bytes.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + total_physical_size_bytes: WORKER_METRICS.v8_total_physical_size_bytes.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + used_global_handles_size_bytes: WORKER_METRICS.v8_used_global_handles_size_bytes.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + used_heap_size_bytes: WORKER_METRICS.v8_used_heap_size_bytes.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + heap_size_limit_bytes: WORKER_METRICS.v8_heap_size_limit_bytes.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + external_memory_bytes: WORKER_METRICS.v8_external_memory_bytes.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + native_contexts: WORKER_METRICS.v8_native_contexts.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), + detached_contexts: WORKER_METRICS.v8_detached_contexts.with_label_values( + database_identity, + &worker_kind, + &instance_id, + ), } } - fn adjust_by(&self, delta: V8HeapSnapshot) { - adjust_gauge(&self.total_heap_size_bytes, delta.total_heap_size_bytes); - adjust_gauge(&self.total_physical_size_bytes, delta.total_physical_size_bytes); - adjust_gauge( - &self.used_global_handles_size_bytes, - delta.used_global_handles_size_bytes, - ); - adjust_gauge(&self.used_heap_size_bytes, delta.used_heap_size_bytes); - adjust_gauge(&self.heap_size_limit_bytes, delta.heap_size_limit_bytes); - adjust_gauge(&self.external_memory_bytes, delta.external_memory_bytes); - adjust_gauge(&self.native_contexts, delta.native_contexts); - adjust_gauge(&self.detached_contexts, delta.detached_contexts); - } - fn observe(&mut self, stats: &v8::HeapStatistics) { - let next = V8HeapSnapshot::from_stats(stats); - self.adjust_by(V8HeapSnapshot { - total_heap_size_bytes: next.total_heap_size_bytes - self.last_observed.total_heap_size_bytes, - total_physical_size_bytes: next.total_physical_size_bytes - self.last_observed.total_physical_size_bytes, - used_global_handles_size_bytes: next.used_global_handles_size_bytes - - self.last_observed.used_global_handles_size_bytes, - used_heap_size_bytes: next.used_heap_size_bytes - self.last_observed.used_heap_size_bytes, - heap_size_limit_bytes: next.heap_size_limit_bytes - self.last_observed.heap_size_limit_bytes, - external_memory_bytes: next.external_memory_bytes - self.last_observed.external_memory_bytes, - native_contexts: next.native_contexts - self.last_observed.native_contexts, - detached_contexts: next.detached_contexts - self.last_observed.detached_contexts, - }); - self.last_observed = next; + self.total_heap_size_bytes.set(stats.total_heap_size() as i64); + self.total_physical_size_bytes.set(stats.total_physical_size() as i64); + self.used_global_handles_size_bytes + .set(stats.used_global_handles_size() as i64); + self.used_heap_size_bytes.set(stats.used_heap_size() as i64); + self.heap_size_limit_bytes.set(stats.heap_size_limit() as i64); + self.external_memory_bytes.set(stats.external_memory() as i64); + self.native_contexts.set(stats.number_of_native_contexts() as i64); + self.detached_contexts.set(stats.number_of_detached_contexts() as i64); } } impl Drop for V8HeapMetrics { fn drop(&mut self) { - self.adjust_by(V8HeapSnapshot { - total_heap_size_bytes: -self.last_observed.total_heap_size_bytes, - total_physical_size_bytes: -self.last_observed.total_physical_size_bytes, - used_global_handles_size_bytes: -self.last_observed.used_global_handles_size_bytes, - used_heap_size_bytes: -self.last_observed.used_heap_size_bytes, - heap_size_limit_bytes: -self.last_observed.heap_size_limit_bytes, - external_memory_bytes: -self.last_observed.external_memory_bytes, - native_contexts: -self.last_observed.native_contexts, - detached_contexts: -self.last_observed.detached_contexts, - }); + WORKER_METRICS.v8_total_heap_size_bytes.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); + WORKER_METRICS.v8_total_physical_size_bytes.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); + WORKER_METRICS.v8_used_global_handles_size_bytes.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); + WORKER_METRICS.v8_heap_size_limit_bytes.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); + WORKER_METRICS.v8_external_memory_bytes.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); + WORKER_METRICS.v8_native_contexts.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); + WORKER_METRICS.v8_detached_contexts.remove_label_values( + &self.database_identity, + &self.worker_kind, + &self.instance_id, + ); } } @@ -1193,6 +1243,9 @@ where f(&mut guard) } +const MAIN_WORKER_INSTANCE_ID: u64 = 0; +const FIRST_PROCEDURE_WORKER_INSTANCE_ID: u64 = MAIN_WORKER_INSTANCE_ID + 1; + async fn spawn_main_instance_worker( program: Arc, module_or_mcc: Either, @@ -1208,6 +1261,7 @@ async fn spawn_main_instance_worker( core_pinner, heap_policy, metrics, + MAIN_WORKER_INSTANCE_ID, ) .await } @@ -1219,6 +1273,7 @@ async fn spawn_procedure_instance_worker( core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, metrics: InstanceManagerMetrics, + instance_id: u64, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( program, @@ -1227,6 +1282,7 @@ async fn spawn_procedure_instance_worker( core_pinner, heap_policy, metrics, + instance_id, ) .await } @@ -1506,6 +1562,9 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// /// `load_balance_guard` and `core_pinner` should both be from the same /// [`AllocatedJobCore`], and are used to manage the core pinning of this thread. +/// +/// `instance_kind` should be either [`V8_WORKER_KIND_MAIN`] or [`V8_WORKER_KIND_PROCEDURE`]. +/// `instance_id` should be a unique integer sourced async fn spawn_instance_worker( program: Arc, module_or_mcc: Either, @@ -1513,6 +1572,7 @@ async fn spawn_instance_worker( mut core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, instance_metrics: InstanceManagerMetrics, + instance_id: u64, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where W: JsWorkerSpec + 'static, @@ -1625,9 +1685,7 @@ where let info = &module_common.info(); let mut instance_common = InstanceCommon::new(&module_common); let replica_ctx: &Arc = module_common.replica_ctx(); - let mut heap_metrics = worker_kind - .checks_heap() - .then(|| V8HeapMetrics::new(&info.database_identity)); + let mut heap_metrics = V8HeapMetrics::new(&info.database_identity, worker_kind, instance_id); let mut inst = V8Instance { scope, @@ -1639,9 +1697,7 @@ where .with_label_values(&info.database_identity), initial_heap_limit: heap_policy.heap_limit_bytes, }; - if let Some(heap_metrics) = heap_metrics.as_mut() { - let _initial_heap_stats = sample_heap_stats(inst.scope, heap_metrics); - } + let _initial_heap_stats = sample_heap_stats(inst.scope, &mut heap_metrics); // Process requests to the worker. // @@ -1655,9 +1711,7 @@ where let mut outcome = W::handle_request(request, &mut instance_common, &mut inst, &module_common, replica_ctx); - if let WorkerRequestOutcome::Continue = outcome - && let Some(heap_metrics) = heap_metrics.as_mut() - { + if let WorkerRequestOutcome::Continue = outcome { let request_check_due = heap_policy.heap_check_request_interval.is_some_and(|interval| { requests_since_heap_check += 1; requests_since_heap_check >= interval @@ -1669,7 +1723,7 @@ where requests_since_heap_check = 0; last_heap_check_at = Instant::now(); if let Some((used, limit)) = - should_retire_worker_for_heap(inst.scope, heap_metrics, heap_policy) + should_retire_worker_for_heap(inst.scope, &mut heap_metrics, heap_policy) { outcome = outcome.recreate_instance(); log::warn!( diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index c1847fa6d1c..f955da398a2 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -1,6 +1,6 @@ -use crate::hash::Hash; use crate::messages::control_db::HostType; use crate::subscription::row_list_builder_pool::BsatnRowListBuilderPool; +use crate::{hash::Hash, host::v8::JsWorkerKind}; use once_cell::sync::Lazy; use prometheus::{GaugeVec, HistogramVec, IntCounterVec, IntGaugeVec}; use spacetimedb_datastore::execution_context::WorkloadType; @@ -288,28 +288,28 @@ metrics_group!( pub wasm_memory_bytes: IntGaugeVec, #[name = spacetime_worker_v8_total_heap_size_bytes] - #[help = "The total size of the V8 heap for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The total size of the V8 heap for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_total_heap_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_total_physical_size_bytes] - #[help = "The total committed physical V8 heap memory for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The total committed physical V8 heap memory for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_total_physical_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_used_global_handles_size_bytes] - #[help = "The used size of V8 global handles for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The used size of V8 global handles for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_used_global_handles_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_used_heap_size_bytes] - #[help = "The live V8 heap size for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The live V8 heap size for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_used_heap_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_heap_size_limit_bytes] - #[help = "The V8 heap size limit for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The V8 heap size limit for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_heap_size_limit_bytes: IntGaugeVec, #[name = spacetime_worker_v8_heap_limit_hit] @@ -318,18 +318,18 @@ metrics_group!( pub v8_heap_limit_hit: IntCounterVec, #[name = spacetime_worker_v8_external_memory_bytes] - #[help = "The external memory tracked by V8 for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The external memory tracked by V8 for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_external_memory_bytes: IntGaugeVec, #[name = spacetime_worker_v8_native_contexts] - #[help = "The number of native V8 contexts for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The number of native V8 contexts for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_native_contexts: IntGaugeVec, #[name = spacetime_worker_v8_detached_contexts] - #[help = "The number of detached V8 contexts for a database's tracked JS worker kind (currently main only)"] - #[labels(database_identity: Identity, worker_kind: str)] + #[help = "The number of detached V8 contexts for a database's tracked JS worker kind"] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] pub v8_detached_contexts: IntGaugeVec, #[name = spacetime_worker_v8_request_queue_length] From 38f192865031627043b74ffe067976c04f8b331c Mon Sep 17 00:00:00 2001 From: Phoebe Goldman Date: Tue, 26 May 2026 13:40:22 -0400 Subject: [PATCH 2/4] Clippy Explicitly ignore `Result` from `remove_label_values`, delete some dead code. --- crates/core/src/host/v8/mod.rs | 55 +++++----------------------------- 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 8fae748111c..dcd227c6d06 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -186,12 +186,6 @@ impl AsRef for JsWorkerKind { } } -impl JsWorkerKind { - const fn checks_heap(self) -> bool { - matches!(self, Self::Main) - } -} - /// The actual V8 runtime, with initialization of V8. struct V8RuntimeInner { _priv: (), @@ -983,33 +977,6 @@ struct V8HeapMetrics { detached_contexts: IntGauge, } -#[derive(Clone, Copy, Default)] -struct V8HeapSnapshot { - total_heap_size_bytes: i64, - total_physical_size_bytes: i64, - used_global_handles_size_bytes: i64, - used_heap_size_bytes: i64, - heap_size_limit_bytes: i64, - external_memory_bytes: i64, - native_contexts: i64, - detached_contexts: i64, -} - -impl V8HeapSnapshot { - fn from_stats(stats: &v8::HeapStatistics) -> Self { - Self { - total_heap_size_bytes: stats.total_heap_size() as i64, - total_physical_size_bytes: stats.total_physical_size() as i64, - used_global_handles_size_bytes: stats.used_global_handles_size() as i64, - used_heap_size_bytes: stats.used_heap_size() as i64, - heap_size_limit_bytes: stats.heap_size_limit() as i64, - external_memory_bytes: stats.external_memory() as i64, - native_contexts: stats.number_of_native_contexts() as i64, - detached_contexts: stats.number_of_detached_contexts() as i64, - } - } -} - impl V8HeapMetrics { fn new(database_identity: &Identity, worker_kind: JsWorkerKind, instance_id: u64) -> Self { Self { @@ -1074,37 +1041,37 @@ impl V8HeapMetrics { impl Drop for V8HeapMetrics { fn drop(&mut self) { - WORKER_METRICS.v8_total_heap_size_bytes.remove_label_values( + let _ = WORKER_METRICS.v8_total_heap_size_bytes.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, ); - WORKER_METRICS.v8_total_physical_size_bytes.remove_label_values( + let _ = WORKER_METRICS.v8_total_physical_size_bytes.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, ); - WORKER_METRICS.v8_used_global_handles_size_bytes.remove_label_values( + let _ = WORKER_METRICS.v8_used_global_handles_size_bytes.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, ); - WORKER_METRICS.v8_heap_size_limit_bytes.remove_label_values( + let _ = WORKER_METRICS.v8_heap_size_limit_bytes.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, ); - WORKER_METRICS.v8_external_memory_bytes.remove_label_values( + let _ = WORKER_METRICS.v8_external_memory_bytes.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, ); - WORKER_METRICS.v8_native_contexts.remove_label_values( + let _ = WORKER_METRICS.v8_native_contexts.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, ); - WORKER_METRICS.v8_detached_contexts.remove_label_values( + let _ = WORKER_METRICS.v8_detached_contexts.remove_label_values( &self.database_identity, &self.worker_kind, &self.instance_id, @@ -1112,14 +1079,6 @@ impl Drop for V8HeapMetrics { } } -fn adjust_gauge(gauge: &IntGauge, delta: i64) { - if delta > 0 { - gauge.add(delta); - } else if delta < 0 { - gauge.sub(-delta); - } -} - fn sample_heap_stats(scope: &mut PinScope<'_, '_>, metrics: &mut V8HeapMetrics) -> v8::HeapStatistics { // Whenever we sample heap statistics, we cache them on the isolate so that // the per-call execution stats can avoid querying them on each invocation. From d4f4b9aea1ed47febfc32a5f0222198fad53b80a Mon Sep 17 00:00:00 2001 From: Phoebe Goldman Date: Tue, 26 May 2026 18:01:31 -0400 Subject: [PATCH 3/4] Remove `instance_id` label; restore inc/dec of shared label values Review flagged cardinality of these metrics as a concern, as even when we properly clean up unused entries with `remove_label_values`, all values remain live in the remote Prometheus database. As such, this commit reverts part of the PR's previous changes, so that all of the procedure workers for a given database share a single set of label values, and incrementally update the measurements in that single set of metric entries. --- crates/core/src/host/host_controller.rs | 3 +- crates/core/src/host/v8/mod.rs | 260 +++++++++++++----------- crates/core/src/worker_metrics/mod.rs | 16 +- 3 files changed, 152 insertions(+), 127 deletions(-) diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index d9c287d4841..2fec6f79ebf 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,5 +1,6 @@ use super::module_host::{EventStatus, ModuleHost, ModuleInfo, NoSuchModule}; use super::scheduler::SchedulerStarter; +use super::v8::V8HeapMetrics; use super::wasmtime::WasmtimeRuntime; use super::{Scheduler, UpdateDatabaseResult}; use crate::client::{ClientActorId, ClientName}; @@ -1412,7 +1413,7 @@ where .remove_label_values(db); let _ = WORKER_METRICS.wasm_memory_bytes.remove_label_values(db); - // N.b.: the V8 heap metrics that are handled by `V8HeapMetrics` will be cleaned up by that type's `drop` method. + V8HeapMetrics::remove_all_metric_label_values_for_database(db); let _ = WORKER_METRICS.v8_request_queue_length.remove_label_values(db); } diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index dcd227c6d06..63c8e409424 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -109,7 +109,6 @@ use std::cell::Cell; use std::num::NonZeroUsize; use std::os::raw::c_void; use std::panic::{self, AssertUnwindSafe}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, LazyLock}; use std::time::Instant; use tokio::sync::{mpsc, oneshot}; @@ -261,7 +260,6 @@ impl V8RuntimeInner { procedure_instance_pool_size: config.procedure_instance_pool_size, heap_policy: config.heap_policy, metrics, - instance_id_source: Arc::new(AtomicU64::new(FIRST_PROCEDURE_WORKER_INSTANCE_ID)), }; Ok(ModuleWithInstance::Js { module, init_inst }) @@ -277,11 +275,6 @@ pub struct JsModule { procedure_instance_pool_size: NonZeroUsize, heap_policy: V8HeapPolicyConfig, metrics: InstanceManagerMetrics, - - /// Source for instance IDs, which are used as metrics labels. - /// - /// When creating a new instance, [`AtomicU64::fetch_add`] from this with [`Ordering::Relaxed`]. - instance_id_source: Arc, } impl JsModule { @@ -313,9 +306,6 @@ impl JsModule { let heap_policy = self.heap_policy; let metrics = self.metrics.clone(); - // Relaxed ordering fine: this is a source of unique IDs, not a synch primitive. - let instance_id = self.instance_id_source.fetch_add(1, Ordering::Relaxed); - // This has to be done in a blocking context because of `blocking_recv`. let (_, instance) = spawn_procedure_instance_worker( program, @@ -324,7 +314,6 @@ impl JsModule { core_pinner, heap_policy, metrics, - instance_id, ) .await .expect("`spawn_procedure_instance_worker` should succeed when passed `ModuleCommon`"); @@ -959,14 +948,7 @@ fn handle_detached_worker_request( } } -struct V8HeapMetrics { - /// So we can `remove_label_values` during `drop`. - database_identity: Identity, - /// So we can `remove_label_values` during `drop`. - worker_kind: JsWorkerKind, - /// So we can `remove_label_values` during `drop`. - instance_id: u64, - +pub(in crate::host) struct V8HeapMetrics { total_heap_size_bytes: IntGauge, total_physical_size_bytes: IntGauge, used_global_handles_size_bytes: IntGauge, @@ -975,107 +957,159 @@ struct V8HeapMetrics { external_memory_bytes: IntGauge, native_contexts: IntGauge, detached_contexts: IntGauge, + + /// Previous values observed by this instance. + /// + /// In [`Self::observe`], we use this to compute deltas against the new instance's values, + /// then increment/decrement the metric values by those deltas. + /// We do this rather than `set`ting the metric values as multiple instances may coexist + /// and share the same metric label values. + /// This happens when a database has multiple procedure workers running, + /// and during a module update, as there is a period when the new version has already been created + /// but the old version has not yet shut down. + last_observed: V8HeapSnapshot, +} + +#[derive(Clone, Copy, Default)] +struct V8HeapSnapshot { + total_heap_size_bytes: i64, + total_physical_size_bytes: i64, + used_global_handles_size_bytes: i64, + used_heap_size_bytes: i64, + heap_size_limit_bytes: i64, + external_memory_bytes: i64, + native_contexts: i64, + detached_contexts: i64, +} + +impl V8HeapSnapshot { + fn from_stats(stats: &v8::HeapStatistics) -> Self { + Self { + total_heap_size_bytes: stats.total_heap_size() as i64, + total_physical_size_bytes: stats.total_physical_size() as i64, + used_global_handles_size_bytes: stats.used_global_handles_size() as i64, + used_heap_size_bytes: stats.used_heap_size() as i64, + heap_size_limit_bytes: stats.heap_size_limit() as i64, + external_memory_bytes: stats.external_memory() as i64, + native_contexts: stats.number_of_native_contexts() as i64, + detached_contexts: stats.number_of_detached_contexts() as i64, + } + } } impl V8HeapMetrics { - fn new(database_identity: &Identity, worker_kind: JsWorkerKind, instance_id: u64) -> Self { + pub(in crate::host) fn remove_all_metric_label_values_for_database(database_identity: &Identity) { + for worker_kind in [JsWorkerKind::Main, JsWorkerKind::Procedure] { + let _ = WORKER_METRICS + .v8_total_heap_size_bytes + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_total_physical_size_bytes + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_used_global_handles_size_bytes + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_used_heap_size_bytes + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_heap_size_limit_bytes + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_external_memory_bytes + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_native_contexts + .remove_label_values(database_identity, &worker_kind); + let _ = WORKER_METRICS + .v8_detached_contexts + .remove_label_values(database_identity, &worker_kind); + } + } + + fn new(database_identity: &Identity, worker_kind: JsWorkerKind) -> Self { Self { - database_identity: *database_identity, - worker_kind, - instance_id, - total_heap_size_bytes: WORKER_METRICS.v8_total_heap_size_bytes.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - total_physical_size_bytes: WORKER_METRICS.v8_total_physical_size_bytes.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - used_global_handles_size_bytes: WORKER_METRICS.v8_used_global_handles_size_bytes.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - used_heap_size_bytes: WORKER_METRICS.v8_used_heap_size_bytes.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - heap_size_limit_bytes: WORKER_METRICS.v8_heap_size_limit_bytes.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - external_memory_bytes: WORKER_METRICS.v8_external_memory_bytes.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - native_contexts: WORKER_METRICS.v8_native_contexts.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), - detached_contexts: WORKER_METRICS.v8_detached_contexts.with_label_values( - database_identity, - &worker_kind, - &instance_id, - ), + total_heap_size_bytes: WORKER_METRICS + .v8_total_heap_size_bytes + .with_label_values(database_identity, &worker_kind), + total_physical_size_bytes: WORKER_METRICS + .v8_total_physical_size_bytes + .with_label_values(database_identity, &worker_kind), + used_global_handles_size_bytes: WORKER_METRICS + .v8_used_global_handles_size_bytes + .with_label_values(database_identity, &worker_kind), + used_heap_size_bytes: WORKER_METRICS + .v8_used_heap_size_bytes + .with_label_values(database_identity, &worker_kind), + heap_size_limit_bytes: WORKER_METRICS + .v8_heap_size_limit_bytes + .with_label_values(database_identity, &worker_kind), + external_memory_bytes: WORKER_METRICS + .v8_external_memory_bytes + .with_label_values(database_identity, &worker_kind), + native_contexts: WORKER_METRICS + .v8_native_contexts + .with_label_values(database_identity, &worker_kind), + detached_contexts: WORKER_METRICS + .v8_detached_contexts + .with_label_values(database_identity, &worker_kind), + last_observed: V8HeapSnapshot::default(), } } + fn adjust_by(&self, delta: V8HeapSnapshot) { + adjust_gauge(&self.total_heap_size_bytes, delta.total_heap_size_bytes); + adjust_gauge(&self.total_physical_size_bytes, delta.total_physical_size_bytes); + adjust_gauge( + &self.used_global_handles_size_bytes, + delta.used_global_handles_size_bytes, + ); + adjust_gauge(&self.used_heap_size_bytes, delta.used_heap_size_bytes); + adjust_gauge(&self.heap_size_limit_bytes, delta.heap_size_limit_bytes); + adjust_gauge(&self.external_memory_bytes, delta.external_memory_bytes); + adjust_gauge(&self.native_contexts, delta.native_contexts); + adjust_gauge(&self.detached_contexts, delta.detached_contexts); + } + fn observe(&mut self, stats: &v8::HeapStatistics) { - self.total_heap_size_bytes.set(stats.total_heap_size() as i64); - self.total_physical_size_bytes.set(stats.total_physical_size() as i64); - self.used_global_handles_size_bytes - .set(stats.used_global_handles_size() as i64); - self.used_heap_size_bytes.set(stats.used_heap_size() as i64); - self.heap_size_limit_bytes.set(stats.heap_size_limit() as i64); - self.external_memory_bytes.set(stats.external_memory() as i64); - self.native_contexts.set(stats.number_of_native_contexts() as i64); - self.detached_contexts.set(stats.number_of_detached_contexts() as i64); + // See doc comment on `Self::last_observed` for why we compute a delta and apply it to the metrics value + // rather than directly calling `set`. + let next = V8HeapSnapshot::from_stats(stats); + self.adjust_by(V8HeapSnapshot { + total_heap_size_bytes: next.total_heap_size_bytes - self.last_observed.total_heap_size_bytes, + total_physical_size_bytes: next.total_physical_size_bytes - self.last_observed.total_physical_size_bytes, + used_global_handles_size_bytes: next.used_global_handles_size_bytes + - self.last_observed.used_global_handles_size_bytes, + used_heap_size_bytes: next.used_heap_size_bytes - self.last_observed.used_heap_size_bytes, + heap_size_limit_bytes: next.heap_size_limit_bytes - self.last_observed.heap_size_limit_bytes, + external_memory_bytes: next.external_memory_bytes - self.last_observed.external_memory_bytes, + native_contexts: next.native_contexts - self.last_observed.native_contexts, + detached_contexts: next.detached_contexts - self.last_observed.detached_contexts, + }); + self.last_observed = next; } } impl Drop for V8HeapMetrics { fn drop(&mut self) { - let _ = WORKER_METRICS.v8_total_heap_size_bytes.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); - let _ = WORKER_METRICS.v8_total_physical_size_bytes.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); - let _ = WORKER_METRICS.v8_used_global_handles_size_bytes.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); - let _ = WORKER_METRICS.v8_heap_size_limit_bytes.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); - let _ = WORKER_METRICS.v8_external_memory_bytes.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); - let _ = WORKER_METRICS.v8_native_contexts.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); - let _ = WORKER_METRICS.v8_detached_contexts.remove_label_values( - &self.database_identity, - &self.worker_kind, - &self.instance_id, - ); + self.adjust_by(V8HeapSnapshot { + total_heap_size_bytes: -self.last_observed.total_heap_size_bytes, + total_physical_size_bytes: -self.last_observed.total_physical_size_bytes, + used_global_handles_size_bytes: -self.last_observed.used_global_handles_size_bytes, + used_heap_size_bytes: -self.last_observed.used_heap_size_bytes, + heap_size_limit_bytes: -self.last_observed.heap_size_limit_bytes, + external_memory_bytes: -self.last_observed.external_memory_bytes, + native_contexts: -self.last_observed.native_contexts, + detached_contexts: -self.last_observed.detached_contexts, + }); + } +} + +fn adjust_gauge(gauge: &IntGauge, delta: i64) { + if delta > 0 { + gauge.add(delta); + } else if delta < 0 { + gauge.sub(-delta); } } @@ -1202,9 +1236,6 @@ where f(&mut guard) } -const MAIN_WORKER_INSTANCE_ID: u64 = 0; -const FIRST_PROCEDURE_WORKER_INSTANCE_ID: u64 = MAIN_WORKER_INSTANCE_ID + 1; - async fn spawn_main_instance_worker( program: Arc, module_or_mcc: Either, @@ -1220,7 +1251,6 @@ async fn spawn_main_instance_worker( core_pinner, heap_policy, metrics, - MAIN_WORKER_INSTANCE_ID, ) .await } @@ -1232,7 +1262,6 @@ async fn spawn_procedure_instance_worker( core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, metrics: InstanceManagerMetrics, - instance_id: u64, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( program, @@ -1241,7 +1270,6 @@ async fn spawn_procedure_instance_worker( core_pinner, heap_policy, metrics, - instance_id, ) .await } @@ -1521,9 +1549,6 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// /// `load_balance_guard` and `core_pinner` should both be from the same /// [`AllocatedJobCore`], and are used to manage the core pinning of this thread. -/// -/// `instance_kind` should be either [`V8_WORKER_KIND_MAIN`] or [`V8_WORKER_KIND_PROCEDURE`]. -/// `instance_id` should be a unique integer sourced async fn spawn_instance_worker( program: Arc, module_or_mcc: Either, @@ -1531,7 +1556,6 @@ async fn spawn_instance_worker( mut core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, instance_metrics: InstanceManagerMetrics, - instance_id: u64, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where W: JsWorkerSpec + 'static, @@ -1644,7 +1668,7 @@ where let info = &module_common.info(); let mut instance_common = InstanceCommon::new(&module_common); let replica_ctx: &Arc = module_common.replica_ctx(); - let mut heap_metrics = V8HeapMetrics::new(&info.database_identity, worker_kind, instance_id); + let mut heap_metrics = V8HeapMetrics::new(&info.database_identity, worker_kind); let mut inst = V8Instance { scope, diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index f955da398a2..e446679f824 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -289,27 +289,27 @@ metrics_group!( #[name = spacetime_worker_v8_total_heap_size_bytes] #[help = "The total size of the V8 heap for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_total_heap_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_total_physical_size_bytes] #[help = "The total committed physical V8 heap memory for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_total_physical_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_used_global_handles_size_bytes] #[help = "The used size of V8 global handles for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_used_global_handles_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_used_heap_size_bytes] #[help = "The live V8 heap size for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_used_heap_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_heap_size_limit_bytes] #[help = "The V8 heap size limit for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_heap_size_limit_bytes: IntGaugeVec, #[name = spacetime_worker_v8_heap_limit_hit] @@ -319,17 +319,17 @@ metrics_group!( #[name = spacetime_worker_v8_external_memory_bytes] #[help = "The external memory tracked by V8 for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_external_memory_bytes: IntGaugeVec, #[name = spacetime_worker_v8_native_contexts] #[help = "The number of native V8 contexts for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_native_contexts: IntGaugeVec, #[name = spacetime_worker_v8_detached_contexts] #[help = "The number of detached V8 contexts for a database's tracked JS worker kind"] - #[labels(database_identity: Identity, worker_kind: JsWorkerKind, instance_id: u64)] + #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_detached_contexts: IntGaugeVec, #[name = spacetime_worker_v8_request_queue_length] From 25f0a4092e9da9ffd8f6971577576ecebfc4365c Mon Sep 17 00:00:00 2001 From: Phoebe Goldman Date: Tue, 26 May 2026 18:03:37 -0400 Subject: [PATCH 4/4] Update metric help text further The phrase "a database's tracked JS worker kind" no longer makes sense, as all possible JS worker kinds are tracked. --- crates/core/src/worker_metrics/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index e446679f824..9246022ced2 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -288,27 +288,27 @@ metrics_group!( pub wasm_memory_bytes: IntGaugeVec, #[name = spacetime_worker_v8_total_heap_size_bytes] - #[help = "The total size of the V8 heap for a database's tracked JS worker kind"] + #[help = "The total size of the V8 heap for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_total_heap_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_total_physical_size_bytes] - #[help = "The total committed physical V8 heap memory for a database's tracked JS worker kind"] + #[help = "The total committed physical V8 heap memory for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_total_physical_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_used_global_handles_size_bytes] - #[help = "The used size of V8 global handles for a database's tracked JS worker kind"] + #[help = "The used size of V8 global handles for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_used_global_handles_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_used_heap_size_bytes] - #[help = "The live V8 heap size for a database's tracked JS worker kind"] + #[help = "The live V8 heap size for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_used_heap_size_bytes: IntGaugeVec, #[name = spacetime_worker_v8_heap_size_limit_bytes] - #[help = "The V8 heap size limit for a database's tracked JS worker kind"] + #[help = "The V8 heap size limit for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_heap_size_limit_bytes: IntGaugeVec, @@ -318,17 +318,17 @@ metrics_group!( pub v8_heap_limit_hit: IntCounterVec, #[name = spacetime_worker_v8_external_memory_bytes] - #[help = "The external memory tracked by V8 for a database's tracked JS worker kind"] + #[help = "The external memory tracked by V8 for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_external_memory_bytes: IntGaugeVec, #[name = spacetime_worker_v8_native_contexts] - #[help = "The number of native V8 contexts for a database's tracked JS worker kind"] + #[help = "The number of native V8 contexts for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_native_contexts: IntGaugeVec, #[name = spacetime_worker_v8_detached_contexts] - #[help = "The number of detached V8 contexts for a database's tracked JS worker kind"] + #[help = "The number of detached V8 contexts for a database's JS workers"] #[labels(database_identity: Identity, worker_kind: JsWorkerKind)] pub v8_detached_contexts: IntGaugeVec,