From 6e45a778defd118d8bc134f3afc506329d115dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Tue, 5 May 2026 21:55:59 +0200 Subject: [PATCH 1/7] Add JFR periodic event for DuckDB memory consumption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operating DuckDB-backed Java applications at scale currently offers no in-process, low-overhead way to observe DuckDB's internal memory usage over time. Users either sit inside the JVM's heap metrics (which don't capture native allocations) or run ad-hoc SELECT * FROM duckdb_memory() queries by hand. JFR is the standard, always-available observability sink on modern JVMs, so emitting DuckDB memory usage as a periodic JFR event lets operators diagnose memory growth, per-tag breakdowns, and temporary-storage spill with the same tooling already used for the rest of the JVM — enabled or disabled via JFR recording settings, sampled at the period the consumer chooses, and opt-in per database via a single JDBC connection property. --- duckdb_java.def | 1 + duckdb_java.exp | 1 + duckdb_java.map | 1 + src/jni/duckdb_java.cpp | 5 + src/jni/functions.cpp | 10 + src/jni/functions.hpp | 4 + .../java/org/duckdb/DuckDBConnection.java | 47 +++- src/main/java/org/duckdb/DuckDBDriver.java | 9 + .../java/org/duckdb/DuckDBMemoryEvent.java | 63 +++++ .../java/org/duckdb/DuckDBMemoryMonitor.java | 227 ++++++++++++++++++ src/main/java/org/duckdb/DuckDBNative.java | 3 + .../java/org/duckdb/JfrMemoryMonitor.java | 83 +++++++ src/test/java/org/duckdb/TestDuckDBJDBC.java | 175 ++++++++++++++ 13 files changed, 626 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/duckdb/DuckDBMemoryEvent.java create mode 100644 src/main/java/org/duckdb/DuckDBMemoryMonitor.java create mode 100644 src/main/java/org/duckdb/JfrMemoryMonitor.java diff --git a/duckdb_java.def b/duckdb_java.def index 7ae4d081b..4593c41f5 100644 --- a/duckdb_java.def +++ b/duckdb_java.def @@ -26,6 +26,7 @@ Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1connect Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref +Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute diff --git a/duckdb_java.exp b/duckdb_java.exp index 1e92c19c7..8ca137161 100644 --- a/duckdb_java.exp +++ b/duckdb_java.exp @@ -23,6 +23,7 @@ _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1connect _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref +_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute diff --git a/duckdb_java.map b/duckdb_java.map index f3b476f8c..c780493dc 100644 --- a/duckdb_java.map +++ b/duckdb_java.map @@ -25,6 +25,7 @@ DUCKDB_JAVA { Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref; + Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute; diff --git a/src/jni/duckdb_java.cpp b/src/jni/duckdb_java.cpp index b49036531..45d8a047d 100644 --- a/src/jni/duckdb_java.cpp +++ b/src/jni/duckdb_java.cpp @@ -95,6 +95,11 @@ jobject _duckdb_jdbc_create_db_ref(JNIEnv *env, jclass, jobject conn_ref_buf) { return env->NewDirectByteBuffer(db_ref, 0); } +jlong _duckdb_jdbc_db_address(JNIEnv *env, jclass, jobject conn_ref_buf) { + auto conn_ref = get_connection_ref(env, conn_ref_buf); + return (jlong)conn_ref->db.get(); +} + void _duckdb_jdbc_destroy_db_ref(JNIEnv *env, jclass, jobject db_ref_buf) { if (nullptr == db_ref_buf) { return; diff --git a/src/jni/functions.cpp b/src/jni/functions.cpp index 2bff3ee86..160c0f71a 100644 --- a/src/jni/functions.cpp +++ b/src/jni/functions.cpp @@ -37,6 +37,16 @@ JNIEXPORT jobject JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_ } } +JNIEXPORT jlong JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address(JNIEnv * env, jclass param0, jobject param1) { + try { + return _duckdb_jdbc_db_address(env, param0, param1); + } catch (const std::exception &e) { + duckdb::ErrorData error(e); + ThrowJNI(env, error.Message().c_str()); + + } +} + JNIEXPORT void JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref(JNIEnv * env, jclass param0, jobject param1) { try { return _duckdb_jdbc_destroy_db_ref(env, param0, param1); diff --git a/src/jni/functions.hpp b/src/jni/functions.hpp index e92e92bfc..d6b2c452f 100644 --- a/src/jni/functions.hpp +++ b/src/jni/functions.hpp @@ -21,6 +21,10 @@ jobject _duckdb_jdbc_create_db_ref(JNIEnv * env, jclass param0, jobject param1); JNIEXPORT jobject JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref(JNIEnv * env, jclass param0, jobject param1); +jlong _duckdb_jdbc_db_address(JNIEnv * env, jclass param0, jobject param1); + +JNIEXPORT jlong JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address(JNIEnv * env, jclass param0, jobject param1); + void _duckdb_jdbc_destroy_db_ref(JNIEnv * env, jclass param0, jobject param1); JNIEXPORT void JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref(JNIEnv * env, jclass param0, jobject param1); diff --git a/src/main/java/org/duckdb/DuckDBConnection.java b/src/main/java/org/duckdb/DuckDBConnection.java index d51c0c00e..80201f643 100644 --- a/src/main/java/org/duckdb/DuckDBConnection.java +++ b/src/main/java/org/duckdb/DuckDBConnection.java @@ -48,6 +48,21 @@ public final class DuckDBConnection implements java.sql.Connection { private final boolean readOnly; private final String sessionInitSQL; + /** + * User-supplied identifier for JFR memory monitoring (the value of the + * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property). {@code null} or empty + * means this connection does not participate in monitoring — either the user + * did not opt in, or this connection IS the monitor connection. + */ + final String monitorName; + + /** + * Native address of the underlying DuckDB instance. Captured once at construction so that + * {@link #close()} can notify {@link JfrMemoryMonitor} without an additional JNI call + * and so the JFR event can expose it as a secondary disambiguator. + */ + final long dbAddress; + public static DuckDBConnection newConnection(String url, boolean readOnly, Properties properties) throws Exception { return newConnection(url, readOnly, null, properties); } @@ -60,19 +75,25 @@ public static DuckDBConnection newConnection(String url, boolean readOnly, Strin String dbName = dbNameFromUrl(url); String autoCommitStr = removeOption(properties, JDBC_AUTO_COMMIT); boolean autoCommit = isStringTruish(autoCommitStr, true); + String monitorName = removeOption(properties, DuckDBDriver.JDBC_JFR_MEMORY_MONITOR); ByteBuffer nativeReference = DuckDBNative.duckdb_jdbc_startup(dbName.getBytes(UTF_8), readOnly, properties); - return new DuckDBConnection(nativeReference, url, readOnly, sessionInitSQL, autoCommit); + return new DuckDBConnection(nativeReference, url, readOnly, sessionInitSQL, autoCommit, monitorName); } private DuckDBConnection(ByteBuffer connectionReference, String url, boolean readOnly, String sessionInitSQL, - boolean autoCommit) throws SQLException { + boolean autoCommit, String monitorName) throws SQLException { this.connRef = connectionReference; this.url = url; this.readOnly = readOnly; this.autoCommit = autoCommit; this.sessionInitSQL = sessionInitSQL; + this.monitorName = (monitorName != null && !monitorName.isEmpty()) ? monitorName : null; + this.dbAddress = DuckDBNative.duckdb_jdbc_db_address(connectionReference); // Hardcoded 'true' here is intentional, autocommit is handled in stmt#execute() DuckDBNative.duckdb_jdbc_set_auto_commit(connectionReference, true); + if (this.monitorName != null) { + JfrMemoryMonitor.connectionOpened(this); + } } public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) @@ -99,12 +120,24 @@ public Statement createStatement() throws SQLException { } public DuckDBConnection duplicate() throws SQLException { + return duplicate(this.monitorName); + } + + /** + * Creates a duplicate connection that is invisible to the JFR memory monitor. + * Used exclusively by the monitor itself to avoid re-entrant lifecycle callbacks. + */ + DuckDBConnection duplicateForMonitor() throws SQLException { + return duplicate(null); + } + + private DuckDBConnection duplicate(String monitorName) throws SQLException { checkOpen(); connRefLock.lock(); try { checkOpen(); ByteBuffer dupRef = DuckDBNative.duckdb_jdbc_connect(connRef); - return new DuckDBConnection(dupRef, url, readOnly, sessionInitSQL, autoCommit); + return new DuckDBConnection(dupRef, url, readOnly, sessionInitSQL, autoCommit, monitorName); } finally { connRefLock.unlock(); } @@ -128,6 +161,10 @@ public void close() throws SQLException { if (isClosed()) { return; } + // Notify the memory monitor only from the call that actually performed + // the disconnect, and after releasing connRefLock so the monitor's own + // native disconnect does not block the caller under our lock. + boolean notifyMonitor = false; connRefLock.lock(); try { if (isClosed()) { @@ -173,9 +210,13 @@ public void close() throws SQLException { DuckDBNative.duckdb_jdbc_disconnect(connRef); connRef = null; + notifyMonitor = (monitorName != null); } finally { connRefLock.unlock(); } + if (notifyMonitor) { + JfrMemoryMonitor.connectionClosed(dbAddress); + } } public boolean isClosed() throws SQLException { diff --git a/src/main/java/org/duckdb/DuckDBDriver.java b/src/main/java/org/duckdb/DuckDBDriver.java index 1b606600c..ddd0c9b08 100644 --- a/src/main/java/org/duckdb/DuckDBDriver.java +++ b/src/main/java/org/duckdb/DuckDBDriver.java @@ -31,6 +31,7 @@ public class DuckDBDriver implements java.sql.Driver { public static final String JDBC_AUTO_COMMIT = "jdbc_auto_commit"; public static final String JDBC_PIN_DB = "jdbc_pin_db"; public static final String JDBC_IGNORE_UNSUPPORTED_OPTIONS = "jdbc_ignore_unsupported_options"; + public static final String JDBC_JFR_MEMORY_MONITOR = "jdbc_jfr_memory_monitor"; static final String DUCKDB_URL_PREFIX = "jdbc:duckdb:"; static final String MEMORY_DB = ":memory:"; @@ -75,6 +76,11 @@ public Thread newThread(Runnable r) { } catch (SQLException e) { throw new RuntimeException(e); } + // Eagerly register the JFR periodic memory-usage event (when JFR is available) + // so that recordings started before any monitored connection is opened see the + // event type and honor its period setting. On JVMs without JFR (e.g. Java 8) + // this is a silent no-op. + JfrMemoryMonitor.init(); } public Connection connect(String url, Properties info) throws SQLException { @@ -165,6 +171,9 @@ public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws "Do not close the DB instance after all connections to it are closed")); list.add(createDriverPropInfo(JDBC_IGNORE_UNSUPPORTED_OPTIONS, "", "Silently discard unsupported connection options")); + list.add(createDriverPropInfo( + JDBC_JFR_MEMORY_MONITOR, "", + "User-assigned identifier under which this connection's DuckDB instance is tracked in the duckdb.MemoryUsage JFR event. Leave empty to disable monitoring. JFR controls the event's enabled state and period via recording settings. Requires a JFR-capable JVM.")); list.sort((o1, o2) -> o1.name.compareToIgnoreCase(o2.name)); return list.toArray(new DriverPropertyInfo[0]); } diff --git a/src/main/java/org/duckdb/DuckDBMemoryEvent.java b/src/main/java/org/duckdb/DuckDBMemoryEvent.java new file mode 100644 index 000000000..7577b560e --- /dev/null +++ b/src/main/java/org/duckdb/DuckDBMemoryEvent.java @@ -0,0 +1,63 @@ +package org.duckdb; + +import jdk.jfr.Category; +import jdk.jfr.Description; +import jdk.jfr.Event; +import jdk.jfr.Label; +import jdk.jfr.Name; +import jdk.jfr.StackTrace; + +/** + * JFR event that records DuckDB memory usage for a single memory tag. + * + *

One event is emitted per memory tag per firing interval. Emission is + * driven by JFR's periodic-event machinery: a single hook is registered via + * {@link jdk.jfr.FlightRecorder#addPeriodicEvent} in + * {@link DuckDBMemoryMonitor}, and JFR invokes it at the period configured on + * the recording. Configure both the enabled state and the period in a JFR + * configuration file or via JMC: + * + * 

{@code
+ * 
+ *   true
+ *   1 s
+ * 
+ * }
+ * + *

Participation is opt-in per connection: set the + * {@link DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} connection property to the + * identifier under which this connection's DuckDB instance should be tracked. + * An absent or empty value disables emission for that connection. The JDBC + * property is a pure enable/label switch; JFR controls whether and how often + * the event fires. + */ +@Name("duckdb.MemoryUsage") +@Label("DuckDB Memory Usage") +@Description("Periodic snapshot of DuckDB internal memory consumption per tag") +@Category("DuckDB") +@StackTrace(false) +public class DuckDBMemoryEvent extends Event { + + @Label("Name") + @Description( + "User-assigned identifier of the DuckDB instance (value of the jdbc_jfr_memory_monitor connection property)") + String name; + + @Label("Tag") + @Description("DuckDB internal memory tag (e.g. \"Base\", \"Hash Table\", \"Buffer Manager\")") + String tag; + + @Label("Database URL") + @Description("JDBC URL or database name of the DuckDB instance emitting this event") + String dbUrl; + + @Label("Database Address") + @Description("Native address of the underlying DuckDB instance; disambiguates databases when names collide") + long dbAddress; + + @Label("Memory Usage") @Description("Bytes currently allocated for this tag") long memoryUsageBytes; + + @Label("Temporary Storage Usage") + @Description("Bytes spilled to the temporary storage for this tag") + long temporaryStorageBytes; +} diff --git a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java new file mode 100644 index 000000000..b64b23d2d --- /dev/null +++ b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java @@ -0,0 +1,227 @@ +package org.duckdb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import jdk.jfr.FlightRecorder; + +/** + * Manages per-database JFR memory monitors. + * + *

Activation

+ *

Monitoring is opt-in per connection. The caller supplies a user-assigned identifier + * via the {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} connection property; that value + * becomes the {@code name} field of every {@link DuckDBMemoryEvent} emitted for the + * connection's DuckDB instance. A monitor is created for a DuckDB instance when the first + * opted-in connection to it is opened and destroyed when the last such connection is closed. + * + *

Event scheduling

+ *

This class does not own a scheduler. It registers a single + * {@linkplain FlightRecorder#addPeriodicEvent periodic JFR hook} for + * {@link DuckDBMemoryEvent}; JFR invokes the hook at the period configured on + * the recording (e.g. {@code 1 s}) and only + * while at least one active recording has the event enabled. Consequently, the + * JDBC property is a pure enable/label switch and JFR alone governs the + * sampling rate and the enabled state of the event. + * + *

Attribution model

+ *

The monitor registry is keyed on the native DuckDB instance address so that multiple + * connections to the same underlying database share a single sample stream — avoiding + * double-counting of shared memory. The user-supplied name is captured from the first + * opted-in connection and emitted on every event for that monitor. When attributing memory + * to distinct application components, use a distinct DuckDB instance per component and + * give each one a unique {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} name. + * + *

Thread safety

+ *

Lifecycle transitions (start+insert, stop+remove) are performed inside + * {@link ConcurrentHashMap#compute}, which provides per-key mutual exclusion. + * The JFR periodic hook iterates {@link ConcurrentHashMap#values} without + * locking and relies on volatile reads in {@link PerDbMonitor} for visibility. + */ +final class DuckDBMemoryMonitor { + + private static final Logger logger = Logger.getLogger(DuckDBMemoryMonitor.class.getName()); + + /** Registry: native DuckDB* address -> per-database monitor. */ + private static final ConcurrentHashMap monitors = new ConcurrentHashMap<>(); + + private static boolean initialized; + + // Non-instantiable + private DuckDBMemoryMonitor() { + } + + /** + * Registers the periodic JFR hook for {@link DuckDBMemoryEvent}. Idempotent + * and called from {@link DuckDBDriver}'s static initializer so that recordings + * started before the first monitored connection still see the event type. + * Iterating an empty monitor map at each tick is cheap, so there is no + * downside to registering unconditionally. + */ + static synchronized void init() { + if (initialized) { + return; + } + FlightRecorder.addPeriodicEvent(DuckDBMemoryEvent.class, DuckDBMemoryMonitor::firePeriodicEvent); + initialized = true; + } + + /** + * Called when a new connection is opened with JFR memory monitoring enabled. + * The caller guarantees {@code conn.monitorName} is non-null. + */ + static void connectionOpened(DuckDBConnection conn) { + monitors.compute(conn.dbAddress, (k, existing) -> { + PerDbMonitor m = (existing != null) ? existing : new PerDbMonitor(); + m.open(conn); + return m; + }); + } + + /** + * Called when a monitored connection is closed. + * + * @param dbAddress the native db address captured at construction time + */ + static void connectionClosed(long dbAddress) { + monitors.compute(dbAddress, (k, existing) -> { + if (existing == null) { + return null; + } + return existing.close() ? null : existing; + }); + } + + /** + * JFR-invoked hook. Runs on a JFR thread at the period configured on the + * active recording. Must never throw. + */ + private static void firePeriodicEvent() { + // Gate all per-monitor sampling on a single probe. When the event is not + // being recorded (disabled, filtered by threshold, etc.), skip the + // duckdb_memory() query entirely for every monitored instance. + // Under normal operation JFR only invokes this hook when the event is + // enabled in some recording, so shouldCommit() will typically be true. + if (!new DuckDBMemoryEvent().shouldCommit()) { + return; + } + for (PerDbMonitor m : monitors.values()) { + try { + m.sample(); + } catch (Throwable t) { + // Defensive: a failure in one monitor must not prevent emission for others. + } + } + } + + /** + * Per-name monitor state. Mutating methods ({@link #open}, {@link #close}) + * are invoked inside {@link ConcurrentHashMap#compute} on {@link #monitors} + * and are serialized per key. {@link #sample} runs on the JFR hook thread + * without the compute lock and reads {@code volatile} fields. + */ + static final class PerDbMonitor { + + private static final String QUERY = + "SELECT tag, memory_usage_bytes, temporary_storage_bytes FROM duckdb_memory()"; + + /** Guarded by compute()-serialization. */ + private int openConnections = 0; + + /** Written under compute(); read without lock by the JFR hook. */ + private volatile DuckDBConnection monitorConn; + private volatile String name; + private volatile String dbUrl; + private volatile long dbAddress; + + /** + * Opens (or re-attempts opening) the monitor connection and increments + * the ref count. Failure to create the monitor connection is logged + * and the ref count is still incremented so close-balance is preserved; + * a subsequent {@code open()} will retry while {@code monitorConn == null}. + */ + void open(DuckDBConnection conn) { + if (monitorConn == null) { + try { + DuckDBConnection mc = conn.duplicateForMonitor(); + name = conn.monitorName; + dbUrl = conn.url; + dbAddress = conn.dbAddress; + // Publish monitorConn last so readers see fully-populated state. + monitorConn = mc; + } catch (SQLException e) { + logger.log(Level.WARNING, "Failed to open JFR memory-monitor connection; will retry on next open()", + e); + } + } + openConnections++; + } + + /** + * Decrements the ref count; when it reaches zero, releases the monitor + * connection and signals the caller to remove the map entry. + * + * @return {@code true} when the entry should be removed + */ + boolean close() { + if (--openConnections <= 0) { + DuckDBConnection mc = monitorConn; + monitorConn = null; + if (mc != null) { + try { + mc.close(); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to close JFR memory-monitor connection", e); + } + } + return true; + } + return false; + } + + /** Invoked by the JFR periodic hook. Must not throw. */ + void sample() { + DuckDBConnection mc = monitorConn; + if (mc == null) { + return; + } + try { + if (mc.isClosed()) { + return; + } + } catch (SQLException ignored) { + return; + } + String nameSnap = name; + String url = dbUrl; + long addr = dbAddress; + try (Statement stmt = mc.createStatement(); ResultSet rs = stmt.executeQuery(QUERY)) { + while (rs.next()) { + String tag = rs.getString(1); + long memoryUsageBytes = rs.getLong(2); + long temporaryStorageBytes = rs.getLong(3); + emitEvent(nameSnap, url, addr, tag, memoryUsageBytes, temporaryStorageBytes); + } + } catch (Exception ignored) { + // Propagating would break JFR's periodic dispatch; swallow. + } + } + + private static void emitEvent(String name, String url, long addr, String tag, long memoryUsageBytes, + long temporaryStorageBytes) { + // shouldCommit() was already gated at the top of firePeriodicEvent(). + DuckDBMemoryEvent event = new DuckDBMemoryEvent(); + event.begin(); + event.name = name; + event.tag = tag; + event.dbUrl = url; + event.dbAddress = addr; + event.memoryUsageBytes = memoryUsageBytes; + event.temporaryStorageBytes = temporaryStorageBytes; + event.commit(); + } + } +} diff --git a/src/main/java/org/duckdb/DuckDBNative.java b/src/main/java/org/duckdb/DuckDBNative.java index 2267bae40..30b1c0e22 100644 --- a/src/main/java/org/duckdb/DuckDBNative.java +++ b/src/main/java/org/duckdb/DuckDBNative.java @@ -149,6 +149,9 @@ private static void loadFromCurrentJarDir(String libName) throws Exception { static native void duckdb_jdbc_destroy_db_ref(ByteBuffer db_ref) throws SQLException; + /** Returns the native address of the underlying DuckDB instance as a stable identity key. */ + static native long duckdb_jdbc_db_address(ByteBuffer conn_ref) throws SQLException; + static native void duckdb_jdbc_set_auto_commit(ByteBuffer conn_ref, boolean auto_commit) throws SQLException; static native boolean duckdb_jdbc_get_auto_commit(ByteBuffer conn_ref) throws SQLException; diff --git a/src/main/java/org/duckdb/JfrMemoryMonitor.java b/src/main/java/org/duckdb/JfrMemoryMonitor.java new file mode 100644 index 000000000..727825ee6 --- /dev/null +++ b/src/main/java/org/duckdb/JfrMemoryMonitor.java @@ -0,0 +1,83 @@ +package org.duckdb; + +import java.lang.reflect.Method; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Indirection over {@link DuckDBMemoryMonitor} that is safe to reference on JVMs without JFR + * support (e.g. Java 8). When {@code jdk.jfr.FlightRecorder} is not available at runtime, every + * method on this class is a silent no-op. + * + *

Only this class may reference {@code DuckDBMemoryMonitor} or {@code DuckDBMemoryEvent}; any + * direct reference from the other main classes would trigger class resolution at load time and + * fail verification on a non-JFR JVM. + */ +final class JfrMemoryMonitor { + + private static final Logger logger = Logger.getLogger(JfrMemoryMonitor.class.getName()); + + private static final Method INIT; + private static final Method CONNECTION_OPENED; + private static final Method CONNECTION_CLOSED; + + static { + Method init = null; + Method opened = null; + Method closed = null; + try { + // Probe for JFR first: on Java 8 this throws ClassNotFoundException + // and we never touch DuckDBMemoryMonitor (which imports jdk.jfr.*). + Class.forName("jdk.jfr.FlightRecorder"); + Class impl = Class.forName("org.duckdb.DuckDBMemoryMonitor"); + init = impl.getDeclaredMethod("init"); + opened = impl.getDeclaredMethod("connectionOpened", DuckDBConnection.class); + closed = impl.getDeclaredMethod("connectionClosed", long.class); + init.setAccessible(true); + opened.setAccessible(true); + closed.setAccessible(true); + } catch (Throwable t) { + // JFR unavailable; every method becomes a silent no-op. + logger.log(Level.FINE, "JFR memory monitor is not available on this JVM", t); + } + INIT = init; + CONNECTION_OPENED = opened; + CONNECTION_CLOSED = closed; + } + + private JfrMemoryMonitor() { + } + + static void init() { + if (INIT == null) { + return; + } + try { + INIT.invoke(null); + } catch (Throwable t) { + logger.log(Level.FINE, "JFR memory monitor init failed", t); + } + } + + static void connectionOpened(DuckDBConnection conn) { + if (CONNECTION_OPENED == null) { + return; + } + try { + CONNECTION_OPENED.invoke(null, conn); + } catch (Throwable t) { + logger.log(Level.FINE, "JFR connectionOpened failed", t); + } + } + + static void connectionClosed(long dbAddress) { + if (CONNECTION_CLOSED == null) { + return; + } + try { + CONNECTION_CLOSED.invoke(null, dbAddress); + } catch (Throwable t) { + logger.log(Level.FINE, "JFR connectionClosed failed", t); + } + } +} diff --git a/src/test/java/org/duckdb/TestDuckDBJDBC.java b/src/test/java/org/duckdb/TestDuckDBJDBC.java index 3d9572acc..14dfde1db 100644 --- a/src/test/java/org/duckdb/TestDuckDBJDBC.java +++ b/src/test/java/org/duckdb/TestDuckDBJDBC.java @@ -40,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.Callable; @@ -50,6 +51,9 @@ import java.util.logging.Logger; import javax.sql.rowset.CachedRowSet; import javax.sql.rowset.RowSetProvider; +import jdk.jfr.Recording; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordingFile; import org.duckdb.test.TempDirectory; public class TestDuckDBJDBC { @@ -2276,6 +2280,177 @@ public static void DISABLED_test_extension_excel() throws Exception { } } + /** + * Verifies that: + *

+ */ + public static void test_jfr_memory_event() throws Exception { + // --- Part 1: no property -> no events --- + try (Recording recOff = new Recording()) { + recOff.enable("duckdb.MemoryUsage").withPeriod(Duration.ofMillis(100)); + recOff.start(); + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + Thread.sleep(300); + } + recOff.stop(); + List events = dumpEvents(recOff); + assertTrue(events.isEmpty(), "Expected no events when monitor property is not set"); + } + + // --- Part 2: property set -> events emitted under each supplied name --- + Properties propsA = new Properties(); + propsA.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, "component-a"); + Properties propsB = new Properties(); + propsB.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, "component-b"); + + try (Recording rec = new Recording()) { + rec.enable("duckdb.MemoryUsage").withPeriod(Duration.ofMillis(100)); + rec.start(); + + // Two independent in-memory databases, each with its own component name. + try (Connection conn1 = DriverManager.getConnection(JDBC_URL, propsA); + Connection conn2 = DriverManager.getConnection(JDBC_URL, propsB)) { + Thread.sleep(400); + } + + rec.stop(); + List events = dumpEvents(rec); + assertFalse(events.isEmpty(), "Expected at least one DuckDBMemory JFR event"); + + Set names = new HashSet<>(); + for (RecordedEvent event : events) { + String name = event.getString("name"); + assertTrue(name != null && !name.isEmpty(), "name field must be non-empty"); + names.add(name); + + String tag = event.getString("tag"); + assertTrue(tag != null && !tag.isEmpty(), "tag field must be non-empty"); + + String dbUrl = event.getString("dbUrl"); + assertTrue(dbUrl != null && dbUrl.startsWith("jdbc:duckdb:"), + "dbUrl must be a JDBC URL, was: " + dbUrl); + + long dbAddress = event.getLong("dbAddress"); + assertTrue(dbAddress != 0L, "dbAddress must be non-zero"); + + long memUsage = event.getLong("memoryUsageBytes"); + assertTrue(memUsage >= 0, "memoryUsageBytes must be >= 0"); + long tmpStorage = event.getLong("temporaryStorageBytes"); + assertTrue(tmpStorage >= 0, "temporaryStorageBytes must be >= 0"); + } + + // Each component must emit events under its own name. + assertTrue(names.contains("component-a") && names.contains("component-b"), + "Expected events for both component names, got: " + names); + } + } + + /** + * After the last monitored connection is closed, the monitor entry must be removed so + * a subsequent recording observes no events. + */ + public static void test_jfr_memory_event_cleanup_after_close() throws Exception { + Properties props = new Properties(); + props.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, "cleanup-test"); + + // Prime: open and close a monitored connection outside of any recording. + try (Connection conn = DriverManager.getConnection(JDBC_URL, props)) { + // no-op + } + + // Start a fresh recording with no monitored connection open. + try (Recording rec = new Recording()) { + rec.enable("duckdb.MemoryUsage").withPeriod(Duration.ofMillis(100)); + rec.start(); + Thread.sleep(400); + rec.stop(); + List events = dumpEvents(rec); + assertTrue(events.isEmpty(), + "Expected no events after all monitored connections were closed, got " + events.size()); + } + } + + /** + * For a file-based database, two monitored connections share a single underlying DuckDB + * instance, so the monitor samples it once and events carry a single {@code dbAddress}. + * The monitor must remain active while at least one monitored connection is open. + */ + public static void test_jfr_memory_event_file_db_refcount() throws Exception { + try (TempDirectory dir = new TempDirectory()) { + Path dbFile = dir.path().resolve("refcount.db"); + String url = JDBC_URL + dbFile; + Properties monitored = new Properties(); + monitored.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, "shared-db"); + + try (Recording rec = new Recording()) { + rec.enable("duckdb.MemoryUsage").withPeriod(Duration.ofMillis(100)); + rec.start(); + + Connection conn1 = DriverManager.getConnection(url, monitored); + Connection conn2 = DriverManager.getConnection(url, monitored); + try { + Thread.sleep(250); + // Close one connection; monitor must stay alive via conn2. + conn1.close(); + Thread.sleep(250); + } finally { + conn2.close(); + } + + rec.stop(); + List events = dumpEvents(rec); + assertFalse(events.isEmpty(), "Expected events for the shared file-based DB"); + + Set addresses = new HashSet<>(); + Set names = new HashSet<>(); + for (RecordedEvent e : events) { + addresses.add(e.getLong("dbAddress")); + names.add(e.getString("name")); + } + assertEquals(addresses.size(), 1, + "Two connections to the same file DB must share dbAddress, got " + addresses); + assertEquals(names, new HashSet<>(singletonList("shared-db")), + "All events must be tagged with the supplied name, got: " + names); + } + } + } + + /** + * An empty {@code jdbc_jfr_memory_monitor} value must be treated as "not set" and emit + * no events, mirroring the absent-property behaviour. + */ + public static void test_jfr_memory_event_empty_property_disables() throws Exception { + Properties props = new Properties(); + props.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, ""); + + try (Recording rec = new Recording()) { + rec.enable("duckdb.MemoryUsage").withPeriod(Duration.ofMillis(100)); + rec.start(); + try (Connection conn = DriverManager.getConnection(JDBC_URL, props)) { + Thread.sleep(300); + } + rec.stop(); + List events = dumpEvents(rec); + assertTrue(events.isEmpty(), + "Expected no events when jdbc_jfr_memory_monitor is empty, got " + events.size()); + } + } + + private static List dumpEvents(Recording rec) throws Exception { + Path jfrPath = Files.createTempFile("duckdb-jfr-", ".jfr"); + try { + rec.dump(jfrPath); + return RecordingFile.readAllEvents(jfrPath); + } finally { + Files.deleteIfExists(jfrPath); + } + } + public static void main(String[] args) throws Exception { String arg1 = args.length > 0 ? args[0] : ""; final int statusCode; From 7211c6f792c97d07f53b908dbaa6e0c1bdba36aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Wed, 6 May 2026 09:25:46 +0200 Subject: [PATCH 2/7] Refine JFR memory usage tracking --- JFR.md | 150 ++++++++++++++++++ README.md | 2 + scripts/verify-jfr-java8.sh | 88 ++++++++++ scripts/verify-jfr.sh | 74 +++++++++ .../java/org/duckdb/DuckDBConnection.java | 2 +- src/main/java/org/duckdb/DuckDBDriver.java | 9 +- .../java/org/duckdb/DuckDBMemoryEvent.java | 17 +- .../java/org/duckdb/DuckDBMemoryMonitor.java | 85 ++++++---- .../java/org/duckdb/JfrMemoryMonitor.java | 67 +++----- src/test/java/org/duckdb/TestDuckDBJDBC.java | 30 ++-- 10 files changed, 415 insertions(+), 109 deletions(-) create mode 100644 JFR.md create mode 100755 scripts/verify-jfr-java8.sh create mode 100755 scripts/verify-jfr.sh diff --git a/JFR.md b/JFR.md new file mode 100644 index 000000000..a84abe003 --- /dev/null +++ b/JFR.md @@ -0,0 +1,150 @@ +# JFR Memory Monitoring + +The driver can publish DuckDB memory-usage statistics as a periodic +Java Flight Recorder (JFR) event, so any JFR-aware tool (JMC, `jfr` +CLI, async-profiler, Datadog/New Relic continuous profilers, …) can +ingest DuckDB memory metrics alongside the rest of the JVM signal. + +The feature is **strictly opt-in** per connection and is a silent +no-op on JVMs without JFR support. Nothing is emitted unless the +application both sets the JDBC property on a connection and has an +active JFR recording that enables the event. + +## Enabling emission + +Pass `jdbc_jfr_memory_monitor=` when opening a +connection: + +```java +Properties props = new Properties(); +props.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, "pricing-service"); +Connection conn = DriverManager.getConnection("jdbc:duckdb:/tmp/pricing.db", props); +``` + +…or in the URL: + +``` +jdbc:duckdb:/tmp/pricing.db;jdbc_jfr_memory_monitor=pricing-service +``` + +The `` is an arbitrary label chosen by the application; +it is attached to every event as the `component` field so operators +can attribute memory to logical components in dashboards and queries. + +Rules: + +| Property value | Effect | +| ------------------------- | ---------------------------------------------- | +| absent | no events emitted for this connection | +| empty string | no events emitted (same as absent) | +| non-empty string | events emitted, tagged with the given value | + +The JDBC property is purely an opt-in switch and a label. It does +**not** control whether JFR is actually recording, nor the sampling +period — those are governed by JFR recording settings (see below). + +## Controlling period and enabled state + +Sampling rate and enabled state are JFR-native settings. Configure +them in a `.jfc` profile, via JMC, or programmatically: + +```java +try (Recording r = new Recording()) { + r.enable("duckdb.MemoryUsage").withPeriod(Duration.ofSeconds(1)); + r.start(); + // ... application work ... + r.stop(); + r.dump(Path.of("app.jfr")); +} +``` + +Equivalent `.jfc` snippet: + +```xml + + true + 1 s + +``` + +When no recording enables the event, the driver performs zero work — +the DuckDB `duckdb_memory()` query is never issued. + +## Event schema + +Event name: **`duckdb.MemoryUsage`** — one event per memory tag per +JFR tick. + +| Field | Type | Meaning | +| ----------------------- | ------ | ----------------------------------------------------------------- | +| `component` | String | Application-supplied identifier (the JDBC property value). | +| `tag` | String | DuckDB memory tag (e.g. `IN_MEMORY_TABLE`, `HASH_TABLE`, `ALLOCATOR`). | +| `dbAddress` | long | Native address of the DuckDB instance — stable per-instance id. | +| `memoryUsageBytes` | long | Bytes currently allocated for this tag. | +| `temporaryStorageBytes` | long | Bytes spilled to temporary storage for this tag. | + +Plus the standard JFR fields `startTime`, `duration`, `eventThread` +(stack traces are disabled for this event). + +## Attribution model + +The monitor is keyed on the **native DuckDB instance address**, not +on the JDBC connection. This matters when multiple connections share +a DuckDB instance (multiple `DriverManager.getConnection` calls +against the same file DB, or `conn.duplicate()`): + +- One sample stream per distinct DuckDB instance — no double-counting + of shared memory. +- `component` is captured from the first opted-in connection to an + instance; later opted-in connections to the same instance do not + change the label. +- The monitor is created when the first opted-in connection opens and + torn down when the last one closes; a subsequent `getConnection` + starts a fresh monitor. + +To attribute memory to distinct logical components, open each against +a distinct DuckDB instance and give each one a unique +`jdbc_jfr_memory_monitor` value. + +## Requirements + +A JFR-capable JVM: + +- OpenJDK/HotSpot 11 and newer: JFR is included. +- Amazon Corretto 8, OpenJDK 8u272+, and several other Java 8 + distributions: JFR backport included (`jdk.jfr` package). +- JVMs without `jdk.jfr` (e.g. some stripped Java 8 builds): the + feature is a silent no-op; the `jdbc_jfr_memory_monitor` property + is ignored and no classes that depend on `jdk.jfr` are loaded. + +No additional JVM flags are required. + +## Inspecting a recording + +With the `jfr` CLI bundled with the JDK: + +``` +jfr summary app.jfr | grep duckdb.MemoryUsage +jfr print --events duckdb.MemoryUsage app.jfr | head -12 +jfr metadata app.jfr | sed -n '/class MemoryUsage/,/^}/p' +``` + +Or open `app.jfr` in JMC for an interactive view. + +## Manual verification + +Two shell scripts reproduce the above end-to-end and are the +recommended way to sanity-check a new build: + +``` +./scripts/verify-jfr.sh # Java >= 9 +./scripts/verify-jfr-java8.sh # Java 8 (covers both JFR and no-JFR paths) +``` + +Switch the active JDK first (for example +`sdk u java 25.0.3-amzn` or `sdk u java 8.0.462-amzn`). Each script +builds any missing artifacts, runs the four `test_jfr_memory_event*` +unit tests, and — on Java ≥ 9 — captures a live recording and +verifies the event with `jfr summary` and `jfr print`. The Java 8 +script additionally asserts the JFR-less fallback by running the +driver with `jfr.jar` stripped from the bootclasspath. diff --git a/README.md b/README.md index 3f1b60c52..6f06e4353 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,5 @@ java -cp "build/release/duckdb_jdbc_tests.jar:build/release/duckdb_jdbc.jar" or ``` Scalar function usage examples: [UDF.MD](UDF.MD) + +JFR memory monitoring usage: [JFR.md](JFR.md) diff --git a/scripts/verify-jfr-java8.sh b/scripts/verify-jfr-java8.sh new file mode 100755 index 000000000..0104a0b8a --- /dev/null +++ b/scripts/verify-jfr-java8.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Manual verification of the JFR memory-monitoring feature on Java 8. +# +# Usage: ./scripts/verify-jfr-java8.sh +# Assumes: `java`, `javac` on PATH point at JDK 8 (e.g. `sdk u java 8.0.462-amzn`). +# `make release` has been run so that the platform-specific native +# library is available; this script rebuilds only the Java jars. + +set -euo pipefail + +REPO="$(cd "$(dirname "$0")/.." && pwd)" +RELEASE="$REPO/build/release" +BUILD="$REPO/build/java8" +WORK="$(mktemp -d -t duckdb-jfr-verify8.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +die() { echo "error: $*" >&2; exit 1; } +step() { printf '\n== %s ==\n' "$*"; } + +# Preconditions --------------------------------------------------------------- + +java_major=$(java -version 2>&1 | awk -F\" '/version/ {split($2,a,"."); print (a[1]=="1")?a[2]:a[1]; exit}') +[[ "$java_major" == "8" ]] || die "need Java 8, got $java_major (use verify-jfr.sh for Java >= 9)" +[[ -d "$RELEASE" ]] || die "run 'make release' first (native library not found)" +NATIVE_LIB=$(ls "$RELEASE"/libduckdb_java.so_* 2>/dev/null | head -n1) \ + || die "no libduckdb_java.so_* under $RELEASE" + +# Build Java 8 jars (idempotent) --------------------------------------------- + +if [[ ! -f "$BUILD/duckdb_jdbc_tests.jar" ]]; then + step "building Java 8 jars" + mkdir -p "$BUILD" + (cd "$BUILD" \ + && cmake -DCMAKE_BUILD_TYPE=Release "$REPO" >/dev/null \ + && cmake --build . --target duckdb_jdbc_tests >/dev/null) + cp "$BUILD/duckdb_jdbc_nolib.jar" "$BUILD/duckdb_jdbc.jar" + jar uf "$BUILD/duckdb_jdbc.jar" -C "$(dirname "$NATIVE_LIB")" "$(basename "$NATIVE_LIB")" +fi + +JAR="$BUILD/duckdb_jdbc.jar" +TESTS="$BUILD/duckdb_jdbc_tests.jar" + +# Confirm bytecode 52 (Java 8) ------------------------------------------------ + +bc_hex=$(unzip -p "$JAR" org/duckdb/DuckDBMemoryEvent.class | od -An -N8 -tx1 | awk '{print $8}') +[[ "$bc_hex" == "34" ]] || die "expected bytecode 0x34 (Java 8), got 0x$bc_hex" + +# 1. Unit tests on Java 8 + JFR ---------------------------------------------- + +step "running JFR unit tests on Java 8 (jdk.jfr backport present)" +java -cp "$TESTS:$JAR" org/duckdb/TestDuckDBJDBC test_jfr_memory + +# 2. Fallback path: jfr.jar stripped from the bootclasspath ------------------- + +step "verifying the JFR-absent fallback path" +cat > "$WORK/NoJfrDemo.java" <<'EOF' +import java.lang.reflect.Method; +import java.sql.*; +import java.util.Properties; + +public class NoJfrDemo { + public static void main(String[] a) throws Exception { + try { Class.forName("jdk.jfr.FlightRecorder"); throw new AssertionError("JFR present"); } + catch (ClassNotFoundException ok) {} + Class.forName("org.duckdb.DuckDBDriver"); + Properties p = new Properties(); + p.setProperty("jdbc_jfr_memory_monitor", "ignored"); + try (Connection c = DriverManager.getConnection("jdbc:duckdb:", p); + Statement s = c.createStatement(); + ResultSet r = s.executeQuery("SELECT 42")) { r.next(); } + Method f = ClassLoader.class.getDeclaredMethod("findLoadedClass", String.class); + f.setAccessible(true); + ClassLoader cl = ClassLoader.getSystemClassLoader(); + if (f.invoke(cl, "org.duckdb.DuckDBMemoryMonitor") != null + || f.invoke(cl, "org.duckdb.DuckDBMemoryEvent") != null) + throw new AssertionError("JFR-dependent class was loaded"); + System.out.println("OK"); + } +} +EOF +javac -d "$WORK" -cp "$JAR" "$WORK/NoJfrDemo.java" + +JRE_LIB="$JAVA_HOME/jre/lib" +BOOT="$JRE_LIB/rt.jar:$JRE_LIB/jsse.jar:$JRE_LIB/jce.jar:$JRE_LIB/charsets.jar" +java -Xbootclasspath:"$BOOT" -cp "$WORK:$JAR" NoJfrDemo + +printf '\nOK\n' diff --git a/scripts/verify-jfr.sh b/scripts/verify-jfr.sh new file mode 100755 index 000000000..3ab6c3df8 --- /dev/null +++ b/scripts/verify-jfr.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# Manual verification of the JFR memory-monitoring feature on Java >= 9. +# +# Usage: ./scripts/verify-jfr.sh +# Assumes: `java`, `javac`, `jfr` on PATH point at Java 9+ (same major version). +# `make release` has been run (artifacts under build/release). + +set -euo pipefail + +REPO="$(cd "$(dirname "$0")/.." && pwd)" +JAR="$REPO/build/release/duckdb_jdbc.jar" +TESTS="$REPO/build/release/duckdb_jdbc_tests.jar" +WORK="$(mktemp -d -t duckdb-jfr-verify.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +die() { echo "error: $*" >&2; exit 1; } +step() { printf '\n== %s ==\n' "$*"; } + +# Preconditions --------------------------------------------------------------- + +java_major=$(java -version 2>&1 | awk -F\" '/version/ {split($2,a,"."); print (a[1]=="1")?a[2]:a[1]; exit}') +[[ "$java_major" -ge 9 ]] || die "need Java >= 9, got $java_major (use verify-jfr-java8.sh for Java 8)" +command -v jfr >/dev/null || die "'jfr' CLI not found on PATH" +[[ -f "$JAR" && -f "$TESTS" ]] || die "build artifacts missing; run 'make release' first" + +echo "java $java_major -- $JAR" + +# 1. Unit tests --------------------------------------------------------------- + +step "running JFR unit tests" +java --enable-native-access=ALL-UNNAMED \ + -cp "$TESTS:$JAR" \ + org/duckdb/TestDuckDBJDBC test_jfr_memory + +# 2. End-to-end demo + jfr CLI inspection ------------------------------------ + +step "capturing a live recording" +cat > "$WORK/JfrDemo.java" <<'EOF' +import java.nio.file.*; +import java.sql.*; +import java.time.Duration; +import java.util.Properties; +import jdk.jfr.Recording; + +public class JfrDemo { + public static void main(String[] a) throws Exception { + try (Recording r = new Recording()) { + r.enable("duckdb.MemoryUsage").withPeriod(Duration.ofMillis(500)); + r.start(); + Properties p = new Properties(); + p.setProperty("jdbc_jfr_memory_monitor", "verify-jfr"); + try (Connection c = DriverManager.getConnection("jdbc:duckdb:", p); + Statement s = c.createStatement()) { + s.execute("CREATE TABLE t AS SELECT range AS i FROM range(2000000)"); + Thread.sleep(2000); + } + r.stop(); + r.dump(Path.of(a[0])); + } + } +} +EOF +javac -d "$WORK" -cp "$JAR" "$WORK/JfrDemo.java" +java --enable-native-access=ALL-UNNAMED -cp "$WORK:$JAR" JfrDemo "$WORK/demo.jfr" + +step "jfr summary (expect a non-zero count for duckdb.MemoryUsage)" +jfr summary "$WORK/demo.jfr" | grep duckdb.MemoryUsage \ + || die "duckdb.MemoryUsage event not found in recording" + +step "first event (expect component=verify-jfr, non-zero dbAddress)" +jfr print --events duckdb.MemoryUsage "$WORK/demo.jfr" | sed -n '1,12p' + +printf '\nOK\n' diff --git a/src/main/java/org/duckdb/DuckDBConnection.java b/src/main/java/org/duckdb/DuckDBConnection.java index 80201f643..1a894650d 100644 --- a/src/main/java/org/duckdb/DuckDBConnection.java +++ b/src/main/java/org/duckdb/DuckDBConnection.java @@ -59,7 +59,7 @@ public final class DuckDBConnection implements java.sql.Connection { /** * Native address of the underlying DuckDB instance. Captured once at construction so that * {@link #close()} can notify {@link JfrMemoryMonitor} without an additional JNI call - * and so the JFR event can expose it as a secondary disambiguator. + * and so the JFR event can expose it as a stable per-instance identifier. */ final long dbAddress; diff --git a/src/main/java/org/duckdb/DuckDBDriver.java b/src/main/java/org/duckdb/DuckDBDriver.java index ddd0c9b08..31938bdb6 100644 --- a/src/main/java/org/duckdb/DuckDBDriver.java +++ b/src/main/java/org/duckdb/DuckDBDriver.java @@ -171,9 +171,12 @@ public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws "Do not close the DB instance after all connections to it are closed")); list.add(createDriverPropInfo(JDBC_IGNORE_UNSUPPORTED_OPTIONS, "", "Silently discard unsupported connection options")); - list.add(createDriverPropInfo( - JDBC_JFR_MEMORY_MONITOR, "", - "User-assigned identifier under which this connection's DuckDB instance is tracked in the duckdb.MemoryUsage JFR event. Leave empty to disable monitoring. JFR controls the event's enabled state and period via recording settings. Requires a JFR-capable JVM.")); + list.add( + createDriverPropInfo(JDBC_JFR_MEMORY_MONITOR, "", + "User-assigned identifier under which this connection's DuckDB instance is tracked" + + " in the duckdb.MemoryUsage JFR event. Leave empty to disable monitoring." + + " JFR controls the event's enabled state and period via recording settings." + + " Requires a JFR-capable JVM.")); list.sort((o1, o2) -> o1.name.compareToIgnoreCase(o2.name)); return list.toArray(new DriverPropertyInfo[0]); } diff --git a/src/main/java/org/duckdb/DuckDBMemoryEvent.java b/src/main/java/org/duckdb/DuckDBMemoryEvent.java index 7577b560e..e57fb0972 100644 --- a/src/main/java/org/duckdb/DuckDBMemoryEvent.java +++ b/src/main/java/org/duckdb/DuckDBMemoryEvent.java @@ -1,6 +1,7 @@ package org.duckdb; import jdk.jfr.Category; +import jdk.jfr.DataAmount; import jdk.jfr.Description; import jdk.jfr.Event; import jdk.jfr.Label; @@ -36,28 +37,28 @@ @Description("Periodic snapshot of DuckDB internal memory consumption per tag") @Category("DuckDB") @StackTrace(false) -public class DuckDBMemoryEvent extends Event { +final class DuckDBMemoryEvent extends Event { - @Label("Name") + @Label("Component") @Description( "User-assigned identifier of the DuckDB instance (value of the jdbc_jfr_memory_monitor connection property)") - String name; + String component; @Label("Tag") @Description("DuckDB internal memory tag (e.g. \"Base\", \"Hash Table\", \"Buffer Manager\")") String tag; - @Label("Database URL") - @Description("JDBC URL or database name of the DuckDB instance emitting this event") - String dbUrl; - @Label("Database Address") @Description("Native address of the underlying DuckDB instance; disambiguates databases when names collide") long dbAddress; - @Label("Memory Usage") @Description("Bytes currently allocated for this tag") long memoryUsageBytes; + @Label("Memory Usage") + @Description("Bytes currently allocated for this tag") + @DataAmount(DataAmount.BYTES) + long memoryUsageBytes; @Label("Temporary Storage Usage") @Description("Bytes spilled to the temporary storage for this tag") + @DataAmount(DataAmount.BYTES) long temporaryStorageBytes; } diff --git a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java index b64b23d2d..d74294f47 100644 --- a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java +++ b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java @@ -1,8 +1,8 @@ package org.duckdb; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -14,7 +14,7 @@ *

Activation

*

Monitoring is opt-in per connection. The caller supplies a user-assigned identifier * via the {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} connection property; that value - * becomes the {@code name} field of every {@link DuckDBMemoryEvent} emitted for the + * becomes the {@code component} field of every {@link DuckDBMemoryEvent} emitted for the * connection's DuckDB instance. A monitor is created for a DuckDB instance when the first * opted-in connection to it is opened and destroyed when the last such connection is closed. * @@ -30,10 +30,10 @@ *

Attribution model

*

The monitor registry is keyed on the native DuckDB instance address so that multiple * connections to the same underlying database share a single sample stream — avoiding - * double-counting of shared memory. The user-supplied name is captured from the first - * opted-in connection and emitted on every event for that monitor. When attributing memory - * to distinct application components, use a distinct DuckDB instance per component and - * give each one a unique {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} name. + * double-counting of shared memory. The user-supplied component identifier is captured from + * the first opted-in connection and emitted on every event for that monitor. When attributing + * memory to distinct application components, use a distinct DuckDB instance per component and + * give each one a unique {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} value. * *

Thread safety

*

Lifecycle transitions (start+insert, stop+remove) are performed inside @@ -100,14 +100,6 @@ static void connectionClosed(long dbAddress) { * active recording. Must never throw. */ private static void firePeriodicEvent() { - // Gate all per-monitor sampling on a single probe. When the event is not - // being recorded (disabled, filtered by threshold, etc.), skip the - // duckdb_memory() query entirely for every monitored instance. - // Under normal operation JFR only invokes this hook when the event is - // enabled in some recording, so shouldCommit() will typically be true. - if (!new DuckDBMemoryEvent().shouldCommit()) { - return; - } for (PerDbMonitor m : monitors.values()) { try { m.sample(); @@ -131,45 +123,72 @@ static final class PerDbMonitor { /** Guarded by compute()-serialization. */ private int openConnections = 0; - /** Written under compute(); read without lock by the JFR hook. */ + /** + * Written under compute(); read without lock by the JFR hook. + * + *

The prepared statement is parsed + planned once and executed on every tick, + * avoiding the per-tick parse overhead of a fresh {@code createStatement}. It is + * only touched from the JFR hook thread (JFR serialises periodic callbacks), so + * no additional synchronisation is required for re-execution. + */ private volatile DuckDBConnection monitorConn; - private volatile String name; - private volatile String dbUrl; + private volatile PreparedStatement sampleStmt; + private volatile String component; private volatile long dbAddress; /** * Opens (or re-attempts opening) the monitor connection and increments - * the ref count. Failure to create the monitor connection is logged - * and the ref count is still incremented so close-balance is preserved; - * a subsequent {@code open()} will retry while {@code monitorConn == null}. + * the ref count. Failure to create the monitor connection or prepare the + * sampling statement is logged and the ref count is still incremented so + * close-balance is preserved; a subsequent {@code open()} will retry + * while {@code monitorConn == null}. */ void open(DuckDBConnection conn) { if (monitorConn == null) { + DuckDBConnection mc = null; try { - DuckDBConnection mc = conn.duplicateForMonitor(); - name = conn.monitorName; - dbUrl = conn.url; + mc = conn.duplicateForMonitor(); + PreparedStatement ps = mc.prepareStatement(QUERY); + component = conn.monitorName; dbAddress = conn.dbAddress; + sampleStmt = ps; // Publish monitorConn last so readers see fully-populated state. monitorConn = mc; } catch (SQLException e) { logger.log(Level.WARNING, "Failed to open JFR memory-monitor connection; will retry on next open()", e); + if (mc != null) { + try { + mc.close(); + } catch (SQLException ce) { + // best-effort cleanup on setup failure + } + } } } openConnections++; } /** - * Decrements the ref count; when it reaches zero, releases the monitor - * connection and signals the caller to remove the map entry. + * Decrements the ref count; when it reaches zero, releases the cached + * statement and monitor connection, and signals the caller to remove + * the map entry. * * @return {@code true} when the entry should be removed */ boolean close() { if (--openConnections <= 0) { + PreparedStatement ps = sampleStmt; DuckDBConnection mc = monitorConn; + sampleStmt = null; monitorConn = null; + if (ps != null) { + try { + ps.close(); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to close JFR memory-monitor statement", e); + } + } if (mc != null) { try { mc.close(); @@ -185,7 +204,8 @@ boolean close() { /** Invoked by the JFR periodic hook. Must not throw. */ void sample() { DuckDBConnection mc = monitorConn; - if (mc == null) { + PreparedStatement ps = sampleStmt; + if (mc == null || ps == null) { return; } try { @@ -195,29 +215,26 @@ void sample() { } catch (SQLException ignored) { return; } - String nameSnap = name; - String url = dbUrl; + String componentSnap = component; long addr = dbAddress; - try (Statement stmt = mc.createStatement(); ResultSet rs = stmt.executeQuery(QUERY)) { + try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { String tag = rs.getString(1); long memoryUsageBytes = rs.getLong(2); long temporaryStorageBytes = rs.getLong(3); - emitEvent(nameSnap, url, addr, tag, memoryUsageBytes, temporaryStorageBytes); + emitEvent(componentSnap, addr, tag, memoryUsageBytes, temporaryStorageBytes); } } catch (Exception ignored) { // Propagating would break JFR's periodic dispatch; swallow. } } - private static void emitEvent(String name, String url, long addr, String tag, long memoryUsageBytes, + private static void emitEvent(String component, long addr, String tag, long memoryUsageBytes, long temporaryStorageBytes) { - // shouldCommit() was already gated at the top of firePeriodicEvent(). DuckDBMemoryEvent event = new DuckDBMemoryEvent(); event.begin(); - event.name = name; + event.component = component; event.tag = tag; - event.dbUrl = url; event.dbAddress = addr; event.memoryUsageBytes = memoryUsageBytes; event.temporaryStorageBytes = temporaryStorageBytes; diff --git a/src/main/java/org/duckdb/JfrMemoryMonitor.java b/src/main/java/org/duckdb/JfrMemoryMonitor.java index 727825ee6..83e7325d7 100644 --- a/src/main/java/org/duckdb/JfrMemoryMonitor.java +++ b/src/main/java/org/duckdb/JfrMemoryMonitor.java @@ -1,83 +1,58 @@ package org.duckdb; -import java.lang.reflect.Method; import java.util.logging.Level; import java.util.logging.Logger; /** * Indirection over {@link DuckDBMemoryMonitor} that is safe to reference on JVMs without JFR - * support (e.g. Java 8). When {@code jdk.jfr.FlightRecorder} is not available at runtime, every - * method on this class is a silent no-op. + * support (e.g. stripped Java 8 builds). When {@code jdk.jfr.FlightRecorder} is not available + * at runtime, every method on this class is a silent no-op and {@code DuckDBMemoryMonitor} — + * which imports {@code jdk.jfr.*} — is never resolved. * - *

Only this class may reference {@code DuckDBMemoryMonitor} or {@code DuckDBMemoryEvent}; any - * direct reference from the other main classes would trigger class resolution at load time and - * fail verification on a non-JFR JVM. + *

This relies on the JVM's lazy class resolution: an {@code invokestatic} against + * {@link DuckDBMemoryMonitor} only triggers resolution of that class when the instruction + * actually executes. The {@link #AVAILABLE} guard therefore prevents the JFR-dependent class + * from ever being loaded on non-JFR JVMs. + * + *

Only this class may reference {@code DuckDBMemoryMonitor} or {@code DuckDBMemoryEvent}; + * any direct reference from the other main classes would risk eager resolution on class load. */ final class JfrMemoryMonitor { private static final Logger logger = Logger.getLogger(JfrMemoryMonitor.class.getName()); - private static final Method INIT; - private static final Method CONNECTION_OPENED; - private static final Method CONNECTION_CLOSED; + private static final boolean AVAILABLE; static { - Method init = null; - Method opened = null; - Method closed = null; + boolean available; try { - // Probe for JFR first: on Java 8 this throws ClassNotFoundException - // and we never touch DuckDBMemoryMonitor (which imports jdk.jfr.*). Class.forName("jdk.jfr.FlightRecorder"); - Class impl = Class.forName("org.duckdb.DuckDBMemoryMonitor"); - init = impl.getDeclaredMethod("init"); - opened = impl.getDeclaredMethod("connectionOpened", DuckDBConnection.class); - closed = impl.getDeclaredMethod("connectionClosed", long.class); - init.setAccessible(true); - opened.setAccessible(true); - closed.setAccessible(true); + available = true; } catch (Throwable t) { - // JFR unavailable; every method becomes a silent no-op. + available = false; logger.log(Level.FINE, "JFR memory monitor is not available on this JVM", t); } - INIT = init; - CONNECTION_OPENED = opened; - CONNECTION_CLOSED = closed; + AVAILABLE = available; } private JfrMemoryMonitor() { } static void init() { - if (INIT == null) { - return; - } - try { - INIT.invoke(null); - } catch (Throwable t) { - logger.log(Level.FINE, "JFR memory monitor init failed", t); + if (AVAILABLE) { + DuckDBMemoryMonitor.init(); } } static void connectionOpened(DuckDBConnection conn) { - if (CONNECTION_OPENED == null) { - return; - } - try { - CONNECTION_OPENED.invoke(null, conn); - } catch (Throwable t) { - logger.log(Level.FINE, "JFR connectionOpened failed", t); + if (AVAILABLE) { + DuckDBMemoryMonitor.connectionOpened(conn); } } static void connectionClosed(long dbAddress) { - if (CONNECTION_CLOSED == null) { - return; - } - try { - CONNECTION_CLOSED.invoke(null, dbAddress); - } catch (Throwable t) { - logger.log(Level.FINE, "JFR connectionClosed failed", t); + if (AVAILABLE) { + DuckDBMemoryMonitor.connectionClosed(dbAddress); } } } diff --git a/src/test/java/org/duckdb/TestDuckDBJDBC.java b/src/test/java/org/duckdb/TestDuckDBJDBC.java index 14dfde1db..9386b7f0f 100644 --- a/src/test/java/org/duckdb/TestDuckDBJDBC.java +++ b/src/test/java/org/duckdb/TestDuckDBJDBC.java @@ -2285,7 +2285,7 @@ public static void DISABLED_test_extension_excel() throws Exception { *

    *
  • No events are emitted when the {@code jdbc_jfr_memory_monitor} property is absent.
    • *
  • Events are emitted when the property is set, for each independent in-memory database, - * tagged with the user-supplied name.
    • + * tagged with the user-supplied component identifier. *
  • Each event carries a non-empty tag and non-negative memory values.
    • *
*/ @@ -2302,7 +2302,7 @@ public static void test_jfr_memory_event() throws Exception { assertTrue(events.isEmpty(), "Expected no events when monitor property is not set"); } - // --- Part 2: property set -> events emitted under each supplied name --- + // --- Part 2: property set -> events emitted under each supplied component --- Properties propsA = new Properties(); propsA.setProperty(DuckDBDriver.JDBC_JFR_MEMORY_MONITOR, "component-a"); Properties propsB = new Properties(); @@ -2322,19 +2322,15 @@ public static void test_jfr_memory_event() throws Exception { List events = dumpEvents(rec); assertFalse(events.isEmpty(), "Expected at least one DuckDBMemory JFR event"); - Set names = new HashSet<>(); + Set components = new HashSet<>(); for (RecordedEvent event : events) { - String name = event.getString("name"); - assertTrue(name != null && !name.isEmpty(), "name field must be non-empty"); - names.add(name); + String component = event.getString("component"); + assertTrue(component != null && !component.isEmpty(), "component field must be non-empty"); + components.add(component); String tag = event.getString("tag"); assertTrue(tag != null && !tag.isEmpty(), "tag field must be non-empty"); - String dbUrl = event.getString("dbUrl"); - assertTrue(dbUrl != null && dbUrl.startsWith("jdbc:duckdb:"), - "dbUrl must be a JDBC URL, was: " + dbUrl); - long dbAddress = event.getLong("dbAddress"); assertTrue(dbAddress != 0L, "dbAddress must be non-zero"); @@ -2344,9 +2340,9 @@ public static void test_jfr_memory_event() throws Exception { assertTrue(tmpStorage >= 0, "temporaryStorageBytes must be >= 0"); } - // Each component must emit events under its own name. - assertTrue(names.contains("component-a") && names.contains("component-b"), - "Expected events for both component names, got: " + names); + // Each component must emit events under its own identifier. + assertTrue(components.contains("component-a") && components.contains("component-b"), + "Expected events for both component identifiers, got: " + components); } } @@ -2407,15 +2403,15 @@ public static void test_jfr_memory_event_file_db_refcount() throws Exception { assertFalse(events.isEmpty(), "Expected events for the shared file-based DB"); Set addresses = new HashSet<>(); - Set names = new HashSet<>(); + Set components = new HashSet<>(); for (RecordedEvent e : events) { addresses.add(e.getLong("dbAddress")); - names.add(e.getString("name")); + components.add(e.getString("component")); } assertEquals(addresses.size(), 1, "Two connections to the same file DB must share dbAddress, got " + addresses); - assertEquals(names, new HashSet<>(singletonList("shared-db")), - "All events must be tagged with the supplied name, got: " + names); + assertEquals(components, new HashSet<>(singletonList("shared-db")), + "All events must be tagged with the supplied component, got: " + components); } } } From 4ac768692e82cc8cf17002d8358d9a9a8addc670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Wed, 6 May 2026 10:04:19 +0200 Subject: [PATCH 3/7] Use `BufferManager::GetMemoryUsageInfo()` instead of a `duckdb_memory()` The last row is the real trade-off: we now depend on `BufferManager::GetMemoryUsageInfo()` and the `MemoryTag` enum layout. Both have been stable for a long time and the `MEMORY_TAGS` length-min guard in sample() lets an older JAR cope with a DuckDB that adds new tags. --- JFR.md | 5 +- duckdb_java.def | 2 + duckdb_java.exp | 2 + duckdb_java.map | 2 + src/jni/duckdb_java.cpp | 62 ++++++ src/jni/functions.cpp | 23 +++ src/jni/functions.hpp | 8 + .../java/org/duckdb/DuckDBConnection.java | 22 +-- .../java/org/duckdb/DuckDBMemoryMonitor.java | 182 ++++++++---------- src/main/java/org/duckdb/DuckDBNative.java | 17 ++ 10 files changed, 210 insertions(+), 115 deletions(-) diff --git a/JFR.md b/JFR.md index a84abe003..493a35dc8 100644 --- a/JFR.md +++ b/JFR.md @@ -68,7 +68,10 @@ Equivalent `.jfc` snippet: ``` When no recording enables the event, the driver performs zero work — -the DuckDB `duckdb_memory()` query is never issued. +the native buffer-manager counters are not read. When enabled, each +JFR tick performs one lock-free read of the per-tag counters straight +from `BufferManager::GetMemoryUsageInfo()`; no SQL is issued and no +connection-level lock is taken. ## Event schema diff --git a/duckdb_java.def b/duckdb_java.def index 4593c41f5..34d30e857 100644 --- a/duckdb_java.def +++ b/duckdb_java.def @@ -27,6 +27,8 @@ Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address +Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot +Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute diff --git a/duckdb_java.exp b/duckdb_java.exp index 8ca137161..55556c85a 100644 --- a/duckdb_java.exp +++ b/duckdb_java.exp @@ -24,6 +24,8 @@ _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address +_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot +_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute diff --git a/duckdb_java.map b/duckdb_java.map index c780493dc..df8dfaa6c 100644 --- a/duckdb_java.map +++ b/duckdb_java.map @@ -26,6 +26,8 @@ DUCKDB_JAVA { Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address; + Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot; + Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute; diff --git a/src/jni/duckdb_java.cpp b/src/jni/duckdb_java.cpp index 45d8a047d..516b50ed5 100644 --- a/src/jni/duckdb_java.cpp +++ b/src/jni/duckdb_java.cpp @@ -5,6 +5,8 @@ extern "C" { #include "duckdb.hpp" #include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/common/arrow/result_arrow_wrapper.hpp" +#include "duckdb/common/enum_util.hpp" +#include "duckdb/common/enums/memory_tag.hpp" #include "duckdb/common/operator/cast_operators.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/vector/array_vector.hpp" @@ -19,6 +21,8 @@ extern "C" { #include "duckdb/main/db_instance_cache.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/parser/parsed_data/create_type_info.hpp" +#include "duckdb/storage/buffer/temporary_file_information.hpp" +#include "duckdb/storage/buffer_manager.hpp" #include "functions.hpp" #include "holders.hpp" #include "refs.hpp" @@ -100,6 +104,64 @@ jlong _duckdb_jdbc_db_address(JNIEnv *env, jclass, jobject conn_ref_buf) { return (jlong)conn_ref->db.get(); } +jlongArray _duckdb_jdbc_memory_snapshot(JNIEnv *env, jclass, jobject db_ref_buf) { + if (nullptr == db_ref_buf) { + throw std::runtime_error("db_ref is null"); + } + auto db_ref = (DBHolder *)env->GetDirectBufferAddress(db_ref_buf); + if (!db_ref || !db_ref->db) { + throw std::runtime_error("invalid db_ref"); + } + auto &instance = *db_ref->db->instance; + auto infos = BufferManager::GetBufferManager(instance).GetMemoryUsageInfo(); + + // Packed layout: [tag_0, size_0, evicted_0, tag_1, size_1, evicted_1, ...]. + // The tag index is emitted explicitly so the Java side does not depend on + // the order in which GetMemoryUsageInfo() returns entries. Any entry index + // beyond MEMORY_TAG_COUNT (not expected from the current native impl) is + // dropped; Java would ignore unknown tag indices in any case. + jsize n = (jsize)infos.size(); + if (n > (jsize)MEMORY_TAG_COUNT) { + n = (jsize)MEMORY_TAG_COUNT; + } + jlongArray result = env->NewLongArray(n * 3); + if (!result) { + return nullptr; // OOM: a pending OutOfMemoryError is already set by the JVM. + } + jlong buf[MEMORY_TAG_COUNT * 3]; + for (jsize i = 0; i < n; i++) { + buf[i * 3 + 0] = (jlong)infos[i].tag; + buf[i * 3 + 1] = (jlong)infos[i].size; + buf[i * 3 + 2] = (jlong)infos[i].evicted_data; + } + env->SetLongArrayRegion(result, 0, n * 3, buf); + return result; +} + +jobjectArray _duckdb_jdbc_memory_tags(JNIEnv *env, jclass) { + const jsize n = (jsize)MEMORY_TAG_COUNT; + jclass string_cls = env->FindClass("java/lang/String"); + if (!string_cls) { + return nullptr; + } + jobjectArray result = env->NewObjectArray(n, string_cls, nullptr); + env->DeleteLocalRef(string_cls); + if (!result) { + return nullptr; + } + for (jsize i = 0; i < n; i++) { + const char *name = EnumUtil::ToChars(MemoryTag(i)); + jstring s = env->NewStringUTF(name); + if (!s) { + env->DeleteLocalRef(result); + return nullptr; + } + env->SetObjectArrayElement(result, i, s); + env->DeleteLocalRef(s); + } + return result; +} + void _duckdb_jdbc_destroy_db_ref(JNIEnv *env, jclass, jobject db_ref_buf) { if (nullptr == db_ref_buf) { return; diff --git a/src/jni/functions.cpp b/src/jni/functions.cpp index 160c0f71a..f648567f6 100644 --- a/src/jni/functions.cpp +++ b/src/jni/functions.cpp @@ -44,6 +44,29 @@ JNIEXPORT jlong JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address(J duckdb::ErrorData error(e); ThrowJNI(env, error.Message().c_str()); + return -1; + } +} + +JNIEXPORT jlongArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot(JNIEnv * env, jclass param0, jobject param1) { + try { + return _duckdb_jdbc_memory_snapshot(env, param0, param1); + } catch (const std::exception &e) { + duckdb::ErrorData error(e); + ThrowJNI(env, error.Message().c_str()); + + return nullptr; + } +} + +JNIEXPORT jobjectArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags(JNIEnv * env, jclass param0) { + try { + return _duckdb_jdbc_memory_tags(env, param0); + } catch (const std::exception &e) { + duckdb::ErrorData error(e); + ThrowJNI(env, error.Message().c_str()); + + return nullptr; } } diff --git a/src/jni/functions.hpp b/src/jni/functions.hpp index d6b2c452f..cbc05d73a 100644 --- a/src/jni/functions.hpp +++ b/src/jni/functions.hpp @@ -25,6 +25,14 @@ jlong _duckdb_jdbc_db_address(JNIEnv * env, jclass param0, jobject param1); JNIEXPORT jlong JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address(JNIEnv * env, jclass param0, jobject param1); +jlongArray _duckdb_jdbc_memory_snapshot(JNIEnv * env, jclass param0, jobject param1); + +JNIEXPORT jlongArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot(JNIEnv * env, jclass param0, jobject param1); + +jobjectArray _duckdb_jdbc_memory_tags(JNIEnv * env, jclass param0); + +JNIEXPORT jobjectArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags(JNIEnv * env, jclass param0); + void _duckdb_jdbc_destroy_db_ref(JNIEnv * env, jclass param0, jobject param1); JNIEXPORT void JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref(JNIEnv * env, jclass param0, jobject param1); diff --git a/src/main/java/org/duckdb/DuckDBConnection.java b/src/main/java/org/duckdb/DuckDBConnection.java index 1a894650d..5688afc79 100644 --- a/src/main/java/org/duckdb/DuckDBConnection.java +++ b/src/main/java/org/duckdb/DuckDBConnection.java @@ -50,9 +50,9 @@ public final class DuckDBConnection implements java.sql.Connection { /** * User-supplied identifier for JFR memory monitoring (the value of the - * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property). {@code null} or empty - * means this connection does not participate in monitoring — either the user - * did not opt in, or this connection IS the monitor connection. + * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property), or {@code null} when the property + * was absent or empty. A non-null value opts this connection into monitoring and becomes the + * {@code component} field on every emitted {@link DuckDBMemoryEvent}. */ final String monitorName; @@ -120,18 +120,6 @@ public Statement createStatement() throws SQLException { } public DuckDBConnection duplicate() throws SQLException { - return duplicate(this.monitorName); - } - - /** - * Creates a duplicate connection that is invisible to the JFR memory monitor. - * Used exclusively by the monitor itself to avoid re-entrant lifecycle callbacks. - */ - DuckDBConnection duplicateForMonitor() throws SQLException { - return duplicate(null); - } - - private DuckDBConnection duplicate(String monitorName) throws SQLException { checkOpen(); connRefLock.lock(); try { @@ -162,8 +150,8 @@ public void close() throws SQLException { return; } // Notify the memory monitor only from the call that actually performed - // the disconnect, and after releasing connRefLock so the monitor's own - // native disconnect does not block the caller under our lock. + // the disconnect, and only after releasing connRefLock so the potentially + // slow DB-instance teardown does not run while we hold the lock. boolean notifyMonitor = false; connRefLock.lock(); try { diff --git a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java index d74294f47..9fa334883 100644 --- a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java +++ b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java @@ -1,7 +1,6 @@ package org.duckdb; -import java.sql.PreparedStatement; -import java.sql.ResultSet; +import java.nio.ByteBuffer; import java.sql.SQLException; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; @@ -18,33 +17,45 @@ * connection's DuckDB instance. A monitor is created for a DuckDB instance when the first * opted-in connection to it is opened and destroyed when the last such connection is closed. * + *

Sampling model

+ *

The monitor does not hold a JDBC connection. It pins the underlying + * DuckDB instance via {@link DuckDBNative#duckdb_jdbc_create_db_ref} and, on each JFR tick, + * calls {@link DuckDBNative#duckdb_jdbc_memory_snapshot} to read the per-tag memory counters + * straight from the native {@code BufferManager}. This avoids running SQL, allocating result + * sets, and taking any connection-level lock that might interfere with user traffic. + * *

Event scheduling

- *

This class does not own a scheduler. It registers a single - * {@linkplain FlightRecorder#addPeriodicEvent periodic JFR hook} for - * {@link DuckDBMemoryEvent}; JFR invokes the hook at the period configured on - * the recording (e.g. {@code 1 s}) and only - * while at least one active recording has the event enabled. Consequently, the - * JDBC property is a pure enable/label switch and JFR alone governs the - * sampling rate and the enabled state of the event. + *

This class registers a single {@linkplain FlightRecorder#addPeriodicEvent periodic JFR hook} + * for {@link DuckDBMemoryEvent}; JFR invokes it at the period configured on the recording + * and only while at least one active recording has the event enabled. * *

Attribution model

*

The monitor registry is keyed on the native DuckDB instance address so that multiple * connections to the same underlying database share a single sample stream — avoiding * double-counting of shared memory. The user-supplied component identifier is captured from - * the first opted-in connection and emitted on every event for that monitor. When attributing - * memory to distinct application components, use a distinct DuckDB instance per component and - * give each one a unique {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} value. + * the first opted-in connection and emitted on every event for that monitor. * *

Thread safety

*

Lifecycle transitions (start+insert, stop+remove) are performed inside * {@link ConcurrentHashMap#compute}, which provides per-key mutual exclusion. - * The JFR periodic hook iterates {@link ConcurrentHashMap#values} without - * locking and relies on volatile reads in {@link PerDbMonitor} for visibility. + * The JFR periodic hook iterates {@link ConcurrentHashMap#values} without locking and + * relies on a single volatile read of {@code dbRef} for visibility. */ final class DuckDBMemoryMonitor { private static final Logger logger = Logger.getLogger(DuckDBMemoryMonitor.class.getName()); + /** Memory-tag names in enum order; fetched once from the native side and cached. */ + private static final String[] MEMORY_TAGS = loadMemoryTags(); + + private static String[] loadMemoryTags() { + try { + return DuckDBNative.duckdb_jdbc_memory_tags(); + } catch (SQLException e) { + throw new ExceptionInInitializerError(e); + } + } + /** Registry: native DuckDB* address -> per-database monitor. */ private static final ConcurrentHashMap monitors = new ConcurrentHashMap<>(); @@ -55,11 +66,9 @@ private DuckDBMemoryMonitor() { } /** - * Registers the periodic JFR hook for {@link DuckDBMemoryEvent}. Idempotent - * and called from {@link DuckDBDriver}'s static initializer so that recordings - * started before the first monitored connection still see the event type. - * Iterating an empty monitor map at each tick is cheap, so there is no - * downside to registering unconditionally. + * Registers the periodic JFR hook for {@link DuckDBMemoryEvent}. Idempotent and called + * from {@link DuckDBDriver}'s static initializer so that recordings started before any + * monitored connection is opened still see the event type. */ static synchronized void init() { if (initialized) { @@ -96,8 +105,10 @@ static void connectionClosed(long dbAddress) { } /** - * JFR-invoked hook. Runs on a JFR thread at the period configured on the - * active recording. Must never throw. + * JFR-invoked hook. Runs on a JFR thread at the period configured on the active recording. + * JFR invokes it only when the event is enabled in some recording; the per-event + * {@link jdk.jfr.Event#commit()} performs the final {@code shouldCommit()} check. + * Must never throw. */ private static void firePeriodicEvent() { for (PerDbMonitor m : monitors.values()) { @@ -110,122 +121,99 @@ private static void firePeriodicEvent() { } /** - * Per-name monitor state. Mutating methods ({@link #open}, {@link #close}) - * are invoked inside {@link ConcurrentHashMap#compute} on {@link #monitors} - * and are serialized per key. {@link #sample} runs on the JFR hook thread - * without the compute lock and reads {@code volatile} fields. + * Per-instance monitor state. Mutating methods ({@link #open}, {@link #close}) are + * invoked inside {@link ConcurrentHashMap#compute} on {@link #monitors} and are + * serialized per key. {@link #sample} runs on the JFR hook thread without the compute + * lock and reads the {@code volatile dbRef} first to establish happens-before for the + * other fields. */ static final class PerDbMonitor { - private static final String QUERY = - "SELECT tag, memory_usage_bytes, temporary_storage_bytes FROM duckdb_memory()"; - /** Guarded by compute()-serialization. */ private int openConnections = 0; /** - * Written under compute(); read without lock by the JFR hook. - * - *

The prepared statement is parsed + planned once and executed on every tick, - * avoiding the per-tick parse overhead of a fresh {@code createStatement}. It is - * only touched from the JFR hook thread (JFR serialises periodic callbacks), so - * no additional synchronisation is required for re-execution. + * Pinned DuckDB-instance reference (DBHolder ByteBuffer). Volatile: read lock-free by + * the JFR hook. Writing this last publishes the fully-initialized {@link #component} + * and {@link #dbAddress} fields. */ - private volatile DuckDBConnection monitorConn; - private volatile PreparedStatement sampleStmt; - private volatile String component; - private volatile long dbAddress; + private volatile ByteBuffer dbRef; + private String component; + private long dbAddress; /** - * Opens (or re-attempts opening) the monitor connection and increments - * the ref count. Failure to create the monitor connection or prepare the - * sampling statement is logged and the ref count is still incremented so - * close-balance is preserved; a subsequent {@code open()} will retry - * while {@code monitorConn == null}. + * Opens (or re-attempts opening) the pinned DB reference and increments the ref count. + * Failure to pin is logged; the ref count is still incremented so close-balance is + * preserved and a subsequent {@code open()} will retry while {@code dbRef == null}. */ void open(DuckDBConnection conn) { - if (monitorConn == null) { - DuckDBConnection mc = null; + if (dbRef == null) { try { - mc = conn.duplicateForMonitor(); - PreparedStatement ps = mc.prepareStatement(QUERY); + ByteBuffer ref = DuckDBNative.duckdb_jdbc_create_db_ref(conn.connRef); component = conn.monitorName; dbAddress = conn.dbAddress; - sampleStmt = ps; - // Publish monitorConn last so readers see fully-populated state. - monitorConn = mc; + // Publish dbRef last so readers see fully-populated state. + dbRef = ref; } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to open JFR memory-monitor connection; will retry on next open()", - e); - if (mc != null) { - try { - mc.close(); - } catch (SQLException ce) { - // best-effort cleanup on setup failure - } - } + logger.log(Level.WARNING, "Failed to pin DB for JFR memory monitor; will retry on next open()", e); } } openConnections++; } /** - * Decrements the ref count; when it reaches zero, releases the cached - * statement and monitor connection, and signals the caller to remove - * the map entry. + * Decrements the ref count; when it reaches zero, releases the pinned DB reference + * and signals the caller to remove the map entry. * * @return {@code true} when the entry should be removed */ boolean close() { - if (--openConnections <= 0) { - PreparedStatement ps = sampleStmt; - DuckDBConnection mc = monitorConn; - sampleStmt = null; - monitorConn = null; - if (ps != null) { - try { - ps.close(); - } catch (SQLException e) { - logger.log(Level.FINE, "Failed to close JFR memory-monitor statement", e); - } - } - if (mc != null) { - try { - mc.close(); - } catch (SQLException e) { - logger.log(Level.FINE, "Failed to close JFR memory-monitor connection", e); - } + if (--openConnections > 0) { + return false; + } + // Clamp on imbalance: a stray close() without a matching open() must not leave + // the counter negative so that a future open() starts from a consistent state. + openConnections = 0; + ByteBuffer ref = dbRef; + dbRef = null; + if (ref != null) { + try { + DuckDBNative.duckdb_jdbc_destroy_db_ref(ref); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to release JFR memory-monitor DB ref", e); } - return true; } - return false; + return true; } /** Invoked by the JFR periodic hook. Must not throw. */ void sample() { - DuckDBConnection mc = monitorConn; - PreparedStatement ps = sampleStmt; - if (mc == null || ps == null) { + ByteBuffer ref = dbRef; + if (ref == null) { return; } + long[] snapshot; try { - if (mc.isClosed()) { - return; - } - } catch (SQLException ignored) { + snapshot = DuckDBNative.duckdb_jdbc_memory_snapshot(ref); + } catch (Throwable t) { + // Rare: native snapshot failure. Swallow so JFR's periodic dispatch survives. + return; + } + if (snapshot == null) { return; } String componentSnap = component; long addr = dbAddress; - try (ResultSet rs = ps.executeQuery()) { - while (rs.next()) { - String tag = rs.getString(1); - long memoryUsageBytes = rs.getLong(2); - long temporaryStorageBytes = rs.getLong(3); - emitEvent(componentSnap, addr, tag, memoryUsageBytes, temporaryStorageBytes); + // Snapshot is packed as [tag, size, evicted] triples; the tag index is + // emitted explicitly by the native side, so the order of entries does + // not matter. An unknown tag index (e.g. from a future native lib that + // added tags Java does not know about) is skipped rather than misnamed. + for (int i = 0; i + 3 <= snapshot.length; i += 3) { + int tagIdx = (int) snapshot[i]; + if (tagIdx < 0 || tagIdx >= MEMORY_TAGS.length) { + continue; } - } catch (Exception ignored) { - // Propagating would break JFR's periodic dispatch; swallow. + emitEvent(componentSnap, addr, MEMORY_TAGS[tagIdx], snapshot[i + 1], snapshot[i + 2]); } } diff --git a/src/main/java/org/duckdb/DuckDBNative.java b/src/main/java/org/duckdb/DuckDBNative.java index 30b1c0e22..ed71c7715 100644 --- a/src/main/java/org/duckdb/DuckDBNative.java +++ b/src/main/java/org/duckdb/DuckDBNative.java @@ -152,6 +152,23 @@ private static void loadFromCurrentJarDir(String libName) throws Exception { /** Returns the native address of the underlying DuckDB instance as a stable identity key. */ static native long duckdb_jdbc_db_address(ByteBuffer conn_ref) throws SQLException; + /** + * Reads DuckDB memory counters directly via {@code BufferManager::GetMemoryUsageInfo()}. + * Returns a packed {@code long[]} of length {@code 3 * N} where N is the number of memory + * entries reported: {@code [tag_0, size_0, evicted_0, tag_1, size_1, evicted_1, ...]}. Each + * {@code tag} is an index into the {@link #duckdb_jdbc_memory_tags} array; entries may be + * returned in any order. The {@code db_ref} argument must be a live pinned DB reference + * created with {@link #duckdb_jdbc_create_db_ref}. + */ + static native long[] duckdb_jdbc_memory_snapshot(ByteBuffer db_ref) throws SQLException; + + /** + * Returns the DuckDB memory-tag names indexed by {@code MemoryTag} enum value. Tag indices + * returned by {@link #duckdb_jdbc_memory_snapshot} can be used directly as indices into this + * array. The result is stable for a given native library and safe to cache. + */ + static native String[] duckdb_jdbc_memory_tags() throws SQLException; + static native void duckdb_jdbc_set_auto_commit(ByteBuffer conn_ref, boolean auto_commit) throws SQLException; static native boolean duckdb_jdbc_get_auto_commit(ByteBuffer conn_ref) throws SQLException; From b011206b2bed8f1043f40d02d451991de3bbaff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Wed, 6 May 2026 13:07:22 +0200 Subject: [PATCH 4/7] Revert "Use `BufferManager::GetMemoryUsageInfo()` instead of a `duckdb_memory()`" This reverts commit 4ac768692e82cc8cf17002d8358d9a9a8addc670. --- JFR.md | 5 +- duckdb_java.def | 2 - duckdb_java.exp | 2 - duckdb_java.map | 2 - src/jni/duckdb_java.cpp | 62 ------ src/jni/functions.cpp | 23 --- src/jni/functions.hpp | 8 - .../java/org/duckdb/DuckDBConnection.java | 22 ++- .../java/org/duckdb/DuckDBMemoryMonitor.java | 182 ++++++++++-------- src/main/java/org/duckdb/DuckDBNative.java | 17 -- 10 files changed, 115 insertions(+), 210 deletions(-) diff --git a/JFR.md b/JFR.md index 493a35dc8..a84abe003 100644 --- a/JFR.md +++ b/JFR.md @@ -68,10 +68,7 @@ Equivalent `.jfc` snippet: ``` When no recording enables the event, the driver performs zero work — -the native buffer-manager counters are not read. When enabled, each -JFR tick performs one lock-free read of the per-tag counters straight -from `BufferManager::GetMemoryUsageInfo()`; no SQL is issued and no -connection-level lock is taken. +the DuckDB `duckdb_memory()` query is never issued. ## Event schema diff --git a/duckdb_java.def b/duckdb_java.def index 34d30e857..4593c41f5 100644 --- a/duckdb_java.def +++ b/duckdb_java.def @@ -27,8 +27,6 @@ Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address -Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot -Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute diff --git a/duckdb_java.exp b/duckdb_java.exp index 55556c85a..8ca137161 100644 --- a/duckdb_java.exp +++ b/duckdb_java.exp @@ -24,8 +24,6 @@ _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1appender _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address -_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot -_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute diff --git a/duckdb_java.map b/duckdb_java.map index df8dfaa6c..c780493dc 100644 --- a/duckdb_java.map +++ b/duckdb_java.map @@ -26,8 +26,6 @@ DUCKDB_JAVA { Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1db_1ref; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address; - Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot; - Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1create_1extension_1type; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1disconnect; Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1execute; diff --git a/src/jni/duckdb_java.cpp b/src/jni/duckdb_java.cpp index 516b50ed5..45d8a047d 100644 --- a/src/jni/duckdb_java.cpp +++ b/src/jni/duckdb_java.cpp @@ -5,8 +5,6 @@ extern "C" { #include "duckdb.hpp" #include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/common/arrow/result_arrow_wrapper.hpp" -#include "duckdb/common/enum_util.hpp" -#include "duckdb/common/enums/memory_tag.hpp" #include "duckdb/common/operator/cast_operators.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/vector/array_vector.hpp" @@ -21,8 +19,6 @@ extern "C" { #include "duckdb/main/db_instance_cache.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/parser/parsed_data/create_type_info.hpp" -#include "duckdb/storage/buffer/temporary_file_information.hpp" -#include "duckdb/storage/buffer_manager.hpp" #include "functions.hpp" #include "holders.hpp" #include "refs.hpp" @@ -104,64 +100,6 @@ jlong _duckdb_jdbc_db_address(JNIEnv *env, jclass, jobject conn_ref_buf) { return (jlong)conn_ref->db.get(); } -jlongArray _duckdb_jdbc_memory_snapshot(JNIEnv *env, jclass, jobject db_ref_buf) { - if (nullptr == db_ref_buf) { - throw std::runtime_error("db_ref is null"); - } - auto db_ref = (DBHolder *)env->GetDirectBufferAddress(db_ref_buf); - if (!db_ref || !db_ref->db) { - throw std::runtime_error("invalid db_ref"); - } - auto &instance = *db_ref->db->instance; - auto infos = BufferManager::GetBufferManager(instance).GetMemoryUsageInfo(); - - // Packed layout: [tag_0, size_0, evicted_0, tag_1, size_1, evicted_1, ...]. - // The tag index is emitted explicitly so the Java side does not depend on - // the order in which GetMemoryUsageInfo() returns entries. Any entry index - // beyond MEMORY_TAG_COUNT (not expected from the current native impl) is - // dropped; Java would ignore unknown tag indices in any case. - jsize n = (jsize)infos.size(); - if (n > (jsize)MEMORY_TAG_COUNT) { - n = (jsize)MEMORY_TAG_COUNT; - } - jlongArray result = env->NewLongArray(n * 3); - if (!result) { - return nullptr; // OOM: a pending OutOfMemoryError is already set by the JVM. - } - jlong buf[MEMORY_TAG_COUNT * 3]; - for (jsize i = 0; i < n; i++) { - buf[i * 3 + 0] = (jlong)infos[i].tag; - buf[i * 3 + 1] = (jlong)infos[i].size; - buf[i * 3 + 2] = (jlong)infos[i].evicted_data; - } - env->SetLongArrayRegion(result, 0, n * 3, buf); - return result; -} - -jobjectArray _duckdb_jdbc_memory_tags(JNIEnv *env, jclass) { - const jsize n = (jsize)MEMORY_TAG_COUNT; - jclass string_cls = env->FindClass("java/lang/String"); - if (!string_cls) { - return nullptr; - } - jobjectArray result = env->NewObjectArray(n, string_cls, nullptr); - env->DeleteLocalRef(string_cls); - if (!result) { - return nullptr; - } - for (jsize i = 0; i < n; i++) { - const char *name = EnumUtil::ToChars(MemoryTag(i)); - jstring s = env->NewStringUTF(name); - if (!s) { - env->DeleteLocalRef(result); - return nullptr; - } - env->SetObjectArrayElement(result, i, s); - env->DeleteLocalRef(s); - } - return result; -} - void _duckdb_jdbc_destroy_db_ref(JNIEnv *env, jclass, jobject db_ref_buf) { if (nullptr == db_ref_buf) { return; diff --git a/src/jni/functions.cpp b/src/jni/functions.cpp index f648567f6..160c0f71a 100644 --- a/src/jni/functions.cpp +++ b/src/jni/functions.cpp @@ -44,29 +44,6 @@ JNIEXPORT jlong JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address(J duckdb::ErrorData error(e); ThrowJNI(env, error.Message().c_str()); - return -1; - } -} - -JNIEXPORT jlongArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot(JNIEnv * env, jclass param0, jobject param1) { - try { - return _duckdb_jdbc_memory_snapshot(env, param0, param1); - } catch (const std::exception &e) { - duckdb::ErrorData error(e); - ThrowJNI(env, error.Message().c_str()); - - return nullptr; - } -} - -JNIEXPORT jobjectArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags(JNIEnv * env, jclass param0) { - try { - return _duckdb_jdbc_memory_tags(env, param0); - } catch (const std::exception &e) { - duckdb::ErrorData error(e); - ThrowJNI(env, error.Message().c_str()); - - return nullptr; } } diff --git a/src/jni/functions.hpp b/src/jni/functions.hpp index cbc05d73a..d6b2c452f 100644 --- a/src/jni/functions.hpp +++ b/src/jni/functions.hpp @@ -25,14 +25,6 @@ jlong _duckdb_jdbc_db_address(JNIEnv * env, jclass param0, jobject param1); JNIEXPORT jlong JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1db_1address(JNIEnv * env, jclass param0, jobject param1); -jlongArray _duckdb_jdbc_memory_snapshot(JNIEnv * env, jclass param0, jobject param1); - -JNIEXPORT jlongArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1snapshot(JNIEnv * env, jclass param0, jobject param1); - -jobjectArray _duckdb_jdbc_memory_tags(JNIEnv * env, jclass param0); - -JNIEXPORT jobjectArray JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1memory_1tags(JNIEnv * env, jclass param0); - void _duckdb_jdbc_destroy_db_ref(JNIEnv * env, jclass param0, jobject param1); JNIEXPORT void JNICALL Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1destroy_1db_1ref(JNIEnv * env, jclass param0, jobject param1); diff --git a/src/main/java/org/duckdb/DuckDBConnection.java b/src/main/java/org/duckdb/DuckDBConnection.java index 5688afc79..1a894650d 100644 --- a/src/main/java/org/duckdb/DuckDBConnection.java +++ b/src/main/java/org/duckdb/DuckDBConnection.java @@ -50,9 +50,9 @@ public final class DuckDBConnection implements java.sql.Connection { /** * User-supplied identifier for JFR memory monitoring (the value of the - * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property), or {@code null} when the property - * was absent or empty. A non-null value opts this connection into monitoring and becomes the - * {@code component} field on every emitted {@link DuckDBMemoryEvent}. + * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property). {@code null} or empty + * means this connection does not participate in monitoring — either the user + * did not opt in, or this connection IS the monitor connection. */ final String monitorName; @@ -120,6 +120,18 @@ public Statement createStatement() throws SQLException { } public DuckDBConnection duplicate() throws SQLException { + return duplicate(this.monitorName); + } + + /** + * Creates a duplicate connection that is invisible to the JFR memory monitor. + * Used exclusively by the monitor itself to avoid re-entrant lifecycle callbacks. + */ + DuckDBConnection duplicateForMonitor() throws SQLException { + return duplicate(null); + } + + private DuckDBConnection duplicate(String monitorName) throws SQLException { checkOpen(); connRefLock.lock(); try { @@ -150,8 +162,8 @@ public void close() throws SQLException { return; } // Notify the memory monitor only from the call that actually performed - // the disconnect, and only after releasing connRefLock so the potentially - // slow DB-instance teardown does not run while we hold the lock. + // the disconnect, and after releasing connRefLock so the monitor's own + // native disconnect does not block the caller under our lock. boolean notifyMonitor = false; connRefLock.lock(); try { diff --git a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java index 9fa334883..d74294f47 100644 --- a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java +++ b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java @@ -1,6 +1,7 @@ package org.duckdb; -import java.nio.ByteBuffer; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; @@ -17,45 +18,33 @@ * connection's DuckDB instance. A monitor is created for a DuckDB instance when the first * opted-in connection to it is opened and destroyed when the last such connection is closed. * - *

Sampling model

- *

The monitor does not hold a JDBC connection. It pins the underlying - * DuckDB instance via {@link DuckDBNative#duckdb_jdbc_create_db_ref} and, on each JFR tick, - * calls {@link DuckDBNative#duckdb_jdbc_memory_snapshot} to read the per-tag memory counters - * straight from the native {@code BufferManager}. This avoids running SQL, allocating result - * sets, and taking any connection-level lock that might interfere with user traffic. - * *

Event scheduling

- *

This class registers a single {@linkplain FlightRecorder#addPeriodicEvent periodic JFR hook} - * for {@link DuckDBMemoryEvent}; JFR invokes it at the period configured on the recording - * and only while at least one active recording has the event enabled. + *

This class does not own a scheduler. It registers a single + * {@linkplain FlightRecorder#addPeriodicEvent periodic JFR hook} for + * {@link DuckDBMemoryEvent}; JFR invokes the hook at the period configured on + * the recording (e.g. {@code 1 s}) and only + * while at least one active recording has the event enabled. Consequently, the + * JDBC property is a pure enable/label switch and JFR alone governs the + * sampling rate and the enabled state of the event. * *

Attribution model

*

The monitor registry is keyed on the native DuckDB instance address so that multiple * connections to the same underlying database share a single sample stream — avoiding * double-counting of shared memory. The user-supplied component identifier is captured from - * the first opted-in connection and emitted on every event for that monitor. + * the first opted-in connection and emitted on every event for that monitor. When attributing + * memory to distinct application components, use a distinct DuckDB instance per component and + * give each one a unique {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} value. * *

Thread safety

*

Lifecycle transitions (start+insert, stop+remove) are performed inside * {@link ConcurrentHashMap#compute}, which provides per-key mutual exclusion. - * The JFR periodic hook iterates {@link ConcurrentHashMap#values} without locking and - * relies on a single volatile read of {@code dbRef} for visibility. + * The JFR periodic hook iterates {@link ConcurrentHashMap#values} without + * locking and relies on volatile reads in {@link PerDbMonitor} for visibility. */ final class DuckDBMemoryMonitor { private static final Logger logger = Logger.getLogger(DuckDBMemoryMonitor.class.getName()); - /** Memory-tag names in enum order; fetched once from the native side and cached. */ - private static final String[] MEMORY_TAGS = loadMemoryTags(); - - private static String[] loadMemoryTags() { - try { - return DuckDBNative.duckdb_jdbc_memory_tags(); - } catch (SQLException e) { - throw new ExceptionInInitializerError(e); - } - } - /** Registry: native DuckDB* address -> per-database monitor. */ private static final ConcurrentHashMap monitors = new ConcurrentHashMap<>(); @@ -66,9 +55,11 @@ private DuckDBMemoryMonitor() { } /** - * Registers the periodic JFR hook for {@link DuckDBMemoryEvent}. Idempotent and called - * from {@link DuckDBDriver}'s static initializer so that recordings started before any - * monitored connection is opened still see the event type. + * Registers the periodic JFR hook for {@link DuckDBMemoryEvent}. Idempotent + * and called from {@link DuckDBDriver}'s static initializer so that recordings + * started before the first monitored connection still see the event type. + * Iterating an empty monitor map at each tick is cheap, so there is no + * downside to registering unconditionally. */ static synchronized void init() { if (initialized) { @@ -105,10 +96,8 @@ static void connectionClosed(long dbAddress) { } /** - * JFR-invoked hook. Runs on a JFR thread at the period configured on the active recording. - * JFR invokes it only when the event is enabled in some recording; the per-event - * {@link jdk.jfr.Event#commit()} performs the final {@code shouldCommit()} check. - * Must never throw. + * JFR-invoked hook. Runs on a JFR thread at the period configured on the + * active recording. Must never throw. */ private static void firePeriodicEvent() { for (PerDbMonitor m : monitors.values()) { @@ -121,99 +110,122 @@ private static void firePeriodicEvent() { } /** - * Per-instance monitor state. Mutating methods ({@link #open}, {@link #close}) are - * invoked inside {@link ConcurrentHashMap#compute} on {@link #monitors} and are - * serialized per key. {@link #sample} runs on the JFR hook thread without the compute - * lock and reads the {@code volatile dbRef} first to establish happens-before for the - * other fields. + * Per-name monitor state. Mutating methods ({@link #open}, {@link #close}) + * are invoked inside {@link ConcurrentHashMap#compute} on {@link #monitors} + * and are serialized per key. {@link #sample} runs on the JFR hook thread + * without the compute lock and reads {@code volatile} fields. */ static final class PerDbMonitor { + private static final String QUERY = + "SELECT tag, memory_usage_bytes, temporary_storage_bytes FROM duckdb_memory()"; + /** Guarded by compute()-serialization. */ private int openConnections = 0; /** - * Pinned DuckDB-instance reference (DBHolder ByteBuffer). Volatile: read lock-free by - * the JFR hook. Writing this last publishes the fully-initialized {@link #component} - * and {@link #dbAddress} fields. + * Written under compute(); read without lock by the JFR hook. + * + *

The prepared statement is parsed + planned once and executed on every tick, + * avoiding the per-tick parse overhead of a fresh {@code createStatement}. It is + * only touched from the JFR hook thread (JFR serialises periodic callbacks), so + * no additional synchronisation is required for re-execution. */ - private volatile ByteBuffer dbRef; - private String component; - private long dbAddress; + private volatile DuckDBConnection monitorConn; + private volatile PreparedStatement sampleStmt; + private volatile String component; + private volatile long dbAddress; /** - * Opens (or re-attempts opening) the pinned DB reference and increments the ref count. - * Failure to pin is logged; the ref count is still incremented so close-balance is - * preserved and a subsequent {@code open()} will retry while {@code dbRef == null}. + * Opens (or re-attempts opening) the monitor connection and increments + * the ref count. Failure to create the monitor connection or prepare the + * sampling statement is logged and the ref count is still incremented so + * close-balance is preserved; a subsequent {@code open()} will retry + * while {@code monitorConn == null}. */ void open(DuckDBConnection conn) { - if (dbRef == null) { + if (monitorConn == null) { + DuckDBConnection mc = null; try { - ByteBuffer ref = DuckDBNative.duckdb_jdbc_create_db_ref(conn.connRef); + mc = conn.duplicateForMonitor(); + PreparedStatement ps = mc.prepareStatement(QUERY); component = conn.monitorName; dbAddress = conn.dbAddress; - // Publish dbRef last so readers see fully-populated state. - dbRef = ref; + sampleStmt = ps; + // Publish monitorConn last so readers see fully-populated state. + monitorConn = mc; } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to pin DB for JFR memory monitor; will retry on next open()", e); + logger.log(Level.WARNING, "Failed to open JFR memory-monitor connection; will retry on next open()", + e); + if (mc != null) { + try { + mc.close(); + } catch (SQLException ce) { + // best-effort cleanup on setup failure + } + } } } openConnections++; } /** - * Decrements the ref count; when it reaches zero, releases the pinned DB reference - * and signals the caller to remove the map entry. + * Decrements the ref count; when it reaches zero, releases the cached + * statement and monitor connection, and signals the caller to remove + * the map entry. * * @return {@code true} when the entry should be removed */ boolean close() { - if (--openConnections > 0) { - return false; - } - // Clamp on imbalance: a stray close() without a matching open() must not leave - // the counter negative so that a future open() starts from a consistent state. - openConnections = 0; - ByteBuffer ref = dbRef; - dbRef = null; - if (ref != null) { - try { - DuckDBNative.duckdb_jdbc_destroy_db_ref(ref); - } catch (SQLException e) { - logger.log(Level.FINE, "Failed to release JFR memory-monitor DB ref", e); + if (--openConnections <= 0) { + PreparedStatement ps = sampleStmt; + DuckDBConnection mc = monitorConn; + sampleStmt = null; + monitorConn = null; + if (ps != null) { + try { + ps.close(); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to close JFR memory-monitor statement", e); + } } + if (mc != null) { + try { + mc.close(); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to close JFR memory-monitor connection", e); + } + } + return true; } - return true; + return false; } /** Invoked by the JFR periodic hook. Must not throw. */ void sample() { - ByteBuffer ref = dbRef; - if (ref == null) { + DuckDBConnection mc = monitorConn; + PreparedStatement ps = sampleStmt; + if (mc == null || ps == null) { return; } - long[] snapshot; try { - snapshot = DuckDBNative.duckdb_jdbc_memory_snapshot(ref); - } catch (Throwable t) { - // Rare: native snapshot failure. Swallow so JFR's periodic dispatch survives. - return; - } - if (snapshot == null) { + if (mc.isClosed()) { + return; + } + } catch (SQLException ignored) { return; } String componentSnap = component; long addr = dbAddress; - // Snapshot is packed as [tag, size, evicted] triples; the tag index is - // emitted explicitly by the native side, so the order of entries does - // not matter. An unknown tag index (e.g. from a future native lib that - // added tags Java does not know about) is skipped rather than misnamed. - for (int i = 0; i + 3 <= snapshot.length; i += 3) { - int tagIdx = (int) snapshot[i]; - if (tagIdx < 0 || tagIdx >= MEMORY_TAGS.length) { - continue; + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + String tag = rs.getString(1); + long memoryUsageBytes = rs.getLong(2); + long temporaryStorageBytes = rs.getLong(3); + emitEvent(componentSnap, addr, tag, memoryUsageBytes, temporaryStorageBytes); } - emitEvent(componentSnap, addr, MEMORY_TAGS[tagIdx], snapshot[i + 1], snapshot[i + 2]); + } catch (Exception ignored) { + // Propagating would break JFR's periodic dispatch; swallow. } } diff --git a/src/main/java/org/duckdb/DuckDBNative.java b/src/main/java/org/duckdb/DuckDBNative.java index ed71c7715..30b1c0e22 100644 --- a/src/main/java/org/duckdb/DuckDBNative.java +++ b/src/main/java/org/duckdb/DuckDBNative.java @@ -152,23 +152,6 @@ private static void loadFromCurrentJarDir(String libName) throws Exception { /** Returns the native address of the underlying DuckDB instance as a stable identity key. */ static native long duckdb_jdbc_db_address(ByteBuffer conn_ref) throws SQLException; - /** - * Reads DuckDB memory counters directly via {@code BufferManager::GetMemoryUsageInfo()}. - * Returns a packed {@code long[]} of length {@code 3 * N} where N is the number of memory - * entries reported: {@code [tag_0, size_0, evicted_0, tag_1, size_1, evicted_1, ...]}. Each - * {@code tag} is an index into the {@link #duckdb_jdbc_memory_tags} array; entries may be - * returned in any order. The {@code db_ref} argument must be a live pinned DB reference - * created with {@link #duckdb_jdbc_create_db_ref}. - */ - static native long[] duckdb_jdbc_memory_snapshot(ByteBuffer db_ref) throws SQLException; - - /** - * Returns the DuckDB memory-tag names indexed by {@code MemoryTag} enum value. Tag indices - * returned by {@link #duckdb_jdbc_memory_snapshot} can be used directly as indices into this - * array. The result is stable for a given native library and safe to cache. - */ - static native String[] duckdb_jdbc_memory_tags() throws SQLException; - static native void duckdb_jdbc_set_auto_commit(ByteBuffer conn_ref, boolean auto_commit) throws SQLException; static native boolean duckdb_jdbc_get_auto_commit(ByteBuffer conn_ref) throws SQLException; From fcbcb6a508b40066bc17aa1027eddb4b67db7336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Wed, 6 May 2026 13:18:37 +0200 Subject: [PATCH 5/7] Refine implementation --- .../java/org/duckdb/DuckDBConnection.java | 18 ++- .../java/org/duckdb/DuckDBMemoryMonitor.java | 145 +++++++++++------- 2 files changed, 100 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/duckdb/DuckDBConnection.java b/src/main/java/org/duckdb/DuckDBConnection.java index 1a894650d..0bc04e942 100644 --- a/src/main/java/org/duckdb/DuckDBConnection.java +++ b/src/main/java/org/duckdb/DuckDBConnection.java @@ -50,9 +50,10 @@ public final class DuckDBConnection implements java.sql.Connection { /** * User-supplied identifier for JFR memory monitoring (the value of the - * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property). {@code null} or empty - * means this connection does not participate in monitoring — either the user - * did not opt in, or this connection IS the monitor connection. + * {@value DuckDBDriver#JDBC_JFR_MEMORY_MONITOR} property). Either {@code null} + * (the user did not opt in, or this is the monitor's own internal duplicate + * connection) or a non-empty string; empty or absent property values are + * normalised to {@code null} in the constructor. */ final String monitorName; @@ -69,6 +70,10 @@ public static DuckDBConnection newConnection(String url, boolean readOnly, Prope public static DuckDBConnection newConnection(String url, boolean readOnly, String sessionInitSQL, Properties properties) throws SQLException { + // Ensure the JFR periodic memory-usage event is registered for callers + // that bypass DuckDBDriver (which also calls this in its static init). + // Idempotent and a no-op on JVMs without JFR. + JfrMemoryMonitor.init(); if (null == properties) { properties = new Properties(); } @@ -215,7 +220,12 @@ public void close() throws SQLException { connRefLock.unlock(); } if (notifyMonitor) { - JfrMemoryMonitor.connectionClosed(dbAddress); + try { + JfrMemoryMonitor.connectionClosed(dbAddress); + } catch (Throwable t) { + // The connection is already disconnected at this point; a failure + // in the monitor teardown must not propagate to close() callers. + } } } diff --git a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java index d74294f47..030fc84aa 100644 --- a/src/main/java/org/duckdb/DuckDBMemoryMonitor.java +++ b/src/main/java/org/duckdb/DuckDBMemoryMonitor.java @@ -60,20 +60,41 @@ private DuckDBMemoryMonitor() { * started before the first monitored connection still see the event type. * Iterating an empty monitor map at each tick is cheap, so there is no * downside to registering unconditionally. + * + *

Any failure from JFR (e.g. {@link SecurityException} under a + * {@code SecurityManager}, or an unexpected {@link Error} from a non-standard + * JFR implementation) is caught and logged: the feature must never prevent + * {@link DuckDBDriver} from loading. */ static synchronized void init() { if (initialized) { return; } - FlightRecorder.addPeriodicEvent(DuckDBMemoryEvent.class, DuckDBMemoryMonitor::firePeriodicEvent); initialized = true; + try { + FlightRecorder.addPeriodicEvent(DuckDBMemoryEvent.class, DuckDBMemoryMonitor::firePeriodicEvent); + } catch (Throwable t) { + logger.log(Level.WARNING, "JFR periodic event registration failed; memory monitoring disabled", t); + } } /** * Called when a new connection is opened with JFR memory monitoring enabled. - * The caller guarantees {@code conn.monitorName} is non-null. + * The caller guarantees {@code conn.monitorName} is non-null and that + * {@link #init()} has already been called (which is the contract of + * {@link DuckDBConnection#newConnection}). + * + *

A zero {@code dbAddress} is treated as a "no such instance" sentinel and + * skipped: the native layer is expected to return a non-zero pointer for every + * live DuckDB instance, so a zero value indicates a bug or an unexpected state. + * Registering under key {@code 0} would silently alias every such connection + * into a single monitor entry. */ static void connectionOpened(DuckDBConnection conn) { + if (conn.dbAddress == 0L) { + logger.log(Level.FINE, "Skipping JFR memory monitor registration: native DuckDB address is 0"); + return; + } monitors.compute(conn.dbAddress, (k, existing) -> { PerDbMonitor m = (existing != null) ? existing : new PerDbMonitor(); m.open(conn); @@ -82,11 +103,16 @@ static void connectionOpened(DuckDBConnection conn) { } /** - * Called when a monitored connection is closed. + * Called when a monitored connection is closed. Symmetric with + * {@link #connectionOpened(DuckDBConnection)}: a zero {@code dbAddress} means + * no entry was ever registered, so there is nothing to tear down. * * @param dbAddress the native db address captured at construction time */ static void connectionClosed(long dbAddress) { + if (dbAddress == 0L) { + return; + } monitors.compute(dbAddress, (k, existing) -> { if (existing == null) { return null; @@ -105,36 +131,37 @@ private static void firePeriodicEvent() { m.sample(); } catch (Throwable t) { // Defensive: a failure in one monitor must not prevent emission for others. + logger.log(Level.FINE, "JFR memory sample failed", t); } } } /** - * Per-name monitor state. Mutating methods ({@link #open}, {@link #close}) - * are invoked inside {@link ConcurrentHashMap#compute} on {@link #monitors} - * and are serialized per key. {@link #sample} runs on the JFR hook thread - * without the compute lock and reads {@code volatile} fields. + * Per-database monitor state. + * + *

Two threads may touch an instance: the user thread closing a monitored + * connection (via {@link #close()}, invoked inside {@link ConcurrentHashMap#compute}) + * and the JFR periodic thread sampling the database (via {@link #sample()}). + * They are mutually exclusive under the instance's intrinsic lock so that + * {@link #close()} can never free a {@link PreparedStatement} that + * {@link #sample()} is executing against. + * + *

{@link #open(DuckDBConnection)} is only invoked inside {@code compute}, which + * already serialises it against itself and {@link #close()} per key, but it still + * synchronises on {@code this} to establish a happens-before with any concurrent + * {@link #sample()}. */ static final class PerDbMonitor { private static final String QUERY = "SELECT tag, memory_usage_bytes, temporary_storage_bytes FROM duckdb_memory()"; - /** Guarded by compute()-serialization. */ + // All fields below are guarded by the instance's intrinsic lock. private int openConnections = 0; - - /** - * Written under compute(); read without lock by the JFR hook. - * - *

The prepared statement is parsed + planned once and executed on every tick, - * avoiding the per-tick parse overhead of a fresh {@code createStatement}. It is - * only touched from the JFR hook thread (JFR serialises periodic callbacks), so - * no additional synchronisation is required for re-execution. - */ - private volatile DuckDBConnection monitorConn; - private volatile PreparedStatement sampleStmt; - private volatile String component; - private volatile long dbAddress; + private DuckDBConnection monitorConn; + private PreparedStatement sampleStmt; + private String component; + private long dbAddress; /** * Opens (or re-attempts opening) the monitor connection and increments @@ -143,20 +170,19 @@ static final class PerDbMonitor { * close-balance is preserved; a subsequent {@code open()} will retry * while {@code monitorConn == null}. */ - void open(DuckDBConnection conn) { + synchronized void open(DuckDBConnection conn) { if (monitorConn == null) { DuckDBConnection mc = null; try { mc = conn.duplicateForMonitor(); - PreparedStatement ps = mc.prepareStatement(QUERY); + sampleStmt = mc.prepareStatement(QUERY); component = conn.monitorName; dbAddress = conn.dbAddress; - sampleStmt = ps; - // Publish monitorConn last so readers see fully-populated state. monitorConn = mc; } catch (SQLException e) { logger.log(Level.WARNING, "Failed to open JFR memory-monitor connection; will retry on next open()", e); + sampleStmt = null; if (mc != null) { try { mc.close(); @@ -172,47 +198,46 @@ void open(DuckDBConnection conn) { /** * Decrements the ref count; when it reaches zero, releases the cached * statement and monitor connection, and signals the caller to remove - * the map entry. + * the map entry. Blocks any in-flight {@link #sample()} until it + * completes, ensuring the statement is never closed mid-execution. * * @return {@code true} when the entry should be removed */ - boolean close() { - if (--openConnections <= 0) { - PreparedStatement ps = sampleStmt; - DuckDBConnection mc = monitorConn; - sampleStmt = null; - monitorConn = null; - if (ps != null) { - try { - ps.close(); - } catch (SQLException e) { - logger.log(Level.FINE, "Failed to close JFR memory-monitor statement", e); - } + synchronized boolean close() { + if (--openConnections > 0) { + return false; + } + PreparedStatement ps = sampleStmt; + DuckDBConnection mc = monitorConn; + sampleStmt = null; + monitorConn = null; + if (ps != null) { + try { + ps.close(); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to close JFR memory-monitor statement", e); } - if (mc != null) { - try { - mc.close(); - } catch (SQLException e) { - logger.log(Level.FINE, "Failed to close JFR memory-monitor connection", e); - } + } + if (mc != null) { + try { + mc.close(); + } catch (SQLException e) { + logger.log(Level.FINE, "Failed to close JFR memory-monitor connection", e); } - return true; } - return false; + return true; } - /** Invoked by the JFR periodic hook. Must not throw. */ - void sample() { - DuckDBConnection mc = monitorConn; + /** + * Invoked by the JFR periodic hook. Must not throw. Holds the monitor's + * intrinsic lock for the duration so that {@link #close()} cannot release + * the cached statement mid-execution. The query against {@code duckdb_memory()} + * is a quick system-table scan; any contention with a concurrent connection + * close is bounded by its runtime. + */ + synchronized void sample() { PreparedStatement ps = sampleStmt; - if (mc == null || ps == null) { - return; - } - try { - if (mc.isClosed()) { - return; - } - } catch (SQLException ignored) { + if (ps == null) { return; } String componentSnap = component; @@ -224,8 +249,10 @@ void sample() { long temporaryStorageBytes = rs.getLong(3); emitEvent(componentSnap, addr, tag, memoryUsageBytes, temporaryStorageBytes); } - } catch (Exception ignored) { - // Propagating would break JFR's periodic dispatch; swallow. + } catch (Throwable t) { + // Propagating would break JFR's periodic dispatch; log at FINE so + // operators can diagnose silent emission gaps without flooding logs. + logger.log(Level.FINE, "JFR memory sample query failed", t); } } From 3a350fc0bf0eda529363893ea95fa7ce540f5f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Wed, 6 May 2026 14:52:21 +0200 Subject: [PATCH 6/7] Update DuckDBMemoryEvent.java --- src/main/java/org/duckdb/DuckDBMemoryEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/duckdb/DuckDBMemoryEvent.java b/src/main/java/org/duckdb/DuckDBMemoryEvent.java index e57fb0972..44f9750e7 100644 --- a/src/main/java/org/duckdb/DuckDBMemoryEvent.java +++ b/src/main/java/org/duckdb/DuckDBMemoryEvent.java @@ -45,7 +45,7 @@ final class DuckDBMemoryEvent extends Event { String component; @Label("Tag") - @Description("DuckDB internal memory tag (e.g. \"Base\", \"Hash Table\", \"Buffer Manager\")") + @Description("DuckDB internal memory tag (e.g. \"BASE_TABLE\", \"HASH_TABLE\", \"ALLOCATOR\")") String tag; @Label("Database Address") From aa41a1ca5be94d102154037fa6ac9e71741df199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Thu, 7 May 2026 10:23:14 +0200 Subject: [PATCH 7/7] Update JFR.md --- JFR.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/JFR.md b/JFR.md index a84abe003..3516e18a2 100644 --- a/JFR.md +++ b/JFR.md @@ -78,7 +78,7 @@ JFR tick. | Field | Type | Meaning | | ----------------------- | ------ | ----------------------------------------------------------------- | | `component` | String | Application-supplied identifier (the JDBC property value). | -| `tag` | String | DuckDB memory tag (e.g. `IN_MEMORY_TABLE`, `HASH_TABLE`, `ALLOCATOR`). | +| `tag` | String | DuckDB memory tag (e.g. `BASE_TABLE`, `HASH_TABLE`, `ALLOCATOR`). | | `dbAddress` | long | Native address of the DuckDB instance — stable per-instance id. | | `memoryUsageBytes` | long | Bytes currently allocated for this tag. | | `temporaryStorageBytes` | long | Bytes spilled to temporary storage for this tag. | @@ -88,10 +88,8 @@ Plus the standard JFR fields `startTime`, `duration`, `eventThread` ## Attribution model -The monitor is keyed on the **native DuckDB instance address**, not -on the JDBC connection. This matters when multiple connections share -a DuckDB instance (multiple `DriverManager.getConnection` calls -against the same file DB, or `conn.duplicate()`): +The monitor is keyed on the **native DuckDB instance address** +(exposed as `dbAddress`), not on the JDBC connection: - One sample stream per distinct DuckDB instance — no double-counting of shared memory. @@ -102,8 +100,20 @@ against the same file DB, or `conn.duplicate()`): torn down when the last one closes; a subsequent `getConnection` starts a fresh monitor. -To attribute memory to distinct logical components, open each against -a distinct DuckDB instance and give each one a unique +### When two `getConnection` calls share an instance + +| URL / operation | Same `dbAddress`? | +| ---------------------------------------- | -------------------------------- | +| `jdbc:duckdb:` (unnamed in-memory) | **No** — fresh instance per call | +| `jdbc:duckdb::memory:` | Yes, when `` matches | +| `jdbc:duckdb:/path/to/file.db` | Yes, when the path matches | +| `conn.duplicate()` | Yes, always | + +Connections that share an instance share a `dbAddress` and therefore +a single `component` label (the one supplied by the first opted-in +connection). For per-component attribution, open each component +against a URL that yields a fresh instance — unnamed in-memory URLs +are the simplest choice — and give each connection a unique `jdbc_jfr_memory_monitor` value. ## Requirements