Skip to content

Extract DuckDBPyConnection module state into DuckDBPyModuleState #515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
evertlammerts merged 1 commit into from
Jun 26, 2026
+28 −24
Merged

Extract DuckDBPyConnection module state into DuckDBPyModuleState #515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions src/duckdb_py/include/duckdb_python/pyconnection/pyconnection.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,11 +361,6 @@ struct DuckDBPyConnection : public std::enable_shared_from_this<DuckDBPyConnecti
void EnableProfiling();
void DisableProfiling();

//! Default connection to an in-memory database
static DefaultConnectionHolder default_connection;
//! Caches and provides an interface to get frequently used modules+subtypes
static std::shared_ptr<PythonImportCache> import_cache;

static bool IsPandasDataframe(const py::object &object);
static PyArrowObjectType GetArrowType(const py::handle &obj);
static bool IsAcceptedArrowObject(const py::object &object);
Expand All @@ -383,8 +378,6 @@ struct DuckDBPyConnection : public std::enable_shared_from_this<DuckDBPyConnecti
bool side_effects);
vector<unique_ptr<SQLStatement>> GetStatements(const py::object &query);

static PythonEnvironmentType environment;
static std::string formatted_python_version;
static void DetectEnvironment();
};

Expand Down
45 changes: 28 additions & 17 deletions src/duckdb_py/pyconnection.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,21 @@

namespace duckdb {

DefaultConnectionHolder DuckDBPyConnection::default_connection; // NOLINT: allow global
DBInstanceCache instance_cache; // NOLINT: allow global
std::shared_ptr<PythonImportCache> DuckDBPyConnection::import_cache = nullptr; // NOLINT: allow global
PythonEnvironmentType DuckDBPyConnection::environment = PythonEnvironmentType::NORMAL; // NOLINT: allow global
std::string DuckDBPyConnection::formatted_python_version = "";
// All process-global module state lives in one struct, reached only through GetModuleState().
// This is the single seam to retarget for PEP 489 multi-phase init (per-module state via
// PyModule_GetState); call sites never touch the storage directly.
struct DuckDBPyModuleState {
DefaultConnectionHolder default_connection;
DBInstanceCache instance_cache;
std::shared_ptr<PythonImportCache> import_cache;
PythonEnvironmentType environment = PythonEnvironmentType::NORMAL;
std::string formatted_python_version;
};

static DuckDBPyModuleState &GetModuleState() {
static DuckDBPyModuleState state; // NOLINT: allow global - sole module-state seam (future: PyModule_GetState)
return state;
}

DuckDBPyConnection::~DuckDBPyConnection() {
try {
Expand Down Expand Up @@ -90,14 +100,14 @@ void DuckDBPyConnection::DetectEnvironment() {
py::object version_info = sys.attr("version_info");
int major = py::cast<int>(version_info.attr("major"));
int minor = py::cast<int>(version_info.attr("minor"));
DuckDBPyConnection::formatted_python_version = std::to_string(major) + "." + std::to_string(minor);
GetModuleState().formatted_python_version = std::to_string(major) + "." + std::to_string(minor);

// If __main__ does not have a __file__ attribute, we are in interactive mode
auto main_module = py::module_::import("__main__");
if (py::hasattr(main_module, "__file__")) {
return;
}
DuckDBPyConnection::environment = PythonEnvironmentType::INTERACTIVE;
GetModuleState().environment = PythonEnvironmentType::INTERACTIVE;
if (!ModuleIsLoaded<IpythonCacheItem>()) {
return;
}
Expand All @@ -115,7 +125,7 @@ void DuckDBPyConnection::DetectEnvironment() {
}
py::dict ipython_config = ipython.attr("config");
if (ipython_config.contains("IPKernelApp")) {
DuckDBPyConnection::environment = PythonEnvironmentType::JUPYTER;
GetModuleState().environment = PythonEnvironmentType::JUPYTER;
}
return;
}
Expand All @@ -126,11 +136,11 @@ bool DuckDBPyConnection::DetectAndGetEnvironment() {
}

bool DuckDBPyConnection::IsJupyter() {
return DuckDBPyConnection::environment == PythonEnvironmentType::JUPYTER;
return GetModuleState().environment == PythonEnvironmentType::JUPYTER;
}

std::string DuckDBPyConnection::FormattedPythonVersion() {
return DuckDBPyConnection::formatted_python_version;
return GetModuleState().formatted_python_version;
}

// NOTE: this function is generated by tools/pythonpkg/scripts/generate_connection_methods.py.
Expand Down Expand Up @@ -2198,8 +2208,8 @@ static std::shared_ptr<DuckDBPyConnection> FetchOrCreateInstance(const string &d
D_ASSERT(py::gil_check());
py::gil_scoped_release release;
unique_lock<std::recursive_mutex> lock(res->py_connection_lock);
auto database =
instance_cache.GetOrCreateInstance(database_path, config, cache_instance, InstantiateNewInstance);
auto database = GetModuleState().instance_cache.GetOrCreateInstance(database_path, config, cache_instance,
InstantiateNewInstance);
res->con.SetDatabase(std::move(database));
res->con.SetConnection(make_uniq<Connection>(res->con.GetDatabase()));
}
Expand Down Expand Up @@ -2284,14 +2294,15 @@ identifier_map_t<BoundParameterData> DuckDBPyConnection::TransformPythonParamDic
}

std::shared_ptr<DuckDBPyConnection> DuckDBPyConnection::DefaultConnection() {
return default_connection.Get();
return GetModuleState().default_connection.Get();
}

void DuckDBPyConnection::SetDefaultConnection(std::shared_ptr<DuckDBPyConnection> connection) {
return default_connection.Set(std::move(connection));
return GetModuleState().default_connection.Set(std::move(connection));
}

PythonImportCache *DuckDBPyConnection::ImportCache() {
auto &import_cache = GetModuleState().import_cache;
if (!import_cache) {
import_cache = std::make_shared<PythonImportCache>();
}
Expand All @@ -2315,7 +2326,7 @@ ModifiedMemoryFileSystem &DuckDBPyConnection::GetObjectFileSystem() {
}

bool DuckDBPyConnection::IsInteractive() {
return DuckDBPyConnection::environment != PythonEnvironmentType::NORMAL;
return GetModuleState().environment != PythonEnvironmentType::NORMAL;
}

std::shared_ptr<DuckDBPyConnection> DuckDBPyConnection::Enter() {
Expand All @@ -2333,8 +2344,8 @@ void DuckDBPyConnection::Exit(DuckDBPyConnection &self, const py::object &exc_ty
}

void DuckDBPyConnection::Cleanup() {
default_connection.Set(nullptr);
import_cache.reset();
GetModuleState().default_connection.Set(nullptr);
GetModuleState().import_cache.reset();
}

bool DuckDBPyConnection::IsPandasDataframe(const py::object &object) {
Expand Down
Loading