From d51548934274c9b13d55b7400489524cc51b702d Mon Sep 17 00:00:00 2001 From: Raaif-Yousuf Date: Sun, 20 Sep 2026 14:29:37 -0500 Subject: [PATCH] Quote the path in DataFrameReader.load when no format is given load() without a format built its query as select * from {path}, splicing the raw path straight into the SQL text, so any path with a directory separator in it fails to parse. That is every path except a bare filename in the working directory, and on Windows the backslash fails as well. Quote it as a string literal instead, the same way the rest of the reader and writer methods hand paths to DuckDB, and escape embedded single quotes by doubling them. --- duckdb/experimental/spark/sql/readwriter.py | 3 +- tests/fast/spark/test_spark_read_load.py | 38 +++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/fast/spark/test_spark_read_load.py diff --git a/duckdb/experimental/spark/sql/readwriter.py b/duckdb/experimental/spark/sql/readwriter.py index 230d5d2a..0c9b9396 100644 --- a/duckdb/experimental/spark/sql/readwriter.py +++ b/duckdb/experimental/spark/sql/readwriter.py @@ -116,7 +116,8 @@ def load( # noqa: D102 else: raise ContributionsAcceptedError else: - rel = self.session.conn.sql(f"select * from {path}") + escaped_path = path.replace("'", "''") + rel = self.session.conn.sql(f"select * from '{escaped_path}'") df = DataFrame(rel, self.session) if schema: if not isinstance(schema, StructType): diff --git a/tests/fast/spark/test_spark_read_load.py b/tests/fast/spark/test_spark_read_load.py new file mode 100644 index 00000000..7da2cefa --- /dev/null +++ b/tests/fast/spark/test_spark_read_load.py @@ -0,0 +1,38 @@ +import pytest + +_ = pytest.importorskip("duckdb.experimental.spark") + + +from spark_namespace import USE_ACTUAL_SPARK +from spark_namespace.sql.types import Row + + +@pytest.mark.skipif( + USE_ACTUAL_SPARK, + reason="load() without a format reads whatever the extension says here, while Spark defaults to parquet", +) +class TestSparkReadLoad: + def test_read_load_no_format(self, spark, tmp_path): + # 'load' without a 'format' falls back to building a bare SQL query out of the + # path, which must be quoted as a string literal or it cannot survive a path + # that contains anything the parser treats as syntax: a separator, a space, a colon. + sub_dir = tmp_path / "dir with space" + sub_dir.mkdir() + file_path = sub_dir / "basic.csv" + file_path.write_text("a,b\n1,2\n3,4\n") + + df = spark.read.load(file_path.as_posix()) + res = df.collect() + + assert sorted(res) == sorted([Row(a=1, b=2), Row(a=3, b=4)]) + + def test_read_load_no_format_quote_in_path(self, spark, tmp_path): + sub_dir = tmp_path / "dir's" + sub_dir.mkdir() + file_path = sub_dir / "basic.csv" + file_path.write_text("a,b\n1,2\n") + + df = spark.read.load(file_path.as_posix()) + res = df.collect() + + assert res == [Row(a=1, b=2)]