Give the Parquet byte source a vtable, ahead of object storage (#393 M1)

Makefile

@@ -27,7 +27,8 @@ OBJS = \
 	src/columnar_projection.o \
 	src/columnar_parquet_reader.o \
 	src/columnar_parallel_copy.o \
-	src/columnar_parallel_export.o
+	src/columnar_parallel_export.o \
+	src/columnar_objstore.o
 
 EXTENSION = pgcolumnar
 DATA = pgcolumnar--1.0-alpha.sql pgcolumnar--1.0-dev--1.0-alpha.sql
@@ -113,3 +114,24 @@ PG_CFLAGS += -std=gnu17
 endif
 
 include $(PGXS)
+
+# The object-store module is a SEPARATE shared library, built and installed
+# alongside this one but never linked into it. See src/columnar_objstore.h: this
+# extension is preloaded, so anything it links reaches the postmaster.
+#
+# A build failure there must not be silent, so these do not use the `-` prefix.
+OBJSTORE_DIR = $(realpath $(dir $(firstword $(MAKEFILE_LIST))))/objstore
+
+all: objstore-all
+objstore-all:
+	$(MAKE) -C $(OBJSTORE_DIR) PG_CONFIG=$(PG_CONFIG) all
+
+install: objstore-install
+objstore-install:
+	$(MAKE) -C $(OBJSTORE_DIR) PG_CONFIG=$(PG_CONFIG) install
+
+clean: objstore-clean
+objstore-clean:
+	$(MAKE) -C $(OBJSTORE_DIR) PG_CONFIG=$(PG_CONFIG) clean
+
+.PHONY: objstore-all objstore-install objstore-clean

docs/ARCHITECTURE.md

@@ -320,6 +320,62 @@ outside the buffer, an incorrect value, or work without a bound.
 Each of those guards has a crafted-file test verified to fail when that specific
 guard is removed; `test/mutate_guard.py` is the harness that does the removing.
 
+Every read of a Parquet file goes through one positional read, `pq_source_read`.
+Its callers have already bounded the offset. That is the seam a non-local byte
+source replaces, and it is why object storage touches three functions rather
+than the reader.
+
+The seam dispatches through a vtable that is never null. The local file path is
+an implementation like any other, not a case the dispatcher tests for. Every
+Parquet test in the suite therefore exercises the dispatch, rather than only the
+first remote source to arrive.
+
+### columnar_objstore.c
+The loader for the object-store module, and the only file in the main library
+that knows the module exists. Object storage lives in a **separate, non-preloaded
+shared library**, `pgcolumnar_objstore`, built from `objstore/`.
+
+The reason is measured rather than stylistic. pgColumnar loads through
+`shared_preload_libraries`. Everything it links is therefore mapped into the
+postmaster and inherited by every backend through `fork`, whether or not any
+query reads a remote file. libcurl alone resolves to 30 shared objects against
+this extension's 4, including two TLS implementations. An OpenSSL-linked client
+would bring a TLS stack into a postmaster that may have none.
+
+PostgreSQL made the same call for its own libcurl dependency. That shape is
+configure default off, a separate library, runtime `dlopen` behind a frozen ABI,
+and a separate package.
+
+The module is loaded with `load_external_function` on the first read of a remote
+path and never before. A build or an install without it is fully functional for
+local files. A remote path then reports that the module is not installed. That
+is a different report from a scheme the installed module does not handle, because
+the two ask the operator for different things.
+
+The ABI is frozen in `columnar_objstore.h` and version-checked on load. A
+mismatch is refused rather than called through. A stale module and a new main
+library agree on the symbol name and disagree on the struct.
+
+Two ordering constraints in the loader are load-bearing, and each has a test.
+
+A missing library is raised by `internal_load_library` before the symbol lookup.
+So `signalNotFound = false` never gets a say, and the miss needs a `PG_TRY`.
+
+"Already tried" is recorded only after the attempt completes, with nothing that
+can raise between the assignment and its return. Setting it first lets an
+`ereport` unwind past the assignment while the static keeps its new value. The
+first remote read of a session then reports one error, and every later one
+reports a different and more plausible error.
+
+Missing and corrupt cannot be told apart by SQLSTATE. `internal_load_library`
+uses `errcode_for_file_access()` for both the `stat` failure and the `dlopen`
+failure, so both arrive as 58P01. The discriminator is file presence.
+
+Object storage reads **exact object keys**. A glob metacharacter in a remote path
+is refused rather than expanded. Expanding one needs a LIST call, and its paged
+response would be a third parser written here for input an outside party
+controls.
+
 ### columnar_visibilitymap.c
 Index-only-scan support (gap 28). A columnar visibility-map fork records which
 synthetic blocks (chunk groups) are all-visible. Lazy `VACUUM`

objstore/Makefile

@@ -0,0 +1,33 @@
+# pgColumnar object-store module (#393).
+#
+# A SEPARATE shared library on purpose, and not part of MODULE_big. pgColumnar
+# loads through shared_preload_libraries, so anything the main library links is
+# mapped into the postmaster and inherited by every backend. This one is opened
+# with load_external_function on the first read of a remote path and never
+# before, so a cluster that reads only local files never maps it.
+#
+# PostgreSQL packages its own libcurl dependency this way (libpq-oauth): separate
+# library, runtime dlopen, separate package.
+MODULE_big = pgcolumnar_objstore
+
+OBJS = columnar_objstore_module.o
+
+PGFILEDESC = "pgColumnar object-store byte source"
+
+PG_CPPFLAGS = -I$(realpath $(dir $(firstword $(MAKEFILE_LIST)))../src)
+
+PG_CONFIG ?= pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+
+# Match the language standard the main library uses, for the same reason: PG19's
+# headers are C23 and GCC 13 spells it gnu2x while GCC 14 spells it gnu23.
+PG_MAJORVERSION := $(shell $(PG_CONFIG) --version | sed -E 's/[^0-9]*([0-9]+).*/\1/')
+ifeq ($(shell test "$(PG_MAJORVERSION)" -ge 19 && echo yes),yes)
+C23_STD := $(shell echo 'int main(void){return 0;}' \
+	| cc -std=gnu23 -x c -c -o /dev/null - >/dev/null 2>&1 && echo gnu23 || echo gnu2x)
+PG_CFLAGS += -std=$(C23_STD)
+else
+PG_CFLAGS += -std=gnu17
+endif
+
+include $(PGXS)

objstore/columnar_objstore_module.c

@@ -0,0 +1,73 @@
+/*-------------------------------------------------------------------------
+ * columnar_objstore_module.c
+ *		Object-store byte source for pgColumnar (#393).
+ *
+ * Loaded on demand by PgColumnarObjStoreGet, never preloaded. See
+ * src/columnar_objstore.h for why it is a separate library and for the ABI.
+ *
+ * This commit establishes the module, its build, and the ABI. The protocol
+ * itself arrives next: a range GET over HTTP/1.1 driven from a WaitEventSet, then
+ * SigV4 signing, then TLS. Until then every scheme reports unsupported, which is
+ * a better failure than the reader silently treating an s3:// URL as a filename.
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "columnar_objstore.h"
+
+PG_MODULE_MAGIC;
+
+PGDLLEXPORT const PgColumnarObjStoreApi *pgcolumnar_objstore_init(void);
+
+static bool
+objstore_handles_url(const char *url)
+{
+	/*
+	 * Nothing is handled yet. Deliberately not returning true for s3:// before
+	 * the protocol exists: the reader asks this question so it can report an
+	 * unsupported scheme without attempting a connection, and answering yes here
+	 * would turn a clear error into a failure inside open().
+	 */
+	(void) url;
+	return false;
+}
+
+static PgColumnarObjHandle *
+objstore_open(const char *url, int64 *len)
+{
+	(void) len;
+	ereport(ERROR,
+			(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+			 errmsg("columnar: object storage is not implemented yet"),
+			 errdetail("The object-store module is installed but carries no "
+					   "protocol implementation for \"%s\".", url)));
+	return NULL;				/* unreachable */
+}
+
+static void
+objstore_read(PgColumnarObjHandle *h, int64 off, void *buf, size_t n)
+{
+	(void) h; (void) off; (void) buf; (void) n;
+	elog(ERROR, "columnar: object-store read reached an unopened handle");
+}
+
+static void
+objstore_close(PgColumnarObjHandle *h)
+{
+	(void) h;
+}
+
+static const PgColumnarObjStoreApi objstore_api = {
+	.abi_version = PGCOLUMNAR_OBJSTORE_ABI,
+	.handles_url = objstore_handles_url,
+	.open = objstore_open,
+	.read = objstore_read,
+	.close = objstore_close,
+};
+
+const PgColumnarObjStoreApi *
+pgcolumnar_objstore_init(void)
+{
+	return &objstore_api;
+}

src/columnar_objstore.c

@@ -0,0 +1,169 @@
+/*-------------------------------------------------------------------------
+ * columnar_objstore.c
+ *		Loader for the object-store module (#393).
+ *
+ * The module is a separate, non-preloaded shared library. See
+ * columnar_objstore.h for why. This file is the only thing in the main library
+ * that knows it exists, and it never loads it until a remote path is read.
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "utils/elog.h"
+#include "miscadmin.h"
+#include <sys/stat.h>
+
+#include "columnar.h"
+#include "columnar_objstore.h"
+
+/*
+ * Cached across the session, INCLUDING the negative result. Whether the module is
+ * installed is a property of the installation, so a miss will not become a hit
+ * mid-session, and retrying a dlopen per read would be a per-read cost for
+ * something that cannot change.
+ */
+static const PgColumnarObjStoreApi *objstore_api = NULL;
+static bool objstore_tried = false;
+
+/*
+ * Is the module file actually there? Resolves $libdir the way the server does,
+ * because the SQLSTATE cannot distinguish a missing library from a broken one.
+ */
+static bool
+objstore_module_present(void)
+{
+	char		libdir[MAXPGPATH];
+	char		path[MAXPGPATH];
+	struct stat st;
+
+	get_pkglib_path(my_exec_path, libdir);
+	snprintf(path, sizeof(path), "%s/pgcolumnar_objstore%s", libdir, DLSUFFIX);
+	return stat(path, &st) == 0;
+}
+
+bool
+PgColumnarPathIsRemote(const char *path)
+{
+	static const char *const schemes[] = {"s3://", "gs://", "az://", "https://",
+	"http://", NULL};
+	int			i;
+
+	if (path == NULL)
+		return false;
+	for (i = 0; schemes[i] != NULL; i++)
+		if (pg_strncasecmp(path, schemes[i], strlen(schemes[i])) == 0)
+			return true;
+	return false;
+}
+
+const PgColumnarObjStoreApi *
+PgColumnarObjStoreGet(void)
+{
+	PgColumnarObjStoreInitFn init;
+	const PgColumnarObjStoreApi *api;
+	MemoryContext loadcxt;
+
+	if (objstore_tried)
+		return objstore_api;
+
+	/*
+	 * signalNotFound = false suppresses a missing SYMBOL. It does NOT suppress a
+	 * missing LIBRARY: internal_load_library raises before the symbol lookup
+	 * happens, so that argument never gets a say and the caller sees
+	 * 'could not access file "pgcolumnar_objstore"'. An installation without the
+	 * module is a supported configuration, not an error, so catch it.
+	 *
+	 * Only the load is inside the PG_TRY, so nothing else is swallowed.
+	 *
+	 * objstore_tried is set AFTER the attempt, not before. Setting it first looks
+	 * equivalent and is not: the ereport unwinds past the assignment while the
+	 * static keeps its new value, so the FIRST remote read of a session reported
+	 * the raw load failure and every later one reported the documented message.
+	 * Two identical queries in one session gave two different errors, and the
+	 * second was the plausible-looking one.
+	 */
+	loadcxt = CurrentMemoryContext;
+	init = NULL;
+	PG_TRY();
+	{
+		init = (PgColumnarObjStoreInitFn)
+			load_external_function("$libdir/pgcolumnar_objstore",
+								   "pgcolumnar_objstore_init", false, NULL);
+	}
+	PG_CATCH();
+	{
+		MemoryContext ecxt = MemoryContextSwitchTo(loadcxt);
+
+		/*
+		 * "Not installed" and "installed but broken" are different situations and
+		 * only the first is supported. Swallowing both reports a truncated
+		 * library or a permission problem as an unsupported scheme, which sends
+		 * the operator looking in the wrong place for a fault that is theirs.
+		 *
+		 * The SQLSTATE cannot tell them apart, which is the trap here and the
+		 * reason the first attempt at this was wrong. internal_load_library uses
+		 * errcode_for_file_access() for BOTH the stat failure and the dlopen
+		 * failure, so a missing file and "file too short" both arrive as
+		 * ERRCODE_UNDEFINED_FILE (58P01). Measured, not assumed.
+		 *
+		 * So ask the filesystem instead. Absent means not installed; present
+		 * means the load failed for a reason the operator needs to see, and the
+		 * original error is re-raised with its own message intact.
+		 */
+		if (objstore_module_present())
+		{
+			MemoryContextSwitchTo(ecxt);
+			PG_RE_THROW();
+		}
+
+		MemoryContextSwitchTo(ecxt);
+		FlushErrorState();
+		init = NULL;
+	}
+	PG_END_TRY();
+
+	/*
+	 * The flag is set on each return path below, never once up here, and nothing
+	 * that can raise sits between an assignment and its return.
+	 *
+	 * This is the same trap as the one above, one step later. init() is a call
+	 * into a separately built library: it can raise, and if the flag were already
+	 * set the unwind would leave the verdict cached as "tried, nothing found", so
+	 * the first remote read of a session would report init()'s error and every
+	 * later one would report "requires the object-store module" — the same two
+	 * identical queries, two different errors, the plausible one second. It is
+	 * latent while init() only returns a pointer to a static struct. It stops
+	 * being latent the moment the module has anything to set up, which is the
+	 * commit after this one.
+	 */
+	if (init == NULL)
+	{
+		objstore_tried = true;
+		return NULL;
+	}
+
+	api = init();
+
+	/*
+	 * Refuse a mismatch rather than calling through it. A stale module and a new
+	 * main library agree on the symbol name and disagree on the struct, and the
+	 * failure that produces is a wrong function through a valid-looking pointer.
+	 */
+	if (api == NULL || api->abi_version != PGCOLUMNAR_OBJSTORE_ABI)
+	{
+		ereport(WARNING,
+				(errmsg("columnar: ignoring object-store module with ABI version %d",
+						api ? api->abi_version : -1),
+				 errdetail("This build expects ABI version %d.",
+						   PGCOLUMNAR_OBJSTORE_ABI),
+				 errhint("Reinstall pgcolumnar_objstore from the same build as "
+						 "pgcolumnar.")));
+		objstore_tried = true;
+		return NULL;
+	}
+
+	objstore_api = api;
+	objstore_tried = true;
+	return objstore_api;
+}