diff --git a/Makefile b/Makefile index f206f2b..3f26fcb 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,8 @@ OBJS = \ src/columnar_projection.o \ src/columnar_parquet_reader.o \ src/columnar_parallel_copy.o \ - src/columnar_parallel_export.o + src/columnar_parallel_export.o \ + src/columnar_objstore.o EXTENSION = pgcolumnar DATA = pgcolumnar--1.0-alpha.sql pgcolumnar--1.0-dev--1.0-alpha.sql @@ -113,3 +114,24 @@ PG_CFLAGS += -std=gnu17 endif include $(PGXS) + +# The object-store module is a SEPARATE shared library, built and installed +# alongside this one but never linked into it. See src/columnar_objstore.h: this +# extension is preloaded, so anything it links reaches the postmaster. +# +# A build failure there must not be silent, so these do not use the `-` prefix. +OBJSTORE_DIR = $(realpath $(dir $(firstword $(MAKEFILE_LIST))))/objstore + +all: objstore-all +objstore-all: + $(MAKE) -C $(OBJSTORE_DIR) PG_CONFIG=$(PG_CONFIG) all + +install: objstore-install +objstore-install: + $(MAKE) -C $(OBJSTORE_DIR) PG_CONFIG=$(PG_CONFIG) install + +clean: objstore-clean +objstore-clean: + $(MAKE) -C $(OBJSTORE_DIR) PG_CONFIG=$(PG_CONFIG) clean + +.PHONY: objstore-all objstore-install objstore-clean diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8804d3a..364f0b3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -320,6 +320,62 @@ outside the buffer, an incorrect value, or work without a bound. Each of those guards has a crafted-file test verified to fail when that specific guard is removed; `test/mutate_guard.py` is the harness that does the removing. +Every read of a Parquet file goes through one positional read, `pq_source_read`. +Its callers have already bounded the offset. That is the seam a non-local byte +source replaces, and it is why object storage touches three functions rather +than the reader. + +The seam dispatches through a vtable that is never null. The local file path is +an implementation like any other, not a case the dispatcher tests for. Every +Parquet test in the suite therefore exercises the dispatch, rather than only the +first remote source to arrive. + +### columnar_objstore.c +The loader for the object-store module, and the only file in the main library +that knows the module exists. Object storage lives in a **separate, non-preloaded +shared library**, `pgcolumnar_objstore`, built from `objstore/`. + +The reason is measured rather than stylistic. pgColumnar loads through +`shared_preload_libraries`. Everything it links is therefore mapped into the +postmaster and inherited by every backend through `fork`, whether or not any +query reads a remote file. libcurl alone resolves to 30 shared objects against +this extension's 4, including two TLS implementations. An OpenSSL-linked client +would bring a TLS stack into a postmaster that may have none. + +PostgreSQL made the same call for its own libcurl dependency. That shape is +configure default off, a separate library, runtime `dlopen` behind a frozen ABI, +and a separate package. + +The module is loaded with `load_external_function` on the first read of a remote +path and never before. A build or an install without it is fully functional for +local files. A remote path then reports that the module is not installed. That +is a different report from a scheme the installed module does not handle, because +the two ask the operator for different things. + +The ABI is frozen in `columnar_objstore.h` and version-checked on load. A +mismatch is refused rather than called through. A stale module and a new main +library agree on the symbol name and disagree on the struct. + +Two ordering constraints in the loader are load-bearing, and each has a test. + +A missing library is raised by `internal_load_library` before the symbol lookup. +So `signalNotFound = false` never gets a say, and the miss needs a `PG_TRY`. + +"Already tried" is recorded only after the attempt completes, with nothing that +can raise between the assignment and its return. Setting it first lets an +`ereport` unwind past the assignment while the static keeps its new value. The +first remote read of a session then reports one error, and every later one +reports a different and more plausible error. + +Missing and corrupt cannot be told apart by SQLSTATE. `internal_load_library` +uses `errcode_for_file_access()` for both the `stat` failure and the `dlopen` +failure, so both arrive as 58P01. The discriminator is file presence. + +Object storage reads **exact object keys**. A glob metacharacter in a remote path +is refused rather than expanded. Expanding one needs a LIST call, and its paged +response would be a third parser written here for input an outside party +controls. + ### columnar_visibilitymap.c Index-only-scan support (gap 28). A columnar visibility-map fork records which synthetic blocks (chunk groups) are all-visible. Lazy `VACUUM` diff --git a/objstore/Makefile b/objstore/Makefile new file mode 100644 index 0000000..2e51d9d --- /dev/null +++ b/objstore/Makefile @@ -0,0 +1,33 @@ +# pgColumnar object-store module (#393). +# +# A SEPARATE shared library on purpose, and not part of MODULE_big. pgColumnar +# loads through shared_preload_libraries, so anything the main library links is +# mapped into the postmaster and inherited by every backend. This one is opened +# with load_external_function on the first read of a remote path and never +# before, so a cluster that reads only local files never maps it. +# +# PostgreSQL packages its own libcurl dependency this way (libpq-oauth): separate +# library, runtime dlopen, separate package. +MODULE_big = pgcolumnar_objstore + +OBJS = columnar_objstore_module.o + +PGFILEDESC = "pgColumnar object-store byte source" + +PG_CPPFLAGS = -I$(realpath $(dir $(firstword $(MAKEFILE_LIST)))../src) + +PG_CONFIG ?= pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) + +# Match the language standard the main library uses, for the same reason: PG19's +# headers are C23 and GCC 13 spells it gnu2x while GCC 14 spells it gnu23. +PG_MAJORVERSION := $(shell $(PG_CONFIG) --version | sed -E 's/[^0-9]*([0-9]+).*/\1/') +ifeq ($(shell test "$(PG_MAJORVERSION)" -ge 19 && echo yes),yes) +C23_STD := $(shell echo 'int main(void){return 0;}' \ + | cc -std=gnu23 -x c -c -o /dev/null - >/dev/null 2>&1 && echo gnu23 || echo gnu2x) +PG_CFLAGS += -std=$(C23_STD) +else +PG_CFLAGS += -std=gnu17 +endif + +include $(PGXS) diff --git a/objstore/columnar_objstore_module.c b/objstore/columnar_objstore_module.c new file mode 100644 index 0000000..0c6753b --- /dev/null +++ b/objstore/columnar_objstore_module.c @@ -0,0 +1,73 @@ +/*------------------------------------------------------------------------- + * columnar_objstore_module.c + * Object-store byte source for pgColumnar (#393). + * + * Loaded on demand by PgColumnarObjStoreGet, never preloaded. See + * src/columnar_objstore.h for why it is a separate library and for the ABI. + * + * This commit establishes the module, its build, and the ABI. The protocol + * itself arrives next: a range GET over HTTP/1.1 driven from a WaitEventSet, then + * SigV4 signing, then TLS. Until then every scheme reports unsupported, which is + * a better failure than the reader silently treating an s3:// URL as a filename. + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "columnar_objstore.h" + +PG_MODULE_MAGIC; + +PGDLLEXPORT const PgColumnarObjStoreApi *pgcolumnar_objstore_init(void); + +static bool +objstore_handles_url(const char *url) +{ + /* + * Nothing is handled yet. Deliberately not returning true for s3:// before + * the protocol exists: the reader asks this question so it can report an + * unsupported scheme without attempting a connection, and answering yes here + * would turn a clear error into a failure inside open(). + */ + (void) url; + return false; +} + +static PgColumnarObjHandle * +objstore_open(const char *url, int64 *len) +{ + (void) len; + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("columnar: object storage is not implemented yet"), + errdetail("The object-store module is installed but carries no " + "protocol implementation for \"%s\".", url))); + return NULL; /* unreachable */ +} + +static void +objstore_read(PgColumnarObjHandle *h, int64 off, void *buf, size_t n) +{ + (void) h; (void) off; (void) buf; (void) n; + elog(ERROR, "columnar: object-store read reached an unopened handle"); +} + +static void +objstore_close(PgColumnarObjHandle *h) +{ + (void) h; +} + +static const PgColumnarObjStoreApi objstore_api = { + .abi_version = PGCOLUMNAR_OBJSTORE_ABI, + .handles_url = objstore_handles_url, + .open = objstore_open, + .read = objstore_read, + .close = objstore_close, +}; + +const PgColumnarObjStoreApi * +pgcolumnar_objstore_init(void) +{ + return &objstore_api; +} diff --git a/src/columnar_objstore.c b/src/columnar_objstore.c new file mode 100644 index 0000000..8dd4df8 --- /dev/null +++ b/src/columnar_objstore.c @@ -0,0 +1,169 @@ +/*------------------------------------------------------------------------- + * columnar_objstore.c + * Loader for the object-store module (#393). + * + * The module is a separate, non-preloaded shared library. See + * columnar_objstore.h for why. This file is the only thing in the main library + * that knows it exists, and it never loads it until a remote path is read. + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "utils/elog.h" +#include "miscadmin.h" +#include + +#include "columnar.h" +#include "columnar_objstore.h" + +/* + * Cached across the session, INCLUDING the negative result. Whether the module is + * installed is a property of the installation, so a miss will not become a hit + * mid-session, and retrying a dlopen per read would be a per-read cost for + * something that cannot change. + */ +static const PgColumnarObjStoreApi *objstore_api = NULL; +static bool objstore_tried = false; + +/* + * Is the module file actually there? Resolves $libdir the way the server does, + * because the SQLSTATE cannot distinguish a missing library from a broken one. + */ +static bool +objstore_module_present(void) +{ + char libdir[MAXPGPATH]; + char path[MAXPGPATH]; + struct stat st; + + get_pkglib_path(my_exec_path, libdir); + snprintf(path, sizeof(path), "%s/pgcolumnar_objstore%s", libdir, DLSUFFIX); + return stat(path, &st) == 0; +} + +bool +PgColumnarPathIsRemote(const char *path) +{ + static const char *const schemes[] = {"s3://", "gs://", "az://", "https://", + "http://", NULL}; + int i; + + if (path == NULL) + return false; + for (i = 0; schemes[i] != NULL; i++) + if (pg_strncasecmp(path, schemes[i], strlen(schemes[i])) == 0) + return true; + return false; +} + +const PgColumnarObjStoreApi * +PgColumnarObjStoreGet(void) +{ + PgColumnarObjStoreInitFn init; + const PgColumnarObjStoreApi *api; + MemoryContext loadcxt; + + if (objstore_tried) + return objstore_api; + + /* + * signalNotFound = false suppresses a missing SYMBOL. It does NOT suppress a + * missing LIBRARY: internal_load_library raises before the symbol lookup + * happens, so that argument never gets a say and the caller sees + * 'could not access file "pgcolumnar_objstore"'. An installation without the + * module is a supported configuration, not an error, so catch it. + * + * Only the load is inside the PG_TRY, so nothing else is swallowed. + * + * objstore_tried is set AFTER the attempt, not before. Setting it first looks + * equivalent and is not: the ereport unwinds past the assignment while the + * static keeps its new value, so the FIRST remote read of a session reported + * the raw load failure and every later one reported the documented message. + * Two identical queries in one session gave two different errors, and the + * second was the plausible-looking one. + */ + loadcxt = CurrentMemoryContext; + init = NULL; + PG_TRY(); + { + init = (PgColumnarObjStoreInitFn) + load_external_function("$libdir/pgcolumnar_objstore", + "pgcolumnar_objstore_init", false, NULL); + } + PG_CATCH(); + { + MemoryContext ecxt = MemoryContextSwitchTo(loadcxt); + + /* + * "Not installed" and "installed but broken" are different situations and + * only the first is supported. Swallowing both reports a truncated + * library or a permission problem as an unsupported scheme, which sends + * the operator looking in the wrong place for a fault that is theirs. + * + * The SQLSTATE cannot tell them apart, which is the trap here and the + * reason the first attempt at this was wrong. internal_load_library uses + * errcode_for_file_access() for BOTH the stat failure and the dlopen + * failure, so a missing file and "file too short" both arrive as + * ERRCODE_UNDEFINED_FILE (58P01). Measured, not assumed. + * + * So ask the filesystem instead. Absent means not installed; present + * means the load failed for a reason the operator needs to see, and the + * original error is re-raised with its own message intact. + */ + if (objstore_module_present()) + { + MemoryContextSwitchTo(ecxt); + PG_RE_THROW(); + } + + MemoryContextSwitchTo(ecxt); + FlushErrorState(); + init = NULL; + } + PG_END_TRY(); + + /* + * The flag is set on each return path below, never once up here, and nothing + * that can raise sits between an assignment and its return. + * + * This is the same trap as the one above, one step later. init() is a call + * into a separately built library: it can raise, and if the flag were already + * set the unwind would leave the verdict cached as "tried, nothing found", so + * the first remote read of a session would report init()'s error and every + * later one would report "requires the object-store module" — the same two + * identical queries, two different errors, the plausible one second. It is + * latent while init() only returns a pointer to a static struct. It stops + * being latent the moment the module has anything to set up, which is the + * commit after this one. + */ + if (init == NULL) + { + objstore_tried = true; + return NULL; + } + + api = init(); + + /* + * Refuse a mismatch rather than calling through it. A stale module and a new + * main library agree on the symbol name and disagree on the struct, and the + * failure that produces is a wrong function through a valid-looking pointer. + */ + if (api == NULL || api->abi_version != PGCOLUMNAR_OBJSTORE_ABI) + { + ereport(WARNING, + (errmsg("columnar: ignoring object-store module with ABI version %d", + api ? api->abi_version : -1), + errdetail("This build expects ABI version %d.", + PGCOLUMNAR_OBJSTORE_ABI), + errhint("Reinstall pgcolumnar_objstore from the same build as " + "pgcolumnar."))); + objstore_tried = true; + return NULL; + } + + objstore_api = api; + objstore_tried = true; + return objstore_api; +} diff --git a/src/columnar_objstore.h b/src/columnar_objstore.h new file mode 100644 index 0000000..4360e6b --- /dev/null +++ b/src/columnar_objstore.h @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------- + * columnar_objstore.h + * The ABI between pgColumnar and its object-store module (#393). + * + * Object-store support lives in a SEPARATE, non-preloaded shared library. The + * reason is measured rather than stylistic: pgColumnar loads through + * shared_preload_libraries, so anything it links is mapped into the postmaster + * and inherited by every backend through fork, whether or not any query ever + * reads a remote file. libcurl alone resolves to 30 shared objects against this + * extension's 4, including two TLS implementations, and an OpenSSL-linked client + * brings a TLS stack into a postmaster that may have none. + * + * PostgreSQL made the same call for its own libcurl dependency: configure + * default off, a separate shared library, runtime dlopen behind a frozen ABI, + * and a separate distribution package. This follows that shape. + * + * The module is loaded with load_external_function on the first read of a remote + * path and never before. A build or an install without it is fully functional for + * local files. + * + * A missing module reports the remote path as unsupported. That needs a PG_TRY in + * the loader and not just signalNotFound = false, because signalNotFound + * suppresses a missing SYMBOL while a missing LIBRARY is raised earlier, by + * internal_load_library, before the symbol lookup happens. + * + * VERSIONING. Bump PGCOLUMNAR_OBJSTORE_ABI whenever the meaning or the order of + * anything below changes. The loader refuses a mismatch, because the failure it + * prevents is a wrong function called through a stale pointer. + *------------------------------------------------------------------------- + */ +#ifndef COLUMNAR_OBJSTORE_H +#define COLUMNAR_OBJSTORE_H + +#include "postgres.h" + +#define PGCOLUMNAR_OBJSTORE_ABI 1 + +/* An open remote object. The module owns everything behind this. */ +typedef struct PgColumnarObjHandle PgColumnarObjHandle; + +typedef struct PgColumnarObjStoreApi +{ + int abi_version; /* must equal PGCOLUMNAR_OBJSTORE_ABI */ + + /* + * Does this module handle `url`? Called before open so the reader can report + * an unsupported scheme without a connection attempt. + */ + bool (*handles_url) (const char *url); + + /* + * Open `url` and report its size. Raises on failure, like every other read + * path in this extension. `len` is required: the Parquet footer is located + * from the end of the object, so a source that cannot report a length cannot + * be read. + */ + PgColumnarObjHandle *(*open) (const char *url, int64 *len); + + /* + * Read exactly `n` bytes at `off`. The caller has already bounded the range + * against the length reported by open. A short read is an error, not a + * partial success, because the reader has no way to make progress from one. + */ + void (*read) (PgColumnarObjHandle *h, int64 off, void *buf, size_t n); + + void (*close) (PgColumnarObjHandle *h); +} PgColumnarObjStoreApi; + +/* + * The module's single exported entry point. Returns a pointer to a static API + * struct owned by the module. + */ +typedef const PgColumnarObjStoreApi *(*PgColumnarObjStoreInitFn) (void); + +/* + * Resolve the module, or return NULL when it is not installed. Cached after the + * first call, including the negative result: a missing module is a property of + * the installation and will not appear mid-session. + */ +extern const PgColumnarObjStoreApi *PgColumnarObjStoreGet(void); + +/* Does `path` look like a remote URL at all? Cheap, no module load. */ +extern bool PgColumnarPathIsRemote(const char *path); + +#endif /* COLUMNAR_OBJSTORE_H */ diff --git a/src/columnar_parquet_reader.c b/src/columnar_parquet_reader.c index ba56caf..4bb684f 100644 --- a/src/columnar_parquet_reader.c +++ b/src/columnar_parquet_reader.c @@ -18,6 +18,7 @@ *------------------------------------------------------------------------- */ #include "columnar.h" +#include "columnar_objstore.h" #include "columnar_parquet_format.h" #include "columnar_thrift.h" #include "columnar_parquet_codec.h" @@ -1407,29 +1408,54 @@ decode_plain_bools(const uint8 *buf, size_t buflen, int n, Datum *out) } /* - * An open Parquet file, read on demand. + * An open Parquet file, read on demand, behind a byte source (#393). * * Only the footer is held: `meta` is the serialized file metadata, and the * parsed PqFile's chunk statistics (PqChunk.stat_min and stat_max) point into * it, so it must outlive every consumer of those pointers, which means the whole * scan of this file. Page bytes are read as they are needed, so peak memory does * not scale with the file. + * + * Every one of those reads goes through pq_source_read, a positional read whose + * callers have already bounded the offset. That makes it the one seam a remote + * source has to replace, and the reason object storage touches three functions + * rather than the reader. + * + * `ops` dispatches, and is never NULL: the local file path is an implementation + * like any other, not a fallback the dispatcher tests for. That is deliberate. + * A sentinel would leave the dispatch branch unexecuted by every existing test + * until a remote source appeared, so the seam would ship unproven and the first + * thing to exercise it would be network code. With pq_local_ops installed, every + * Parquet read in the suite runs through the vtable. A remote implementation + * lives in a separate, non-preloaded module so that nothing an object-store + * client links is mapped into the postmaster. See PgColumnarObjStoreApi. */ -typedef struct PqSource +typedef struct PqSource PqSource; + +typedef struct PqSourceOps { - FILE *f; /* AllocateFile handle (buffered) */ + const char *name; /* for error messages: "file", "s3", ... */ + void (*read) (PqSource *src, int64 off, void *buf, size_t n); + void (*close) (PqSource *src); +} PqSourceOps; + +struct PqSource +{ + const PqSourceOps *ops; /* the implementation; never NULL once opened */ + FILE *f; /* AllocateFile handle (buffered), local only */ + void *priv; /* remote implementation's own state */ const char *path; /* for error messages; palloc'd by the caller */ int64 len; /* file length in bytes */ uint8 *meta; /* serialized footer metadata */ uint32 metalen; -} PqSource; +}; /* * Read `n` bytes at `off` into `buf`. Every caller has already bounded `off` and * `n` against src->len; this reports the I/O failure that is left. */ static void -pq_source_read(PqSource *src, int64 off, void *buf, size_t n) +pq_source_read_local(PqSource *src, int64 off, void *buf, size_t n) { Assert(off >= 0 && n <= (size_t) (src->len - off)); if (fseeko(src->f, (off_t) off, SEEK_SET) != 0) @@ -1453,6 +1479,34 @@ pq_source_read(PqSource *src, int64 off, void *buf, size_t n) } } +static void +pq_source_close_local(PqSource *src) +{ + if (src->f != NULL) + { + FreeFile(src->f); + src->f = NULL; + } +} + +/* + * The local file implementation. Named "file" because that is what appears in an + * error message beside "s3". + */ +static const PqSourceOps pq_local_ops = { + .name = "file", + .read = pq_source_read_local, + .close = pq_source_close_local, +}; + +/* The dispatchers every caller uses. */ +static void +pq_source_read(PqSource *src, int64 off, void *buf, size_t n) +{ + src->ops->read(src, off, buf, n); +} + + /* * Open a Parquet file and parse its footer. Reads the two magics and the footer, * never the body. The error texts match what the whole-file reader raised, so a @@ -1466,6 +1520,53 @@ pq_source_open(const char *path, PqSource *src, PqFile *pf) memset(src, 0, sizeof(*src)); src->path = path; + + /* + * Set before anything below can raise, so an error path never leaves a + * source whose dispatch table is NULL. + */ + src->ops = &pq_local_ops; + + /* + * A remote path is not a filename (#393). Without this, AllocateFile reports + * "No such file or directory" for s3://bucket/key, which is true of the + * filesystem and useless to the reader. + * + * The module is loaded here and only here, on the first remote read. + * + * The two failures below are REPORTED SEPARATELY, and that is not cosmetic. + * They ask the operator for different things: one is "install a package", + * the other is "this build will never read that URL". Collapsing them also + * made the absent-module test inert, because a module that handles nothing + * yet produces the same message as no module at all, so the arm that moves + * the library aside passed whether or not the move succeeded. + */ + if (PgColumnarPathIsRemote(path)) + { + const PgColumnarObjStoreApi *api = PgColumnarObjStoreGet(); + + if (api == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("columnar: reading \"%s\" requires the object-store module", + path), + errdetail("Object storage support is a separate library, " + "pgcolumnar_objstore, which is not installed."), + errhint("Install the pgcolumnar object-store package, or " + "use a local filesystem path."))); + if (!api->handles_url(path)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("columnar: reading \"%s\" is not supported", path), + errdetail("The installed object-store module handles no " + "such URL scheme."), + errhint("Use a local filesystem path."))); + /* the remote source is wired in the next commit */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("columnar: object storage is not implemented yet"))); + } + src->f = AllocateFile(path, PG_BINARY_R); if (src->f == NULL) ereport(ERROR, @@ -1522,13 +1623,10 @@ pq_source_open(const char *path, PqSource *src, PqFile *pf) static void pq_source_close(PqSource *src) { - if (src->f != NULL) - { - FreeFile(src->f); - src->f = NULL; - } + src->ops->close(src); } + /* * Page headers are small thrift structures, but a v2 header can carry column * statistics whose min and max are values from the column, so the size is @@ -2486,6 +2584,14 @@ pq_walk_dir(const char *path, int depth, List **files, int *skipped) * the normal "could not open file" error for a genuine typo. * An empty directory or a non-matching glob is an error: the user named a set and * meant to read something. Returns a List of palloc'd cstrings. + * + * A REMOTE path is none of those things and must not reach any of them (#393). + * This runs ahead of pq_source_open at every entry point, so without the check + * below an s3:// key containing a glob metacharacter went to glob() against the + * LOCAL filesystem and came back "no files match pattern" — the filesystem-miss + * report for a remote path that the byte source exists to eliminate, arriving one + * layer above the byte source. `*`, `?` and `[` are all legal in an S3 key, so + * this is an ordinary key, not a crafted one. */ static List * pq_resolve_paths(const char *path) @@ -2493,6 +2599,27 @@ pq_resolve_paths(const char *path) struct stat st; List *files = NIL; + if (PgColumnarPathIsRemote(path)) + { + /* + * v1 reads exact object keys. Expanding a pattern needs a LIST call, + * whose paged XML or JSON response is a third hand-rolled parser over + * input an outside party controls, which is the shape that produced #210 + * and #228. Refusing here is a decision, so it says so; treating the + * metacharacter as a literal key byte would be the silent alternative + * and would surprise anyone who typed a pattern on purpose. + */ + if (pq_has_glob_meta(path)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("columnar: cannot expand a pattern in the object-storage path \"%s\"", + path), + errdetail("Patterns are expanded on the local filesystem " + "only. Object storage is read by exact key."), + errhint("Name each object explicitly."))); + return list_make1(pstrdup(path)); + } + if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) { int skipped = 0; diff --git a/test/objstore_module.sh b/test/objstore_module.sh new file mode 100755 index 0000000..e8c31cc --- /dev/null +++ b/test/objstore_module.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +# +# The object-store module's packaging and loader (#393). +# +# The module is a SEPARATE, non-preloaded shared library, because pgColumnar loads +# through shared_preload_libraries and anything the main library links is mapped into +# the postmaster and inherited by every backend. This suite asserts that separation +# holds, because it is the property the whole design rests on and it is easy to lose +# by accident: adding one object to the main OBJS list would undo it silently. +# +# It also asserts a remote path reports a remote error. Before this, s3://bucket/key +# reached AllocateFile and reported "No such file or directory", which is true of the +# filesystem and useless to the reader. +# +# Usage: test/objstore_module.sh [PG_CONFIG] +# Written fresh for pgColumnar. +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +# nm reads the symbol tables the separation checks rest on; stat sizes the +# stand-in module. Without this, a missing nm makes "the symbol is absent from +# the main library" true for the wrong reason, which is the whole failure this +# suite exists to catch, turned on itself. +pgc_require_tools nm stat || { pgc_summary; exit 1; } + +LIBDIR="$("$PGC_PG_CONFIG" --pkglibdir 2>/dev/null || echo "")" +[ -n "$LIBDIR" ] || LIBDIR="$(pgc_pg "pg_config --pkglibdir" | tail -1 | tr -d '\r')" +MOD="$LIBDIR/pgcolumnar_objstore.so" + +# Recover from a stash left by an interrupted run, and decide which kind it is. +# +# An interrupt between a move and its restore leaves a .probe or .away behind. The +# dangerous case is real: inside the broken-module arm below, this run's STAND-IN +# is installed as the module and the real one is parked at .away, so a later run +# that moved the stand-in aside would overwrite the only real copy with 19 bytes +# of garbage and destroy the installation. Running alone does not prevent it, +# because the two runs are sequential. +# +# But refusing on PRESENCE alone refuses forever. Every matrix leg runs +# `make install`, so the ordinary leftover is debris sitting beside a module that +# is already fine, and this suite then stays red on that major until somebody +# moves a file by hand. On 2026-08-06 a .probe from 15:09 did exactly that to PG17 +# in a five-major matrix, while the other 119 suites passed. +# +# What tells the two apart is the module BESIDE the stash, not the stash. nm is +# required above, so this discriminator cannot go quiet the way a bare -e test can. +stash_is_debris() { + [ -e "$MOD" ] || return 1 + [ "$(nm -D --defined-only "$MOD" 2>/dev/null | + grep -c ' T pgcolumnar_objstore_init')" -ge 1 ] +} +for stash in "$MOD.away" "$MOD.probe"; do + [ -e "$stash" ] || continue + if stash_is_debris; then + echo "NOTE $stash was left by an interrupted run. The installed module is" + echo " valid, so the stash is debris; removing it and continuing." + rm -f "$stash" + continue + fi + # Nothing valid is installed, so this stash is the only surviving copy. Put it + # back. The old advice was to do this by hand, which is why an interrupt on one + # run reddened every later run on that major until somebody read the message. + echo "NOTE $stash was left by an interrupted run, and the module beside it is" + echo " missing or not a module, so the stash is the only surviving copy." + echo " Restoring it and continuing." + rm -f "$MOD" + mv "$stash" "$MOD" + # Restoring garbage is not recovery. If the stash was not a module either, + # every check below would run against a broken installation and report the + # confusing half of the truth, so stop here and say which file to look at. + if ! stash_is_debris; then + echo "FAIL restored $stash to $MOD, but that is not a module either." + echo " This installation needs 'make install' before the suite can run." + PGC_CHECKS=$((PGC_CHECKS + 1)) + PGC_FAIL=1 + pgc_summary + exit 1 + fi +done + +check "the main library is installed" \ + "$(pgc_pg "test -f '$LIBDIR/pgcolumnar.so' && echo yes || echo no" | tail -1)" "yes" +check "the object-store module is installed BESIDE it, not inside it" \ + "$(pgc_pg "test -f '$MOD' && echo yes || echo no" | tail -1)" "yes" + +# ---- the separation the design rests on --------------------------------------- +# +# The question is whether the module's code is INSIDE the preloaded library. The +# obvious check does not answer it: `readelf -d pgcolumnar.so | grep -c objstore` +# counts DT_NEEDED entries, and the failure mode we care about -- someone adds +# the module's objects to the main OBJS list -- links them STATICALLY and emits no +# DT_NEEDED entry at all. That check reads 0 whether or not the mistake was made. +# It also reads 0 when readelf is absent, or when the .so is not there. +# +# Ask about the symbol instead, as a PAIR. The entry point must be absent from +# the main library and present in the module. A broken or missing instrument +# fails the positive half, so the pair cannot go quiet the way a lone count of +# zero can. +check_num "the module's entry point is NOT inside the preloaded library" \ + "$(pgc_pg "nm -D --defined-only '$LIBDIR/pgcolumnar.so' | grep -c pgcolumnar_objstore_init" | tail -1)" "0" +check_num "positive control: it IS defined in the module, so nm really looked" \ + "$(pgc_pg "nm -D --defined-only '$MOD' | grep -c ' T pgcolumnar_objstore_init'" | tail -1)" "1" + +# A remote path must report a remote error, from the reader, without a connection. +for url in "s3://bucket/key.parquet" "gs://bucket/key.parquet" "https://host/key.parquet"; do + out=$(psql_run "SELECT * FROM pgcolumnar.read_parquet('$url') AS (a int)" 2>&1) + check "a $(cut -d: -f1 <<<"$url") URL reports an object-storage error, not a missing file" \ + "$([ "$(grep -c 'object storage is not implemented\|is not supported\|requires the object-store module' <<<"$out")" -ge 1 ] && echo yes || echo no)" "yes" + check_num "and does NOT report it as a missing file" \ + "$(grep -c 'No such file or directory' <<<"$out")" "0" +done + +# ---- a remote path must not be expanded against the local filesystem ---------- +# +# pq_resolve_paths runs AHEAD of the byte source at every entry point, so before +# #393's fix an s3:// key containing a glob metacharacter went to glob() against +# the LOCAL filesystem and came back "no files match pattern". That is exactly the +# filesystem-miss-for-a-remote-path report the byte source exists to remove, +# arriving one layer above it. `*`, `?` and `[` are all legal in an S3 key, so +# this is an ordinary key and not a crafted one. +for pat in "s3://bucket/a*.parquet" "s3://bucket/a?.parquet" "s3://bucket/a[0-9].parquet"; do + out=$(psql_run "SELECT * FROM pgcolumnar.read_parquet('$pat') AS (a int)" 2>&1) + check "a glob character in an object key is refused as a pattern, not expanded" \ + "$([ "$(grep -c 'cannot expand a pattern in the object-storage path' <<<"$out")" -ge 1 ] && echo yes || echo no)" "yes" + check_num "and it is NOT reported as a local filesystem miss" \ + "$(grep -c 'no files match pattern\|matched no regular files' <<<"$out")" "0" +done + +# A local path must be entirely unaffected, which is the regression this could cause. +psql_run "CREATE TABLE lp (a int) USING pgcolumnar; INSERT INTO lp VALUES (1),(2),(3);" >/dev/null 2>&1 +check_num "a local path still works" "$(q 'SELECT count(*) FROM lp')" "3" +# A RELATIVE path, which is what the label says. The previous version of this +# passed an absolute one, so the case it named was never exercised. +check_num "a relative path is not mistaken for a URL" \ + "$(psql_run "SELECT * FROM pgcolumnar.read_parquet('nonexistent-dir/x.parquet') AS (a int)" 2>&1 | + grep -c 'No such file or directory\|could not open')" "1" + +# ---- everything below MOVES the installed module ------------------------------ +# +# That needs write permission on its directory, which the suite has when it runs as +# the installing user and may not otherwise. Skip visibly rather than fail: a suite +# that cannot perform its manipulation has not found a defect, and a red gate here +# would be about the environment. +# +# This suite also runs ALONE in the matrix, because the file it moves is shared with +# every other suite in the run. +# + +# Armed BEFORE the writability probe below, which is itself a move. The first +# version armed it after, leaving a two-syscall window in which an interrupt left +# the real module at $MOD.probe with nothing at $MOD: restore_module knew only +# .away, so the next run sailed past the start guard, found no module to move, +# and reported SKIP on an installation that was itself broken and stayed broken +# until somebody noticed the .probe file. Same shape as the defect this suite was +# written to catch, one move earlier. Both suffixes are handled here and above. +restore_module() { + local stash + for stash in "$MOD.away" "$MOD.probe"; do + [ -e "$stash" ] || continue + rm -f "$MOD" + mv "$stash" "$MOD" + done +} +# Chained, not replacing. pgc_setup installs `trap pgc_teardown EXIT`, and a bare +# `trap restore_module EXIT` overwrites it: the module came back but the cluster +# was never stopped and its workdir never removed. Every run of this suite then +# left a live postmaster holding a port. Measured at 32 orphaned postmasters from +# one matrix plus one gate, which is enough to exhaust the port band -- and a +# suite that cannot get a port fails with 8 start attempts, which reads exactly +# like a real red. replication.sh's sb_teardown already chains this way. +objstore_teardown() { restore_module; pgc_teardown; } +trap objstore_teardown EXIT INT TERM + +if ! mv "$MOD" "$MOD.probe" 2>/dev/null; then + echo "SKIP cannot move $MOD, so the absent and broken paths are untested here" + pgc_summary + exit 0 +fi +mv "$MOD.probe" "$MOD" 2>/dev/null + +# ---- present, absent, and broken must be three DIFFERENT reports --------------- +# +# Three states, three messages, and the checks below are written so that each one +# fails if the state it names is not the state that occurred. +# +# The earlier version could not do that. A module that handles no scheme yet and a +# module that is not installed at all both produced "is not supported", so the +# absent arm passed whether or not the mv succeeded: deleting both mv lines left +# every check green. That is why the loader now names the missing module +# separately -- it is a real distinction for the operator ("install a package" +# against "this build will never read that URL"), and it is what makes this arm +# able to fail. +# +# What is still asserted here from before: +# +# 1 a missing module must report the remote path against the missing module, not +# 'could not access file "pgcolumnar_objstore"'. signalNotFound = false +# suppresses a missing SYMBOL; a missing LIBRARY is raised earlier by +# internal_load_library and needs a PG_TRY. +# +# 2 the SAME query must give the SAME error twice in one session. The cache flag +# used to be set before the load, so the ereport unwound past the assignment +# while the static kept its new value: the first read reported the raw load +# failure and every later one reported the documented message. Two identical +# queries, one session, two different errors, the second one plausible. +# +# Both reads run in ONE psql session on purpose. Separate sessions cannot see it. +Q="SELECT * FROM pgcolumnar.read_parquet('s3://bucket/key.parquet') AS (a int)" +two_reads() { + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -At -q -c "$Q" -c "$Q" 2>&1 +} + +both=$(two_reads) +check_num "with the module present, both reads report an unhandled scheme" \ + "$(grep -c 'is not supported' <<<"$both")" "2" +check_num "and neither claims the module is missing" \ + "$(grep -c 'requires the object-store module' <<<"$both")" "0" + +# ---- the module ABSENT, which is a supported configuration -------------------- +mv "$MOD" "$MOD.away" 2>/dev/null +# Recorded DURING the run, not after the restore. The previous premise ran after +# the module was back and asserted it was present, so it could only fail if the +# RESTORE failed -- it said nothing about the state the reads actually saw. +absent_state=$([ -e "$MOD" ] && echo present || echo absent) +absent=$(two_reads) +mv "$MOD.away" "$MOD" 2>/dev/null + +check "premise: the module really was absent while those two reads ran" \ + "$absent_state" "absent" +check "premise: and the real module is back afterwards" \ + "$([ -e "$MOD" ] && echo yes || echo no)" "yes" +check_num "with the module absent, neither read leaks the loader's own error" \ + "$(grep -c 'could not access file' <<<"$absent")" "0" +check_num "both reads name the missing module, identically" \ + "$(grep -c 'requires the object-store module' <<<"$absent")" "2" +check_num "and neither downgrades it to an unhandled scheme" \ + "$(grep -c 'is not supported' <<<"$absent")" "0" + +# ---- installed but BROKEN is not the same as not installed -------------------- +# +# Only "not installed" is supported. A truncated library or a permission problem is the +# operator's to fix, and reporting it as an unsupported scheme sends them elsewhere. +# +# TWO premises, because the first version of this test had neither and passed while doing +# nothing: +# 1 the module must actually BE corrupt during the run. Overwriting it in place fails +# silently: it is root-owned 0755 and this runs as postgres, which can write the +# DIRECTORY but not that file. So move it aside and CREATE a new one. +# 2 the SQLSTATE cannot be used to tell the two cases apart. internal_load_library uses +# errcode_for_file_access() for both the stat failure and the dlopen failure, so +# missing and "file too short" both arrive as 58P01. Measured. The check is on file +# presence for that reason. +mv "$MOD" "$MOD.away" 2>/dev/null && printf 'not a shared object' > "$MOD" 2>/dev/null +check_num "premise: the module really is corrupt for this run, not merely intended to be" \ + "$(stat -c %s "$MOD" 2>/dev/null)" "19" +broken=$(two_reads) +rm -f "$MOD" 2>/dev/null; mv "$MOD.away" "$MOD" 2>/dev/null +check "premise: the real module was restored afterwards" \ + "$([ "$(stat -c %s "$MOD" 2>/dev/null || echo 0)" -gt 1000 ] && echo yes || echo no)" "yes" +check_num "a broken module does NOT masquerade as an unhandled scheme" \ + "$(grep -c 'is not supported' <<<"$broken")" "0" +check_num "nor as a module that was never installed" \ + "$(grep -c 'requires the object-store module' <<<"$broken")" "0" +check "and the loader's own reason survives" \ + "$([ "$(grep -ci 'could not load library' <<<"$broken")" -ge 1 ] && echo yes || echo no)" "yes" + +pgc_summary diff --git a/test/objstore_stash_recovery.sh b/test/objstore_stash_recovery.sh new file mode 100755 index 0000000..51c583f --- /dev/null +++ b/test/objstore_stash_recovery.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +# Recovery from a stash left behind by an interrupted objstore_module run (#393). +# +# objstore_module.sh moves the installed module aside to reach the absent and the +# broken paths. An interrupt between a move and its restore leaves a +# pgcolumnar_objstore.so.probe or .away in pkglibdir, and that suite's start guard +# then refuses to run. +# +# Refusing is right when the stash is the only surviving copy of the module: +# continuing would overwrite it with the run's 19-byte stand-in and destroy the +# installation. Refusing FOREVER is not. Every matrix leg runs `make install`, so +# the ordinary leftover is debris sitting beside a perfectly good module, and the +# suite stays red on that major until a person notices and moves a file by hand. +# +# It is not hypothetical. On 2026-08-06 a .probe from 15:09 turned PG17 red in a +# five-major matrix while 119 other suites passed, and the run before it had been +# interrupted hours earlier. The guard's own advice -- `mv "$stash" "$MOD"` -- was +# by then wrong as well: make install had already restored the module, so +# following it would have replaced a freshly built module with a stale one. +# +# What separates the two cases is not whether a stash is PRESENT. It is whether +# the module beside it is VALID. Presence alone cannot tell debris from the only +# real copy, and an interrupt inside objstore_module's broken-module arm leaves a +# 19-byte stand-in installed with the real module at .away -- deleting the stash +# there is the destructive move the guard exists to prevent. +# +# Tested at the only seam that matters: run the suite, then look at its exit +# status and at what it left in pkglibdir. Nothing here reaches inside the guard, +# so the guard can be rewritten without touching this file. +# +# Usage: test/objstore_stash_recovery.sh [PG_CONFIG] +# Written fresh for pgColumnar. +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +PG_CONFIG="${1:-/usr/local/pg17/bin/pg_config}" +SRCDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SUITE="$SRCDIR/test/objstore_module.sh" + +# nm decides "valid module" for every check below. Without it the discriminator +# silently answers "not valid" for everything, which would make the destructive +# branch look correct. +pgc_require_tools nm || { pgc_summary; exit 1; } + +echo "== pgColumnar test: $(basename "$0") ==" +echo "PG_CONFIG=$PG_CONFIG" + +# No cluster of our own: this suite asserts on files and on another suite's exit +# status, and objstore_module.sh stands up its own. It does need the module +# installed, so build once here and let each invocation skip it. +if [ -z "${PGC_SKIP_BUILD:-}" ]; then + echo "-- building" + make -C "$SRCDIR" PG_CONFIG="$PG_CONFIG" >/dev/null || { + echo "FAIL build failed, so nothing below measures the guard" + PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAIL=1; pgc_summary + } + echo "-- installing" + make -C "$SRCDIR" install PG_CONFIG="$PG_CONFIG" >/dev/null || { + echo "FAIL install failed, so nothing below measures the guard" + PGC_CHECKS=$((PGC_CHECKS + 1)); PGC_FAIL=1; pgc_summary + } +fi + +LIBDIR="$("$PG_CONFIG" --pkglibdir)" +MOD="$LIBDIR/pgcolumnar_objstore.so" + +# The discriminator under test, computed here independently of the suite's copy +# of it. A module is valid when it defines the entry point, which is the same +# question objstore_module.sh asks with nm at its positive control. +mod_is_valid() { + [ -e "$1" ] || return 1 + [ "$(nm -D --defined-only "$1" 2>/dev/null | grep -c ' T pgcolumnar_objstore_init')" -ge 1 ] +} + +# Safety net, and deliberately NOT the logic under test: a pristine copy taken +# before anything moves. If the fix is wrong and leaves the installation broken, +# this restores it from that copy rather than from a rule this suite is asserting. +# Without it, a failing run of this suite leaves the box in exactly the state it +# exists to recover from, and poisons every later run on this major. +SAFE="" +cleanup() { + rm -f "$MOD.probe" "$MOD.away" + if [ -n "$SAFE" ] && [ -e "$SAFE" ]; then + mod_is_valid "$MOD" || cp -p "$SAFE" "$MOD" + rm -f "$SAFE" + fi +} +trap cleanup EXIT INT TERM + +if ! mod_is_valid "$MOD"; then + echo "SKIP no valid module installed at $MOD, so there is no state to arrange" + pgc_summary +fi +SAFE="$(mktemp /tmp/pgc-objstore-safe.XXXXXX)" +cp -p "$MOD" "$SAFE" + +# ---- debris beside a good installation must not stop the suite ---------------- +# +# The PG17 case, exactly: a stash left by an interrupted run, and a valid module +# reinstalled beside it by the next `make install`. +check "premise: the installed module is valid before this slice arranges anything" \ + "$(mod_is_valid "$MOD" && echo yes || echo no)" "yes" + +cp -p "$MOD" "$MOD.probe" +check "premise: a debris stash really is in place for the run below" \ + "$([ -e "$MOD.probe" ] && echo yes || echo no)" "yes" + +out="$(PGC_SKIP_BUILD=1 PGC_PORT="$(pgc_pick_port)" bash "$SUITE" "$PG_CONFIG" 2>&1)" +rc=$? + +check_num "the suite runs to completion with debris beside a valid module" "$rc" "0" +check_num "and it did not stop at the start guard" \ + "$(grep -c 'interrupted mid-move' <<<"$out")" "0" +check "the debris is gone afterwards" \ + "$([ -e "$MOD.probe" ] && echo present || echo absent)" "absent" +check "and the module left installed is still valid" \ + "$(mod_is_valid "$MOD" && echo yes || echo no)" "yes" + +# ---- a stash that is the only surviving copy must be restored, not refused ---- +# +# The interrupted state itself: the module moved aside and nothing put back. The +# suite must not proceed past this by treating the stash as debris -- there is no +# module to fall back on -- and it must not leave the box needing a human either. +mv "$MOD" "$MOD.away" +check "premise: nothing is installed at the module's path for this slice" \ + "$([ -e "$MOD" ] && echo present || echo absent)" "absent" +check "premise: and the stash beside it is the real module" \ + "$(mod_is_valid "$MOD.away" && echo yes || echo no)" "yes" + +out="$(PGC_SKIP_BUILD=1 PGC_PORT="$(pgc_pick_port)" bash "$SUITE" "$PG_CONFIG" 2>&1)" +rc=$? + +check_num "the suite recovers the only surviving copy and runs to completion" "$rc" "0" +check "the module is installed again afterwards, and valid" \ + "$(mod_is_valid "$MOD" && echo yes || echo no)" "yes" +check "and the stash is not left behind for the next run to trip on" \ + "$([ -e "$MOD.away" ] && echo present || echo absent)" "absent" + +# ---- a stand-in installed over the real module is the destructive case -------- +# +# An interrupt inside objstore_module's broken-module arm leaves this exact state: +# 19 bytes of text installed AS the module, with the real one at .away. A stash +# rule written on presence alone deletes .away here and destroys the installation, +# and it looks correct in every other arrangement. This is the arm that says the +# discriminator has to be validity. +mv "$MOD" "$MOD.away" +printf 'not a shared object' > "$MOD" +check_num "premise: a 19-byte stand-in really is installed as the module" \ + "$(stat -c %s "$MOD" 2>/dev/null)" "19" +check "premise: which is not a module" \ + "$(mod_is_valid "$MOD" && echo yes || echo no)" "no" +check "premise: and the real module is the thing parked at the stash" \ + "$(mod_is_valid "$MOD.away" && echo yes || echo no)" "yes" + +out="$(PGC_SKIP_BUILD=1 PGC_PORT="$(pgc_pick_port)" bash "$SUITE" "$PG_CONFIG" 2>&1)" +rc=$? + +check_num "the suite replaces the stand-in with the real module and completes" "$rc" "0" +check "the real module survived, rather than being deleted with the stash" \ + "$(mod_is_valid "$MOD" && echo yes || echo no)" "yes" +check "and the stash is cleared" \ + "$([ -e "$MOD.away" ] && echo present || echo absent)" "absent" + +pgc_summary diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 727425a..3bdc579 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -208,7 +208,7 @@ SRCDIR="${PGC_RUN_SRCDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" SUITES=(harness_selftest docs_style smoke phase2 phase3 phase4 phase5 phase6 audit concurrency unique_conc \ differential recovery replication native_backend_crash fuzz fuzz_parquet fuzz_arrow hardening concurrent_diff parallel sorted_projection \ arrow_export parquet_export read_stream corruption \ - generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_fastdecode native_zonemap write_minmax_fastpath write_fsst_compressed fsst_margin encode_invariants encode_effort native_skip pushdown_report zonemap_cost native_agg native_agg_deletes native_agg_addcolumn native_groupagg ungrouped_vector_agg parallel_vector_agg native_bloom bloom_setting bloom_lazy native_vecskip native_index native_index_projection native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster pg19_vacuum_options native_repack native_compact native_recluster recluster_extent native_vacuum_race native_sort_by sort_status native_reclaim native_ownership drop_cleanup pg_dump_roundtrip native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_format native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening server_file_privilege native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel cancel_decode wal_envelope decode_interrupts import_exclusion import_deferred parallel_copy parallel_export_parquet fk_referencing row_triggers native_lazy_slot native_ctas native_fetch_cache native_fetch_bigcap native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection column_projection advisory_lock_class logical_subscriber parallel_degree isolation) + generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_fastdecode native_zonemap write_minmax_fastpath write_fsst_compressed fsst_margin encode_invariants encode_effort native_skip pushdown_report zonemap_cost native_agg native_agg_deletes native_agg_addcolumn native_groupagg ungrouped_vector_agg parallel_vector_agg native_bloom bloom_setting bloom_lazy native_vecskip native_index native_index_projection native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster pg19_vacuum_options native_repack native_compact native_recluster recluster_extent native_vacuum_race native_sort_by sort_status native_reclaim native_ownership drop_cleanup pg_dump_roundtrip native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_format native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening server_file_privilege native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel cancel_decode wal_envelope decode_interrupts import_exclusion import_deferred parallel_copy parallel_export_parquet fk_referencing row_triggers native_lazy_slot native_ctas native_fetch_cache native_fetch_bigcap native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection column_projection advisory_lock_class logical_subscriber parallel_degree objstore_module objstore_stash_recovery isolation) # Default matrix: one assert-enabled pg_config per major, 15 through 19. DEFAULT_CONFIGS=( @@ -334,6 +334,16 @@ is_timing_suite() { runs_alone() { case "$1" in replication) return 0 ;; + # objstore_module moves the INSTALLED module aside to test the + # not-installed and broken paths. That file is shared by every suite in + # the run, so doing it while others execute would break them, and the + # breakage would look like a defect in whichever suite happened to load + # the extension at the wrong moment. + objstore_module) return 0 ;; + # objstore_stash_recovery arranges that same shared file into the states + # an interrupted run leaves behind, and runs objstore_module against each. + # It moves the module for the same reason and must be alone for it. + objstore_stash_recovery) return 0 ;; *) is_timing_suite "$1" ;; esac }