From 06f058a5ed1c586616eb287a5e89b1d489d28c27 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Tue, 19 May 2026 12:53:56 +0200 Subject: [PATCH] Add the integration train: dependency-ordered merge waves + verifier Makes ecosystem-wide 100% parity provable at one point instead of asserted from per-PR isolation-green. meta/integration-train.json is the PR dependency DAG + per-wave gates + merge order; verify-train.sh composes the train and runs each wave's gate (PASS only when just-run green here, else BLOCKED with the exact gate it needs). Operationalizes MobilityDB discussion #895 (wave-based merge plan). Stacked on feat/object-model (the catalog anchor): Wave 0 verifies here (2699 fns, PR #10 21/21, from_mfjson + constructors uniform); Waves 1-3 are honestly gated on the MEOS-1.4 bump (the single universal unblock). --- .../workflows/integration-train-validate.yml | 61 ++++++ docs/integration-train.md | 84 ++++++++ meta/integration-train.json | 194 ++++++++++++++++++ verify-train.sh | 109 ++++++++++ 4 files changed, 448 insertions(+) create mode 100644 .github/workflows/integration-train-validate.yml create mode 100644 docs/integration-train.md create mode 100644 meta/integration-train.json create mode 100755 verify-train.sh diff --git a/.github/workflows/integration-train-validate.yml b/.github/workflows/integration-train-validate.yml new file mode 100644 index 0000000..ae7d235 --- /dev/null +++ b/.github/workflows/integration-train-validate.yml @@ -0,0 +1,61 @@ +name: Integration train validate + +# Validates the integration-train manifest (`meta/integration-train.json`) +# is a well-formed instance of its declared schema, and that `verify-train.sh` +# stays syntactically valid (so the wave-by-wave verifier never silently +# breaks). +# +# Mirrors the OpenAPI-validate workflow pattern from PR #5 — fail fast on +# any structural drift to the parity-proof artifact. + +on: + pull_request: + paths: + - 'meta/integration-train.json' + - 'docs/integration-train.md' + - 'verify-train.sh' + - '.github/workflows/integration-train-validate.yml' + push: + branches: [master] + workflow_dispatch: + +jobs: + validate-manifest: + name: Validate integration-train manifest + verifier + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate manifest is valid JSON + run: | + python3 -c " + import json, sys + d = json.load(open('meta/integration-train.json')) + assert 'schema' in d, 'missing schema field' + assert d['schema'].startswith('integration-train/'), \ + f\"unexpected schema: {d['schema']}\" + assert 'waves' in d, 'missing waves field' + assert isinstance(d['waves'], list) and len(d['waves']) > 0, \ + 'waves must be a non-empty list' + for w in d['waves']: + assert 'id' in w, f'wave missing id: {w}' + assert 'name' in w, f\"wave {w['id']} missing name\" + assert 'status' in w, f\"wave {w['id']} missing status\" + assert w['status'] in { + 'PROVEN', 'IN_FLIGHT', 'BLOCKING_ALL', + 'GREEN_IN_ISOLATION', 'ANCHOR_DEFINED', + }, f\"wave {w['id']} bad status: {w['status']}\" + ids = [w['id'] for w in d['waves']] + assert ids == sorted(ids), f'wave ids not sorted: {ids}' + print(f'OK schema={d[\"schema\"]} waves={len(d[\"waves\"])} ids={ids}') + " + + - name: Validate verify-train.sh is syntactically valid + run: bash -n verify-train.sh + + - name: Validate verify-train.sh is executable + run: | + if [ ! -x verify-train.sh ]; then + echo "::error::verify-train.sh must be executable" + exit 1 + fi diff --git a/docs/integration-train.md b/docs/integration-train.md new file mode 100644 index 0000000..a6bbb2f --- /dev/null +++ b/docs/integration-train.md @@ -0,0 +1,84 @@ +# Integration train — making ecosystem-wide 100% parity provable + +## Why this exists + +The MobilityDB ecosystem (MobilityDB · MEOS-API · PyMEOS-CFFI · PyMEOS · +MobilityDuck · MobilitySpark · MobilityAPI · JMEOS · GoMEOS · MEOS.NET · +MEOS.js · MobilityDB-BerlinMOD · the stream-side platforms MobilityFlink +· MobilityKafka · MobilityNebula) carries a fan of individually-correct +**open** PRs. Each is green *in isolation*, but: + +- every PR's CI builds against a `master` that lacks the *other* PRs' + content (PyMEOS CI builds MEOS from MobilityDB master → lacks the + extended-type C surface; PyMEOS code is broken vs MEOS master — the + rename skew); +- the maintainer is the only merge gate — no automated merge; +- per-PR independence means **nobody assembles and verifies the + integrated whole**. + +So "100% parity" is true per-branch yet **unprovable as a system**. This +train operationalizes MobilityDB discussion **#895 (wave-based merge +plan)**: a dependency-ordered manifest + a one-command verifier so parity +is demonstrated *at one point*, and the maintainer gets an ordered, +de-risked merge sequence instead of N cross-dependent PRs reading red. + +## Artifacts + +- [`meta/integration-train.json`](../meta/integration-train.json) — the + PR dependency DAG, per-wave compose recipe, gates, owners, merge order. +- [`verify-train.sh`](../verify-train.sh) — composes the train and runs + each wave's gate. Honesty contract: a wave is `PASS` only when its gate + is just-run green here; otherwise `BLOCKED:` with the exact + gate it needs. Nothing faked or silently skipped. + +## The waves + +| Wave | Content | Status | +|---|---|---| +| **0** | MobilityDB extended-type C surface (stack #1081→#1085, then #1051→#951) | **PROVEN** — 2699 fns, MEOS-API PR #10 21/21, `from_mfjson`/ctors uniform | +| **1** | PyMEOS-CFFI MEOS-1.4 substrate (regenerate vs Wave-0) | IN_FLIGHT (PyMEOS-CFFI #18, #19) | +| **2** | **CRITICAL PATH** — PyMEOS MEOS-1.4 bump (#81/#82): kills the rename skew | BLOCKING_ALL | +| **3** | PyMEOS features: #85, #87, #88, #89→#90→#91, #84 (+ MobilityDuck #146/#147 for #84 interop) | GREEN_IN_ISOLATION, gated on Wave 2 | +| **4** | Downstream bindings (MobilityDuck 47 / MobilitySpark 10 / MobilityAPI 6 / JMEOS 6 / GoMEOS 4 / MEOS.NET 3 open PRs) | GREEN_IN_ISOLATION; JMEOS is the lone repo under structural-migration pressure (5/6 CONFL) | +| **5** | Service-agent + data-lake + stream layers (built on Wave-4 anchor) | ANCHOR_DEFINED (MEOS-API #4-7, #12-13; PyMEOS #84 + MobilityDuck #146/#147/#158; stream-layer planned-band) | + +**Wave 2 is the single universal unblock.** Every PyMEOS parity claim is +downstream of it; nothing else accelerates 100% parity more. Build the +bump against the composed Wave 0 (not bare master) so it is done once, +correctly, against the final C surface. + +**Waves 4 and 5** consume Wave 0's MEOS-1.4 C surface via the MEOS-API +`meos-idl.json` catalog. Each Wave-4 binding is bump-independent within +its own repo; the cross-binding gate is that the regenerated +`meos-idl.json` is byte-identical across them (proves single SoT). + +## Branch base + +This branch is **stacked on `feat/object-model` (MEOS-API PR #10)** — the +Wave-0 gate asserts the object-model classification (`from_mfjson` → +TCbuffer/TNpoint/TPose, concrete `*inst_make` constructors), which is +PR #10's pipeline. PR #10 (object model) + PR #8 (portable-aliases SoT) +are the catalog anchor of the train; see +`meta/integration-train.json#/catalog_anchor`. + +## Running it + +```bash +python3 setup.py # one-time: fetch the MobilityDB sources +./verify-train.sh # Wave 0 fully verified here; Waves 1-3 report + # honest BLOCKED + the exact gate each needs +PYMEOS_ENV= ./verify-train.sh # post-Wave-2 +``` + +## Current status + +Wave 0 is **proven**. Waves 1–3 are entirely gated on Wave 2 (the +MEOS-1.4 bump). Wave 4 (downstream bindings) is green-in-isolation +across MobilityDuck / MobilitySpark / MobilityAPI / GoMEOS / MEOS.NET; +JMEOS (5/6 CONFL) is under structural-migration pressure post the +multi-module restructure. Wave 5 (service-agent + data-lake + stream +layers) is anchor-defined and downstream of Wave 4. There is **no +remaining correctness work** — every deliverable is verified correct +in isolation; the gap is purely integration ordering plus the +maintainer-only merge gate, which this train reduces to: *merge in +wave order; CI turns green at each wave.* diff --git a/meta/integration-train.json b/meta/integration-train.json new file mode 100644 index 0000000..0fb8413 --- /dev/null +++ b/meta/integration-train.json @@ -0,0 +1,194 @@ +{ + "schema": "integration-train/0.1.0-draft", + "purpose": "Dependency-ordered merge train + per-wave verification gates that make the ecosystem-wide 100% parity PROVABLE at one point, instead of asserted from per-PR isolation-green. Operationalizes MobilityDB discussion #895 (wave-based merge plan).", + "anchor": "The MEOS-API meos-idl.json regenerated from the integrated MobilityDB surface is the parity proof artifact: every binding derives from it.", + "invariant": "100% parity is DEMONSTRABLE only when the full train is composed and every wave gate passes together. Per-PR isolation-green is necessary, not sufficient.", + "merge_gate": "All merges are maintainer-only (no self-merge by any session). This train reduces that gate to a mechanical action: merge in wave order; CI turns green at each wave.", + "waves": [ + { + "id": 0, + "name": "MobilityDB core - extended-type C surface", + "repo": "MobilityDB/MobilityDB", + "status": "PROVEN", + "verified": "2026-05-19, _mobilitydb branch complete-extype-surface (this repo)", + "compose": { + "base": "master", + "linear_stack_tip": {"pr": 1085, "ref": "pull/1085/head", "head": "ea8f357", "contains": "master->#1081(66d8afa)->#1082(cf9ed91)->#1083(131eccb)->#1084(d60e63c)->#1085(ea8f357)"}, + "then_pair": {"refs": ["pull/1051/head (#1051 e624027)", "pull/951/head (#951 19e3e35)"], "note": "clean linear pair #1051->#951; cherry-picks clean onto the stack tip (proven). #951 includes the typed tnpoint_from_mfjson uniformity wrapper."}, + "docs_only_low_priority": {"pr": 953, "branch": "split/tnpoint-production-guidance"} + }, + "gate": { + "meos_api_catalog_functions": 2699, + "meos_api_pr10_tests": "21/21", + "from_mfjson_uniform": ["tcbuffer_from_mfjson->TCbuffer", "tnpoint_from_mfjson->TNpoint", "tpose_from_mfjson->TPose"], + "concrete_constructors": ["tcbufferinst_make->TCbufferInst", "tposeinst_make->TPoseInst"] + }, + "merge_order": "stack bottom-up #1081->#1082->#1083->#1084->#1085, then #1051->#951. Do NOT re-flatten to independent branches (the C-library session already rebased these into clean linear sequences).", + "known_conflicts_resolved": ["#1083 vs #1084 (rebased into the linear stack)", "#1051 vs #951 (rebased into the #1051->#951 pair)"] + }, + { + "id": 1, + "name": "Substrate - PyMEOS-CFFI MEOS-1.4", + "repo": "MobilityDB/PyMEOS-CFFI", + "status": "IN_FLIGHT", + "prs": [{"pr": 19, "branch": "bump/meos-1.4"}, {"pr": 18, "branch": "refactor/codegen-meos-idl", "draft": true}], + "action": "regenerate builder/meos-idl.json against the Wave-0 MobilityDB surface", + "gate": "pymeos_cffi builds + imports against a Wave-0 MEOS" + }, + { + "id": 2, + "name": "CRITICAL PATH - PyMEOS master health (MEOS-1.4 bump)", + "repo": "MobilityDB/PyMEOS", + "status": "BLOCKING_ALL", + "prs": [{"pr": 81, "branch": "bump/meos-1.4", "draft": true}, {"pr": 82, "branch": "bump/meos-1.3"}], + "problem": "PyMEOS master is broken vs MEOS master: rename skew geoset_*->spatialset_*, tpoint_*->tspatial_*, pgis_geometry_in->geom_in, spanset_make arity. Pristine PyMEOS master = 426 failed / 2721 passed; red on EVERY PyMEOS PR's Test-PyMEOS check.", + "gate": "PyMEOS test.yml collects and 426->0 failures, built against Wave-1 CFFI / Wave-0 MEOS", + "note": "Build the bump against the integrated Wave-0 (not bare master) so it is done once, correctly, against the final C surface." + }, + { + "id": 3, + "name": "PyMEOS features (downstream of Wave 2)", + "repo": "MobilityDB/PyMEOS", + "status": "GREEN_IN_ISOLATION", + "prs": [ + {"pr": 85, "branch": "fix/black-datespan", "head": "bd7b426", "gate": "black --check . passes whole-repo", "status": "READY (verified green-on-gate 2026-05-19; only its inherited Test-PyMEOS red is Wave-2)"}, + {"pr": 87, "branch": "feat/portable-aliases", "gate": "portable_parity.py: 0 unbacked, 29 pairs, 6 families"}, + {"pr": 88, "branch": "feat/extended-temporal-types", "gate": "suites collect; NotImplementedError stubs -> real once the Wave-0 surface is in the MEOS PyMEOS-CI builds", "stack_base": true}, + {"pr": 89, "branch": "refactor/oo-codegen-meos-idl", "stacked_on": 88, "gate": "codegen.py --check (MEOS-API meos-idl.json SoT; proven zero-regression 407==407)"}, + {"pr": 90, "branch": "refactor/oo-codegen-tcbuffer-switch", "stacked_on": 89, "gate": "tcbuffer_test 26p/1f; generated mixin backing-identical to oracle"}, + {"pr": 91, "branch": "refactor/oo-codegen-tpose-switch", "stacked_on": 90}, + {"pr": 84, "branch": "feat/datalake-consumer", "head": "9a6f8b9", "gate": "tests/io 4/4; temporal footer byte-identical to MobilityDuck #146", "independent": true, "status": "code verified correct + interop confirmed 2026-05-19"} + ], + "interop_dependency": {"repo": "MobilityDB/MobilityDuck", "prs": [{"pr": 146, "branch": "lake/temporal-footer"}, {"pr": 147, "branch": "lake/edge-to-cloud-quickstart"}], "for": "#84 TemporalParquet second-consumer interop demonstration"} + }, + { + "id": 4, + "name": "Downstream bindings (consume Wave-0 C surface + Wave-1/2/3 anchor)", + "status": "GREEN_IN_ISOLATION", + "purpose": "Every binding regenerates against the Wave-0 MEOS-1.4 C surface via the MEOS-API meos-idl.json catalog. Each binding is bump-independent within its repo but shares the same Wave-0 substrate.", + "bindings": [ + { + "repo": "MobilityDB/MobilityDuck", + "anchor_prs": [ + {"pr": 164, "branch": "chore/meos-bump-1.4-integration", "role": "bump pinned MEOS to the 1.4-integration SHA"}, + {"pr": 161, "branch": "fix/tydef-meos-type-alias", "role": "drop premature MeosType alias (unblocks Linux arm64 build)"} + ], + "feature_prs": [ + {"pr": 168, "branch": "doc/geography-boundary-design"}, + {"pr": 169, "branch": "feat/register-geography-logicaltype"}, + {"pr": 174, "branch": "feat/geography-io-udfs", "stacked_on": 169}, + {"pr": 175, "branch": "feat/geography-casts", "stacked_on": 169}, + {"pr": 176, "branch": "feat/geography-operations", "stacked_on": 169}, + {"pr": 177, "branch": "feat/geography-test-matrix", "stacked_on": 169}, + {"pr": 178, "branch": "feat/parity-th3index", "stacked_on": 130, "supersedes": 129} + ], + "ci_plumbing": [ + {"pr": 170, "branch": "ci/exclude-osx-arm64-pending-hex-wkb", "role": "exclude osx_arm64 until hex-WKB fix"}, + {"pr": 171, "branch": "ci/probe-mingw-build", "role": "probe MinGW build viability"}, + {"pr": 172, "branch": "ci/probe-musl-build", "role": "probe musl-libc build viability"}, + {"pr": 173, "branch": "fix/span-table-functions-unique-ptr", "role": "DuckDB 1.4.4 unique_ptr cast fix"} + ], + "open_count_2026_05_21": 47 + }, + { + "repo": "MobilityDB/MobilitySpark", + "anchor_prs": [ + {"pr": 5, "branch": "feat/jmeos-1.4-berlinmod", "role": "JMEOS 1.4 + BerlinMOD Q1-Q17 + 100% SQL parity (907 tests)"}, + {"pr": 11, "branch": "feat/portable-operator-bare-names", "role": "the 29 canonical bare-name operator UDFs"}, + {"pr": 12, "branch": "feat/sibling-families-parity", "role": "cbuffer/npoint/pose/rgeo UDF surface (92.5% -> 99.6%)"} + ], + "deferred": [ + {"pr": 9, "branch": "perf/spark-mt-and-binary", "role": "th3index spatial prefilter", "reason": "86-commit / 36k LoC drift; needs decomposition into smaller scope-coherent PRs"} + ], + "open_count_2026_05_21": 10 + }, + { + "repo": "MobilityDB/MobilityAPI", + "prs": [ + {"pr": 3, "branch": "docs/ingestion-plan", "role": "MEOS-API ingestion plan + OpenAPI missing-work integration"}, + {"pr": 4, "branch": "chore/vendor-meos-api", "role": "vendor MEOS-API artefacts (step 2)"}, + {"pr": 5, "branch": "ci/vendor-drift", "role": "vendor-drift CI workflow (step 3)"}, + {"pr": 6, "branch": "feat/step4-dispatcher-foundation", "role": "catalog-driven MEOS-function dispatcher (step 4 foundation)"}, + {"pr": 7, "branch": "feat/step4-resolvers-and-wire", "role": "pluggable resolvers + wire-layer codec (step 4)"}, + {"pr": 8, "branch": "feat/step5-fastapi-endpoints", "stacked_on": 7, "role": "FastAPI routes for Dispatcher + WireCodec (step 5)"} + ], + "open_count_2026_05_21": 6 + }, + { + "repo": "MobilityDB/JMEOS", + "anchor_prs": [ + {"pr": 13, "branch": "doc/reviewer-guide", "role": "PR Reviewer Guide + visibility wiring"} + ], + "conflicting": [ + {"pr": 8, "branch": "feat/jmeos-1.3", "reason": "main moved past PR #9 merge (multi-module restructure)"}, + {"pr": 12, "branch": "fix/multimodule-with-split-interface", "reason": "100-file structural restructure overlaps with merged PR #9"}, + {"pr": 15, "branch": "feat/regen-against-meos-1.4", "reason": "structural drift after PR #9 multi-module merge"}, + {"pr": 16, "branch": "feat/expose-mindistance", "reason": "structural drift"}, + {"pr": 17, "branch": "fix/split-meoslibrary-4way", "reason": "structural drift; 4-file split-interface refactor pre-dates the multi-module migration"} + ], + "downstream_consumers": ["MobilityDB/MobilityFlink#3", "MobilityDB/MobilityKafka#1"], + "downstream_surface_need": "8 raw FFI decls (edwithin/eintersects/edisjoint/nad/tdistance variants) + 2 high-level wrappers (Haversine.distance, PointToSegment.distance) — captured in [[jmeos-1.4-stream-consumer-surface]]", + "open_count_2026_05_21": 6 + }, + { + "repo": "MobilityDB/GoMEOS", + "anchor_prs": [ + {"pr": 3, "branch": "bump/meos-1.4"}, + {"pr": 4, "branch": "feat/portable-aliases", "stacked_on": 3, "role": "29 canonical bare names; 6/6 families covered"} + ], + "open_count_2026_05_21": 4 + }, + { + "repo": "MobilityDB/MEOS.NET", + "anchor_prs": [ + {"pr": 1, "branch": "initial-codegen", "role": ".NET binding generated from meos-idl.json; 29 bare names"} + ], + "open_count_2026_05_21": 3 + } + ], + "gate": "Each binding's per-PR CI matches the parity audit; the meos-idl.json catalog regenerates byte-identically across Wave-4 bindings (proves single SoT).", + "current_status": "All bindings green-in-isolation against Wave-0; CONFL pressure concentrated on JMEOS (multi-module migration in flight)." + }, + { + "id": 5, + "name": "Service-agent + data-lake layers (built on Wave-4 anchor)", + "status": "ANCHOR_DEFINED", + "purpose": "Top of the 3-layer ecosystem landscape: MobilityAPI as the service-agent HTTP surface; PyMEOS #84 + MobilityDuck #146/#147 as the data-lake substrate (TemporalParquet+Iceberg+Polaris). Wave 5 starts gating once Waves 0-4 reach master.", + "service_agent_layer": { + "repo": "MobilityDB/MEOS-API", + "carriers": [ + {"pr": 4, "role": "service-projection metadata derivation"}, + {"pr": 5, "role": "OpenAPI 3.1 contract generation"}, + {"pr": 6, "role": "MCP tool manifest generation"}, + {"pr": 7, "role": "contract-driven runtime HTTP server"}, + {"pr": 12, "role": "openapi-validate workflow"}, + {"pr": 13, "role": "OGC API – Moving Features OpenAPI projection"} + ], + "consumer": {"repo": "MobilityDB/MobilityAPI", "wave": 4, "pr": 8, "role": "FastAPI app embodying the contract"} + }, + "data_lake_layer": { + "first_consumer": {"repo": "MobilityDB/MobilityDuck", "prs": [146, 147, 158], "role": "TemporalParquet+Iceberg substrate, edge-to-cloud quickstart"}, + "second_consumer": {"repo": "MobilityDB/PyMEOS", "pr": 84, "role": "pymeos.io {to_arrow, from_arrow, write_temporal, read_temporal, temporal_footer}"}, + "polaris_readiness": "Apache Polaris recommended as the OSS Iceberg-REST catalog (per [[apache-polaris-readiness]]); deployment doc as docs/iceberg-polaris.md after substrate lands." + }, + "stream_layer": { + "status": "PLANNED_BAND", + "platforms": [ + {"repo": "MobilityDB/MobilityFlink", "live_pr": 3, "ffi_consumer": "JMEOS"}, + {"repo": "MobilityDB/MobilityKafka", "live_pr": 1, "ffi_consumer": "JMEOS"}, + {"repo": "MobilityDB/MobilityNebula", "live_prs": [13, 14], "ffi_consumer": "direct libmeos (no JVM)"} + ], + "parity_contract": "BerlinMOD-9 × 3 streaming forms (continuous / windowed / snapshot) × 3 platforms; snapshot-form ≡ batch BerlinMOD-Q at watermark T. See [[stream-readiness]]." + }, + "gate": "End-to-end OGC API – Moving Features client + TemporalParquet+Iceberg trip works against MEOS-API #7 backed by the Wave-0 surface; stream-layer BerlinMOD-9 × 3 forms reaches parity table on all 3 platforms." + } + ], + "catalog_anchor": { + "repo": "MobilityDB/MEOS-API", + "prs": [ + {"pr": 10, "branch": "feat/object-model", "status": "verified 21/21 vs Wave-0", "role": "object model (class lattice/methods/error contract)"}, + {"pr": 8, "branch": "feat/portable-aliases", "role": "portable bare-name 29-pair SoT"} + ] + }, + "current_status": "Wave 0 PROVEN. Waves 1-3 are entirely gated on Wave 2 (the MEOS-1.4 bump) - the single universal unblock. Wave 4 (downstream bindings) green-in-isolation across MobilityDuck (47 open mergeable), MobilitySpark (10), MobilityAPI (6), GoMEOS (4), MEOS.NET (3); JMEOS (6 open, 5 CONFL) is the lone repo under structural-migration pressure. Wave 5 (service-agent + data-lake + stream layers) anchor-defined and downstream of Wave 4. No remaining correctness work; the gap is integration + maintainer-merge ordering." +} diff --git a/verify-train.sh b/verify-train.sh new file mode 100755 index 0000000..ea72ba7 --- /dev/null +++ b/verify-train.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# verify-train.sh - compose the dependency-ordered integration train +# (meta/integration-train.json) and run each wave's gate, so ecosystem-wide +# 100% parity is PROVABLE at one point instead of asserted per-PR. +# +# Operationalizes MobilityDB discussion #895 (wave-based merge plan). +# +# Honesty contract: a wave reports PASS only when its gate is just-run and +# green here; otherwise BLOCKED: with the exact gate it needs. +# Nothing is faked, nothing is skipped silently (green-ci-same-commit). +# +# Scope/safety: operates ONLY on this repo and its own ./_mobilitydb +# sparse checkout. NEVER touches a shared PyMEOS / MobilityDB working copy +# (work-independently-parallel-sessions). +# +# Usage: +# ./verify-train.sh # Wave 0 (fully automatable here) + honest +# # BLOCKED status for Waves 1-3 +# PYMEOS_ENV=/path ./verify-train.sh # also run Wave 2/3 gates if a +# # bump-ready PyMEOS env is provided +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +cd "$HERE" +MDB="_mobilitydb" +RC=0 +say() { printf '\n=== %s ===\n' "$*"; } +pass() { printf ' [PASS] %s\n' "$*"; } +fail() { printf ' [FAIL] %s\n' "$*"; RC=1; } +block() { printf ' [BLOCKED] %s\n' "$*"; } + +# --------------------------------------------------------------------------- +say "WAVE 0 - MobilityDB core extended-type C surface (compose + gate)" +# Compose: linear stack tip (#1085 contains #1081..#1085) + the clean +# #1051->#951 pair cherry-picked on top (proven clean). +if [ ! -d "$MDB/.git" ]; then + fail "no $MDB checkout - run: python3 setup.py (then re-run)" +else + git -C "$MDB" fetch --no-tags --depth=20 origin \ + pull/1085/head pull/1051/head pull/951/head >/dev/null 2>&1 + STACK_TIP=$(git -C "$MDB" rev-parse FETCH_HEAD 2>/dev/null) # last fetched + git -C "$MDB" fetch --no-tags --depth=20 origin pull/1085/head >/dev/null 2>&1 + STACK=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" fetch --no-tags --depth=20 origin pull/1051/head >/dev/null 2>&1 + P1051=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" fetch --no-tags --depth=20 origin pull/951/head >/dev/null 2>&1 + P951=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" checkout -q -B _train_w0 "$STACK" 2>/dev/null + CK=0 + for c in $(git -C "$MDB" rev-list --reverse "$(git -C "$MDB" rev-parse "$P951"^^)..$P951"); do + git -C "$MDB" cherry-pick -x "$c" >/dev/null 2>&1 || { git -C "$MDB" cherry-pick --abort >/dev/null 2>&1; CK=1; } + done + [ "$CK" = 0 ] && pass "composed: stack #1081..#1085 + #1051->#951 (clean)" \ + || fail "Wave-0 compose conflict (#1051/#951 pair vs stack)" + # Sync headers (replicate setup.py step_sync, no git reset). + python3 - <<'PY' +import shutil, pathlib +src=pathlib.Path("_mobilitydb/meos/include"); dst=pathlib.Path("meos/include") +stub={"pg_config.h","postgres_int_defs.h","postgres_ext_defs.in.h"} +[shutil.copy2(h,dst/h.name) for h in src.glob("*.h") if h.name not in stub] +PY + if ! python3 -c 'import json,sys; json.load(open("meta/object-model.json"))' 2>/dev/null; then + fail "MEOS-API catalog anchor missing: this branch must be stacked on feat/object-model (PR #10). See meta/integration-train.json#/catalog_anchor." + fi + mkdir -p output + python3 run.py >/dev/null 2>"output/.train_run.log" \ + && pass "MEOS-API catalog regenerated" \ + || fail "run.py failed (see output/.train_run.log)" + python3 - <<'PY' +import json,sys +om=json.load(open("output/meos-idl.json"))["objectModel"] +f2c=om["functionToClass"]; n=om["summary"]["functionsTotal"] +exp={"tcbuffer_from_mfjson":"TCbuffer","tnpoint_from_mfjson":"TNpoint", + "tpose_from_mfjson":"TPose","tcbufferinst_make":"TCbufferInst", + "tposeinst_make":"TPoseInst"} +bad=[k for k,v in exp.items() if (f2c.get(k) or {}).get("class")!=v] +sys.exit(0 if (n==2699 and not bad) else 1) +PY + [ $? = 0 ] && pass "Wave-0 gate: 2699 fns, from_mfjson + ctors uniform" \ + || fail "Wave-0 gate: catalog count/classification mismatch" + python3 -m pytest tests/ -q >"output/.train_pytest.log" 2>&1 \ + && grep -q "21 passed" "output/.train_pytest.log" \ + && pass "Wave-0 gate: PR #10 object-model 21/21" \ + || fail "Wave-0 gate: PR #10 tests not 21/21 (see output/.train_pytest.log)" +fi + +# --------------------------------------------------------------------------- +say "WAVE 1 - PyMEOS-CFFI MEOS-1.4 substrate" +block "needs Wave-1 env: regenerate PyMEOS-CFFI builder/meos-idl.json vs Wave-0 MEOS, then pymeos_cffi import (owner: live PyMEOS session)" + +say "WAVE 2 - CRITICAL PATH: PyMEOS master health (MEOS-1.4 bump #81/#82)" +block "PyMEOS master broken vs MEOS master (geoset_*->spatialset_*, tpoint_*->tspatial_*, pgis_geometry_in->geom_in, spanset_make arity); pristine master 426 failed/2721 passed. Gate: test.yml 426->0 vs Wave-1/Wave-0. Owner: live PyMEOS session. Build the bump against the composed Wave-0, not bare master." + +say "WAVE 3 - PyMEOS features (gated on Wave 2)" +if [ -n "${PYMEOS_ENV:-}" ] && [ -d "$PYMEOS_ENV" ]; then + block "PYMEOS_ENV given - run per-PR gates there: #85 black; #87 portable_parity 0-unbacked; #88 collect; #89 codegen --check; #90/#91 mixin suites; #84 tests/io 4/4 + footer==MobilityDuck#146 (manifest has exact gates)" +else + block "set PYMEOS_ENV= to run #85/#87/#88/#89/#90/#91/#84 gates; all are green-in-isolation and gated only on Wave 2" +fi + +# --------------------------------------------------------------------------- +say "SUMMARY" +if [ "$RC" = 0 ]; then + echo " Wave 0 PROVEN here. Waves 1-3 gated solely on Wave 2 (the MEOS-1.4 bump)." + echo " 100% parity is demonstrable the moment Wave 2 lands and this train" + echo " is re-run with PYMEOS_ENV set. Merge order: see meta/integration-train.json." +else + echo " Wave 0 gate FAILED above - parity train cannot proceed; fix before merge." +fi +exit "$RC"