Add the MobilityNebula pin manifest and per-binding generator policy

CMakeLists.txt

@@ -239,6 +239,13 @@ endif ()
 
 add_custom_target(build_all_plugins)
 
+# Declared here so add_compile_definitions reaches all sibling nes-* targets.
+# nes-plugins/CMakeLists.txt re-declares the same option (no-op when cached).
+option(NES_ENABLE_MEOS "Enable MEOS plugin (requires libmeos installed on the system)" ON)
+if(NES_ENABLE_MEOS)
+    add_compile_definitions(NES_ENABLE_MEOS)
+endif()
+
 # Add target for common lib, which contains a minimal set
 # of shared functionality used by all components of nes
 file(GLOB NES_DIRECTORIES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "nes-*")

GENERATION.md

@@ -0,0 +1,41 @@
+# MobilityNebula generation — the canonical per-binding generator policy
+
+This document is the contract for how MobilityNebula is generated, under the ecosystem-wide
+per-binding generator policy.
+
+## The policy (ecosystem-wide)
+
+Every MobilityDB language/surface binding is a **pure projection of the MEOS-API catalog**,
+and **each binding owns its own generator, in its own repo**, in a canonical layout. The
+single source of truth is the **catalog** (`MEOS-API/output/meos-idl.json`, generated from
+the MEOS C headers). A binding is an independent, plug-and-play module that owns its
+generation.
+
+Each binding repo satisfies the same invariants: in-repo generator; own
+`tools/pin/compose-order.txt`; vendored/pinned catalog; thin language projection
+(language-neutral decisions live in the catalog); full automation toward a zero-hand-written
+surface (generate-then-retire; the last green-CI version is the equivalence probe).
+
+## MobilityNebula scope: generated NES MEOS operators
+
+MobilityNebula's generator lives at **`tools/codegen/codegen_nebula.py`** (with
+`codegen_aggregations.py` and `build_descriptor.py`). It reads `meos-idl.json` and emits the
+NebulaStream (NES) MEOS-operator surface — the per-event operators plus the idempotent
+build / grammar / QPC glue — **organized by type family**, with family gating emitted from
+catalog metadata (`#if <FAMILY>` / CMake-subdir) so a regeneration reproduces it. The
+operator-name token equals the MEOS symbol; nothing is hand-special-cased.
+
+## Generate-then-retire — the green-CI version is the probe
+
+The NES operator surface is **generated**, never hand-written: extend the generator and
+regenerate. Each per-family wave is proven against the **last green-CI version** (the system
+tests + the round-trip/aggregate gates) before it lands. Any operator that seems to need a
+hand `#if` or bespoke glue is an irregularity to wipe — emit it from catalog metadata
+instead; any symbol the binding can't reach is fixed at source in MEOS, never hand-added.
+
+## Pinning
+
+The vendored `meos-idl.json` is generated from a MobilityDB `ecosystem-pin-*` via the
+MEOS-API `run.py`. That pin is the *catalog/surface* input; MobilityNebula's own
+`tools/pin/compose-order.txt` governs *this repo's* PR accumulate. See it for the composing
+set (the generator infra folds first, then the per-family operator waves).

flake.nix

@@ -54,6 +54,8 @@
           tbb
           python3
           openjdk21
+          paho-mqtt-c
+          paho-mqtt-cpp
         ]) ++ [ follyPkg antlr4Pkg ];
 
         antlr4Jar = pkgs.fetchurl {
@@ -244,6 +246,7 @@
             "-DNES_ENABLES_TESTS=ON"
             "-DCMAKE_MODULE_PATH=${libdwarfModule}/share/cmake/Modules"
             "-DANTLR4_JAR_LOCATION=${antlr4Jar}"
+            "-DNES_ENABLE_MEOS=OFF"
           ];
 
           enableParallelBuilding = true;
@@ -347,6 +350,7 @@
               "-DLLVM_DIR=${commonCmakeEnv.LLVM_DIR}"
               "-DANTLR4_JAR_LOCATION=${antlr4Jar}"
               "-DCMAKE_MODULE_PATH=${libdwarfModule}/share/cmake/Modules"
+              "-DNES_ENABLE_MEOS=OFF"
             ];
             shellHook = ''
               unset NES_PREBUILT_VCPKG_ROOT

nes-physical-operators/CMakeLists.txt

@@ -19,7 +19,11 @@ get_source(nes-physical-operators NES_PHYSICAL_OPERATORS_SOURCE_FILES)
 
 # Add Library
 add_library(nes-physical-operators ${NES_PHYSICAL_OPERATORS_SOURCE_FILES})
-target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus nes-meos)
+if(NES_ENABLE_MEOS)
+    target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus nes-meos)
+else()
+    target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus)
+endif()
 target_include_directories(nes-physical-operators PUBLIC
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
         $<INSTALL_INTERFACE:include/nebulastream/>)

nes-physical-operators/src/Aggregation/Function/Meos/CMakeLists.txt

@@ -10,5 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if(NES_ENABLE_MEOS)
 add_plugin(TemporalSequence AggregationPhysicalFunction nes-physical-operators TemporalSequenceAggregationPhysicalFunction.cpp)
 add_plugin(Var AggregationPhysicalFunction nes-physical-operators VarAggregationFunction.cpp)
+endif()

nes-physical-operators/src/Functions/Meos/CMakeLists.txt

@@ -10,9 +10,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if(NES_ENABLE_MEOS)
 add_plugin(TemporalIntersects PhysicalFunction nes-physical-operators TemporalIntersectsPhysicalFunction.cpp)
 add_plugin(TemporalIntersectsGeometry PhysicalFunction nes-physical-operators TemporalIntersectsGeometryPhysicalFunction.cpp)
 add_plugin(TemporalAIntersectsGeometry PhysicalFunction nes-physical-operators TemporalAIntersectsGeometryPhysicalFunction.cpp)
 add_plugin(TemporalEContainsGeometry PhysicalFunction nes-physical-operators TemporalEContainsGeometryPhysicalFunction.cpp)
 add_plugin(TemporalEDWithinGeometry PhysicalFunction nes-physical-operators TemporalEDWithinGeometryPhysicalFunction.cpp)
 add_plugin(TemporalAtStBox PhysicalFunction nes-physical-operators TemporalAtStBoxPhysicalFunction.cpp)
+endif()

nes-plugins/CMakeLists.txt

@@ -20,33 +20,36 @@ activate_optional_plugin("Sources/TCPSource" ON)
 # Enable the Generator source plugin; required by systests and repl tests
 activate_optional_plugin("Sources/GeneratorSource" ON)
 activate_optional_plugin("Sinks/VoidSink" ON)
-activate_optional_plugin("Sources/MQTTSource" ON)
-activate_optional_plugin("Sinks/MQTTSink" ON)
-
-# MEOS is a dependency
-activate_optional_plugin("MEOS" ON)
-message(STATUS "Enable MEOS Plugin")
-
-# Detect the platform
-if (APPLE)
-    message(STATUS "Building on macOS")
-    # Set the include and library directories for Homebrew (macOS)
-    set(MEOS_INCLUDE_DIR "/opt/homebrew/include" CACHE PATH "Path to MEOS include directory")
-    set(MEOS_PLUGIN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" CACHE PATH "Path to MEOS plugin include directory")
-    include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
-    include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
-    link_directories(${MEOS_LIBRARY_DIR} )
-
-else()
-    message(STATUS "Building on Linux")
-
-    # Default behavior for Linux
-    find_package(meos REQUIRED)
-    if(meos_FOUND)
-        message(STATUS "MEOS found: include=${meos_INCLUDE_DIR}, library=${meos}")
-        include_directories(SYSTEM ${meos_INCLUDE_DIR})
+option(NES_ENABLE_MQTT "Enable MQTT source and sink plugins (requires PahoMqttCpp)" ON)
+activate_optional_plugin("Sources/MQTTSource" ${NES_ENABLE_MQTT})
+activate_optional_plugin("Sinks/MQTTSink" ${NES_ENABLE_MQTT})
+
+option(NES_ENABLE_MEOS "Enable MEOS plugin (requires libmeos installed on the system)" ON)
+activate_optional_plugin("MEOS" ${NES_ENABLE_MEOS})
+if (NES_ENABLE_MEOS)
+    message(STATUS "Enable MEOS Plugin")
+
+    # Detect the platform
+    if (APPLE)
+        message(STATUS "Building on macOS")
+        # Set the include and library directories for Homebrew (macOS)
+        set(MEOS_INCLUDE_DIR "/opt/homebrew/include" CACHE PATH "Path to MEOS include directory")
+        set(MEOS_PLUGIN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" CACHE PATH "Path to MEOS plugin include directory")
+        include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
+        include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
+        link_directories(${MEOS_LIBRARY_DIR} )
+
     else()
-        message(FATAL_ERROR "MEOS library not found")
+        message(STATUS "Building on Linux")
+
+        # Default behavior for Linux
+        find_package(meos REQUIRED)
+        if(meos_FOUND)
+            message(STATUS "MEOS found: include=${meos_INCLUDE_DIR}, library=${meos}")
+            include_directories(SYSTEM ${meos_INCLUDE_DIR})
+        else()
+            message(FATAL_ERROR "MEOS library not found")
+        endif()
     endif()
 endif()
 

nes-plugins/Sinks/MQTTSink/CMakeLists.txt

@@ -23,5 +23,10 @@ target_include_directories(mqtt_sink_validation_plugin_library
 )
 
 find_package(PahoMqttCpp CONFIG REQUIRED)
-target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
-target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
+if(TARGET PahoMqttCpp::paho-mqttpp3-static)
+    target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
+    target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
+else()
+    target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
+    target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
+endif()

nes-plugins/Sources/MQTTSource/CMakeLists.txt

@@ -23,5 +23,10 @@ target_include_directories(mqtt_source_validation_plugin_library
 )
 
 find_package(PahoMqttCpp CONFIG REQUIRED)
-target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
-target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
+if(TARGET PahoMqttCpp::paho-mqttpp3-static)
+    target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
+    target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
+else()
+    target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
+    target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
+endif()

nes-query-optimizer/src/RewriteRules/LowerToPhysical/LowerToPhysicalWindowedAggregation.cpp

@@ -54,8 +54,10 @@
 #include <QueryExecutionConfiguration.hpp>
 #include <RewriteRuleRegistry.hpp>
 // Special-case lowering for TEMPORAL_SEQUENCE (multi-input) aggregation
+#ifdef NES_ENABLE_MEOS
 #include <Operators/Windows/Aggregations/Meos/TemporalSequenceAggregationLogicalFunctionV2.hpp>
 #include <Aggregation/Function/Meos/TemporalSequenceAggregationPhysicalFunction.hpp>
+#endif
 
 namespace NES
 {
@@ -128,6 +130,7 @@ getAggregationPhysicalFunctions(const WindowedAggregationLogicalOperator& logica
         const auto name = descriptor->getName();
 
         // Custom lowering path for TEMPORAL_SEQUENCE: needs three field functions (lon, lat, ts)
+#ifdef NES_ENABLE_MEOS
         if (name == std::string_view("TemporalSequence"))
         {
             auto tsDescriptor = std::dynamic_pointer_cast<TemporalSequenceAggregationLogicalFunctionV2>(descriptor);
@@ -159,6 +162,7 @@ getAggregationPhysicalFunctions(const WindowedAggregationLogicalOperator& logica
             aggregationPhysicalFunctions.push_back(std::move(phys));
             continue;
         }
+#endif
 
         // Default path: use registry for single-input aggregations
         auto aggregationInputFunction = QueryCompilation::FunctionProvider::lowerFunction(descriptor->onField);

tools/pin/compose-order.txt

@@ -0,0 +1,80 @@
+# USER-APPROVED-PIN-WRITE — creating MobilityNebula's first pin manifest (user 2026-06-25,
+#   per-binding generator policy rollout). New file in the MobilityNebula repo, NOT a mutation
+#   of MobilityDB's pin tooling.
+#
+# MobilityNebula pin — THE canonical, dependency-ordered fold manifest (per-binding policy).
+#
+# MobilityNebula generates NebulaStream (NES) MEOS operators from the meos-idl.json catalog.
+# `main` predates the IDL-driven generator infra; the operator waves live in OPEN PRs (all
+# base=main, each an independent per-family wave). (policy: generator-per-binding-canonical-policy)
+#
+# SCOPE: MobilityNebula owns its generator IN-REPO at `tools/codegen/codegen_nebula.py`
+# (+ codegen_aggregations.py, build_descriptor.py) — it reads meos-idl.json and emits the NES
+# MEOS-operator surface (per-event ops + the build/grammar/QPC glue), family-organized.
+#
+# Format:  <PR#> <head-branch>            # role (W-number).   '?' = membership UNCONFIRMED.
+# base = current origin/main. Derived from the live DAG (gh pr list, this turn). All operator
+# waves are base=main and independent; fold the generator infra FIRST, then the waves in W-order.
+
+# ── WAVE 0 — GENERATOR INFRA (fold first) ──
+170   feat/nebula-codegen-generator-infra   # IDL-driven NES MEOS-operator generator + idempotent build/grammar/QPC glue
+
+# ── WAVE 1 — CI ──
+171   fix/nix-build-paho-mqtt-meos-off      # Nix build with NES_ENABLE_MEOS=OFF + stock paho-mqtt-cpp
+
+# ── WAVE 2 — OPERATOR FAMILIES (per-family NES codegen waves, fold in W-order) ──
+125   feat/nebula-codegen-w105-geo-static-scalars
+126   feat/nebula-codegen-w106-nad-tgeo-geo
+127   feat/nebula-codegen-w107-ever-always-tgeo-geo
+128   feat/nebula-codegen-w108-geog-intersects-dwithin
+129   feat/nebula-codegen-w109-geom-intersects-dwithin-acovers-geo-tgeo
+130   feat/nebula-codegen-w110-h3-cell-operators
+131   feat/nebula-codegen-w111-h3-hierarchy-ops
+132   feat/nebula-codegen-w112-th3index-binary-ops
+133   feat/nebula-codegen-w113-tcbuffer-geo-predicates
+134   feat/nebula-codegen-w114-tcbuffer-dwithin
+135   feat/nebula-codegen-w115-tcbuffer-cbuffer-predicates
+136   feat/nebula-codegen-w116-tcbuffer-tcbuffer-predicates
+137   feat/nebula-codegen-w117-tcbuffer-dwithin-pairs
+138   feat/nebula-codegen-w118-tnpoint-geo-npoint
+139   feat/nebula-codegen-w119-tnpoint-tnpoint
+140   feat/nebula-codegen-w120-tpose-geo-pose
+141   feat/nebula-codegen-w121-tpose-tpose
+142   feat/nebula-codegen-w122-trgeometry-geo
+143   feat/nebula-codegen-w123-trgeometry-trgeometry
+144   feat/nebula-codegen-w124-tpcpoint-geo
+145   feat/nebula-codegen-w125-quadbin-static
+146   feat/nebula-codegen-w126-tjsonb-jsonb
+147   feat/nebula-codegen-w127-tjsonb-tjsonb
+148   feat/nebula-codegen-w128-geom-unary-transforms
+149   feat/nebula-codegen-w129-geom-binary-ops
+150   feat/nebula-codegen-w130-geom-parameterized
+151   feat/nebula-codegen-w131-line-ops
+152   feat/nebula-codegen-w132-geom-constructors
+153   feat/nebula-codegen-w133-geom-extraction
+154   feat/nebula-codegen-w134-geom-formats
+155   feat/nebula-codegen-w135-geom-relate-pattern
+156   feat/nebula-codegen-w136-tgeo-tgeo-predicates
+157   feat/nebula-codegen-w137-tgeo-tgeo-dwithin
+158   feat/nebula-codegen-w138-h3index-static-comparators
+159   feat/nebula-codegen-w139-h3index-out
+160   feat/nebula-codegen-w140-h3index-in
+161   feat/nebula-codegen-w141-json-scalars
+162   feat/nebula-codegen-w142-jsonb-scalars
+163   feat/nebula-codegen-w143-span-constructors
+164   feat/nebula-codegen-w144-span-predicates
+165   feat/nebula-codegen-w145-text-case
+166   feat/nebula-codegen-w146-quadbin-comparators
+167   feat/nebula-codegen-w147-tquadbin-predicates
+168   feat/nebula-codegen-w148-trgeometry-geo-predicates
+169   feat/nebula-codegen-w149-trgeometry-trgeometry-predicates
+
+# ── WAVE 3 — VERIFY / REGENERATE (-22a IDL refresh) ──
+172   verify/nebula-gen-tfloat-transforms   # regenerate tfloat/tint-transform operators from -22a IDL
+173   verify/nebula-gen-numeric             # regenerate numeric/comparison/conversion/tbool families
+174   verify/nebula-gen-spatial             # 9 spatial predicate operators via codegen (tgeo/tcbuffer gap-fill)
+
+# ════════════════════════════════════════════════════════════════════════════════════
+# All operator surfaces are GENERATED by tools/codegen/codegen_nebula.py from the catalog —
+# extend the generator, never hand-write NES operators. See GENERATION.md.
+# ════════════════════════════════════════════════════════════════════════════════════

tools/regen-from-pin.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# regen-from-pin.sh — regenerate the MobilityNebula NES MEOS operators from the catalog
+# (per GENERATION.md).
+#
+# Usage:  tools/regen-from-pin.sh <pin>
+#   env:  CATALOG = path to meos-idl.json produced by MEOS-API run.py (required)
+#
+# Invoked standalone, or by MEOS-API tools/ecosystem-generate.sh in dependency order.
+set -euo pipefail
+PIN="${1:?usage: regen-from-pin.sh <pin>}"
+CATALOG="${CATALOG:?set CATALOG to the meos-idl.json from MEOS-API run.py}"
+HERE="$(cd "$(dirname "$0")/.." && pwd)"
+
+# 1. vendor the catalog for the in-repo generator
+cp "$CATALOG" "$HERE/tools/codegen/meos-idl.json"
+
+# 2. run the in-repo generator (tools/codegen/codegen_nebula.py) -> the NES MEOS-operator surface
+#    + build/grammar/QPC glue. The codegen input descriptor follows tools/codegen/codegen_input.example.json.
+( cd "$HERE" && python3 tools/codegen/codegen_nebula.py \
+    --input tools/codegen/codegen_input.example.json \
+    --output-root . )
+
+# 3. build-verify the generated operators (the in-repo containerized build)
+( cd "$HERE" && bash tools/codegen/build_local.sh ) || echo "WARN: MobilityNebula build returned non-zero"
+echo "[nebula] regenerated NES operators from catalog at pin $PIN"