From 346f02963a23f9f6846bae5a0f5afeb93657538d Mon Sep 17 00:00:00 2001 From: Sam-Bolling Date: Wed, 6 May 2026 09:00:04 -0400 Subject: [PATCH 1/4] fix(bootstrap): two-step encoding for procedures and deployments ensure_procedure and ensure_deployment now mirror ensure_system: POST a geo+json stub (uid/name/description/geometry/featureType/validTime), then optionally PUT a SensorML body with Content-Type: application/sml+json against /resource/{id}. Also adds _warn_if_sml_fields_in_stub: a closed-set guardrail that warns (or raises, when OS4CSAPI_STRICT_BOOTSTRAP=1) if a stub body still carries SensorML-only fields under properties. force_sml=True now applies to procedures and deployments as well as systems, allowing in-place recovery for records that were created with the old single-POST shape. Background: pre-strict CSAPI servers returned HTTP 201 and silently dropped SensorML metadata on procedures/deployments. Strict upstream (connected-systems-go after a467aba) returns HTTP 400. Either way, the bug was on the client. Refs: OS4CSAPI/OSHConnect-Python#5 --- publishers/bootstrap_helpers.py | 221 +++++++++++++++++++++++++++++--- 1 file changed, 202 insertions(+), 19 deletions(-) diff --git a/publishers/bootstrap_helpers.py b/publishers/bootstrap_helpers.py index 50d9a81..fbd1d5a 100644 --- a/publishers/bootstrap_helpers.py +++ b/publishers/bootstrap_helpers.py @@ -8,12 +8,32 @@ Functions: find_by_uid() — Lookup resource by UID in a collection find_datastream() — Lookup datastream by outputName under a system - ensure_procedure() — Create procedure if not exists - ensure_system() — Create system (geo+json stub POST → SensorML PUT) + ensure_procedure() — Create procedure (geo+json stub POST → optional SensorML PUT) + ensure_system() — Create system (geo+json stub POST → optional SensorML PUT) ensure_datastream() — Create datastream with SWE DataRecord schema - ensure_deployment() — Create deployment node (with optional parent) + ensure_deployment() — Create deployment node (geo+json stub POST → optional SensorML PUT) clean_resource() — Delete resource by UID if it exists api_get/post/put/delete() — Low-level HTTP helpers with retry + +Content-type contract (CSAPI Part 1, OGC 23-001): + - application/geo+json → spatial-discovery view; carries uid/name/description + (+ geometry) only. SensorML metadata is INTENTIONALLY + stripped server-side. + - application/sml+json → full SensorML metadata view; carries keywords, + identifiers, classifiers, characteristics, capabilities, + contacts, documentation/documents, history, + securityConstraints, legalConstraints, etc. + +Bootstrap pattern for procedures, systems, and deployments: + 1. POST a small geo+json stub (Content-Type: application/json — server + interprets as application/geo+json on these endpoints). + 2. PUT the full SensorML body (Content-Type: application/sml+json) against + the just-created /resource/{id} path. + +This module enforces the contract via _warn_if_sml_fields_in_stub(): if a +caller passes a "stub" with SensorML-only fields under properties, a loud +warning is emitted. Set OS4CSAPI_STRICT_BOOTSTRAP=1 to elevate the warning +to an exception (recommended for tests and CI). """ import argparse @@ -239,27 +259,142 @@ def find_datastream(base_url: str, auth: str, system_id: str, return None +# ═══════════════════════════════════════════════════════════════════════════ +# Encoding-contract guardrail +# ═══════════════════════════════════════════════════════════════════════════ + +# SensorML-only fields that the CSAPI server silently strips when it sees +# them under `properties` of a geo+json POST. Any of these fields appearing +# in a "stub" body indicates the caller has not split GeoJSON encoding from +# SensorML encoding properly — the stub will be accepted (HTTP 201) but +# the listed fields will be DROPPED on the server side. +# +# Background: pre-strict CSAPI servers returned 201 + silent drop. Strict +# servers (post-`a467aba` upstream) return HTTP 400. Either way, the bug +# is on the client. See docs/engineering/2026-05-silent-sensorml-field-loss.md +SML_ONLY_FIELDS = frozenset({ + "keywords", + "identifiers", + "classifiers", + "characteristics", + "capabilities", + "contacts", + "documentation", # OGC links-array form (not the SensorML `documents` form) + "documents", # SensorML form + "history", + "securityConstraints", + "legalConstraints", + "lineage", + "usageConstraints", + "typeOf", + "configuration", + "modes", + "parameters", + "inputs", + "outputs", + "components", + "connections", + "localReferenceFrames", + "localTimeFrames", + "method", +}) + +_STRICT_BOOTSTRAP = os.environ.get("OS4CSAPI_STRICT_BOOTSTRAP", "").lower() in ("1", "true", "yes") + + +def _warn_if_sml_fields_in_stub(stub: dict, label: str) -> None: + """Loud warning (or exception in strict mode) if a caller passes a 'stub' + body whose `properties` contain SensorML-only fields. + + These fields will be silently dropped server-side on a geo+json POST. + Callers must split SensorML metadata out into a separate ``sml_body`` + and let the helper PUT it with ``Content-Type: application/sml+json``. + + Set OS4CSAPI_STRICT_BOOTSTRAP=1 to convert the warning to RuntimeError — + recommended for tests and CI. + """ + if not isinstance(stub, dict): + return + props = stub.get("properties", stub) + if not isinstance(props, dict): + return + leaked = sorted(SML_ONLY_FIELDS & set(props.keys())) + if not leaked: + return + msg = ( + f"[ENCODING-CONTRACT] {label}: stub body carries SensorML-only " + f"field(s) under `properties`: {leaked}. These will be silently " + f"dropped (or 400-rejected by strict servers) on the geo+json POST. " + f"Move them into a separate sml_body argument." + ) + if _STRICT_BOOTSTRAP: + raise RuntimeError(msg) + print(f" [WARN] {msg}") + + # ═══════════════════════════════════════════════════════════════════════════ # Idempotent resource creation # ═══════════════════════════════════════════════════════════════════════════ -def ensure_procedure(base_url: str, auth: str, uid: str, body: dict, - *, dry_run: bool = False, stats: dict = None) -> str | None: - """Create a procedure if it doesn't already exist. Returns server ID.""" +def ensure_procedure(base_url: str, auth: str, uid: str, stub_body: dict, + sml_body: dict | None = None, + *, dry_run: bool = False, stats: dict = None, + force_sml: bool = False) -> str | None: + """Create a procedure if it doesn't already exist. Returns server ID. + + Two-step encoding-correct pattern (mirrors ``ensure_system``): + + 1. POST ``stub_body`` (geo+json Feature: uid/name/description + optional + geometry) with ``Content-Type: application/json``. The server + interprets this as ``application/geo+json`` on the procedures + endpoint. + 2. If ``sml_body`` is provided, PUT it against the new resource path + with ``Content-Type: application/sml+json`` to populate full + SensorML metadata (keywords, identifiers, classifiers, + characteristics, capabilities, contacts, documents, history, + securityConstraints, legalConstraints, …). + + When ``force_sml`` is True and the procedure already exists, the + SensorML body is PUT again — useful for correcting previously-broken + payloads after this fix lands. + + Callers MUST keep SensorML metadata out of the stub. The + ``_warn_if_sml_fields_in_stub`` guardrail catches accidental leakage. + """ + _warn_if_sml_fields_in_stub(stub_body, f"ensure_procedure({uid})") + existing = find_by_uid(base_url, auth, "procedures", uid) if existing: - print(f" [SKIP] Procedure {uid} already exists (id={existing})") - if stats: - stats.setdefault("skipped", 0) - stats["skipped"] += 1 + if force_sml and sml_body: + if dry_run: + print(f" [DRY] Would force-PUT SML for procedure {uid} (id={existing})") + else: + api_put(base_url, f"procedures/{existing}", sml_body, auth, + content_type="application/sml+json") + print(f" [SML] Force-PUT SensorML for procedure {uid} (id={existing})") + if stats: + stats.setdefault("sml_updated", 0) + stats["sml_updated"] += 1 + else: + print(f" [SKIP] Procedure {uid} already exists (id={existing})") + if stats: + stats.setdefault("skipped", 0) + stats["skipped"] += 1 return existing if dry_run: print(f" [DRY] Would create procedure: {uid}") return None - result = api_post(base_url, "procedures", body, auth) + # Step 1: POST geo+json stub + result = api_post(base_url, "procedures", stub_body, auth) new_id = result.get("id") if result else None + + # Step 2: PUT SensorML if provided + if new_id and sml_body: + api_put(base_url, f"procedures/{new_id}", sml_body, auth, + content_type="application/sml+json") + print(f" [OK] Created procedure {uid} → id={new_id}") if stats: stats.setdefault("created", 0) @@ -278,6 +413,8 @@ def ensure_system(base_url: str, auth: str, uid: str, stub_body: dict, When *force_sml* is True and the system already exists, the SML body is PUT again (useful for correcting previously-broken SML payloads). """ + _warn_if_sml_fields_in_stub(stub_body, f"ensure_system({uid})") + existing = find_by_uid(base_url, auth, "systems", uid) if existing: if force_sml and sml_body: @@ -345,10 +482,38 @@ def ensure_datastream(base_url: str, auth: str, system_id: str, return new_id -def ensure_deployment(base_url: str, auth: str, uid: str, body: dict, +def ensure_deployment(base_url: str, auth: str, uid: str, stub_body: dict, + sml_body: dict | None = None, parent_id: str | None = None, - *, dry_run: bool = False, stats: dict = None) -> str | None: - """Create a deployment node if it doesn't exist. Returns server ID.""" + *, dry_run: bool = False, stats: dict = None, + force_sml: bool = False) -> str | None: + """Create a deployment node if it doesn't exist. Returns server ID. + + Two-step encoding-correct pattern (mirrors ``ensure_system`` and + ``ensure_procedure``): + + 1. POST ``stub_body`` (geo+json Feature: uid/name/description + + optional geometry, validTime, deployment-tree links) with + ``Content-Type: application/json``. Server interprets as + ``application/geo+json``. + 2. If ``sml_body`` is provided, PUT it against the new resource path + with ``Content-Type: application/sml+json`` to populate full + SensorML metadata (keywords, identifiers, classifiers, + characteristics, capabilities, contacts, documents, history, + securityConstraints, legalConstraints, …). + + When ``parent_id`` is given, the create path is + ``deployments/{parent_id}/subdeployments``; the SML PUT still targets + the canonical ``deployments/{new_id}`` path. + + When ``force_sml`` is True and the deployment already exists, the + SensorML body is PUT again. + + Callers MUST keep SensorML metadata out of the stub. The + ``_warn_if_sml_fields_in_stub`` guardrail catches accidental leakage. + """ + _warn_if_sml_fields_in_stub(stub_body, f"ensure_deployment({uid})") + # Check top-level deployments first existing = find_by_uid(base_url, auth, "deployments", uid) if not existing and parent_id: @@ -356,22 +521,40 @@ def ensure_deployment(base_url: str, auth: str, uid: str, body: dict, existing = find_by_uid(base_url, auth, f"deployments/{parent_id}/subdeployments", uid) if existing: - print(f" [SKIP] Deployment {uid} already exists (id={existing})") - if stats: - stats.setdefault("skipped", 0) - stats["skipped"] += 1 + if force_sml and sml_body: + if dry_run: + print(f" [DRY] Would force-PUT SML for deployment {uid} (id={existing})") + else: + api_put(base_url, f"deployments/{existing}", sml_body, auth, + content_type="application/sml+json") + print(f" [SML] Force-PUT SensorML for deployment {uid} (id={existing})") + if stats: + stats.setdefault("sml_updated", 0) + stats["sml_updated"] += 1 + else: + print(f" [SKIP] Deployment {uid} already exists (id={existing})") + if stats: + stats.setdefault("skipped", 0) + stats["skipped"] += 1 return existing if dry_run: print(f" [DRY] Would create deployment: {uid}") return None + # Step 1: POST geo+json stub at the (possibly nested) create path path = "deployments" if parent_id: path = f"deployments/{parent_id}/subdeployments" - result = api_post(base_url, path, body, auth) + result = api_post(base_url, path, stub_body, auth) new_id = result.get("id") if result else None + + # Step 2: PUT SensorML against the canonical /deployments/{id} path + if new_id and sml_body: + api_put(base_url, f"deployments/{new_id}", sml_body, auth, + content_type="application/sml+json") + print(f" [OK] Created deployment {uid} → id={new_id}") if stats: stats.setdefault("created", 0) From 2ef4e9557ce0e11bd9ef569d33ec3e45fdfcf776 Mon Sep 17 00:00:00 2001 From: Sam-Bolling Date: Wed, 6 May 2026 09:00:25 -0400 Subject: [PATCH 2/4] fix(nws): split procedure and deployment bodies into stub + SensorML Replaces PROCEDURE_BODY (single mixed-encoding dict) with _procedure_stub and _procedure_sml; strips documentation arrays from _deploy_root and _deploy_group stubs and adds matching _deploy_root_sml / _deploy_group_sml; threads force_sml through procedure and deployment create calls so --force-sml now repairs them in place. NWS is the canonical example for the same refactor that needs to land across the other 9 publishers (E.2 follow-up). Refs: OS4CSAPI/OSHConnect-Python#5 --- publishers/nws/bootstrap_nws.py | 215 +++++++++++++++++++++++++------- 1 file changed, 168 insertions(+), 47 deletions(-) diff --git a/publishers/nws/bootstrap_nws.py b/publishers/nws/bootstrap_nws.py index 4febf42..3eff936 100644 --- a/publishers/nws/bootstrap_nws.py +++ b/publishers/nws/bootstrap_nws.py @@ -105,7 +105,7 @@ def _points_url(lat: float, lon: float) -> str: # Resource definitions # ═══════════════════════════════════════════════════════════════════════════ -PROCEDURE_BODY = { +PROCEDURE_STUB = { "type": "Feature", "geometry": None, "properties": { @@ -119,48 +119,128 @@ def _points_url(lat: float, lon: float) -> str: "ASOS/AWOS; nominal station reporting is hourly with special observations as conditions " "warrant, and API publication may lag upstream MADIS availability by up to about 20 minutes." ), + "validTime": [VALID_TIME_START, ".."], + }, +} + + +def _procedure_sml() -> dict: + """SensorML body for the NWS Surface Observation procedure. + + Encoding: ``application/sml+json``. PUT against ``/procedures/{id}`` + after the geo+json stub POST. Field shapes follow the SensorML JSON + encoding (``documents``, ``contacts.organisationName``, + ``contactInfo``) so the CSAPI server persists into the corresponding + procedure-table columns. + """ + return { + "type": "SimpleProcess", + "id": PROC_UID, + "uniqueId": PROC_UID, + "definition": "http://www.opengis.net/def/procedure/observation", + "label": "NWS Surface Observation v1", + "description": ( + "Observing procedure for NWS ASOS/AWOS surface weather stations exposed through " + "api.weather.gov. Latest station observations are normalized to SI-oriented fields " + "(degC, Pa, km/h, m) and published as flat JSON result objects. Nominal station " + "reporting is hourly with special observations as conditions warrant; API publication " + "may lag upstream MADIS availability by up to about 20 minutes." + ), "keywords": [ - "NWS", - "NOAA", - "ASOS", - "AWOS", - "surface weather", - "METAR", - "api.weather.gov", - "aviation weather", + "NWS", "NOAA", "ASOS", "AWOS", + "surface weather", "METAR", + "api.weather.gov", "aviation weather", ], - "documentation": [ - {"title": "NWS API Web Service", "href": NWS_API_DOCS, "rel": "documentation"}, - {"title": "NWS OpenAPI Specification", "href": NWS_OPENAPI, "rel": "describedby"}, - {"title": "NWS ASOS Program", "href": NWS_ASOS_PAGE, "rel": "about"}, - {"title": "NWS ASOS Equipment FAQ", "href": NWS_ASOS_EQUIP, "rel": "related"}, - {"title": "NWS ASOS Contact", "href": NWS_ASOS_CONTACT, "rel": "contact"}, + "identifiers": [ + {"definition": "http://sensorml.com/ont/swe/property/ShortName", + "label": "Short Name", "value": "NWS Surface Observation"}, + {"definition": "http://sensorml.com/ont/swe/property/LongName", + "label": "Long Name", + "value": "National Weather Service ASOS/AWOS Surface Observation Procedure v1"}, + {"definition": "http://sensorml.com/ont/swe/property/UniqueID", + "label": "OS4CSAPI UID", "value": PROC_UID}, + ], + "classifiers": [ + {"definition": "http://sensorml.com/ont/swe/property/IntendedApplication", + "label": "Network", "value": "ASOS / AWOS Surface Weather Observation Network"}, + {"definition": "http://sensorml.com/ont/swe/property/SystemRole", + "label": "Operator", "value": "National Weather Service"}, ], "contacts": [ { - "role": "operator", - "organizationName": "National Weather Service", - "website": NWS_API_BASE, + "role": "http://sensorml.com/ont/swe/property/Operator", + "organisationName": "National Weather Service", + "contactInfo": { + "website": NWS_API_BASE, + }, }, { - "role": "support", - "organizationName": "ASOS Operations and Monitoring Center", - "email": NWS_AOMC_EMAIL, - "phone": NWS_AOMC_PHONE_1, + "role": "http://sensorml.com/ont/swe/property/Maintainer", + "organisationName": "ASOS Operations and Monitoring Center", + "contactInfo": { + "website": NWS_ASOS_CONTACT, + "phone": {"voice": NWS_AOMC_PHONE_1}, + "email": NWS_AOMC_EMAIL, + "address": {"country": "United States"}, + }, + }, + ], + "documents": [ + { + "role": "http://dbpedia.org/resource/Documentation", + "name": "NWS API Web Service", + "link": {"href": NWS_API_DOCS, "type": "text/html"}, + }, + { + "role": "http://dbpedia.org/resource/OpenAPI_Specification", + "name": "NWS OpenAPI Specification", + "link": {"href": NWS_OPENAPI, "type": "application/openapi+json"}, + }, + { + "role": "http://dbpedia.org/resource/Web_page", + "name": "NWS ASOS Program", + "link": {"href": NWS_ASOS_PAGE, "type": "text/html"}, + }, + { + "role": "http://dbpedia.org/resource/FAQ", + "name": "NWS ASOS Equipment FAQ", + "link": {"href": NWS_ASOS_EQUIP, "type": "text/html"}, + }, + { + "role": "http://dbpedia.org/resource/Contact", + "name": "NWS ASOS Contact", + "link": {"href": NWS_ASOS_CONTACT, "type": "text/html"}, + }, + ], + "characteristics": [ + { + "label": "Lineage", + "definition": "http://sensorml.com/ont/swe/property/Lineage", + "characteristic": [ + {"label": "Source", "value": "NOAA / National Weather Service"}, + {"label": "Upstream", + "value": "MADIS-mediated station observations exposed by api.weather.gov"}, + {"label": "Normalization", + "value": "Publisher normalizes selected values to SI-oriented fields (degC, Pa, km/h, m)."}, + ], + }, + { + "label": "Usage Constraints", + "definition": "http://sensorml.com/ont/swe/property/UsageConstraints", + "characteristic": [ + {"label": "User-Agent Required", "value": "true"}, + {"label": "Rate Limit", + "value": "NWS API is open data with unpublished but reasonable rate limits."}, + ], }, ], - "lineage": { - "source": "NOAA / National Weather Service", - "upstream": "MADIS-mediated station observations exposed by api.weather.gov", - "normalization": "Publisher normalizes selected values to SI-oriented fields (degC, Pa, km/h, m).", - }, - "usageConstraints": { - "userAgentRequired": True, - "rateLimitNote": "NWS API is open data with unpublished but reasonable rate limits.", - }, "validTime": [VALID_TIME_START, ".."], - }, -} + } + + +def _procedure_stub() -> dict: + """Geo+json stub for the procedure (stable copy of PROCEDURE_STUB).""" + return PROCEDURE_STUB def _system_stub(station: dict, proc_id: str) -> dict: @@ -382,6 +462,20 @@ def _datastream_schema(station_id: str = "") -> dict: } +_DEPLOY_DOCUMENTS = [ + { + "role": "http://dbpedia.org/resource/Documentation", + "name": "NWS API Web Service", + "link": {"href": NWS_API_DOCS, "type": "text/html"}, + }, + { + "role": "http://dbpedia.org/resource/Web_page", + "name": "NWS ASOS Program", + "link": {"href": NWS_ASOS_PAGE, "type": "text/html"}, + }, +] + + def _deploy_root() -> dict: return { "type": "Feature", @@ -394,15 +488,24 @@ def _deploy_root() -> dict: "featureType": "sosa:Deployment", "name": "NWS Weather Demo", "description": "Demonstration deployment for NWS surface weather observation systems published into CSAPI.", - "documentation": [ - {"title": "NWS API Web Service", "href": NWS_API_DOCS, "rel": "documentation"}, - {"title": "NWS ASOS Program", "href": NWS_ASOS_PAGE, "rel": "about"}, - ], "validTime": [VALID_TIME_START, ".."], }, } +def _deploy_root_sml() -> dict: + """SensorML body for the root demo deployment node.""" + return { + "type": "Deployment", + "id": DEPLOY_ROOT_UID, + "uniqueId": DEPLOY_ROOT_UID, + "label": "NWS Weather Demo", + "description": "Demonstration deployment for NWS surface weather observation systems published into CSAPI.", + "documents": list(_DEPLOY_DOCUMENTS), + "validTime": [VALID_TIME_START, ".."], + } + + def _deploy_group() -> dict: return { "type": "Feature", @@ -415,15 +518,25 @@ def _deploy_group() -> dict: "featureType": "sosa:Deployment", "name": "NWS Weather Stations", "description": "NWS ASOS/AWOS surface observation stations across the United States used for the OS4CSAPI demo.", - "documentation": [ - {"title": "NWS API Web Service", "href": NWS_API_DOCS, "rel": "documentation"}, - {"title": "NWS ASOS Program", "href": NWS_ASOS_PAGE, "rel": "about"}, - ], "validTime": [VALID_TIME_START, ".."], }, } +def _deploy_group_sml() -> dict: + """SensorML body for the NWS Weather Stations deployment grouping.""" + return { + "type": "Deployment", + "id": DEPLOY_GROUP_UID, + "uniqueId": DEPLOY_GROUP_UID, + "label": "NWS Weather Stations", + "description": "NWS ASOS/AWOS surface observation stations across the United States used for the OS4CSAPI demo.", + "keywords": ["NWS", "NOAA", "ASOS", "AWOS", "surface weather"], + "documents": list(_DEPLOY_DOCUMENTS), + "validTime": [VALID_TIME_START, ".."], + } + + def _deploy_station(station: dict, system_server_id: str) -> dict: return { "type": "Feature", @@ -508,8 +621,10 @@ def bootstrap(*, clean: bool = False, clean_only: bool = False, # ── Procedure ───────────────────────────────────────────────────── print(" ── Procedures ──") - proc_id = ensure_procedure(base_url, auth, PROC_UID, PROCEDURE_BODY, - dry_run=dry_run, stats=stats) + proc_id = ensure_procedure(base_url, auth, PROC_UID, + _procedure_stub(), _procedure_sml(), + dry_run=dry_run, stats=stats, + force_sml=force_sml) # ── Systems + Datastreams ───────────────────────────────────────── print(" ── Systems + Datastreams ──") @@ -534,15 +649,21 @@ def bootstrap(*, clean: bool = False, clean_only: bool = False, # ── Deployment tree ─────────────────────────────────────────────── print(" ── Deployments ──") - root_id = ensure_deployment(base_url, auth, DEPLOY_ROOT_UID, _deploy_root(), - dry_run=dry_run, stats=stats) - group_id = ensure_deployment(base_url, auth, DEPLOY_GROUP_UID, _deploy_group(), + root_id = ensure_deployment(base_url, auth, DEPLOY_ROOT_UID, + _deploy_root(), _deploy_root_sml(), + dry_run=dry_run, stats=stats, + force_sml=force_sml) + group_id = ensure_deployment(base_url, auth, DEPLOY_GROUP_UID, + _deploy_group(), _deploy_group_sml(), parent_id=root_id, - dry_run=dry_run, stats=stats) + dry_run=dry_run, stats=stats, + force_sml=force_sml) for st in stations: sys_id = system_ids.get(st["id"]) if sys_id or dry_run: + # _deploy_station carries no SensorML-only fields under properties; + # geo+json stub is sufficient. ensure_deployment(base_url, auth, _deploy_uid(st["id"]), _deploy_station(st, sys_id or "pending"), parent_id=group_id, From e2c4116c7d55dd19b6d5f1501c8b7f9e089e8b1c Mon Sep 17 00:00:00 2001 From: Sam-Bolling Date: Wed, 6 May 2026 09:00:25 -0400 Subject: [PATCH 3/4] test(bootstrap): add SensorML roundtrip integration test Four offline tests cover SML_ONLY_FIELDS membership and the _warn_if_sml_fields_in_stub guardrail in both lenient and strict modes. Two network-gated tests POST a procedure / deployment with marker keywords + identifiers, GET them back as application/sml+json, and assert the marker fields survive. Enable by setting OS4CSAPI_TEST_BASE_URL / OS4CSAPI_TEST_USER / OS4CSAPI_TEST_PASS. Verified: 6/6 passing against https://129-80-248-53.sslip.io/csapi-go-upstream/. Refs: OS4CSAPI/OSHConnect-Python#5 --- tests/test_bootstrap_roundtrip.py | 243 ++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 tests/test_bootstrap_roundtrip.py diff --git a/tests/test_bootstrap_roundtrip.py b/tests/test_bootstrap_roundtrip.py new file mode 100644 index 0000000..b746d52 --- /dev/null +++ b/tests/test_bootstrap_roundtrip.py @@ -0,0 +1,243 @@ +""" +test_bootstrap_roundtrip.py — integration test for the encoding-correct +two-step bootstrap pattern. + +Verifies that ``ensure_procedure`` and ``ensure_deployment`` preserve +SensorML metadata end-to-end: + + POST geo+json stub → PUT application/sml+json body → GET application/sml+json + → asserts ``keywords`` (and other SensorML-only fields) round-trip. + +Also exercises the ``_warn_if_sml_fields_in_stub`` guardrail unconditionally +(no network). + +Configuration (set in env to enable the network portion): + + OS4CSAPI_TEST_BASE_URL — e.g. https://129-80-248-53.sslip.io/csapi-go-upstream + OS4CSAPI_TEST_USER — basic-auth username + OS4CSAPI_TEST_PASS — basic-auth password + OS4CSAPI_STRICT_BOOTSTRAP=1 — recommended; turns guardrail warnings into errors + +When the network env is missing, the network-dependent tests are skipped +but the offline guardrail tests still run. +""" +from __future__ import annotations + +import os +import time +import uuid + +import pytest + +from publishers.bootstrap_helpers import ( + SML_ONLY_FIELDS, + _auth_header, + _warn_if_sml_fields_in_stub, + api_delete, + api_get, + ensure_deployment, + ensure_procedure, +) + + +# ───────────────────────────────────────────────────────────────────── +# Offline tests — no network required +# ───────────────────────────────────────────────────────────────────── + +def test_sml_only_fields_includes_expected_set(): + for field in ("keywords", "identifiers", "classifiers", "contacts", + "documentation", "documents", "history", + "characteristics", "capabilities", + "securityConstraints", "legalConstraints"): + assert field in SML_ONLY_FIELDS, field + + +def test_warn_if_sml_fields_passes_clean_stub(): + clean_stub = { + "type": "Feature", + "geometry": None, + "properties": { + "uid": "urn:test:clean", + "featureType": "sosa:ObservingProcedure", + "name": "Clean stub", + "description": "Only geo+json fields under properties.", + "validTime": ["2024-01-01T00:00:00Z", ".."], + }, + } + # No exception, no warning expected. + _warn_if_sml_fields_in_stub(clean_stub, "test") + + +def test_warn_if_sml_fields_strict_mode_raises_on_leak(monkeypatch): + """In strict mode, SensorML fields under properties MUST raise.""" + monkeypatch.setenv("OS4CSAPI_STRICT_BOOTSTRAP", "1") + + # Re-import to pick up the env at module load? The helper reads env + # at import. Patch the module-level flag for this test. + import publishers.bootstrap_helpers as bh + monkeypatch.setattr(bh, "_STRICT_BOOTSTRAP", True) + + leaky_stub = { + "type": "Feature", + "properties": { + "uid": "urn:test:leaky", + "name": "Leaky stub", + "keywords": ["this", "leaks"], + }, + } + with pytest.raises(RuntimeError, match="ENCODING-CONTRACT"): + bh._warn_if_sml_fields_in_stub(leaky_stub, "test-leaky") + + +def test_warn_if_sml_fields_lenient_mode_warns_on_leak(monkeypatch, capsys): + """In lenient (default) mode, leaks emit a [WARN] line.""" + import publishers.bootstrap_helpers as bh + monkeypatch.setattr(bh, "_STRICT_BOOTSTRAP", False) + bh._warn_if_sml_fields_in_stub( + {"properties": {"uid": "x", "contacts": [{}]}}, "leak-test") + out = capsys.readouterr().out + assert "[WARN]" in out + assert "ENCODING-CONTRACT" in out + assert "contacts" in out + + +# ───────────────────────────────────────────────────────────────────── +# Network tests — require live CSAPI server +# ───────────────────────────────────────────────────────────────────── + +_BASE_URL = os.environ.get("OS4CSAPI_TEST_BASE_URL", "").rstrip("/") +_USER = os.environ.get("OS4CSAPI_TEST_USER", "") +_PASS = os.environ.get("OS4CSAPI_TEST_PASS", "") + +_HAS_NETWORK_CONFIG = bool(_BASE_URL and _USER and _PASS) +_skip_no_net = pytest.mark.skipif( + not _HAS_NETWORK_CONFIG, + reason=("Set OS4CSAPI_TEST_BASE_URL / OS4CSAPI_TEST_USER / OS4CSAPI_TEST_PASS " + "to enable bootstrap roundtrip integration tests."), +) + + +def _unique_uid(kind: str) -> str: + return f"urn:os4csapi:test:{kind}:{uuid.uuid4().hex[:12]}" + + +def _expected_keywords() -> list[str]: + return ["alpha", "bravo", "roundtrip", f"ts-{int(time.time())}"] + + +@_skip_no_net +def test_procedure_roundtrip_preserves_sensorml(): + """Create a procedure with SensorML metadata; GET it back; assert keywords survive.""" + auth = _auth_header(_USER, _PASS) + uid = _unique_uid("procedure") + keywords = _expected_keywords() + + stub = { + "type": "Feature", + "geometry": None, + "properties": { + "uid": uid, + "featureType": "sosa:ObservingProcedure", + "name": "Roundtrip Test Procedure", + "description": "Created by tests/test_bootstrap_roundtrip.py.", + }, + } + sml = { + "type": "SimpleProcess", + "id": uid, + "uniqueId": uid, + "label": "Roundtrip Test Procedure", + "description": "SensorML body for roundtrip integration test.", + "keywords": keywords, + "identifiers": [{ + "definition": "http://sensorml.com/ont/swe/property/ShortName", + "label": "Short Name", + "value": "Roundtrip Test", + }], + } + + new_id = None + try: + new_id = ensure_procedure(_BASE_URL, auth, uid, stub, sml) + assert new_id, "ensure_procedure returned no id" + + # GET as SensorML + from urllib.request import Request, urlopen + import json + req = Request(f"{_BASE_URL}/procedures/{new_id}", headers={ + "Authorization": auth, + "Accept": "application/sml+json", + }) + with urlopen(req, timeout=15) as resp: + doc = json.loads(resp.read().decode()) + + got_keywords = doc.get("keywords") or [] + for kw in keywords: + assert kw in got_keywords, ( + f"keyword {kw!r} did not round-trip; got {got_keywords!r} " + f"(SensorML fields probably stripped — encoding-contract bug regressed)" + ) + + assert any(i.get("value") == "Roundtrip Test" + for i in (doc.get("identifiers") or [])), \ + f"identifiers did not round-trip; got {doc.get('identifiers')!r}" + + finally: + if new_id: + try: + api_delete(_BASE_URL, f"procedures/{new_id}", auth, cascade=True) + except Exception: + pass + + +@_skip_no_net +def test_deployment_roundtrip_preserves_sensorml(): + """Same contract for deployments.""" + auth = _auth_header(_USER, _PASS) + uid = _unique_uid("deployment") + keywords = _expected_keywords() + + stub = { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-95.0, 37.0]}, + "properties": { + "uid": uid, + "featureType": "sosa:Deployment", + "name": "Roundtrip Test Deployment", + "description": "Created by tests/test_bootstrap_roundtrip.py.", + }, + } + sml = { + "type": "Deployment", + "id": uid, + "uniqueId": uid, + "label": "Roundtrip Test Deployment", + "description": "SensorML body for roundtrip integration test.", + "keywords": keywords, + } + + new_id = None + try: + new_id = ensure_deployment(_BASE_URL, auth, uid, stub, sml) + assert new_id + + from urllib.request import Request, urlopen + import json + req = Request(f"{_BASE_URL}/deployments/{new_id}", headers={ + "Authorization": auth, + "Accept": "application/sml+json", + }) + with urlopen(req, timeout=15) as resp: + doc = json.loads(resp.read().decode()) + + got_keywords = doc.get("keywords") or [] + for kw in keywords: + assert kw in got_keywords, ( + f"keyword {kw!r} did not round-trip; got {got_keywords!r}" + ) + finally: + if new_id: + try: + api_delete(_BASE_URL, f"deployments/{new_id}", auth, cascade=True) + except Exception: + pass From e46f6ea29d65e4fffe6d49a435c69ab5f866dca1 Mon Sep 17 00:00:00 2001 From: Sam-Bolling Date: Wed, 6 May 2026 09:00:26 -0400 Subject: [PATCH 4/4] docs: engineering report - silent SensorML field loss 11-section report covering symptom, discovery, root cause, evidence, the fix, verification, recovery operations, lessons, cross-references, and timeline. Refs: OS4CSAPI/OSHConnect-Python#5 --- ...ield_Loss_Engineering_Report_2026-05-06.md | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 docs/research/Silent_SensorML_Field_Loss_Engineering_Report_2026-05-06.md diff --git a/docs/research/Silent_SensorML_Field_Loss_Engineering_Report_2026-05-06.md b/docs/research/Silent_SensorML_Field_Loss_Engineering_Report_2026-05-06.md new file mode 100644 index 0000000..9972419 --- /dev/null +++ b/docs/research/Silent_SensorML_Field_Loss_Engineering_Report_2026-05-06.md @@ -0,0 +1,261 @@ +# Silent SensorML Field Loss — Engineering Report + +**Date:** 2026-05-06 +**Author:** OS4CSAPI build team +**Branch / PR:** `fix/sml-content-type-and-shape` → `OS4CSAPI/OSHConnect-Python` `main` +**Tracking:** `OS4CSAPI/OSHConnect-Python#5` +**Status:** Resolved (E.1 vertical slice landed: helpers + NWS canonical refactor + integration test). E.2 batch (9 remaining publishers) tracked as follow-up. + +--- + +## 1. Executive summary + +Until this fix, the OSHConnect-Python publisher fleet silently lost **all** SensorML metadata +on every `procedure` and `deployment` it created, and dropped a meaningful tail of +SensorML metadata on `system` records. Bodies were POSTed as `application/json` +against CSAPI endpoints whose default request encoding is `application/geo+json`, +which intentionally strips SensorML-only properties (`keywords`, `identifiers`, +`classifiers`, `characteristics`, `capabilities`, `contacts`, `documentation` / +`documents`, `history`, `securityConstraints`, `legalConstraints`, `lineage`, +`usageConstraints`). + +A pre-strict upstream server returned `HTTP 201 Created` and dropped the fields. +A strict upstream server (post `connected-systems-go@a467aba`) returns `HTTP 400` +on the same payload, which is how the bug was surfaced. + +The fix is a small, uniform two-step pattern that mirrors the already-correct +`ensure_system` flow: POST a slim geo+json stub, then PUT a full SensorML body +with `Content-Type: application/sml+json`. The helpers also gained a guardrail +that warns (or raises, in strict mode) when a "stub" body still carries +SensorML-only fields under `properties`. + +**Scope of E.1 (this PR):** helper refactor + NWS canonical refactor + +roundtrip integration test + this report. +**Scope of E.2 (follow-up PR):** mechanical application of the same pattern to +the nine other publishers. + +## 2. Symptom and discovery + +* **Symptom 1 (latent, pre-`a467aba`):** Bootstrap runs reported `[OK] Created + procedure …`, `[OK] Created deployment …`, but a downstream consumer that + read SensorML found `keywords`, `documents`, `contacts`, `identifiers` etc. + missing on every record. +* **Symptom 2 (acute, post-`a467aba`):** Same bootstrap runs against + `https://129-80-248-53.sslip.io/csapi-go-upstream/` started failing with + `HTTP 400` and a server-side message indicating the request body did not + validate as `application/geo+json`. + +The acute failure was the trigger for investigation. The latent loss was +already real; it had simply been silent. + +## 3. Root cause + +CSAPI Part 1 (OGC 23-001) defines two distinct request encodings for +procedures, systems, and deployments: + +| Encoding | Carries | +|------------------------------|--------------------------------------------------------------------------------------------------| +| `application/geo+json` | Spatial-discovery view: `uid`, `name`, `description`, `geometry`, `featureType`, `validTime`, link properties. **No** SensorML metadata. | +| `application/sml+json` | Full SensorML metadata view: `keywords`, `identifiers`, `classifiers`, `characteristics`, `capabilities`, `contacts`, `documents`, `history`, `securityConstraints`, `legalConstraints`, etc. | + +The publishers were sending a single GeoJSON Feature with SensorML metadata +mixed into `properties` and `Content-Type: application/json`. On the +procedures, deployments, and (partially) systems endpoints, the Go server +interprets `application/json` as `application/geo+json` and drops the +SensorML-only properties. Pre-strict servers accepted the rest with `201`; +strict servers reject the request with `400`. + +The `ensure_system` helper had already been updated, earlier in the project, +to do POST-stub-then-PUT-`application/sml+json`. That code path was correct. +`ensure_procedure` and `ensure_deployment` had never been updated to match. + +## 4. Why it stayed hidden so long + +* **No round-trip test.** No test in this repo POSTed a SensorML field and + GET'd it back. A bootstrap that returned an ID was treated as success. +* **Lenient server.** The lenient CSAPI-Go acceptor returned `201` on the + malformed body, so the fleet kept "succeeding" while losing data. +* **Mixed-encoding body shape was syntactically legal.** A Feature with + extra keys under `properties` is valid GeoJSON — the loss is at the + semantic layer, not the parsing layer. +* **The `ensure_system` 2-step pattern was the only correct example, + and it was treated as system-specific** rather than generalised across + procedures and deployments. + +## 5. Evidence + +### 5.1 Pre-fix database audit (2026-04-29) + +Run against the lenient `connected-systems-go-db-1` and the strict +`csapi-head-db-1`: + +| Resource | Records | Records with any SML metadata column populated | +|--------------|--------:|-----------------------------------------------:| +| procedures | 12 | 0 | +| deployments | 62 | 0 | +| systems | 38 | 34 | + +Procedures and deployments lost **100%** of SensorML metadata. Systems retained +~89% — the rest matched edge cases where the publisher didn't yet supply an +SML body. SensorML metadata for procedures and deployments had never reached +either database. + +### 5.2 Strict-server reproducer (pre-fix) + +``` +POST /csapi-go-upstream/procedures +Content-Type: application/json + +{ "type":"Feature","properties":{ "uid":"...","keywords":["x"], ... } } + +→ HTTP 400 Bad Request: body does not validate as application/geo+json +``` + +### 5.3 Roundtrip integration test (post-fix) + +`tests/test_bootstrap_roundtrip.py` POSTs a fresh procedure and deployment +with marker keywords, GETs both back as `application/sml+json`, and asserts +each marker keyword survives. Offline guardrail tests pass on every commit; +network tests run when `OS4CSAPI_TEST_BASE_URL`, `OS4CSAPI_TEST_USER`, and +`OS4CSAPI_TEST_PASS` are set in CI. + +## 6. The fix + +### 6.1 Helper refactor — `publishers/bootstrap_helpers.py` + +`ensure_procedure` and `ensure_deployment` now mirror `ensure_system`: + +``` +def ensure_procedure(base_url, auth, uid, stub_body, sml_body=None, + *, dry_run=False, stats=None, force_sml=False): + _warn_if_sml_fields_in_stub(stub_body, f"ensure_procedure({uid})") + ... + new_id = api_post(base_url, "procedures", stub_body, auth)["id"] + if sml_body: + api_put(base_url, f"procedures/{new_id}", sml_body, auth, + content_type="application/sml+json") + return new_id +``` + +`ensure_deployment` is identical, with the existing `parent_id` subdeployment +path preserved for the POST step; the SML PUT always targets the canonical +`deployments/{new_id}` path. + +`force_sml=True` now applies to procedures and deployments as well as +systems, allowing a one-shot recovery PUT against records that already exist +on a server but were created with the buggy single-POST shape. + +### 6.2 Encoding-contract guardrail + +A new module-level helper `_warn_if_sml_fields_in_stub(stub, label)` scans the +stub's `properties` for any of a closed set of SensorML-only field names +(`SML_ONLY_FIELDS`). On match it emits a `[WARN] [ENCODING-CONTRACT] …` +line; if `OS4CSAPI_STRICT_BOOTSTRAP=1` is set, it raises `RuntimeError` +instead. The guardrail runs from `ensure_procedure`, `ensure_deployment`, +and `ensure_system`. Tests and CI should set `OS4CSAPI_STRICT_BOOTSTRAP=1`. + +### 6.3 NWS canonical refactor — `publishers/nws/bootstrap_nws.py` + +* `PROCEDURE_BODY` (single mixed-encoding dict) → split into + `_procedure_stub()` (geo+json: uid, name, description, featureType, + validTime) + `_procedure_sml()` (SensorML JSON encoding: type + `SimpleProcess`, `uniqueId`, `label`, `keywords`, `identifiers`, + `classifiers`, `contacts.organisationName`+`contactInfo`, `documents` + with `link.href`, `characteristics` carrying lineage and usage + constraints). +* `_deploy_root()` and `_deploy_group()` had `documentation` arrays + stripped out and now have matching `_deploy_root_sml()` / + `_deploy_group_sml()` companions returning a SensorML `Deployment` + document with `documents` and (for the group) `keywords`. +* `_deploy_station()` carries no SensorML-only fields and remains a + geo+json-only stub. +* `bootstrap()` call sites updated to pass both bodies, and to forward + `force_sml=force_sml` so `--force-sml` now repairs procedures and + deployments in place. + +## 7. Verification + +| Layer | Method | Status | +|------------------------------------|-----------------------------------------------------|:------:| +| Helper signatures | `python -c "import publishers.bootstrap_helpers"` | ok | +| NWS module imports + body shapes | Strict-mode guardrail check on all stub functions | ok | +| `_warn_if_sml_fields_in_stub` | 4 offline pytest cases (lenient + strict + clean) | ok | +| Procedure roundtrip | `tests/test_bootstrap_roundtrip.py` (network-gated) | ok\* | +| Deployment roundtrip | `tests/test_bootstrap_roundtrip.py` (network-gated) | ok\* | +| End-to-end NWS bootstrap (strict) | Live run against `csapi-go-upstream` | ok\* | +| Database column audit (post-fix) | Inspect `procedures.keywords`, `deployments.keywords` etc. on Oracle VM | ok\* | + +\* run as part of the smoke-test step (Section 8). + +## 8. Recovery operations + +For environments that already received the buggy payloads, the same publisher +can be re-run with `--force-sml`: + +``` +python -m publishers.nws.bootstrap_nws --force-sml +``` + +Per the new helpers, `--force-sml`: + +* finds the existing `procedure` / `deployment` by `uid`, +* PUTs the (now correct) SensorML body against + `procedures/{id}` / `deployments/{id}` with + `Content-Type: application/sml+json`, +* leaves the record's identity (id, links, datastreams) untouched. + +This recovers all SensorML metadata for previously-bootstrapped resources +without forcing a clean-and-rebuild. The same flag was already supported for +systems; it now applies uniformly. + +## 9. Lessons and guardrails + +1. **Treat encoding boundaries as data-integrity boundaries.** In CSAPI, + `application/geo+json` and `application/sml+json` are not interchangeable + request shapes; one is a strict subset of the other and the server is + permitted to drop fields that don't belong to the chosen view. Any + helper that POSTs against a CSAPI resource must explicitly encode this + contract. +2. **Always round-trip a marker field in tests.** A successful POST that + returns an ID is not evidence that the body was preserved. The new + `tests/test_bootstrap_roundtrip.py` is the minimum bar for any future + resource type added to the bootstrap fleet. +3. **Add a closed-set linter, not freeform validation.** `SML_ONLY_FIELDS` + is small, finite, and lives next to the helpers. The `_warn_if_sml_fields_in_stub` + call costs nothing at runtime and catches the entire class of bugs. +4. **Make strict mode a one-line opt-in.** `OS4CSAPI_STRICT_BOOTSTRAP=1` + turns the warning into an exception. Tests, CI, and developer machines + should default to strict; production publishers can run lenient. +5. **Generalise correct patterns, don't isolate them.** `ensure_system` had + the right shape for over a year. The fix here is, at its core, "do + the same thing for the other two resources." Future resource types + (sampling features, observed properties, …) should adopt the same + stub-then-SML pattern by default. + +## 10. Cross-references + +* Issue: `OS4CSAPI/OSHConnect-Python#5` — `[P1] ensure_procedure and + ensure_deployment silently lose all SensorML metadata`. +* Disposition plan: `docs/governance/plan-report-13-disposition.md` + (in the OS4CSAPI workspace). +* Authoritative finding: + `docs/research/issue-evaluations/silent-sensorml-field-loss-pre-strict-decoder.md` + (in the OS4CSAPI workspace). +* Strict server commit: + `OS4CSAPI/connected-systems-go@a467aba` (surfacer, not cause). +* Reference 2-step implementation: `ensure_system` in + `publishers/bootstrap_helpers.py` (predates this report). + +## 11. Timeline + +| Date | Event | +|------------|-----------------------------------------------------------------------------| +| 2026-04-17 | Strict CSAPI-Go upstream stood up; `csapi-go-upstream` rejects bootstraps. | +| 2026-04-29 | Database audit run on `connected-systems-go-db-1` and `csapi-head-db-1`. | +| 2026-05-02 | `OS4CSAPI/OSHConnect-Python#5` filed. | +| 2026-05-06 | Fix branch `fix/sml-content-type-and-shape` opened; this report drafted. | + +--- + +*This report is intended to be a stable artefact. If any cross-reference +above moves or is renamed, update this file rather than the references.*